stateful db for chaos tests (#1303)

* stateful docker containers, static port ranges, reload config example

.github/workflows/framework-golden-tests.yml

@@ -1,9 +1,6 @@
 name: Framework Golden Tests Examples
 on:
   push:
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-framework-golden-examples
-  cancel-in-progress: true
 
 jobs:
   test:
@@ -18,6 +15,7 @@ jobs:
       id-token: write
       contents: read
     strategy:
+      fail-fast: false
       matrix:
         test:
           - name: TestSmoke
@@ -32,6 +30,10 @@ jobs:
             config: chaos.toml
             count: 1
             timeout: 10m
+          - name: TestReload
+            config: reload.toml
+            count: 1
+            timeout: 10m
     steps:
       - name: Checkout repo
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
@@ -62,9 +64,10 @@ jobs:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: go-modules-${{ hashFiles('**/go.sum') }}-${{ runner.os }}-framework-golden-examples
+          key: go-modules-${{ hashFiles('framework/examples/myproject/go.sum') }}-${{ runner.os }}-framework-golden-examples
           restore-keys: |
-            go-modules-${{ hashFiles('**/go.sum') }}-${{ runner.os }}-framework-golden-examples
+            go-modules-${{ runner.os }}-framework-golden-examples
+            go-modules-${{ runner.os }}
       - name: Install dependencies
         run: go mod download
       - name: Run Docker Component Tests

book/src/SUMMARY.md

@@ -14,8 +14,10 @@
     - [CLI](./framework/cli.md)
     - [Configuration](./framework/configuration.md)
     - [Test Configuration](./framework/test_configuration_overrides.md)
-    - [Caching](framework/components/caching.md)
+    - [Components Persistence](framework/components/state.md)
+    - [Components Caching](framework/components/caching.md)
     - [Secrets]()
+    - [Docker](framework/docker.md)
     - [Observability Stack](framework/observability/observability_stack.md)
       - [Metrics](framework/observability/metrics.md)
       - [Logs](framework/observability/logs.md)

book/src/ci/ci.md

@@ -2,31 +2,65 @@
 
 Here we describe our good practices for structuring different types of tests in Continuous Integration (GitHub Actions).
 
-The simplest flow can look like:
-
 Set up secrets in your GitHub repository
 ```
 gh secret set CTF_SIMULATED_KEY_1 --body "..."
 ```
 
 Add a workflow
 ```yaml
-name: Run E2E tests
-
+name: Framework Golden Tests Examples
 on:
   push:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: framework/examples/myproject
     env:
-      CTF_CONFIGS: smoke.toml
-      CTF_LOG_LEVEL: info
-      CTF_LOKI_STREAM: "false"
-      PRIVATE_KEY: ${{ secrets.CTF_SIMULATED_KEY_1 }}
+      LOKI_TENANT_ID: promtail
+      LOKI_URL: http://localhost:3030/loki/api/v1/push
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          - name: TestSmoke
+            config: smoke.toml
+            count: 1
+            timeout: 10m
+          - name: TestLoad
+            config: load.toml
+            count: 1
+            timeout: 10m
+          - name: TestChaos
+            config: chaos.toml
+            count: 1
+            timeout: 10m
     steps:
-      - name: Check out code
-        uses: actions/checkout@v3
+      - name: Checkout repo
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      - name: Configure AWS credentials using OIDC
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
+        with:
+          role-to-assume: ${{ secrets.PUBLIC_AWS_ECR_ROLE }}
+          aws-region: us-east-1
+      - name: Authenticate to ECR Public
+        id: login-ecr-public
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
+        with:
+          registry-type: public
+      - name: Check for changes in Framework
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        id: changes
+        with:
+          filters: |
+            src:
+              - 'framework/**'
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
@@ -37,14 +71,18 @@ jobs:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: go-modules-${{ hashFiles('**/go.sum') }}-${{ runner.os }}
+          key: go-modules-${{ hashFiles('framework/examples/myproject/go.sum') }}-${{ runner.os }}-framework-golden-examples
           restore-keys: |
-            go-modules-${{ hashFiles('**/go.sum') }}-${{ runner.os }}
+            go-modules-${{ runner.os }}-framework-golden-examples
+            go-modules-${{ runner.os }}
       - name: Install dependencies
         run: go mod download
-      - name: Run tests
-        working-directory: e2e/capabilities
-        run: go test -v -run TestDON
+      - name: Run Docker Component Tests
+        if: steps.changes.outputs.src == 'true'
+        env:
+          CTF_CONFIGS: ${{ matrix.test.config }}
+        run: |
+          go test -timeout ${{ matrix.test.timeout }} -v -count ${{ matrix.test.count }} -run ${{ matrix.test.name }}
 ```
 
 If you need to structure a lot of different end-to-end tests follow [this](https://github.com/smartcontractkit/.github/tree/main/.github/workflows) guide.

book/src/framework/components/caching.md

@@ -1,6 +1,6 @@
 ## Component caching
 
-We use component caching to accelerate test development and enforce idempotent test actions.
+We use component caching to accelerate test development and enforce idempotent test actions development.
 
 Each component is isolated by means of inputs and outputs.
 

book/src/framework/components/chainlink/nodeset.md

@@ -48,6 +48,11 @@ Then configure NodeSet
   # defines how we override configs, either we apply first node fields to all of them
   # or we define each node custom configuration (used in compatibility testing)
   override_mode = "all"
+  # HTTP API port range start, each new node get port incremented (host machine)
+  http_port_range_start = 10000
+  # P2P API port range start, each new node get port incremented (host machine)
+  p2p_port_range_start = 12000
+
 
   [[nodeset.node_specs]]
     # Optional URL for fake data provider URL
@@ -59,6 +64,8 @@ Then configure NodeSet
       image = "postgres:15.6"
       # Pulls the image every time if set to 'true', used like that in CI. Can be set to 'false' to speed up local runs
       pull_image = true
+      # PostgreSQL volume name
+      volume_name = ""
 
     [nodeset.node_specs.node]
       # A list of paths to capability binaries

book/src/framework/components/databases/pg.md

@@ -0,0 +1 @@
+# PostgreSQL

book/src/framework/components/state.md

@@ -0,0 +1,37 @@
+# Components Persistence
+
+We use static port ranges and volumes for all components to simplify Docker port management for developers.
+
+This approach allows us to apply chaos testing to any container, ensuring it reconnects and retains the data needed for your tests.
+
+When deploying a component, you can explicitly configure port ranges if the default ports don’t meet your needs.
+
+Defaults are:
+- [NodeSet](../components/chainlink/nodeset.md) (Node HTTP API): `10000..100XX`
+- [NodeSet](../components/chainlink/nodeset.md) (Node P2P API): `12000..120XX`
+```
+[nodeset]
+  # HTTP API port range start, each new node get port incremented (host machine)
+  http_port_range_start = 10000
+  # P2P API port range start, each new node get port incremented (host machine)
+  p2p_port_range_start = 12000
+```
+- [PostgreSQL](../components/chainlink/nodeset.md): `13000` (we do not allow to have multiple databases for now, for simplicity)
+```
+    [nodeset.node_specs.db]
+      # PostgreSQL volume name
+      volume_name = "a"
+      # PostgreSQL port (host machine)
+      port = 13000
+```
+
+When you run `ctf d rm` database volume will be **removed**.
+
+<div class="warning">
+
+One node set is enough for any kind of testing, if you need more nodes consider extending your existing node set:
+```
+[nodeset]
+  nodes = 10
+```
+</div>

book/src/framework/docker.md

@@ -0,0 +1,7 @@
+# Docker
+
+We are not removing volumes and images when you are working locally to allow you to debug, however, to clean up some space use:
+```
+docker volume prune -f
+docker system prune -f
+```

book/src/framework/first_test.md

@@ -55,6 +55,6 @@ Summary:
 - We defined configuration for `BlockchainNetwork`
 - We've used one CTF component in test and checked if it's working
 
-You can learn more about [component design](./components/overview.md) or proceed with another example of [connecting Chainlink node](./connecting_chainlink_node.md)
+Now let's connect the [Chainlink](./connecting_chainlink_node.md) node!
 
 Learn more about [anvil](./components/blockchains/anvil.md) component.

book/src/framework/nodeset_environment.md

@@ -72,13 +72,9 @@ Run it
 CTF_CONFIGS=smoke.toml go test -v -run TestNodeSet
 ```
 
-You'll see something like, use any URL to access CL node
+Check the logs to access the UI
 ```bash
-6:14PM INF Chainlink node url URL=http://127.0.0.1:34041
-6:14PM INF Chainlink node url URL=http://127.0.0.1:34045
-6:14PM INF Chainlink node url URL=http://127.0.0.1:34044
-6:14PM INF Chainlink node url URL=http://127.0.0.1:34042
-6:14PM INF Chainlink node url URL=http://127.0.0.1:34043
+12:41AM INF UI=["http://127.0.0.1:10000","http://127.0.0.1:10001", ...]
 ```
 
 Use credentials to authorize:

framework/.changeset/v0.1.11.md

@@ -0,0 +1,3 @@
+- Static port ranges for easy chaos testing
+- SharedDBNodeSet config update method
+- Example of rebooting nodes to apply configuration

framework/chaos/chaos.go

@@ -9,9 +9,13 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/testcontainers/testcontainers-go"
 	"strings"
+	"time"
 )
 
-func ExecPumba(command string) (func(), error) {
+// ExecPumba executes Pumba (https://github.com/alexei-led/pumba) command
+// since handling various docker race conditions is hard and there is no easy API for that
+// for now you can provide time to wait until chaos is applied
+func ExecPumba(command string, wait time.Duration) (func(), error) {
 	ctx := context.Background()
 	cmd := strings.Split(command, " ")
 	pumbaReq := testcontainers.ContainerRequest{
@@ -38,7 +42,9 @@ func ExecPumba(command string) (func(), error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to start pumba chaos container: %w", err)
 	}
-	framework.L.Info().Msg("Pumba chaos started")
+	framework.L.Info().Str("Cmd", command).Msg("Pumba chaos has started")
+	time.Sleep(wait)
+	framework.L.Info().Msg("Pumba chaos has finished")
 	return func() {
 		_ = pumbaContainer.Terminate(ctx)
 	}, nil

framework/cmd/blockscout.go

@@ -18,7 +18,6 @@ func blockscoutUp() error {
 	if err != nil {
 		return err
 	}
-	framework.L.Info().Msg("Done")
 	fmt.Println()
 	framework.L.Info().Msgf("Blockscout is up at: %s", "http://localhost")
 	return nil
@@ -40,6 +39,5 @@ func blockscoutDown() error {
 		rm -rf redis-data && \
 		rm -rf stats-db-data
 	`, filepath.Join("blockscout", "services")))
-	framework.L.Info().Msg("Done")
 	return nil
 }

framework/cmd/docker.go

@@ -14,7 +14,8 @@ func cleanDockerResources() error {
 	// Bash command for removing Docker containers and networks with "framework=ctf" label
 	cmd := exec.Command("bash", "-c", `
 		docker ps -aq --filter "label=framework=ctf" | xargs -r docker rm -f && \
-		docker network ls --filter "label=framework=ctf" -q | xargs -r docker network rm
+		docker network ls --filter "label=framework=ctf" -q | xargs -r docker network rm && \
+		docker volume rm postgresql_data
 	`)
 	framework.L.Debug().Msg("Running command")
 	if framework.L.GetLevel() == zerolog.DebugLevel {
@@ -24,7 +25,6 @@ func cleanDockerResources() error {
 	if err != nil {
 		return fmt.Errorf("error running clean command: %s", string(output))
 	}
-	framework.L.Info().Msgf("Done")
 	return nil
 }
 

framework/cmd/interactive.go

@@ -131,14 +131,14 @@ Docker Desktop (https://www.docker.com/products/docker-desktop/)
 				Options(
 					huh.NewOption("Anvil", "anvil"),
 				).
-				Value(&f.Network), // stores the selected network
+				Value(&f.Network),
 
 			huh.NewSelect[int]().
 				Title("How many nodes do you need?").
 				Options(
 					huh.NewOption("5", 5),
 				).
-				Value(&f.Nodes), // stores the selected number of nodes
+				Value(&f.Nodes),
 
 			huh.NewSelect[string]().
 				Title("Choose Chainlink node version").
@@ -147,11 +147,11 @@ Docker Desktop (https://www.docker.com/products/docker-desktop/)
 				Value(&f.CLVersion),
 			huh.NewConfirm().
 				Title("Do you need to spin up an observability stack?").
-				Value(&f.Observability), // stores the observability option
+				Value(&f.Observability),
 
 			huh.NewConfirm().
 				Title("Do you need to spin up a Blockscout stack?").
-				Value(&f.Blockscout), // stores the Blockscout option
+				Value(&f.Blockscout),
 		),
 	)
 
@@ -167,13 +167,12 @@ Docker Desktop (https://www.docker.com/products/docker-desktop/)
 	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
 	go func() {
 		<-sigs
-		fmt.Println("\nReceived Ctrl+C, starting custom cleanup...")
 		err := cleanup(f)
 		if err != nil {
 			panic(err)
 		}
 		os.Exit(0)
 	}()
-	framework.L.Info().Msg("Press Ctrl+C to remove the stack..")
+	framework.L.Info().Msg("Services are up! Press Ctrl+C to remove them..")
 	select {}
 }

framework/cmd/observability.go

@@ -17,11 +17,8 @@ func observabilityUp() error {
 	if err != nil {
 		return err
 	}
-	framework.L.Info().Msg("Done")
 	fmt.Println()
 	framework.L.Info().Msgf("Loki: %s", LocalLogsURL)
-	framework.L.Info().Msgf("All logs: %s", "{job=\"ctf\"}")
-	framework.L.Info().Msgf("By log level: %s", "{job=\"ctf\", container=~\"node.*\"} |= \"WARN|INFO|DEBUG\"")
 	framework.L.Info().Msgf("Pyroscope: %s", LocalPyroScopeURL)
 	return nil
 }
@@ -35,6 +32,5 @@ func observabilityDown() error {
 	if err != nil {
 		return err
 	}
-	framework.L.Info().Msg("Done")
 	return nil
 }