diff --git a/.github/workflows/framework-golden-tests.yml b/.github/workflows/framework-golden-tests.yml index 1ae744165..fe851cffd 100644 --- a/.github/workflows/framework-golden-tests.yml +++ b/.github/workflows/framework-golden-tests.yml @@ -22,6 +22,10 @@ jobs: config: smoke.toml count: 1 timeout: 10m + - name: TestSmoke + config: smoke_limited_resources.toml + count: 1 + timeout: 10m - name: TestSuiSmoke config: smoke_sui.toml count: 1 diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index e3380efad..b55eae482 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -22,6 +22,7 @@ - [Debugging Tests](framework/components/debug.md) - [Components Cleanup](framework/components/cleanup.md) - [Components Caching](framework/components/caching.md) + - [Components Resources](framework/components/resources.md) - [Mocking Services](framework/components/mocking.md) - [Copying Files](framework/copying_files.md) - [External Environment](framework/components/external.md) diff --git a/book/src/framework/components/resources.md b/book/src/framework/components/resources.md new file mode 100644 index 000000000..17d5e051f --- /dev/null +++ b/book/src/framework/components/resources.md @@ -0,0 +1,33 @@ +# Components Resources + +You can use `resources` to limit containers CPU/Memory for `NodeSet`, `Blockchain` and `PostgreSQL` components. + +```toml +[blockchain_a.resources] +cpus = 0.5 +memory_mb = 1048 + +[nodeset.db.resources] +cpus = 2 +memory_mb = 2048 + +[nodeset.node_specs.node.resources] +cpus = 1 +memory_mb = 1048 +``` + +Read more about resource constraints [here](https://docs.docker.com/engine/containers/resource_constraints/). + +We are using `cpu-period` and `cpu-quota` for simplicity, and because it's working with an arbitrary amount of containers, it is absolute. + +How quota and period works: + +- To allocate `1 CPU`, we set `CPUQuota = 100000` and `CPUPeriod = 100000` (1 full period). +- To allocate `0.5 CPU`, we set `CPUQuota = 50000` and `CPUPeriod = 100000`. +- To allocate `2 CPUs`, we set `CPUQuota = 200000` and `CPUPeriod = 100000`. + +Read more about [CFS](https://engineering.squarespace.com/blog/2017/understanding-linux-container-scheduling). + +When the `resources.memory_mb` key is not empty, we disable swap, ensuring the container goes OOM when memory is exhausted, allowing for more precise detection of sudden memory spikes. + +Full configuration [example]() diff --git a/framework/.changeset/v0.4.7.md b/framework/.changeset/v0.4.7.md new file mode 100644 index 000000000..a7502babb --- /dev/null +++ b/framework/.changeset/v0.4.7.md @@ -0,0 +1 @@ +- Set cgroups resource for containers diff --git a/framework/components/blockchain/anvil.go b/framework/components/blockchain/anvil.go index 4c341be25..da7b378b8 100644 --- a/framework/components/blockchain/anvil.go +++ b/framework/components/blockchain/anvil.go @@ -48,6 +48,7 @@ func newAnvil(in *Input) (*Output, error) { ExposedPorts: []string{bindPort}, HostConfigModifier: func(h *container.HostConfig) { h.PortBindings = framework.MapTheSamePort(bindPort) + framework.ResourceLimitsFunc(h, in.ContainerResources) }, Networks: []string{framework.DefaultNetworkName}, NetworkAliases: map[string][]string{ diff --git a/framework/components/blockchain/aptos.go b/framework/components/blockchain/aptos.go index 99bb20e9d..088fd6826 100644 --- a/framework/components/blockchain/aptos.go +++ b/framework/components/blockchain/aptos.go @@ -49,6 +49,7 @@ func newAptos(in *Input) (*Output, error) { }, HostConfigModifier: func(h *container.HostConfig) { h.PortBindings = framework.MapTheSamePort(bindPort) + framework.ResourceLimitsFunc(h, in.ContainerResources) }, ImagePlatform: "linux/amd64", Cmd: []string{ diff --git a/framework/components/blockchain/besu.go b/framework/components/blockchain/besu.go index 68824ba51..6e6f6b95c 100644 --- a/framework/components/blockchain/besu.go +++ b/framework/components/blockchain/besu.go @@ -68,6 +68,7 @@ func newBesu(in *Input) (*Output, error) { Labels: framework.DefaultTCLabels(), HostConfigModifier: func(h *container.HostConfig) { h.PortBindings = framework.MapTheSamePort(bindPortWs, bindPort) + framework.ResourceLimitsFunc(h, in.ContainerResources) }, WaitingFor: wait.ForListeningPort(nat.Port(in.Port)).WithStartupTimeout(15 * time.Second).WithPollInterval(200 * time.Millisecond), Cmd: entryPoint, diff --git a/framework/components/blockchain/blockchain.go b/framework/components/blockchain/blockchain.go index 87629d447..779de8b76 100644 --- a/framework/components/blockchain/blockchain.go +++ b/framework/components/blockchain/blockchain.go @@ -2,6 +2,7 @@ package blockchain import ( "fmt" + "github.com/smartcontractkit/chainlink-testing-framework/framework" "github.com/testcontainers/testcontainers-go" ) @@ -25,7 +26,8 @@ type Input struct { // programs to deploy on solana-test-validator start // a map of program name to program id // there needs to be a matching .so file in contracts_dir - SolanaPrograms map[string]string `toml:"solana_programs"` + SolanaPrograms map[string]string `toml:"solana_programs"` + ContainerResources *framework.ContainerResources `toml:"resources"` } // Output is a blockchain network output, ChainID and one or more nodes that forms the network diff --git a/framework/components/blockchain/geth.go b/framework/components/blockchain/geth.go index 74bd28d68..6709a9695 100644 --- a/framework/components/blockchain/geth.go +++ b/framework/components/blockchain/geth.go @@ -167,6 +167,7 @@ func newGeth(in *Input) (*Output, error) { ExposedPorts: []string{bindPort}, HostConfigModifier: func(h *container.HostConfig) { h.PortBindings = framework.MapTheSamePort(bindPort) + framework.ResourceLimitsFunc(h, in.ContainerResources) h.Mounts = append(h.Mounts, mount.Mount{ Type: mount.TypeBind, Source: keystoreDir, diff --git a/framework/components/blockchain/solana.go b/framework/components/blockchain/solana.go index 47780c1fa..6f244cce6 100644 --- a/framework/components/blockchain/solana.go +++ b/framework/components/blockchain/solana.go @@ -105,6 +105,7 @@ func newSolana(in *Input) (*Output, error) { WithPollInterval(100 * time.Millisecond), HostConfigModifier: func(h *container.HostConfig) { h.PortBindings = framework.MapTheSamePort(bindPort, wsBindPort) + framework.ResourceLimitsFunc(h, in.ContainerResources) h.Mounts = append(h.Mounts, mount.Mount{ Type: mount.TypeBind, Source: contractsDir, diff --git a/framework/components/blockchain/sui.go b/framework/components/blockchain/sui.go index 7054b3aae..8df2765ae 100644 --- a/framework/components/blockchain/sui.go +++ b/framework/components/blockchain/sui.go @@ -104,6 +104,7 @@ func newSui(in *Input) (*Output, error) { }, HostConfigModifier: func(h *container.HostConfig) { h.PortBindings = framework.MapTheSamePort(bindPort, DefaultFaucetPort) + framework.ResourceLimitsFunc(h, in.ContainerResources) }, ImagePlatform: "linux/amd64", Env: map[string]string{ diff --git a/framework/components/blockchain/tron.go b/framework/components/blockchain/tron.go index 0d3bce9ec..9373f9ecd 100644 --- a/framework/components/blockchain/tron.go +++ b/framework/components/blockchain/tron.go @@ -85,6 +85,7 @@ func newTron(in *Input) (*Output, error) { Labels: framework.DefaultTCLabels(), HostConfigModifier: func(h *container.HostConfig) { h.PortBindings = framework.MapTheSamePort(bindPort) + framework.ResourceLimitsFunc(h, in.ContainerResources) }, WaitingFor: wait.ForLog("Mnemonic").WithPollInterval(200 * time.Millisecond).WithStartupTimeout(1 * time.Minute), Files: []testcontainers.ContainerFile{ diff --git a/framework/components/clnode/clnode.go b/framework/components/clnode/clnode.go index a20708fdb..f955c6b82 100644 --- a/framework/components/clnode/clnode.go +++ b/framework/components/clnode/clnode.go @@ -42,21 +42,22 @@ type Input struct { // NodeInput is CL nod container inputs type NodeInput struct { - Image string `toml:"image" validate:"required"` - Name string `toml:"name"` - DockerFilePath string `toml:"docker_file"` - DockerContext string `toml:"docker_ctx"` - PullImage bool `toml:"pull_image"` - CapabilitiesBinaryPaths []string `toml:"capabilities"` - CapabilityContainerDir string `toml:"capabilities_container_dir"` - TestConfigOverrides string `toml:"test_config_overrides"` - UserConfigOverrides string `toml:"user_config_overrides"` - TestSecretsOverrides string `toml:"test_secrets_overrides"` - UserSecretsOverrides string `toml:"user_secrets_overrides"` - HTTPPort int `toml:"port"` - P2PPort int `toml:"p2p_port"` - CustomPorts []string `toml:"custom_ports"` - DebuggerPort int `toml:"debugger_port"` + Image string `toml:"image" validate:"required"` + Name string `toml:"name"` + DockerFilePath string `toml:"docker_file"` + DockerContext string `toml:"docker_ctx"` + PullImage bool `toml:"pull_image"` + CapabilitiesBinaryPaths []string `toml:"capabilities"` + CapabilityContainerDir string `toml:"capabilities_container_dir"` + TestConfigOverrides string `toml:"test_config_overrides"` + UserConfigOverrides string `toml:"user_config_overrides"` + TestSecretsOverrides string `toml:"test_secrets_overrides"` + UserSecretsOverrides string `toml:"user_secrets_overrides"` + HTTPPort int `toml:"port"` + P2PPort int `toml:"p2p_port"` + CustomPorts []string `toml:"custom_ports"` + DebuggerPort int `toml:"debugger_port"` + ContainerResources *framework.ContainerResources `toml:"resources"` } // Output represents Chainlink node output, nodes and databases connection URLs @@ -247,6 +248,7 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) { if in.Node.HTTPPort != 0 && in.Node.P2PPort != 0 { req.HostConfigModifier = func(h *container.HostConfig) { h.PortBindings = portBindings + framework.ResourceLimitsFunc(h, in.Node.ContainerResources) } } files := []tc.ContainerFile{ diff --git a/framework/components/postgres/postgres.go b/framework/components/postgres/postgres.go index befc30eec..1b3f71c15 100644 --- a/framework/components/postgres/postgres.go +++ b/framework/components/postgres/postgres.go @@ -24,14 +24,15 @@ const ( ) type Input struct { - Image string `toml:"image" validate:"required"` - Port int `toml:"port"` - Name string `toml:"name"` - VolumeName string `toml:"volume_name"` - Databases int `toml:"databases"` - JDDatabase bool `toml:"jd_database"` - PullImage bool `toml:"pull_image"` - Out *Output `toml:"out"` + Image string `toml:"image" validate:"required"` + Port int `toml:"port"` + Name string `toml:"name"` + VolumeName string `toml:"volume_name"` + Databases int `toml:"databases"` + JDDatabase bool `toml:"jd_database"` + PullImage bool `toml:"pull_image"` + ContainerResources *framework.ContainerResources `toml:"resources"` + Out *Output `toml:"out"` } type Output struct { @@ -133,6 +134,7 @@ func NewPostgreSQL(in *Input) (*Output, error) { }, }, } + framework.ResourceLimitsFunc(h, in.ContainerResources) } c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, diff --git a/framework/components/simple_node_set/node_set.go b/framework/components/simple_node_set/node_set.go index 7ee3ca866..1ab5867a3 100644 --- a/framework/components/simple_node_set/node_set.go +++ b/framework/components/simple_node_set/node_set.go @@ -147,6 +147,7 @@ func sharedDBSetup(in *Input, bcOut *blockchain.Output) (*Output, error) { UserConfigOverrides: in.NodeSpecs[overrideIdx].Node.UserConfigOverrides, TestSecretsOverrides: in.NodeSpecs[overrideIdx].Node.TestSecretsOverrides, UserSecretsOverrides: in.NodeSpecs[overrideIdx].Node.UserSecretsOverrides, + ContainerResources: in.NodeSpecs[overrideIdx].Node.ContainerResources, }, } diff --git a/framework/docker.go b/framework/docker.go index b8d7cd43f..b10be5a82 100644 --- a/framework/docker.go +++ b/framework/docker.go @@ -320,3 +320,28 @@ func ExecContainer(containerName string, command []string) (string, error) { L.Info().Str("Output", string(output)).Msg("Command output") return string(output), nil } + +type ContainerResources struct { + CPUs float64 `toml:"cpus" validate:"gte=0"` + MemoryMb uint `toml:"memory_mb"` +} + +// ResourceLimitsFunc returns a function to configure container resources based on the human-readable CPUs and memory in Mb +func ResourceLimitsFunc(h *container.HostConfig, resources *ContainerResources) { + if resources == nil { + return + } + if resources.MemoryMb > 0 { + h.Memory = int64(resources.MemoryMb) * 1024 * 1024 // Memory in Mb + h.MemoryReservation = int64(resources.MemoryMb) * 1024 * 1024 // Total memory that can be reserved (soft) in Mb + // https://docs.docker.com/engine/containers/resource_constraints/ if both values are equal swap is off, read the docs + h.MemorySwap = h.Memory + } + if resources.CPUs > 0 { + // Set CPU limits using CPUQuota and CPUPeriod + // we don't use runtime.NumCPU or docker API to get CPUs because h.CPUShares is relative to amount of containers you run + // CPUPeriod and CPUQuota are absolute and easier to control + h.CPUPeriod = 100000 // Default period (100ms) + h.CPUQuota = int64(resources.CPUs * 100000) // Quota in microseconds (e.g., 0.5 CPUs = 50000) + } +} diff --git a/framework/examples/myproject/smoke.toml b/framework/examples/myproject/smoke.toml index f73ac540c..7af8fc9ea 100644 --- a/framework/examples/myproject/smoke.toml +++ b/framework/examples/myproject/smoke.toml @@ -1,9 +1,7 @@ [blockchain_a] - # choose "anvil", "geth" or "besu" - # use docker_cmd_params = ["-b", "1"] for "anvil" - type = "anvil" docker_cmd_params = ["-b", "1"] + type = "anvil" [data_provider] port = 9111 @@ -18,4 +16,4 @@ [[nodeset.node_specs]] [nodeset.node_specs.node] - image = "public.ecr.aws/chainlink/chainlink:v2.17.0" + image = "public.ecr.aws/chainlink/chainlink:v2.17.0" \ No newline at end of file diff --git a/framework/examples/myproject/smoke_limited_resources.toml b/framework/examples/myproject/smoke_limited_resources.toml new file mode 100644 index 000000000..d8007ca09 --- /dev/null +++ b/framework/examples/myproject/smoke_limited_resources.toml @@ -0,0 +1,31 @@ + +[blockchain_a] + docker_cmd_params = ["-b", "1"] + type = "anvil" + + [blockchain_a.resources] + cpus = 1 + memory_mb = 1048 + +[data_provider] + port = 9111 + +[nodeset] + nodes = 5 + override_mode = "all" + + [nodeset.db] + image = "postgres:12.0" + + [nodeset.db.resources] + cpus = 1 + memory_mb = 1048 + + [[nodeset.node_specs]] + + [nodeset.node_specs.node] + image = "public.ecr.aws/chainlink/chainlink:v2.17.0" + + [nodeset.node_specs.node.resources] + cpus = 1 + memory_mb = 1048