Skip to content

Commit

Permalink
[TT-526] Allow Parallel Tests To Work (#10591)
Browse files Browse the repository at this point in the history 
* [TT-526] Allow Parallel Tests To Work
Use unique base config for each env

* bump node counts to run more tests in parallel

* Fix parallel logging issues
  • Loading branch information
@tateexon
tateexon authored Sep 12, 2023
1 parent aec8c75 commit e161477
Show file tree
Hide file tree
Showing 10 changed files with 110 additions and 170 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/integration-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,8 @@ jobs:
id: build-test-matrix-list
run: |
cd ./integration-tests
MATRIX_JSON_AUTOMATION=$(./scripts/buildTestMatrixList.sh ./smoke/automation_test.go automation ubuntu20.04-8cores-32GB)
MATRIX_JSON_KEEPER=$(./scripts/buildTestMatrixList.sh ./smoke/keeper_test.go keeper ubuntu20.04-8cores-32GB)
MATRIX_JSON_AUTOMATION=$(./scripts/buildTestMatrixList.sh ./smoke/automation_test.go automation ubuntu20.04-8cores-32GB 1)
MATRIX_JSON_KEEPER=$(./scripts/buildTestMatrixList.sh ./smoke/keeper_test.go keeper ubuntu20.04-8cores-32GB 1)
COMBINED_ARRAY=$(jq -c -n "$MATRIX_JSON_AUTOMATION + $MATRIX_JSON_KEEPER")
echo "MATRIX_JSON=${COMBINED_ARRAY}" >> $GITHUB_ENV
eth-smoke-tests-matrix-automation:
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/docker/test_env/test_env_builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ func (b *CLTestEnvBuilder) buildNewEnv(cfg *TestEnvConfig) (*CLClusterTestEnv, e
if b.clNodeConfig != nil {
cfg = b.clNodeConfig
} else {
cfg = node.NewConfig(node.BaseConf,
cfg = node.NewConfig(node.NewBaseConfig(),
node.WithOCR1(),
node.WithP2Pv1(),
)
Expand Down
12 changes: 9 additions & 3 deletions integration-tests/scripts/buildTestMatrixList.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ cd "$SCRIPT_DIR"/../ || exit 1
FILENAME=$1
MATRIX_JOB_NAME=$2
NODE_LABEL=$3
NODE_COUNT=$4

# Get list of test names from JSON file
JSONFILE="${FILENAME}_test_list.json"
Expand All @@ -25,13 +26,18 @@ matrix_output() {
local job_name=$2
local test_name=$3
local node_label=$4
local node_count=$5
local counter_out=$(printf "%02d\n" $counter)
echo -n "{\"name\": \"${job_name}-${counter_out}\", \"file\": \"${job_name}\",\"nodes\": 1, \"os\": \"${node_label}\", \"pyroscope_env\": \"ci-smoke-${job_name}-evm-simulated\", \"run\": \"-run '^${test_name}$'\"}"
echo -n "{\"name\": \"${job_name}-${counter_out}\", \"file\": \"${job_name}\",\"nodes\": ${node_count}, \"os\": \"${node_label}\", \"pyroscope_env\": \"ci-smoke-${job_name}-evm-simulated\", \"run\": \"-run '^${test_name}$'\"}"
}

# Read the JSON file and loop through 'tests' and 'run'
jq -c '.tests[]' ${JSONFILE} | while read -r test; do
testName=$(echo ${test} | jq -r '.name')
label=$(echo ${test} | jq -r '.label // empty')
effective_node_label=${label:-$NODE_LABEL}
node_count=$(echo ${test} | jq -r '.nodes // empty')
effective_node_count=${node_count:-$NODE_COUNT}
subTests=$(echo ${test} | jq -r '.run[]?.name // empty')
output=""

Expand All @@ -41,14 +47,14 @@ jq -c '.tests[]' ${JSONFILE} | while read -r test; do
if [ $COUNTER -ne 1 ]; then
echo -n ","
fi
matrix_output $COUNTER $MATRIX_JOB_NAME "${testName}/${subTest}" ${NODE_LABEL}
matrix_output $COUNTER $MATRIX_JOB_NAME "${testName}/${subTest}" ${effective_node_label} ${effective_node_count}
((COUNTER++))
done
else
if [ $COUNTER -ne 1 ]; then
echo -n ","
fi
matrix_output $COUNTER $MATRIX_JOB_NAME "${testName}" ${NODE_LABEL}
matrix_output $COUNTER $MATRIX_JOB_NAME "${testName}" ${effective_node_label} ${effective_node_count}
((COUNTER++))
fi

Expand Down
22 changes: 7 additions & 15 deletions integration-tests/smoke/automation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,6 @@ func TestAutomationBasic(t *testing.T) {

func SetupAutomationBasic(t *testing.T, nodeUpgrade bool) {
t.Parallel()
l := utils.GetTestLogger(t)

registryVersions := map[string]ethereum.KeeperRegistryVersion{
"registry_2_0": ethereum.RegistryVersion_2_0,
"registry_2_1_conditional": ethereum.RegistryVersion_2_1,
Expand All @@ -100,6 +98,7 @@ func SetupAutomationBasic(t *testing.T, nodeUpgrade bool) {
registryVersion := rv
t.Run(name, func(t *testing.T) {
t.Parallel()
l := utils.GetTestLogger(t)

var (
upgradeImage string
Expand Down Expand Up @@ -431,8 +430,6 @@ func TestAutomationAddFunds(t *testing.T) {

func TestAutomationPauseUnPause(t *testing.T) {
t.Parallel()
l := utils.GetTestLogger(t)

registryVersions := map[string]ethereum.KeeperRegistryVersion{
"registry_2_0": ethereum.RegistryVersion_2_0,
"registry_2_1": ethereum.RegistryVersion_2_1,
Expand All @@ -443,6 +440,7 @@ func TestAutomationPauseUnPause(t *testing.T) {
registryVersion := rv
t.Run(name, func(t *testing.T) {
t.Parallel()
l := utils.GetTestLogger(t)
chainClient, _, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker(
t, "pause-unpause", registryVersion, defaultOCRRegistryConfig, false,
)
Expand Down Expand Up @@ -515,8 +513,6 @@ func TestAutomationPauseUnPause(t *testing.T) {

func TestAutomationRegisterUpkeep(t *testing.T) {
t.Parallel()
l := utils.GetTestLogger(t)

registryVersions := map[string]ethereum.KeeperRegistryVersion{
"registry_2_0": ethereum.RegistryVersion_2_0,
"registry_2_1": ethereum.RegistryVersion_2_1,
Expand All @@ -527,6 +523,7 @@ func TestAutomationRegisterUpkeep(t *testing.T) {
registryVersion := rv
t.Run(name, func(t *testing.T) {
t.Parallel()
l := utils.GetTestLogger(t)
chainClient, _, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker(
t, "register-upkeep", registryVersion, defaultOCRRegistryConfig, false,
)
Expand Down Expand Up @@ -588,7 +585,6 @@ func TestAutomationRegisterUpkeep(t *testing.T) {

func TestAutomationPauseRegistry(t *testing.T) {
t.Parallel()

registryVersions := map[string]ethereum.KeeperRegistryVersion{
"registry_2_0": ethereum.RegistryVersion_2_0,
"registry_2_1": ethereum.RegistryVersion_2_1,
Expand Down Expand Up @@ -646,8 +642,6 @@ func TestAutomationPauseRegistry(t *testing.T) {

func TestAutomationKeeperNodesDown(t *testing.T) {
t.Parallel()
l := utils.GetTestLogger(t)

registryVersions := map[string]ethereum.KeeperRegistryVersion{
"registry_2_0": ethereum.RegistryVersion_2_0,
"registry_2_1": ethereum.RegistryVersion_2_1,
Expand All @@ -658,6 +652,7 @@ func TestAutomationKeeperNodesDown(t *testing.T) {
registryVersion := rv
t.Run(name, func(t *testing.T) {
t.Parallel()
l := utils.GetTestLogger(t)
chainClient, chainlinkNodes, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker(
t, "keeper-nodes-down", registryVersion, defaultOCRRegistryConfig, false,
)
Expand Down Expand Up @@ -733,7 +728,6 @@ func TestAutomationKeeperNodesDown(t *testing.T) {

func TestAutomationPerformSimulation(t *testing.T) {
t.Parallel()

registryVersions := map[string]ethereum.KeeperRegistryVersion{
"registry_2_0": ethereum.RegistryVersion_2_0,
"registry_2_1": ethereum.RegistryVersion_2_1,
Expand Down Expand Up @@ -797,8 +791,6 @@ func TestAutomationPerformSimulation(t *testing.T) {

func TestAutomationCheckPerformGasLimit(t *testing.T) {
t.Parallel()
l := utils.GetTestLogger(t)

registryVersions := map[string]ethereum.KeeperRegistryVersion{
"registry_2_0": ethereum.RegistryVersion_2_0,
"registry_2_1": ethereum.RegistryVersion_2_1,
Expand All @@ -809,6 +801,7 @@ func TestAutomationCheckPerformGasLimit(t *testing.T) {
registryVersion := rv
t.Run(name, func(t *testing.T) {
t.Parallel()
l := utils.GetTestLogger(t)
chainClient, chainlinkNodes, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker(
t, "gas-limit", registryVersion, defaultOCRRegistryConfig, false,
)
Expand Down Expand Up @@ -911,8 +904,6 @@ func TestAutomationCheckPerformGasLimit(t *testing.T) {

func TestUpdateCheckData(t *testing.T) {
t.Parallel()
l := utils.GetTestLogger(t)

registryVersions := map[string]ethereum.KeeperRegistryVersion{
"registry_2_0": ethereum.RegistryVersion_2_0,
"registry_2_1": ethereum.RegistryVersion_2_1,
Expand All @@ -923,6 +914,7 @@ func TestUpdateCheckData(t *testing.T) {
registryVersion := rv
t.Run(name, func(t *testing.T) {
t.Parallel()
l := utils.GetTestLogger(t)
chainClient, _, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker(
t, "update-check-data", registryVersion, defaultOCRRegistryConfig, false,
)
Expand Down Expand Up @@ -1003,7 +995,7 @@ func setupAutomationTestDocker(
network := networks.SelectedNetwork

// build the node config
clNodeConfig := node.NewConfig(node.BaseConf)
clNodeConfig := node.NewConfig(node.NewBaseConfig())
syncInterval := models.MustMakeDuration(5 * time.Minute)
clNodeConfig.Feature.LogPoller = it_utils.Ptr[bool](true)
clNodeConfig.OCR2.Enabled = it_utils.Ptr[bool](true)
Expand Down
99 changes: 21 additions & 78 deletions integration-tests/smoke/automation_test.go_test_list.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,108 +2,51 @@
"tests": [
{
"name": "TestAutomationBasic",
"run": [
{
"name": "registry_2_0"
},
{
"name": "registry_2_1_conditional"
},
{
"name": "registry_2_1_logtrigger"
}
]
"label": "ubuntu20.04-32cores-128GB",
"nodes": 3
},
{
"name": "TestSetUpkeepTriggerConfig"
},
{
"name": "TestAutomationAddFunds",
"run": [
{
"name": "registry_2_0"
},
{
"name": "registry_2_1"
}
]
"label": "ubuntu20.04-32cores-128GB",
"nodes": 2
},
{
"name": "TestAutomationPauseUnPause",
"run": [
{
"name": "registry_2_0"
},
{
"name": "registry_2_1"
}
]
"label": "ubuntu20.04-16cores-64GB",
"nodes": 2
},
{
"name": "TestAutomationRegisterUpkeep",
"run": [
{
"name": "registry_2_0"
},
{
"name": "registry_2_1"
}
]
"label": "ubuntu20.04-16cores-64GB",
"nodes": 2
},
{
"name": "TestAutomationPauseRegistry",
"run": [
{
"name": "registry_2_0"
},
{
"name": "registry_2_1"
}
]
"label": "ubuntu20.04-16cores-64GB",
"nodes": 2
},
{
"name": "TestAutomationKeeperNodesDown",
"run": [
{
"name": "registry_2_0"
},
{
"name": "registry_2_1"
}
]
"label": "ubuntu20.04-16cores-64GB",
"nodes": 2
},
{
"name": "TestAutomationPerformSimulation",
"run": [
{
"name": "registry_2_0"
},
{
"name": "registry_2_1"
}
]
"label": "ubuntu20.04-16cores-64GB",
"nodes": 2
},
{
"name": "TestAutomationCheckPerformGasLimit",
"run": [
{
"name": "registry_2_0"
},
{
"name": "registry_2_1"
}
]
"label": "ubuntu20.04-32cores-128GB",
"nodes": 2
},
{
"name": "TestUpdateCheckData",
"run": [
{
"name": "registry_2_0"
},
{
"name": "registry_2_1"
}
]
},
{
"name": "TestSetUpkeepTriggerConfig"
"label": "ubuntu20.04-32cores-128GB",
"nodes": 2
}
]
}
2 changes: 1 addition & 1 deletion integration-tests/smoke/forwarders_ocr2_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestForwarderOCR2Basic(t *testing.T) {
env, err := test_env.NewCLTestEnvBuilder().
WithGeth().
WithMockServer(1).
WithCLNodeConfig(node.NewConfig(node.BaseConf,
WithCLNodeConfig(node.NewConfig(node.NewBaseConfig(),
node.WithOCR2(),
node.WithP2Pv2(),
)).
Expand Down
Loading

0 comments on commit e161477

Please sign in to comment.