diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml new file mode 100644 index 0000000000..7187054b88 --- /dev/null +++ b/.github/workflows/benchmark-test.yml @@ -0,0 +1,32 @@ +--- +name: Benchmark Tests +on: # yamllint disable-line rule:truthy + workflow_call: + outputs: + workflow_output: + description: "Benchmark Tests output" + value: ${{ jobs.benchmark_test.outputs.test_output_failure }} + +jobs: + benchmark_test: + name: Run Benchmark Tests + runs-on: ubuntu-latest + outputs: + test_output_failure: ${{ steps.run_tests_failure.outputs.test_output }} + steps: + - name: Checkout Code + uses: actions/checkout@v4.1.1 + with: + submodules: recursive + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Setup Go + uses: actions/setup-go@v5.0.0 + with: + go-version: 1.20.x + - name: Run Go Test + run: make benchmark-test + - name: Run Go Test Failed + if: failure() + id: run_tests_failure + run: echo "test_output=false" >> $GITHUB_OUTPUT + \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6df137d078..71d06339fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,10 @@ on: # yamllint disable-line rule:truthy description: Fuzz Tests type: boolean default: true + benchmark-test: + description: Benchmark Tests + type: boolean + default: true workflow_call: inputs: build-blade: @@ -65,6 +69,10 @@ on: # yamllint disable-line rule:truthy description: Fuzz Tests type: boolean required: true + benchmark-test: + description: Benchmark Tests + type: boolean + default: true outputs: build-blade: description: Build Blade output @@ -87,6 +95,9 @@ on: # yamllint disable-line rule:truthy fuzz-test: description: Fuzz Tests output value: ${{ jobs.fuzz-test.outputs.workflow_output }} + benchmark-test: + description: Benchmark Tests output + value: ${{ jobs.benchmark-test.outputs.workflow_output }} jobs: build-blade: @@ -144,3 +155,11 @@ jobs: inputs.fuzz-test || github.event_name == 'pull_request' || (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop')) + benchmark-test: + name: Benchmark Tests + uses: ./.github/workflows/benchmark-test.yml + needs: build-blade + if: | + inputs.benchmark-test || + github.event_name == 'pull_request' || + (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop')) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index eac54f51d8..d4bf6a6e75 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -30,6 +30,7 @@ jobs: e2e-legacy-test: true property-polybft-test: true fuzz-test: true + benchmark-test: true deploy_network: name: Deploy Network uses: ./.github/workflows/deploy-network.yml @@ -120,6 +121,7 @@ jobs: e2e_legacy_test_output: ${{ needs.ci.outputs.e2e-legacy-test }} property_polybft_test_output: ${{ needs.ci.outputs.property-polybft-test }} fuzz_test_output: ${{ needs.ci.outputs.fuzz-test }} + benchmark_test_output: ${{ needs.ci.outputs.benchmark-test }} deploy_network_terraform_output: ${{ needs.deploy_network.outputs.terraform_output }} deploy_network_ansible_output: ${{ needs.deploy_network.outputs.ansible_output }} load_test_multiple_eoa_output: ${{ needs.load_test_multiple_eoa.outputs.load_test_output }} diff --git a/.github/workflows/notification-nightly.yml b/.github/workflows/notification-nightly.yml index df39660a01..8bdf8967be 100644 --- a/.github/workflows/notification-nightly.yml +++ b/.github/workflows/notification-nightly.yml @@ -39,6 +39,10 @@ on: # yamllint disable-line rule:truthy description: Fuzz Tests output type: string required: true + benchmark_test_output: + description: Benchmark Tests output + type: string + required: true deploy_network_terraform_output: description: Deploy Network - Terraform output type: string @@ -91,7 +95,7 @@ jobs: { "attachments": [ { - "color": "${{ inputs.build_blade_output == '' && inputs.lint_output == '' && inputs.unit_test_output == '' && inputs.e2e_polybft_test_output == '' && inputs.e2e_legacy_test_output == '' && inputs.property_polybft_test_output == '' && inputs.fuzz_test_output == '' && inputs.deploy_network_terraform_output == '' && inputs.deploy_network_ansible_output == '' && inputs.load_test_multiple_eoa_output == 'true' && inputs.load_test_multiple_erc20_output == 'true' && inputs.destroy_network_logs_output == '' && inputs.destroy_network_terraform_output == '' && env.green_color || env.red_color }}", + "color": "${{ inputs.build_blade_output == '' && inputs.lint_output == '' && inputs.unit_test_output == '' && inputs.e2e_polybft_test_output == '' && inputs.e2e_legacy_test_output == '' && inputs.property_polybft_test_output == '' && inputs.fuzz_test_output == '' && inputs.benchmark_test_output == '' && inputs.deploy_network_terraform_output == '' && inputs.deploy_network_ansible_output == '' && inputs.load_test_multiple_eoa_output == 'true' && inputs.load_test_multiple_erc20_output == 'true' && inputs.destroy_network_logs_output == '' && inputs.destroy_network_terraform_output == '' && env.green_color || env.red_color }}", "blocks": [ { "type": "header", @@ -152,13 +156,13 @@ jobs: ] }, { - "color": "${{ inputs.build_blade_output == '' && inputs.lint_output == '' && inputs.unit_test_output == '' && inputs.fuzz_test_output == '' && inputs.e2e_legacy_test_output == '' && inputs.e2e_polybft_test_output == '' && inputs.property_polybft_test_output == '' && env.green_color || env.red_color }}", + "color": "${{ inputs.build_blade_output == '' && inputs.lint_output == '' && inputs.unit_test_output == '' && inputs.fuzz_test_output == '' && inputs.benchmark_test_output == '' && inputs.e2e_legacy_test_output == '' && inputs.e2e_polybft_test_output == '' && inputs.property_polybft_test_output == '' && env.green_color || env.red_color }}", "blocks": [ { "type": "section", "text": { "type": "mrkdwn", - "text": "*CI*\n${{ inputs.build_blade_output == '' && 'Build Blade' || '~Build Blade~' }}, ${{ inputs.lint_output == '' && 'Lint' || '~Lint~' }}, ${{ inputs.unit_test_output == '' && 'Unit Tests' || '~Unit Tests~' }},\n${{ inputs.fuzz_test_output == '' && 'Fuzz Tests' || '~Fuzz Tests~' }}, ${{ inputs.e2e_legacy_test_output == '' && 'E2E Legacy Tests' || '~E2E Legacy Tests~' }},\n${{ inputs.e2e_polybft_test_output == '' && 'E2E PolyBFT Tests' || '~E2E PolyBFT Tests~' }}, ${{ inputs.property_polybft_test_output == '' && 'Property PolyBFT Tests' || '~Property PolyBFT Tests~' }}" + "text": "*CI*\n${{ inputs.build_blade_output == '' && 'Build' || '~Build~' }}, ${{ inputs.lint_output == '' && 'Lint' || '~Lint~' }}, ${{ inputs.unit_test_output == '' && 'Unit Tests' || '~Unit Tests~' }},\n${{ inputs.fuzz_test_output == '' && 'Fuzz Tests' || '~Fuzz Tests~' }}, ${{ inputs.e2e_legacy_test_output == '' && 'E2E Legacy Tests' || '~E2E Legacy Tests~' }},\n${{ inputs.e2e_polybft_test_output == '' && 'E2E PolyBFT Tests' || '~E2E PolyBFT Tests~' }}, ${{ inputs.property_polybft_test_output == '' && 'Property PolyBFT Tests' || '~Property PolyBFT Tests~' }},\n${{ inputs.benchmark_test_output == '' && 'Benchmark Tests' || '~Benchmark Tests~' }}" } } ] diff --git a/.gitmodules b/.gitmodules index 083c521ccc..086c671948 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,6 @@ [submodule "blade-contracts"] path = blade-contracts url = https://github.com/Ethernal-Tech/blade-contracts +[submodule "tests/evm-benchmarks"] + path = tests/evm-benchmarks + url = https://github.com/ipsilon/evm-benchmarks diff --git a/Makefile b/Makefile index 9a8c5de0af..d4276c28aa 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,10 @@ generate-bsd-licenses: check-git unit-test: check-go go test -race -shuffle=on -coverprofile coverage.out -timeout 20m `go list ./... | grep -v e2e` +.PHONY: benchmark-test +benchmark-test: check-go + go test -bench=. -run=^$ `go list ./... | grep -v /e2e` + .PHONY: fuzz-test fuzz-test: check-go ./scripts/fuzzAll diff --git a/state/executor.go b/state/executor.go index 6331204d6a..4f094c0c86 100644 --- a/state/executor.go +++ b/state/executor.go @@ -649,7 +649,8 @@ func (t *Transition) apply(msg *types.Transaction) (*runtime.ExecutionResult, er // set up initial access list initialAccessList := runtime.NewAccessList() - if t.config.Berlin { // check if berlin fork is activated or not + if t.config.Berlin { + // populate access list in case Berlin fork is active initialAccessList.PrepareAccessList(msg.From(), msg.To(), t.precompiles.Addrs, msg.AccessList()) } @@ -1354,6 +1355,11 @@ func (t *Transition) RevertToSnapshot(snapshot int) error { return nil } +// PopulateAccessList populates access list based on the provided access list +func (t *Transition) PopulateAccessList(from types.Address, to *types.Address, acl types.TxAccessList) { + t.accessList.PrepareAccessList(from, to, t.precompiles.Addrs, acl) +} + func (t *Transition) AddSlotToAccessList(addr types.Address, slot types.Hash) { t.journal.Append(&runtime.AccessListAddSlotChange{Address: addr, Slot: slot}) t.accessList.AddSlot(addr, slot) diff --git a/tests/evm-benchmarks b/tests/evm-benchmarks new file mode 160000 index 0000000000..d8b88f4046 --- /dev/null +++ b/tests/evm-benchmarks @@ -0,0 +1 @@ +Subproject commit d8b88f4046a87d6b902378cef752591f95427b43 diff --git a/tests/evm_benchmark_test.go b/tests/evm_benchmark_test.go new file mode 100644 index 0000000000..01e94bd90d --- /dev/null +++ b/tests/evm_benchmark_test.go @@ -0,0 +1,170 @@ +package tests + +import ( + "encoding/json" + "fmt" + "math/big" + "os" + "testing" + "time" + + "github.com/hashicorp/go-hclog" + "github.com/stretchr/testify/require" + + "github.com/0xPolygon/polygon-edge/chain" + "github.com/0xPolygon/polygon-edge/crypto" + "github.com/0xPolygon/polygon-edge/state" + "github.com/0xPolygon/polygon-edge/types" +) + +const ( + benchmarksDir = "evm-benchmarks/benchmarks" + chainID = 10 +) + +func BenchmarkEVM(b *testing.B) { + folders, err := listFolders([]string{benchmarksDir}) + require.NoError(b, err) + + for _, folder := range folders { + files, err := listFiles(folder, ".json") + require.NoError(b, err) + + for _, file := range files { + name := getTestName(file) + + b.Run(name, func(b *testing.B) { + data, err := os.ReadFile(file) + require.NoError(b, err) + + var testCases map[string]testCase + if err = json.Unmarshal(data, &testCases); err != nil { + b.Fatalf("failed to unmarshal %s: %v", file, err) + } + + for _, tc := range testCases { + for fork, postState := range tc.Post { + forks, exists := Forks[fork] + if !exists { + b.Logf("%s fork is not supported, skipping test case.", fork) + continue + } + + fc := &forkConfig{name: fork, forks: forks} + + for idx, postStateEntry := range postState { + err := runBenchmarkTest(b, tc, fc, postStateEntry) + require.NoError(b, err, fmt.Sprintf("test %s (case#%d) execution failed", name, idx)) + } + } + } + }) + } + } +} + +func runBenchmarkTest(b *testing.B, c testCase, fc *forkConfig, p postEntry) error { + env := c.Env.ToEnv(b) + forks := fc.forks + + var baseFee *big.Int + + if forks.IsActive(chain.London, 0) { + if c.Env.BaseFee != "" { + baseFee = stringToBigIntT(b, c.Env.BaseFee) + } else { + // Retesteth uses `10` for genesis baseFee. Therefore, it defaults to + // parent - 2 : 0xa as the basefee for 'this' context. + baseFee = big.NewInt(testGenesisBaseFee) + } + } + + msg, err := c.Transaction.At(p.Indexes, baseFee) + if err != nil { + return err + } + + s, _, parentRoot, err := buildState(c.Pre) + if err != nil { + return err + } + + currentForks := forks.At(uint64(env.Number)) + + // try to recover tx with current signer + if len(p.TxBytes) != 0 { + tx := &types.Transaction{} + err := tx.UnmarshalRLP(p.TxBytes) + if err != nil { + return err + } + + signer := crypto.NewSigner(currentForks, chainID) + + _, err = signer.Sender(tx) + if err != nil { + return err + } + } + + executor := state.NewExecutor(&chain.Params{ + Forks: forks, + ChainID: chainID, + BurnContract: map[uint64]types.Address{ + 0: types.ZeroAddress, + }, + }, s, hclog.NewNullLogger()) + + executor.GetHash = func(*types.Header) func(i uint64) types.Hash { + return vmTestBlockHash + } + + transition, err := executor.BeginTxn(parentRoot, c.Env.ToHeader(b), env.Coinbase) + if err != nil { + return err + } + + if currentForks.Berlin { + transition.PopulateAccessList(msg.From(), msg.To(), msg.AccessList()) + } + + var ( + gasUsed uint64 + elapsed uint64 + refund uint64 + ) + + b.ResetTimer() + for n := 0; n < b.N; n++ { + snapshotID := transition.Snapshot() + + b.StartTimer() + start := time.Now() + + // execute the message + result := transition.Call2(msg.From(), *msg.To(), msg.Input(), msg.Value(), msg.Gas()) + if result.Err != nil { + return result.Err + } + + b.StopTimer() + elapsed += uint64(time.Since(start)) + refund += transition.GetRefund() + gasUsed += msg.Gas() - result.GasLeft + + err = transition.RevertToSnapshot(snapshotID) + if err != nil { + return err + } + } + + if elapsed < 1 { + elapsed = 1 + } + + // Keep it as uint64, multiply 100 to get two digit float later + mgasps := (100 * 1000 * (gasUsed - refund)) / elapsed + b.ReportMetric(float64(mgasps)/100, "mgas/s") + + return nil +} diff --git a/tests/state_test.go b/tests/state_test.go index 8b32b71c49..e4fe770177 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -6,8 +6,6 @@ import ( "fmt" "math/big" "os" - "path/filepath" - "strings" "testing" "time" @@ -25,8 +23,6 @@ import ( const ( stateTestsDir = "tests/GeneralStateTests" - - testGenesisBaseFee = 0xa ) var ( @@ -116,8 +112,7 @@ func TestState(t *testing.T) { func runSpecificTestCase(t *testing.T, file string, c testCase, fc *forkConfig, index int, p postEntry) error { t.Helper() - testName := filepath.Base(file) - testName = strings.TrimSuffix(testName, ".json") + testName := getTestName(file) env := c.Env.ToEnv(t) forks := fc.forks @@ -231,8 +226,3 @@ func runSpecificTestCase(t *testing.T, file string, c testCase, fc *forkConfig, return nil } - -type forkConfig struct { - name string - forks *chain.Forks -} diff --git a/tests/state_test_util.go b/tests/state_test_util.go index bf105245fd..ef7dc99e1e 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -11,6 +11,7 @@ import ( "strings" "testing" + "github.com/stretchr/testify/require" "github.com/umbracle/fastrlp" "github.com/0xPolygon/polygon-edge/chain" @@ -24,6 +25,10 @@ import ( "github.com/0xPolygon/polygon-edge/types" ) +const ( + testGenesisBaseFee = 0xa +) + type testCase struct { Env *env `json:"env"` Pre map[types.Address]*chain.GenesisAccount `json:"pre"` @@ -57,7 +62,7 @@ type env struct { Timestamp string `json:"currentTimestamp"` } -func (e *env) ToHeader(t *testing.T) *types.Header { +func (e *env) ToHeader(t testing.TB) *types.Header { t.Helper() baseFee := uint64(0) @@ -75,7 +80,7 @@ func (e *env) ToHeader(t *testing.T) *types.Header { } } -func (e *env) ToEnv(t *testing.T) runtime.TxContext { +func (e *env) ToEnv(t testing.TB) runtime.TxContext { t.Helper() baseFee := new(big.Int) @@ -129,35 +134,29 @@ func stringToBigInt(str string) (*big.Int, error) { return n, nil } -func stringToBigIntT(t *testing.T, str string) *big.Int { +func stringToBigIntT(t testing.TB, str string) *big.Int { t.Helper() number, err := stringToBigInt(str) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) return number } -func stringToAddressT(t *testing.T, str string) types.Address { +func stringToAddressT(t testing.TB, str string) types.Address { t.Helper() address, err := stringToAddress(str) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) return address } -func stringToHashT(t *testing.T, str string) types.Hash { +func stringToHashT(t testing.TB, str string) types.Hash { t.Helper() address, err := stringToHash(str) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) return address } @@ -171,24 +170,20 @@ func stringToUint64(str string) (uint64, error) { return n.Uint64(), nil } -func stringToUint64T(t *testing.T, str string) uint64 { +func stringToUint64T(t testing.TB, str string) uint64 { t.Helper() n, err := stringToUint64(str) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) return n } -func stringToInt64T(t *testing.T, str string) int64 { +func stringToInt64T(t testing.TB, str string) int64 { t.Helper() n, err := stringToUint64(str) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) return int64(n) } @@ -689,3 +684,15 @@ func rlpHashLogs(logs []*types.Log) (res types.Hash) { func vmTestBlockHash(n uint64) types.Hash { return types.BytesToHash(crypto.Keccak256([]byte(big.NewInt(int64(n)).String()))) } + +// getTestName extracts test name from the test file path +func getTestName(testFile string) string { + testName := filepath.Base(testFile) + + return strings.TrimSuffix(testName, ".json") +} + +type forkConfig struct { + name string + forks *chain.Forks +}