EVM benchmark tests

.github/workflows/benchmark-test.yml

@@ -0,0 +1,32 @@
+---
+name: Benchmark Tests
+on: # yamllint disable-line rule:truthy
+    workflow_call:
+      outputs:
+        workflow_output:
+          description: "Benchmark Tests output"
+          value: ${{ jobs.benchmark_test.outputs.test_output_failure }}
+
+jobs:
+  benchmark_test:
+    name: Run Benchmark Tests
+    runs-on: ubuntu-latest
+    outputs:
+      test_output_failure: ${{ steps.run_tests_failure.outputs.test_output }}
+    steps:
+      - name: Checkout Code
+        uses: actions/[email protected]
+        with:
+          submodules: recursive
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Setup Go
+        uses: actions/[email protected]
+        with:
+          go-version: 1.20.x
+      - name: Run Go Test
+        run: make benchmark-test
+      - name: Run Go Test Failed
+        if: failure()
+        id: run_tests_failure
+        run: echo "test_output=false" >> $GITHUB_OUTPUT
+

.github/workflows/ci.yml

@@ -36,6 +36,10 @@ on: # yamllint disable-line rule:truthy
         description: Fuzz Tests
         type: boolean
         default: true
+      benchmark-test:
+        description: Benchmark Tests
+        type: boolean
+        default: true
   workflow_call:
     inputs:
       build-blade:
@@ -65,6 +69,10 @@ on: # yamllint disable-line rule:truthy
         description: Fuzz Tests
         type: boolean
         required: true
+      benchmark-test:
+        description: Benchmark Tests
+        type: boolean
+        default: true
     outputs:
       build-blade:
         description: Build Blade output
@@ -87,6 +95,9 @@ on: # yamllint disable-line rule:truthy
       fuzz-test:
         description: Fuzz Tests output
         value: ${{ jobs.fuzz-test.outputs.workflow_output }}
+      benchmark-test:
+        description: Benchmark Tests output
+        value: ${{ jobs.benchmark-test.outputs.workflow_output }}
 
 jobs:
   build-blade:
@@ -144,3 +155,11 @@ jobs:
       inputs.fuzz-test || 
       github.event_name == 'pull_request' || 
       (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop'))
+  benchmark-test:
+    name: Benchmark Tests
+    uses: ./.github/workflows/benchmark-test.yml
+    needs: build-blade
+    if: |
+      inputs.benchmark-test || 
+      github.event_name == 'pull_request' || 
+      (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop'))

.github/workflows/nightly.yml

@@ -30,6 +30,7 @@ jobs:
       e2e-legacy-test: true
       property-polybft-test: true
       fuzz-test: true
+      benchmark-test: true
   deploy_network:
     name: Deploy Network
     uses: ./.github/workflows/deploy-network.yml
@@ -120,6 +121,7 @@ jobs:
       e2e_legacy_test_output: ${{ needs.ci.outputs.e2e-legacy-test }}
       property_polybft_test_output: ${{ needs.ci.outputs.property-polybft-test }}
       fuzz_test_output: ${{ needs.ci.outputs.fuzz-test }}
+      benchmark_test_output: ${{ needs.ci.outputs.benchmark-test }}
       deploy_network_terraform_output: ${{ needs.deploy_network.outputs.terraform_output }}
       deploy_network_ansible_output: ${{ needs.deploy_network.outputs.ansible_output }}
       load_test_multiple_eoa_output: ${{ needs.load_test_multiple_eoa.outputs.load_test_output }}

.github/workflows/notification-nightly.yml

@@ -39,6 +39,10 @@ on: # yamllint disable-line rule:truthy
         description: Fuzz Tests output
         type: string
         required: true
+      benchmark_test_output:
+        description: Benchmark Tests output
+        type: string
+        required: true
       deploy_network_terraform_output:
         description: Deploy Network - Terraform output
         type: string
@@ -91,7 +95,7 @@ jobs:
             {
               "attachments": [
                 {
-                  "color": "${{ inputs.build_blade_output == '' && inputs.lint_output == '' && inputs.unit_test_output == '' && inputs.e2e_polybft_test_output == '' && inputs.e2e_legacy_test_output == '' && inputs.property_polybft_test_output == '' && inputs.fuzz_test_output == '' && inputs.deploy_network_terraform_output == '' && inputs.deploy_network_ansible_output == '' && inputs.load_test_multiple_eoa_output == 'true' && inputs.load_test_multiple_erc20_output == 'true' && inputs.destroy_network_logs_output == '' && inputs.destroy_network_terraform_output == '' && env.green_color || env.red_color }}",
+                  "color": "${{ inputs.build_blade_output == '' && inputs.lint_output == '' && inputs.unit_test_output == '' && inputs.e2e_polybft_test_output == '' && inputs.e2e_legacy_test_output == '' && inputs.property_polybft_test_output == '' && inputs.fuzz_test_output == '' && inputs.benchmark_test_output == '' && inputs.deploy_network_terraform_output == '' && inputs.deploy_network_ansible_output == '' && inputs.load_test_multiple_eoa_output == 'true' && inputs.load_test_multiple_erc20_output == 'true' && inputs.destroy_network_logs_output == '' && inputs.destroy_network_terraform_output == '' && env.green_color || env.red_color }}",
                   "blocks": [
                     {
                       "type": "header",
@@ -152,13 +156,13 @@ jobs:
                   ]
                 },
                 {
-                  "color": "${{ inputs.build_blade_output == '' && inputs.lint_output == '' && inputs.unit_test_output == '' && inputs.fuzz_test_output == '' && inputs.e2e_legacy_test_output == '' && inputs.e2e_polybft_test_output == '' && inputs.property_polybft_test_output == '' && env.green_color || env.red_color }}",
+                  "color": "${{ inputs.build_blade_output == '' && inputs.lint_output == '' && inputs.unit_test_output == '' && inputs.fuzz_test_output == '' && inputs.benchmark_test_output == '' && inputs.e2e_legacy_test_output == '' && inputs.e2e_polybft_test_output == '' && inputs.property_polybft_test_output == '' && env.green_color || env.red_color }}",
                   "blocks": [
                     {
                       "type": "section",
                       "text": {
                         "type": "mrkdwn",
-                        "text": "*CI*\n${{ inputs.build_blade_output == '' && 'Build Blade' || '~Build Blade~' }}, ${{ inputs.lint_output == '' && 'Lint' || '~Lint~' }}, ${{ inputs.unit_test_output == '' && 'Unit Tests' || '~Unit Tests~' }},\n${{ inputs.fuzz_test_output == '' && 'Fuzz Tests' || '~Fuzz Tests~' }}, ${{ inputs.e2e_legacy_test_output == '' && 'E2E Legacy Tests' || '~E2E Legacy Tests~' }},\n${{ inputs.e2e_polybft_test_output == '' && 'E2E PolyBFT Tests' || '~E2E PolyBFT Tests~' }}, ${{ inputs.property_polybft_test_output == '' && 'Property PolyBFT Tests' || '~Property PolyBFT Tests~' }}"
+                        "text": "*CI*\n${{ inputs.build_blade_output == '' && 'Build' || '~Build~' }}, ${{ inputs.lint_output == '' && 'Lint' || '~Lint~' }}, ${{ inputs.unit_test_output == '' && 'Unit Tests' || '~Unit Tests~' }},\n${{ inputs.fuzz_test_output == '' && 'Fuzz Tests' || '~Fuzz Tests~' }}, ${{ inputs.e2e_legacy_test_output == '' && 'E2E Legacy Tests' || '~E2E Legacy Tests~' }},\n${{ inputs.e2e_polybft_test_output == '' && 'E2E PolyBFT Tests' || '~E2E PolyBFT Tests~' }}, ${{ inputs.property_polybft_test_output == '' && 'Property PolyBFT Tests' || '~Property PolyBFT Tests~' }},\n${{ inputs.benchmark_test_output == '' && 'Benchmark Tests' || '~Benchmark Tests~' }}"
                       }
                     }
                   ]

.gitmodules

@@ -5,3 +5,6 @@
 [submodule "blade-contracts"]
 	path = blade-contracts
 	url = https://github.com/Ethernal-Tech/blade-contracts
+[submodule "tests/evm-benchmarks"]
+	path = tests/evm-benchmarks
+	url = https://github.com/ipsilon/evm-benchmarks

Makefile

@@ -60,6 +60,10 @@ generate-bsd-licenses: check-git
 unit-test: check-go
 	go test -race -shuffle=on -coverprofile coverage.out -timeout 20m `go list ./... | grep -v e2e`
 
+.PHONY: benchmark-test
+benchmark-test: check-go
+	go test -bench=. -run=^$ `go list ./... | grep -v /e2e`
+
 .PHONY: fuzz-test
 fuzz-test: check-go
 	./scripts/fuzzAll

state/executor.go

@@ -649,7 +649,8 @@ func (t *Transition) apply(msg *types.Transaction) (*runtime.ExecutionResult, er
 
 	// set up initial access list
 	initialAccessList := runtime.NewAccessList()
-	if t.config.Berlin { // check if berlin fork is activated or not
+	if t.config.Berlin {
+		// populate access list in case Berlin fork is active
 		initialAccessList.PrepareAccessList(msg.From(), msg.To(), t.precompiles.Addrs, msg.AccessList())
 	}
 
@@ -1354,6 +1355,11 @@ func (t *Transition) RevertToSnapshot(snapshot int) error {
 	return nil
 }
 
+// PopulateAccessList populates access list based on the provided access list
+func (t *Transition) PopulateAccessList(from types.Address, to *types.Address, acl types.TxAccessList) {
+	t.accessList.PrepareAccessList(from, to, t.precompiles.Addrs, acl)
+}
+
 func (t *Transition) AddSlotToAccessList(addr types.Address, slot types.Hash) {
 	t.journal.Append(&runtime.AccessListAddSlotChange{Address: addr, Slot: slot})
 	t.accessList.AddSlot(addr, slot)

tests/evm_benchmark_test.go

@@ -0,0 +1,170 @@
+package tests
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/big"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/go-hclog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/0xPolygon/polygon-edge/chain"
+	"github.com/0xPolygon/polygon-edge/crypto"
+	"github.com/0xPolygon/polygon-edge/state"
+	"github.com/0xPolygon/polygon-edge/types"
+)
+
+const (
+	benchmarksDir = "evm-benchmarks/benchmarks"
+	chainID       = 10
+)
+
+func BenchmarkEVM(b *testing.B) {
+	folders, err := listFolders([]string{benchmarksDir})
+	require.NoError(b, err)
+
+	for _, folder := range folders {
+		files, err := listFiles(folder, ".json")
+		require.NoError(b, err)
+
+		for _, file := range files {
+			name := getTestName(file)
+
+			b.Run(name, func(b *testing.B) {
+				data, err := os.ReadFile(file)
+				require.NoError(b, err)
+
+				var testCases map[string]testCase
+				if err = json.Unmarshal(data, &testCases); err != nil {
+					b.Fatalf("failed to unmarshal %s: %v", file, err)
+				}
+
+				for _, tc := range testCases {
+					for fork, postState := range tc.Post {
+						forks, exists := Forks[fork]
+						if !exists {
+							b.Logf("%s fork is not supported, skipping test case.", fork)
+							continue
+						}
+
+						fc := &forkConfig{name: fork, forks: forks}
+
+						for idx, postStateEntry := range postState {
+							err := runBenchmarkTest(b, tc, fc, postStateEntry)
+							require.NoError(b, err, fmt.Sprintf("test %s (case#%d) execution failed", name, idx))
+						}
+					}
+				}
+			})
+		}
+	}
+}
+
+func runBenchmarkTest(b *testing.B, c testCase, fc *forkConfig, p postEntry) error {
+	env := c.Env.ToEnv(b)
+	forks := fc.forks
+
+	var baseFee *big.Int
+
+	if forks.IsActive(chain.London, 0) {
+		if c.Env.BaseFee != "" {
+			baseFee = stringToBigIntT(b, c.Env.BaseFee)
+		} else {
+			// Retesteth uses `10` for genesis baseFee. Therefore, it defaults to
+			// parent - 2 : 0xa as the basefee for 'this' context.
+			baseFee = big.NewInt(testGenesisBaseFee)
+		}
+	}
+
+	msg, err := c.Transaction.At(p.Indexes, baseFee)
+	if err != nil {
+		return err
+	}
+
+	s, _, parentRoot, err := buildState(c.Pre)
+	if err != nil {
+		return err
+	}
+
+	currentForks := forks.At(uint64(env.Number))
+
+	// try to recover tx with current signer
+	if len(p.TxBytes) != 0 {
+		tx := &types.Transaction{}
+		err := tx.UnmarshalRLP(p.TxBytes)
+		if err != nil {
+			return err
+		}
+
+		signer := crypto.NewSigner(currentForks, chainID)
+
+		_, err = signer.Sender(tx)
+		if err != nil {
+			return err
+		}
+	}
+
+	executor := state.NewExecutor(&chain.Params{
+		Forks:   forks,
+		ChainID: chainID,
+		BurnContract: map[uint64]types.Address{
+			0: types.ZeroAddress,
+		},
+	}, s, hclog.NewNullLogger())
+
+	executor.GetHash = func(*types.Header) func(i uint64) types.Hash {
+		return vmTestBlockHash
+	}
+
+	transition, err := executor.BeginTxn(parentRoot, c.Env.ToHeader(b), env.Coinbase)
+	if err != nil {
+		return err
+	}
+
+	if currentForks.Berlin {
+		transition.PopulateAccessList(msg.From(), msg.To(), msg.AccessList())
+	}
+
+	var (
+		gasUsed uint64
+		elapsed uint64
+		refund  uint64
+	)
+
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		snapshotID := transition.Snapshot()
+
+		b.StartTimer()
+		start := time.Now()
+
+		// execute the message
+		result := transition.Call2(msg.From(), *msg.To(), msg.Input(), msg.Value(), msg.Gas())
+		if result.Err != nil {
+			return result.Err
+		}
+
+		b.StopTimer()
+		elapsed += uint64(time.Since(start))
+		refund += transition.GetRefund()
+		gasUsed += msg.Gas() - result.GasLeft
+
+		err = transition.RevertToSnapshot(snapshotID)
+		if err != nil {
+			return err
+		}
+	}
+
+	if elapsed < 1 {
+		elapsed = 1
+	}
+
+	// Keep it as uint64, multiply 100 to get two digit float later
+	mgasps := (100 * 1000 * (gasUsed - refund)) / elapsed
+	b.ReportMetric(float64(mgasps)/100, "mgas/s")
+
+	return nil
+}

tests/state_test.go

@@ -6,8 +6,6 @@ import (
 	"fmt"
 	"math/big"
 	"os"
-	"path/filepath"
-	"strings"
 	"testing"
 	"time"
 
@@ -25,8 +23,6 @@ import (
 
 const (
 	stateTestsDir = "tests/GeneralStateTests"
-
-	testGenesisBaseFee = 0xa
 )
 
 var (
@@ -116,8 +112,7 @@ func TestState(t *testing.T) {
 func runSpecificTestCase(t *testing.T, file string, c testCase, fc *forkConfig, index int, p postEntry) error {
 	t.Helper()
 
-	testName := filepath.Base(file)
-	testName = strings.TrimSuffix(testName, ".json")
+	testName := getTestName(file)
 
 	env := c.Env.ToEnv(t)
 	forks := fc.forks
@@ -231,8 +226,3 @@ func runSpecificTestCase(t *testing.T, file string, c testCase, fc *forkConfig,
 
 	return nil
 }
-
-type forkConfig struct {
-	name  string
-	forks *chain.Forks
-}