From 35bbc4d5c79c39895017758ddbb67ddbce8d0e9d Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Tue, 6 Aug 2024 17:19:01 -0300 Subject: [PATCH 01/95] Add custom native tracer for cargo stylus replay --- execution/gethexec/stylus_tracer.go | 138 +++++++++++++++++++ system_tests/stylus_tracer_test.go | 207 ++++++++++++++++++++++++++++ 2 files changed, 345 insertions(+) create mode 100644 execution/gethexec/stylus_tracer.go create mode 100644 system_tests/stylus_tracer_test.go diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go new file mode 100644 index 00000000000..5c1fc5f3a0e --- /dev/null +++ b/execution/gethexec/stylus_tracer.go @@ -0,0 +1,138 @@ +// Copyright 2024, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE + +package gethexec + +import ( + "encoding/json" + "math/big" + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/log" +) + +func init() { + tracers.DefaultDirectory.Register("stylusTracer", newStylusTracer, false) +} + +// stylusTracer captures Stylus HostIOs and returns them in a structured format to be used in Cargo +// Stylus Replay. +type stylusTracer struct { + open *[]HostioTraceInfo + stack []*[]HostioTraceInfo + interrupt atomic.Bool + reason error +} + +// HostioTraceInfo contains the captured HostIO log returned by stylusTracer. +type HostioTraceInfo struct { + Name string `json:"name"` + Args hexutil.Bytes `json:"args"` + Outs hexutil.Bytes `json:"outs"` + StartInk uint64 `json:"startInk"` + EndInk uint64 `json:"endInk"` + Address *common.Address `json:"address,omitempty"` + Steps *[]HostioTraceInfo `json:"steps,omitempty"` +} + +// nestsHostios contains the hostios with nested calls. +var nestsHostios = map[string]bool{ + "call_contract": true, + "delegate_call_contract": true, + "static_call_contract": true, +} + +func newStylusTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { + var open []HostioTraceInfo + return &stylusTracer{ + open: &open, + }, nil +} + +func (t *stylusTracer) CaptureStylusHostio(name string, args, outs []byte, startInk, endInk uint64) { + if t.interrupt.Load() { + return + } + info := HostioTraceInfo{ + Name: name, + Args: args, + Outs: outs, + StartInk: startInk, + EndInk: endInk, + } + if nestsHostios[name] { + last := pop(t.open) + info.Address = last.Address + info.Steps = last.Steps + } + *t.open = append(*t.open, info) +} + +func (t *stylusTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + if t.interrupt.Load() { + return + } + inner := []HostioTraceInfo{} + info := HostioTraceInfo{ + Address: &to, + Steps: &inner, + } + *t.open = append(*t.open, info) + t.stack = append(t.stack, t.open) // save where we were + t.open = &inner +} + +func (t *stylusTracer) CaptureExit(output []byte, gasUsed uint64, err error) { + if t.interrupt.Load() { + return + } + t.open = pop(&t.stack) +} + +func (t *stylusTracer) GetResult() (json.RawMessage, error) { + if t.reason != nil { + return nil, t.reason + } + msg, err := json.Marshal(t.open) + if err != nil { + return nil, err + } + return msg, nil +} + +func (t *stylusTracer) Stop(err error) { + t.reason = err + t.interrupt.Store(true) +} + +func pop[T any](stack *[]T) T { + if len(*stack) == 0 { + log.Warn("stylusTracer: trying to pop empty stack") + var zeroVal T + return zeroVal + } + i := len(*stack) - 1 + val := (*stack)[i] + *stack = (*stack)[:i] + return val +} + +// Unimplemented EVMLogger interface methods + +func (t *stylusTracer) CaptureArbitrumTransfer(env *vm.EVM, from, to *common.Address, value *big.Int, before bool, purpose string) { +} +func (t *stylusTracer) CaptureArbitrumStorageGet(key common.Hash, depth int, before bool) {} +func (t *stylusTracer) CaptureArbitrumStorageSet(key, value common.Hash, depth int, before bool) {} +func (t *stylusTracer) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { +} +func (t *stylusTracer) CaptureEnd(output []byte, gasUsed uint64, err error) {} +func (t *stylusTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { +} +func (t *stylusTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, _ *vm.ScopeContext, depth int, err error) { +} +func (t *stylusTracer) CaptureTxStart(gasLimit uint64) {} +func (t *stylusTracer) CaptureTxEnd(restGas uint64) {} diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go new file mode 100644 index 00000000000..1623b0ba030 --- /dev/null +++ b/system_tests/stylus_tracer_test.go @@ -0,0 +1,207 @@ +// Copyright 2024, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE + +package arbtest + +import ( + "encoding/binary" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/google/go-cmp/cmp" + "github.com/offchainlabs/nitro/execution/gethexec" + "github.com/offchainlabs/nitro/solgen/go/mocksgen" + "github.com/offchainlabs/nitro/util/testhelpers" +) + +func TestStylusTracer(t *testing.T) { + const jit = false + builder, auth, cleanup := setupProgramTest(t, jit) + ctx := builder.ctx + l2client := builder.L2.Client + l2info := builder.L2Info + rpcClient := builder.L2.Client.Client() + defer cleanup() + + traceTransaction := func(tx common.Hash, tracer string) []gethexec.HostioTraceInfo { + traceOpts := struct { + Tracer string `json:"tracer"` + }{ + Tracer: tracer, + } + var result []gethexec.HostioTraceInfo + err := rpcClient.CallContext(ctx, &result, "debug_traceTransaction", tx, traceOpts) + Require(t, err, "trace transaction") + return result + } + + // Deploy contracts + stylusMulticall := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + evmMulticall, tx, _, err := mocksgen.DeployMultiCallTest(&auth, builder.L2.Client) + Require(t, err, "deploy evm multicall") + _, err = EnsureTxSucceeded(ctx, l2client, tx) + Require(t, err, "ensure evm multicall deployment") + + // Args for tests + key := testhelpers.RandomHash().Bytes() + value := testhelpers.RandomHash().Bytes() + loadStoreArgs := storeLoadMulticallArgs(key, value) + callArgs := argsForMulticall(vm.CALL, stylusMulticall, nil, []byte{0}) + evmCall := argsForMulticall(vm.CALL, evmMulticall, nil, []byte{0}) + + for _, testCase := range []struct { + name string + contract common.Address + args []byte + want []gethexec.HostioTraceInfo + }{ + { + name: "non-recursive hostios", + contract: stylusMulticall, + args: loadStoreArgs, + want: []gethexec.HostioTraceInfo{ + {Name: "user_entrypoint", Args: intToBe32(len(loadStoreArgs)), Outs: []byte{}}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, + {Name: "read_args", Args: []byte{}, Outs: loadStoreArgs}, + {Name: "storage_cache_bytes32", Args: append(key, value...), Outs: []byte{}}, + {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "storage_load_bytes32", Args: key, Outs: value}, + {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "write_result", Args: value, Outs: []byte{}}, + {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + }, + }, + + { + name: "call stylus contract", + contract: stylusMulticall, + args: callArgs, + want: []gethexec.HostioTraceInfo{ + {Name: "user_entrypoint", Outs: []byte{}, Args: intToBe32(len(callArgs))}, + {Name: "pay_for_memory_grow", Outs: []byte{}, Args: []byte{0x00, 0x01}}, + {Name: "read_args", Args: []byte{}, Outs: callArgs}, + { + Name: "call_contract", + Args: append(stylusMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), + Outs: common.Hex2Bytes("0000000000"), + Address: &stylusMulticall, + Steps: &[]gethexec.HostioTraceInfo{ + {Name: "user_entrypoint", Args: intToBe32(1), Outs: []byte{}}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, + {Name: "read_args", Args: []byte{}, Outs: []byte{0x00}}, + {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "write_result", Args: []byte{}, Outs: []byte{}}, + {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + }, + }, + {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "write_result", Args: []byte{}, Outs: []byte{}}, + {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + }, + }, + + { + name: "call evm contract", + contract: stylusMulticall, + args: evmCall, + want: []gethexec.HostioTraceInfo{ + {Name: "user_entrypoint", Args: intToBe32(len(callArgs)), Outs: []byte{}}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, + {Name: "read_args", Args: []byte{}, Outs: evmCall}, + { + Name: "call_contract", + Args: append(evmMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), + Outs: common.Hex2Bytes("0000000000"), + Address: &evmMulticall, + Steps: &[]gethexec.HostioTraceInfo{}, + }, + {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "write_result", Args: []byte{}, Outs: []byte{}}, + {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + tx := l2info.PrepareTxTo("Owner", &testCase.contract, l2info.TransferGas, nil, testCase.args) + err := l2client.SendTransaction(ctx, tx) + Require(t, err, "send transaction") + + nativeResult := traceTransaction(tx.Hash(), "stylusTracer") + clearInk(nativeResult) + if diff := cmp.Diff(testCase.want, nativeResult); diff != "" { + Fatal(t, "native tracer don't match wanted result", diff) + } + + jsResult := traceTransaction(tx.Hash(), jsStylusTracer) + clearInk(jsResult) + if diff := cmp.Diff(jsResult, nativeResult); diff != "" { + Fatal(t, "native tracer don't match js trace", diff) + } + }) + } +} + +func storeLoadMulticallArgs(key, value []byte) []byte { + // Args for storing and loading from storage + const ( + storageKind = 0x10 + storeAction = storageKind | 0x00 + loadAction = storageKind | 0x01 + ) + args := []byte{2} // number of actions + // first action + args = binary.BigEndian.AppendUint32(args, 1+64) // length + args = append(args, storeAction) + args = append(args, key...) + args = append(args, value...) + // second action + args = binary.BigEndian.AppendUint32(args, 1+32) // length + args = append(args, loadAction) + args = append(args, key...) + return args +} + +func intToBe32(v int) []byte { + return binary.BigEndian.AppendUint32(nil, uint32(v)) +} + +func clearInk(trace []gethexec.HostioTraceInfo) { + for i := range trace { + trace[i].StartInk = 0 + trace[i].EndInk = 0 + if trace[i].Steps != nil { + clearInk(*trace[i].Steps) + } + } +} + +var jsStylusTracer = ` +{ + "hostio": function(info) { + info.args = toHex(info.args); + info.outs = toHex(info.outs); + if (this.nests.includes(info.name)) { + Object.assign(info, this.open.pop()); + } + this.open.push(info); + }, + "enter": function(frame) { + let inner = []; + this.open.push({ + address: toHex(frame.getTo()), + steps: inner, + }); + this.stack.push(this.open); // save where we were + this.open = inner; + }, + "exit": function(result) { + this.open = this.stack.pop(); + }, + "result": function() { return this.open; }, + "fault": function() { return this.open; }, + stack: [], + open: [], + nests: ["call_contract", "delegate_call_contract", "static_call_contract"] +} +` From fcd7810ca801d7c1f8b58e7b33aa6dbcf994471a Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Thu, 8 Aug 2024 14:49:50 -0300 Subject: [PATCH 02/95] Fix memory aliasing lint warning --- system_tests/stylus_tracer_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index 1623b0ba030..82353294571 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -123,7 +123,8 @@ func TestStylusTracer(t *testing.T) { }, } { t.Run(testCase.name, func(t *testing.T) { - tx := l2info.PrepareTxTo("Owner", &testCase.contract, l2info.TransferGas, nil, testCase.args) + to := testCase.contract + tx := l2info.PrepareTxTo("Owner", &to, l2info.TransferGas, nil, testCase.args) err := l2client.SendTransaction(ctx, tx) Require(t, err, "send transaction") From 239bad99b17c1d9f89a78a3a17ff1e3e3b165608 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 9 Aug 2024 11:39:17 -0300 Subject: [PATCH 03/95] Add helpers to append load and store to multicall --- system_tests/program_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 5fa5db95c2a..ae34c6c5bbd 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -1617,6 +1617,35 @@ func multicallAppend(calls []byte, opcode vm.OpCode, address common.Address, inn return calls } +func multicallEmptyArgs() []byte { + return []byte{0} // number of actions +} + +func multicallAppendStore(args []byte, key, value common.Hash, emitLog bool) []byte { + var action byte = 0x10 + if emitLog { + action |= 0x08 + } + args[0] += 1 + args = binary.BigEndian.AppendUint32(args, 1+64) // length + args = append(args, action) + args = append(args, key.Bytes()...) + args = append(args, value.Bytes()...) + return args +} + +func multicallAppendLoad(args []byte, key common.Hash, emitLog bool) []byte { + var action byte = 0x11 + if emitLog { + action |= 0x08 + } + args[0] += 1 + args = binary.BigEndian.AppendUint32(args, 1+32) // length + args = append(args, action) + args = append(args, key.Bytes()...) + return args +} + func assertStorageAt( t *testing.T, ctx context.Context, l2client *ethclient.Client, contract common.Address, key, value common.Hash, ) { From 361eff0505408df5b8b45e94a42fc3b288da7953 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 9 Aug 2024 11:53:28 -0300 Subject: [PATCH 04/95] Use helpers to build multicall args in tracer test --- system_tests/stylus_tracer_test.go | 34 +++++++----------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index 82353294571..0c1fc4fca3e 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -44,9 +44,11 @@ func TestStylusTracer(t *testing.T) { Require(t, err, "ensure evm multicall deployment") // Args for tests - key := testhelpers.RandomHash().Bytes() - value := testhelpers.RandomHash().Bytes() - loadStoreArgs := storeLoadMulticallArgs(key, value) + key := testhelpers.RandomHash() + value := testhelpers.RandomHash() + loadStoreArgs := multicallEmptyArgs() + loadStoreArgs = multicallAppendLoad(loadStoreArgs, key, false) + loadStoreArgs = multicallAppendStore(loadStoreArgs, key, value, false) callArgs := argsForMulticall(vm.CALL, stylusMulticall, nil, []byte{0}) evmCall := argsForMulticall(vm.CALL, evmMulticall, nil, []byte{0}) @@ -64,11 +66,11 @@ func TestStylusTracer(t *testing.T) { {Name: "user_entrypoint", Args: intToBe32(len(loadStoreArgs)), Outs: []byte{}}, {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, {Name: "read_args", Args: []byte{}, Outs: loadStoreArgs}, - {Name: "storage_cache_bytes32", Args: append(key, value...), Outs: []byte{}}, + {Name: "storage_cache_bytes32", Args: append(key.Bytes(), value.Bytes()...), Outs: []byte{}}, {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "storage_load_bytes32", Args: key, Outs: value}, + {Name: "storage_load_bytes32", Args: key.Bytes(), Outs: value.Bytes()}, {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "write_result", Args: value, Outs: []byte{}}, + {Name: "write_result", Args: value.Bytes(), Outs: []byte{}}, {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, }, }, @@ -143,26 +145,6 @@ func TestStylusTracer(t *testing.T) { } } -func storeLoadMulticallArgs(key, value []byte) []byte { - // Args for storing and loading from storage - const ( - storageKind = 0x10 - storeAction = storageKind | 0x00 - loadAction = storageKind | 0x01 - ) - args := []byte{2} // number of actions - // first action - args = binary.BigEndian.AppendUint32(args, 1+64) // length - args = append(args, storeAction) - args = append(args, key...) - args = append(args, value...) - // second action - args = binary.BigEndian.AppendUint32(args, 1+32) // length - args = append(args, loadAction) - args = append(args, key...) - return args -} - func intToBe32(v int) []byte { return binary.BigEndian.AppendUint32(nil, uint32(v)) } From dc712eb51decd7b23308a69ed725df135d747b5a Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 9 Aug 2024 13:35:30 -0300 Subject: [PATCH 05/95] Fix tests --- system_tests/stylus_tracer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index 0c1fc4fca3e..93d0d0c6907 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -47,8 +47,8 @@ func TestStylusTracer(t *testing.T) { key := testhelpers.RandomHash() value := testhelpers.RandomHash() loadStoreArgs := multicallEmptyArgs() - loadStoreArgs = multicallAppendLoad(loadStoreArgs, key, false) loadStoreArgs = multicallAppendStore(loadStoreArgs, key, value, false) + loadStoreArgs = multicallAppendLoad(loadStoreArgs, key, false) callArgs := argsForMulticall(vm.CALL, stylusMulticall, nil, []byte{0}) evmCall := argsForMulticall(vm.CALL, evmMulticall, nil, []byte{0}) From 5a0b003e62e6b13a390181efaa0a2a5891b83d84 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 16 Aug 2024 14:32:32 -0300 Subject: [PATCH 06/95] Add stack struct to improve readability --- execution/gethexec/stylus_tracer.go | 62 ++++++++++++++--------------- system_tests/stylus_tracer_test.go | 7 ++-- util/stack/stack.go | 39 ++++++++++++++++++ 3 files changed, 72 insertions(+), 36 deletions(-) create mode 100644 util/stack/stack.go diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index 5c1fc5f3a0e..ac80596e8df 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -12,7 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" - "github.com/ethereum/go-ethereum/log" + "github.com/offchainlabs/nitro/util/stack" ) func init() { @@ -22,21 +22,21 @@ func init() { // stylusTracer captures Stylus HostIOs and returns them in a structured format to be used in Cargo // Stylus Replay. type stylusTracer struct { - open *[]HostioTraceInfo - stack []*[]HostioTraceInfo + open *stack.Stack[HostioTraceInfo] + stack *stack.Stack[*stack.Stack[HostioTraceInfo]] interrupt atomic.Bool reason error } // HostioTraceInfo contains the captured HostIO log returned by stylusTracer. type HostioTraceInfo struct { - Name string `json:"name"` - Args hexutil.Bytes `json:"args"` - Outs hexutil.Bytes `json:"outs"` - StartInk uint64 `json:"startInk"` - EndInk uint64 `json:"endInk"` - Address *common.Address `json:"address,omitempty"` - Steps *[]HostioTraceInfo `json:"steps,omitempty"` + Name string `json:"name"` + Args hexutil.Bytes `json:"args"` + Outs hexutil.Bytes `json:"outs"` + StartInk uint64 `json:"startInk"` + EndInk uint64 `json:"endInk"` + Address *common.Address `json:"address,omitempty"` + Steps *stack.Stack[HostioTraceInfo] `json:"steps,omitempty"` } // nestsHostios contains the hostios with nested calls. @@ -47,9 +47,9 @@ var nestsHostios = map[string]bool{ } func newStylusTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { - var open []HostioTraceInfo return &stylusTracer{ - open: &open, + open: stack.NewStack[HostioTraceInfo](), + stack: stack.NewStack[*stack.Stack[HostioTraceInfo]](), }, nil } @@ -65,32 +65,40 @@ func (t *stylusTracer) CaptureStylusHostio(name string, args, outs []byte, start EndInk: endInk, } if nestsHostios[name] { - last := pop(t.open) + last, err := t.open.Pop() + if err != nil { + t.Stop(err) + return + } info.Address = last.Address info.Steps = last.Steps } - *t.open = append(*t.open, info) + t.open.Push(info) } func (t *stylusTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { if t.interrupt.Load() { return } - inner := []HostioTraceInfo{} + inner := stack.NewStack[HostioTraceInfo]() info := HostioTraceInfo{ Address: &to, - Steps: &inner, + Steps: inner, } - *t.open = append(*t.open, info) - t.stack = append(t.stack, t.open) // save where we were - t.open = &inner + t.open.Push(info) + t.stack.Push(t.open) + t.open = inner } -func (t *stylusTracer) CaptureExit(output []byte, gasUsed uint64, err error) { +func (t *stylusTracer) CaptureExit(output []byte, gasUsed uint64, _ error) { if t.interrupt.Load() { return } - t.open = pop(&t.stack) + var err error + t.open, err = t.stack.Pop() + if err != nil { + t.Stop(err) + } } func (t *stylusTracer) GetResult() (json.RawMessage, error) { @@ -109,18 +117,6 @@ func (t *stylusTracer) Stop(err error) { t.interrupt.Store(true) } -func pop[T any](stack *[]T) T { - if len(*stack) == 0 { - log.Warn("stylusTracer: trying to pop empty stack") - var zeroVal T - return zeroVal - } - i := len(*stack) - 1 - val := (*stack)[i] - *stack = (*stack)[:i] - return val -} - // Unimplemented EVMLogger interface methods func (t *stylusTracer) CaptureArbitrumTransfer(env *vm.EVM, from, to *common.Address, value *big.Int, before bool, purpose string) { diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index 93d0d0c6907..eaa71b4c4f4 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -12,6 +12,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/offchainlabs/nitro/execution/gethexec" "github.com/offchainlabs/nitro/solgen/go/mocksgen" + "github.com/offchainlabs/nitro/util/stack" "github.com/offchainlabs/nitro/util/testhelpers" ) @@ -88,14 +89,14 @@ func TestStylusTracer(t *testing.T) { Args: append(stylusMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), Outs: common.Hex2Bytes("0000000000"), Address: &stylusMulticall, - Steps: &[]gethexec.HostioTraceInfo{ + Steps: (*stack.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ {Name: "user_entrypoint", Args: intToBe32(1), Outs: []byte{}}, {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, {Name: "read_args", Args: []byte{}, Outs: []byte{0x00}}, {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, {Name: "write_result", Args: []byte{}, Outs: []byte{}}, {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, - }, + }), }, {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, {Name: "write_result", Args: []byte{}, Outs: []byte{}}, @@ -116,7 +117,7 @@ func TestStylusTracer(t *testing.T) { Args: append(evmMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), Outs: common.Hex2Bytes("0000000000"), Address: &evmMulticall, - Steps: &[]gethexec.HostioTraceInfo{}, + Steps: stack.NewStack[gethexec.HostioTraceInfo](), }, {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, {Name: "write_result", Args: []byte{}, Outs: []byte{}}, diff --git a/util/stack/stack.go b/util/stack/stack.go new file mode 100644 index 00000000000..1b7ac3f9d94 --- /dev/null +++ b/util/stack/stack.go @@ -0,0 +1,39 @@ +// Copyright 2024, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE + +package stack + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/log" +) + +type Stack[T any] []T + +func NewStack[T any]() *Stack[T] { + return &Stack[T]{} +} + +func (s *Stack[T]) Push(v T) { + if s == nil { + log.Warn("trying to push nil stack") + return + } + *s = append(*s, v) +} + +func (s *Stack[T]) Pop() (T, error) { + if s == nil { + var zeroVal T + return zeroVal, fmt.Errorf("trying to pop nil stack") + } + if len(*s) == 0 { + var zeroVal T + return zeroVal, fmt.Errorf("trying to pop empty stack") + } + i := len(*s) - 1 + val := (*s)[i] + *s = (*s)[:i] + return val, nil +} From 316db16dba7826e868f1d72f66fc42fe47a1c34e Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 16 Aug 2024 14:58:52 -0300 Subject: [PATCH 07/95] Add documentation to clearink function --- system_tests/stylus_tracer_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index eaa71b4c4f4..f69bfc42bc0 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -150,6 +150,9 @@ func intToBe32(v int) []byte { return binary.BigEndian.AppendUint32(nil, uint32(v)) } +// clearInk removes the start and end ink values from the trace so we can compare them. +// In Arbitrum, the gas used by the transaction varies depending on the L1 fees, so the trace +// returns different gas values and we can't hardcode them. func clearInk(trace []gethexec.HostioTraceInfo) { for i := range trace { trace[i].StartInk = 0 From 37475c23c0632962bf9546950134a91c41c10249 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 16 Aug 2024 15:11:52 -0300 Subject: [PATCH 08/95] Add documentation to HostIO trace info fields --- execution/gethexec/stylus_tracer.go | 31 ++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index ac80596e8df..eb43bed080e 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -30,13 +30,30 @@ type stylusTracer struct { // HostioTraceInfo contains the captured HostIO log returned by stylusTracer. type HostioTraceInfo struct { - Name string `json:"name"` - Args hexutil.Bytes `json:"args"` - Outs hexutil.Bytes `json:"outs"` - StartInk uint64 `json:"startInk"` - EndInk uint64 `json:"endInk"` - Address *common.Address `json:"address,omitempty"` - Steps *stack.Stack[HostioTraceInfo] `json:"steps,omitempty"` + // Name of the HostIO. + Name string `json:"name"` + + // Arguments of the HostIO encoded as binary. + // For details about the encoding check the HostIO implemenation on + // arbitrator/wasm-libraries/user-host-trait. + Args hexutil.Bytes `json:"args"` + + // Outputs of the HostIO encoded as binary. + // For details about the encoding check the HostIO implemenation on + // arbitrator/wasm-libraries/user-host-trait. + Outs hexutil.Bytes `json:"outs"` + + // Amount of Ink before executing the HostIO. + StartInk uint64 `json:"startInk"` + + // Amount of Ink after executing the HostIO. + EndInk uint64 `json:"endInk"` + + // For *call HostIOs, the address of the called contract. + Address *common.Address `json:"address,omitempty"` + + // For *call HostIOs, the steps performed by the called contract. + Steps *stack.Stack[HostioTraceInfo] `json:"steps,omitempty"` } // nestsHostios contains the hostios with nested calls. From cfe65120c80f5524ba33b18490df3529fcd9383c Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 16 Aug 2024 16:31:13 -0300 Subject: [PATCH 09/95] Handle EVM calls on stylus tracer --- execution/gethexec/stylus_tracer.go | 33 +++++++++ system_tests/stylus_tracer_test.go | 111 ++++++++++++++++++++-------- 2 files changed, 113 insertions(+), 31 deletions(-) diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index eb43bed080e..16c43d71d86 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -5,7 +5,9 @@ package gethexec import ( "encoding/json" + "fmt" "math/big" + "strings" "sync/atomic" "github.com/ethereum/go-ethereum/common" @@ -87,6 +89,14 @@ func (t *stylusTracer) CaptureStylusHostio(name string, args, outs []byte, start t.Stop(err) return } + if !strings.HasPrefix(last.Name, "evm_") || last.Name[4:] != info.Name { + t.Stop(fmt.Errorf("trace inconsistency for %v: last opcode is %v", info.Name, last.Name)) + return + } + if last.Steps == nil { + t.Stop(fmt.Errorf("trace inconsistency for %v: nil steps", info.Name)) + return + } info.Address = last.Address info.Steps = last.Steps } @@ -97,8 +107,28 @@ func (t *stylusTracer) CaptureEnter(typ vm.OpCode, from common.Address, to commo if t.interrupt.Load() { return } + + // This function adds the prefix evm_ because it assumes the opcode came from the EVM. + // If the opcode comes from WASM, the CaptureStylusHostio function will remove the evm prefix. + var name string + switch typ { + case vm.CALL: + name = "evm_call_contract" + case vm.DELEGATECALL: + name = "evm_delegate_call_contract" + case vm.STATICCALL: + name = "evm_static_call_contract" + case vm.CREATE: + name = "evm_create1" + case vm.CREATE2: + name = "evm_create2" + case vm.SELFDESTRUCT: + name = "evm_self_destruct" + } + inner := stack.NewStack[HostioTraceInfo]() info := HostioTraceInfo{ + Name: name, Address: &to, Steps: inner, } @@ -122,6 +152,9 @@ func (t *stylusTracer) GetResult() (json.RawMessage, error) { if t.reason != nil { return nil, t.reason } + if t.open == nil { + return nil, fmt.Errorf("trace is nil") + } msg, err := json.Marshal(t.open) if err != nil { return nil, err diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index f69bfc42bc0..f7c83c73f67 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -64,15 +64,15 @@ func TestStylusTracer(t *testing.T) { contract: stylusMulticall, args: loadStoreArgs, want: []gethexec.HostioTraceInfo{ - {Name: "user_entrypoint", Args: intToBe32(len(loadStoreArgs)), Outs: []byte{}}, - {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, - {Name: "read_args", Args: []byte{}, Outs: loadStoreArgs}, - {Name: "storage_cache_bytes32", Args: append(key.Bytes(), value.Bytes()...), Outs: []byte{}}, - {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, + {Name: "user_entrypoint", Args: intToBe32(len(loadStoreArgs))}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, + {Name: "read_args", Outs: loadStoreArgs}, + {Name: "storage_cache_bytes32", Args: append(key.Bytes(), value.Bytes()...)}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, {Name: "storage_load_bytes32", Args: key.Bytes(), Outs: value.Bytes()}, - {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "write_result", Args: value.Bytes(), Outs: []byte{}}, - {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, + {Name: "write_result", Args: value.Bytes()}, + {Name: "user_returned", Outs: intToBe32(0)}, }, }, @@ -81,26 +81,26 @@ func TestStylusTracer(t *testing.T) { contract: stylusMulticall, args: callArgs, want: []gethexec.HostioTraceInfo{ - {Name: "user_entrypoint", Outs: []byte{}, Args: intToBe32(len(callArgs))}, - {Name: "pay_for_memory_grow", Outs: []byte{}, Args: []byte{0x00, 0x01}}, - {Name: "read_args", Args: []byte{}, Outs: callArgs}, + {Name: "user_entrypoint", Args: intToBe32(len(callArgs))}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, + {Name: "read_args", Outs: callArgs}, { Name: "call_contract", Args: append(stylusMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), Outs: common.Hex2Bytes("0000000000"), Address: &stylusMulticall, Steps: (*stack.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ - {Name: "user_entrypoint", Args: intToBe32(1), Outs: []byte{}}, - {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, - {Name: "read_args", Args: []byte{}, Outs: []byte{0x00}}, - {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "write_result", Args: []byte{}, Outs: []byte{}}, - {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + {Name: "user_entrypoint", Args: intToBe32(1)}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, + {Name: "read_args", Outs: []byte{0x00}}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, + {Name: "write_result"}, + {Name: "user_returned", Outs: intToBe32(0)}, }), }, - {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "write_result", Args: []byte{}, Outs: []byte{}}, - {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, + {Name: "write_result"}, + {Name: "user_returned", Outs: intToBe32(0)}, }, }, @@ -109,9 +109,9 @@ func TestStylusTracer(t *testing.T) { contract: stylusMulticall, args: evmCall, want: []gethexec.HostioTraceInfo{ - {Name: "user_entrypoint", Args: intToBe32(len(callArgs)), Outs: []byte{}}, - {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}, Outs: []byte{}}, - {Name: "read_args", Args: []byte{}, Outs: evmCall}, + {Name: "user_entrypoint", Args: intToBe32(len(callArgs))}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, + {Name: "read_args", Outs: evmCall}, { Name: "call_contract", Args: append(evmMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), @@ -119,9 +119,29 @@ func TestStylusTracer(t *testing.T) { Address: &evmMulticall, Steps: stack.NewStack[gethexec.HostioTraceInfo](), }, - {Name: "storage_flush_cache", Args: []byte{0x00}, Outs: []byte{}}, - {Name: "write_result", Args: []byte{}, Outs: []byte{}}, - {Name: "user_returned", Args: []byte{}, Outs: intToBe32(0)}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, + {Name: "write_result"}, + {Name: "user_returned", Outs: intToBe32(0)}, + }, + }, + + { + name: "evm contract calling wasm", + contract: evmMulticall, + args: callArgs, + want: []gethexec.HostioTraceInfo{ + { + Name: "evm_call_contract", + Address: &stylusMulticall, + Steps: (*stack.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ + {Name: "user_entrypoint", Args: intToBe32(1)}, + {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, + {Name: "read_args", Outs: []byte{0x00}}, + {Name: "storage_flush_cache", Args: []byte{0x00}}, + {Name: "write_result"}, + {Name: "user_returned", Outs: intToBe32(0)}, + }), + }, }, }, } { @@ -132,13 +152,13 @@ func TestStylusTracer(t *testing.T) { Require(t, err, "send transaction") nativeResult := traceTransaction(tx.Hash(), "stylusTracer") - clearInk(nativeResult) + normalizeHostioTrace(nativeResult) if diff := cmp.Diff(testCase.want, nativeResult); diff != "" { Fatal(t, "native tracer don't match wanted result", diff) } jsResult := traceTransaction(tx.Hash(), jsStylusTracer) - clearInk(jsResult) + normalizeHostioTrace(jsResult) if diff := cmp.Diff(jsResult, nativeResult); diff != "" { Fatal(t, "native tracer don't match js trace", diff) } @@ -150,15 +170,21 @@ func intToBe32(v int) []byte { return binary.BigEndian.AppendUint32(nil, uint32(v)) } -// clearInk removes the start and end ink values from the trace so we can compare them. +// normalize removes the start and end ink values from the trace so we can compare them. // In Arbitrum, the gas used by the transaction varies depending on the L1 fees, so the trace // returns different gas values and we can't hardcode them. -func clearInk(trace []gethexec.HostioTraceInfo) { +func normalizeHostioTrace(trace []gethexec.HostioTraceInfo) { for i := range trace { trace[i].StartInk = 0 trace[i].EndInk = 0 + if len(trace[i].Args) == 0 { + trace[i].Args = nil + } + if len(trace[i].Outs) == 0 { + trace[i].Outs = nil + } if trace[i].Steps != nil { - clearInk(*trace[i].Steps) + normalizeHostioTrace(*trace[i].Steps) } } } @@ -170,14 +196,37 @@ var jsStylusTracer = ` info.outs = toHex(info.outs); if (this.nests.includes(info.name)) { Object.assign(info, this.open.pop()); + info.name = info.name.substring(4) // remove evm_ } this.open.push(info); }, "enter": function(frame) { let inner = []; + let name = ""; + switch (frame.getType()) { + case "CALL": + name = "evm_call_contract"; + break; + case "DELEGATECALL": + name = "evm_delegate_call_contract"; + break; + case "STATICCALL": + name = "evm_static_call_contract"; + break; + case "CREATE": + name = "evm_create1"; + break; + case "CREATE2": + name = "evm_create2"; + break; + case "SELFDESTRUCT": + name = "evm_self_destruct"; + break; + } this.open.push({ address: toHex(frame.getTo()), steps: inner, + name: name, }); this.stack.push(this.open); // save where we were this.open = inner; From d14e10d101f33c1aaff40a1d0ada291589c1ac83 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 19 Aug 2024 14:30:58 +0200 Subject: [PATCH 10/95] change Ke/s to entries/s in conversion summary log message --- cmd/dbconv/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dbconv/main.go b/cmd/dbconv/main.go index c0b5c8f8e41..2d61c965523 100644 --- a/cmd/dbconv/main.go +++ b/cmd/dbconv/main.go @@ -85,7 +85,7 @@ func main() { os.Exit(1) } stats := conv.Stats() - log.Info("Conversion finished.", "entries", stats.Entries(), "MB", stats.Bytes()/1024/1024, "avg Ke/s", stats.AverageEntriesPerSecond()/1000, "avg MB/s", stats.AverageBytesPerSecond()/1024/1024, "elapsed", stats.Elapsed()) + log.Info("Conversion finished.", "entries", stats.Entries(), "MB", stats.Bytes()/1024/1024, "avg entries/s", fmt.Sprintf("%.3e", stats.AverageEntriesPerSecond()/1000), "avg MB/s", stats.AverageBytesPerSecond()/1024/1024, "elapsed", stats.Elapsed()) } if config.Compact { From 5c482ffea39b1232d682670ed00b136ad151ff97 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 19 Aug 2024 14:33:06 +0200 Subject: [PATCH 11/95] specify unit of IdealBatchSize in the help message --- cmd/dbconv/dbconv/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dbconv/dbconv/config.go b/cmd/dbconv/dbconv/config.go index 74623bc2649..917f34261df 100644 --- a/cmd/dbconv/dbconv/config.go +++ b/cmd/dbconv/dbconv/config.go @@ -71,7 +71,7 @@ var DefaultDBConvConfig = DBConvConfig{ func DBConvConfigAddOptions(f *flag.FlagSet) { DBConfigAddOptions("src", f, &DefaultDBConvConfig.Src) DBConfigAddOptions("dst", f, &DefaultDBConvConfig.Dst) - f.Int("ideal-batch-size", DefaultDBConvConfig.IdealBatchSize, "ideal write batch size") + f.Int("ideal-batch-size", DefaultDBConvConfig.IdealBatchSize, "ideal write batch size in bytes") f.Bool("convert", DefaultDBConvConfig.Convert, "enables conversion step") f.Bool("compact", DefaultDBConvConfig.Compact, "enables compaction step") f.String("verify", DefaultDBConvConfig.Verify, "enables verification step (\"\" = disabled, \"keys\" = only keys, \"full\" = keys and values)") From b5a224b26cbab8830f72c7ec88220cc181dd9554 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 19 Aug 2024 14:34:09 +0200 Subject: [PATCH 12/95] don't enable trace logging in TestConversion --- cmd/dbconv/dbconv/dbconv_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/dbconv/dbconv/dbconv_test.go b/cmd/dbconv/dbconv/dbconv_test.go index f31dd68618a..31aa0c39176 100644 --- a/cmd/dbconv/dbconv/dbconv_test.go +++ b/cmd/dbconv/dbconv/dbconv_test.go @@ -4,12 +4,10 @@ import ( "context" "testing" - "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/util/testhelpers" ) func TestConversion(t *testing.T) { - _ = testhelpers.InitTestLog(t, log.LvlTrace) oldDBConfig := DBConfigDefaultSrc oldDBConfig.Data = t.TempDir() From 11d72a3bc9362a1ef9cca9af067350e2bea1a9f4 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 19 Aug 2024 14:49:48 +0200 Subject: [PATCH 13/95] convert_databases.bash: remove unneeded tr command use for srcdir --- scripts/convert-databases.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/convert-databases.bash b/scripts/convert-databases.bash index bd898c2c987..3020b389b46 100755 --- a/scripts/convert-databases.bash +++ b/scripts/convert-databases.bash @@ -182,7 +182,7 @@ fi convert_result= convert () { - srcdir=$(echo $src/$1 | tr -s /) + srcdir="$src"/$1 dstdir=$(echo $dst/$1 | tr -s /) if ! [ -e $dstdir ]; then echo "== Converting $1 db" From cfb4c8736bd7885c12baf01830d7b7f3bf835a4c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Wed, 21 Aug 2024 11:48:50 -0300 Subject: [PATCH 14/95] Ignores gosec G115 --- .golangci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.golangci.yml b/.golangci.yml index 05946701379..e788eca6c0a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -44,6 +44,7 @@ linters-settings: gosec: excludes: - G404 # checks that random numbers are securely generated + - G115 # Potential integer overflow when converting between integer types govet: enable-all: true From dc976f8b113dbae3bdb51133215147e862eecc2a Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 21 Aug 2024 12:16:24 -0500 Subject: [PATCH 15/95] Re-enable G115 lint --- .golangci.yml | 1 - arbcompress/compress_common.go | 4 +++- arbnode/batch_poster.go | 20 +++++++++++++------- arbnode/dataposter/data_poster.go | 7 +++++++ arbnode/dataposter/dbstorage/storage.go | 2 +- arbnode/dataposter/slice/slicestorage.go | 4 ++-- arbnode/dataposter/storage/time.go | 2 ++ arbnode/dataposter/storage_test.go | 1 + arbnode/inbox_reader.go | 4 ++-- arbnode/inbox_tracker.go | 3 +++ arbnode/node.go | 2 ++ arbnode/transaction_streamer.go | 1 + arbos/addressSet/addressSet_test.go | 1 + arbos/addressTable/addressTable.go | 1 + arbos/arbosState/initialization_test.go | 1 + arbos/l1pricing/l1pricing.go | 4 ++-- arbos/l1pricing_test.go | 6 +++--- arbos/l2pricing/l2pricing_test.go | 3 +++ arbos/l2pricing/model.go | 12 ++++++++---- arbos/storage/storage.go | 8 +++----- arbstate/inbox.go | 6 +++--- arbutil/block_message_relation.go | 1 + arbutil/correspondingl1blocknumber.go | 1 + blocks_reexecutor/blocks_reexecutor.go | 3 ++- broadcaster/backlog/backlog.go | 9 +++++++-- broadcaster/backlog/backlog_test.go | 4 ++-- broadcaster/broadcaster.go | 1 + cmd/nitro/nitro.go | 3 ++- cmd/staterecovery/staterecovery.go | 1 + das/aggregator.go | 1 + das/dasRpcClient.go | 1 + das/dasRpcServer.go | 2 ++ das/dastree/dastree.go | 15 +++++++++------ das/db_storage_service.go | 4 +++- das/local_file_storage_service.go | 6 ++++++ das/local_file_storage_service_test.go | 1 + das/s3_storage_service.go | 4 +++- das/sign_after_store_das_writer.go | 1 + das/simple_das_reader_aggregator.go | 12 ++++++------ das/util.go | 2 ++ execution/gethexec/api.go | 1 + execution/gethexec/sequencer.go | 5 +++++ execution/gethexec/tx_pre_checker.go | 1 + execution/nodeInterface/NodeInterface.go | 7 +++++++ precompiles/ArbAddressTable.go | 8 ++++---- precompiles/ArbRetryableTx.go | 7 ++++--- precompiles/ArbSys.go | 4 ++-- relay/relay_stress_test.go | 2 +- staker/block_challenge_backend.go | 2 +- staker/block_validator.go | 7 +++++++ staker/challenge-cache/cache.go | 4 ++-- staker/challenge_manager.go | 2 +- staker/challenge_test.go | 2 +- staker/l1_validator.go | 2 ++ staker/rollup_watcher.go | 2 +- staker/staker.go | 4 ++++ system_tests/block_validator_test.go | 1 + system_tests/forwarder_test.go | 1 + system_tests/initialization_test.go | 1 + system_tests/outbox_test.go | 1 + system_tests/program_recursive_test.go | 1 + system_tests/program_test.go | 5 +++++ system_tests/recreatestate_rpc_test.go | 7 +++++++ system_tests/seq_nonce_test.go | 1 + system_tests/seqinbox_test.go | 6 +++++- system_tests/snap_sync_test.go | 2 ++ system_tests/twonodeslong_test.go | 1 + system_tests/unsupported_txtypes_test.go | 4 ++-- util/arbmath/bips.go | 4 ++-- util/arbmath/math.go | 14 ++++++++++++++ util/arbmath/math_test.go | 3 +++ util/arbmath/uint24.go | 8 +++++--- util/headerreader/header_reader.go | 1 + util/merkletree/merkleTree.go | 4 ++-- util/rpcclient/rpcclient.go | 16 +++++++++------- util/sharedmetrics/sharedmetrics.go | 2 ++ util/testhelpers/testhelpers.go | 1 + validator/client/validation_client.go | 1 + validator/server_arb/execution_run_test.go | 2 +- validator/server_arb/machine_cache.go | 1 + validator/server_arb/validator_spawner.go | 3 +++ validator/server_jit/jit_machine.go | 15 ++++++++++++--- wavmio/stub.go | 6 +++--- wsbroadcastserver/clientconnection.go | 1 + 84 files changed, 247 insertions(+), 91 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index e788eca6c0a..05946701379 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -44,7 +44,6 @@ linters-settings: gosec: excludes: - G404 # checks that random numbers are securely generated - - G115 # Potential integer overflow when converting between integer types govet: enable-all: true diff --git a/arbcompress/compress_common.go b/arbcompress/compress_common.go index a61dd9a171c..997232e7cc8 100644 --- a/arbcompress/compress_common.go +++ b/arbcompress/compress_common.go @@ -17,6 +17,8 @@ func compressedBufferSizeFor(length int) int { return length + (length>>10)*8 + 64 // actual limit is: length + (length >> 14) * 4 + 6 } -func CompressLevel(input []byte, level int) ([]byte, error) { +func CompressLevel(input []byte, level uint64) ([]byte, error) { + // level is trusted and shouldn't be anything crazy + // #nosec G115 return Compress(input, uint32(level), EmptyDictionary) } diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 71239efdbbc..d0764cc783c 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -121,7 +121,7 @@ type BatchPoster struct { nextRevertCheckBlock int64 // the last parent block scanned for reverting batches postedFirstBatch bool // indicates if batch poster has posted the first batch - accessList func(SequencerInboxAccs, AfterDelayedMessagesRead int) types.AccessList + accessList func(SequencerInboxAccs, AfterDelayedMessagesRead uint64) types.AccessList } type l1BlockBound int @@ -374,7 +374,7 @@ func NewBatchPoster(ctx context.Context, opts *BatchPosterOpts) (*BatchPoster, e } // Dataposter sender may be external signer address, so we should initialize // access list after initializing dataposter. - b.accessList = func(SequencerInboxAccs, AfterDelayedMessagesRead int) types.AccessList { + b.accessList = func(SequencerInboxAccs, AfterDelayedMessagesRead uint64) types.AccessList { if !b.config().UseAccessLists || opts.L1Reader.IsParentChainArbitrum() { // Access lists cost gas instead of saving gas when posting to L2s, // because data is expensive in comparison to computation. @@ -433,8 +433,8 @@ type AccessListOpts struct { BridgeAddr common.Address DataPosterAddr common.Address GasRefunderAddr common.Address - SequencerInboxAccs int - AfterDelayedMessagesRead int + SequencerInboxAccs uint64 + AfterDelayedMessagesRead uint64 } // AccessList returns access list (contracts, storage slots) for batchposter. @@ -476,12 +476,12 @@ func AccessList(opts *AccessListOpts) types.AccessList { }, } - for _, v := range []struct{ slotIdx, val int }{ + for _, v := range []struct{ slotIdx, val uint64 }{ {7, opts.SequencerInboxAccs - 1}, // - sequencerInboxAccs[sequencerInboxAccs.length - 1]; (keccak256(7, sequencerInboxAccs.length - 1)) {7, opts.SequencerInboxAccs}, // - sequencerInboxAccs.push(...); (keccak256(7, sequencerInboxAccs.length)) {6, opts.AfterDelayedMessagesRead - 1}, // - delayedInboxAccs[afterDelayedMessagesRead - 1]; (keccak256(6, afterDelayedMessagesRead - 1)) } { - sb := arbutil.SumBytes(arbutil.PaddedKeccak256([]byte{byte(v.slotIdx)}), big.NewInt(int64(v.val)).Bytes()) + sb := arbutil.SumBytes(arbutil.PaddedKeccak256([]byte{byte(v.slotIdx)}), new(big.Int).SetUint64(v.val).Bytes()) l[1].StorageKeys = append(l[1].StorageKeys, common.Hash(sb)) } @@ -603,9 +603,12 @@ func (b *BatchPoster) pollForL1PriceData(ctx context.Context) { l1GasPrice = blobFeePerByte.Uint64() / 16 } } + // #nosec G115 blobGasUsedGauge.Update(int64(*h.BlobGasUsed)) } + // #nosec G115 blockGasUsedGauge.Update(int64(h.GasUsed)) + // #nosec G115 blockGasLimitGauge.Update(int64(h.GasLimit)) suggestedTipCap, err := b.l1Reader.Client().SuggestGasTipCap(ctx) if err != nil { @@ -613,6 +616,7 @@ func (b *BatchPoster) pollForL1PriceData(ctx context.Context) { } else { suggestedTipCapGauge.Update(suggestedTipCap.Int64()) } + // #nosec G115 l1GasPriceGauge.Update(int64(l1GasPrice)) case <-ctx.Done(): return @@ -1176,6 +1180,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) if err != nil { return false, err } + // #nosec G115 firstMsgTime := time.Unix(int64(firstMsg.Message.Header.Timestamp), 0) lastPotentialMsg, err := b.streamer.GetMessage(msgCount - 1) @@ -1403,7 +1408,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) if len(kzgBlobs)*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { return false, fmt.Errorf("produced %v blobs for batch but a block can only hold %v (compressed batch was %v bytes long)", len(kzgBlobs), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob, len(sequencerMsg)) } - accessList := b.accessList(int(batchPosition.NextSeqNum), int(b.building.segments.delayedMsg)) + accessList := b.accessList(batchPosition.NextSeqNum, b.building.segments.delayedMsg) // On restart, we may be trying to estimate gas for a batch whose successor has // already made it into pending state, if not latest state. // In that case, we might get a revert with `DelayedBackwards()`. @@ -1505,6 +1510,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) messagesPerBatch = 1 } backlog := uint64(unpostedMessages) / messagesPerBatch + // #nosec G115 batchPosterEstimatedBatchBacklogGauge.Update(int64(backlog)) if backlog > 10 { logLevel := log.Warn diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index 5630a52947f..6a483929b26 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -359,6 +359,7 @@ func (p *DataPoster) canPostWithNonce(ctx context.Context, nextNonce uint64, thi if err != nil { return fmt.Errorf("getting nonce of a dataposter sender: %w", err) } + // #nosec G115 latestUnconfirmedNonceGauge.Update(int64(unconfirmedNonce)) if nextNonce >= cfg.MaxMempoolTransactions+unconfirmedNonce { return fmt.Errorf("%w: transaction nonce: %d, unconfirmed nonce: %d, max mempool size: %d", ErrExceedsMaxMempoolSize, nextNonce, unconfirmedNonce, cfg.MaxMempoolTransactions) @@ -371,6 +372,7 @@ func (p *DataPoster) canPostWithNonce(ctx context.Context, nextNonce uint64, thi if err != nil { return fmt.Errorf("getting nonce of a dataposter sender: %w", err) } + // #nosec G115 latestUnconfirmedNonceGauge.Update(int64(unconfirmedNonce)) if unconfirmedNonce > nextNonce { return fmt.Errorf("latest on-chain nonce %v is greater than to next nonce %v", unconfirmedNonce, nextNonce) @@ -525,6 +527,7 @@ func (p *DataPoster) feeAndTipCaps(ctx context.Context, nonce uint64, gasLimit u if err != nil { return nil, nil, nil, fmt.Errorf("failed to get latest nonce %v blocks ago (block %v): %w", config.NonceRbfSoftConfs, softConfBlock, err) } + // #nosec G115 latestSoftConfirmedNonceGauge.Update(int64(softConfNonce)) suggestedTip, err := p.client.SuggestGasTipCap(ctx) @@ -1052,6 +1055,7 @@ func (p *DataPoster) updateNonce(ctx context.Context) error { } return nil } + // #nosec G115 latestFinalizedNonceGauge.Update(int64(nonce)) log.Info("Data poster transactions confirmed", "previousNonce", p.nonce, "newNonce", nonce, "previousL1Block", p.lastBlock, "newL1Block", header.Number) if len(p.errorCount) > 0 { @@ -1132,6 +1136,7 @@ func (p *DataPoster) Start(ctxIn context.Context) { log.Warn("Failed to get latest nonce", "err", err) return minWait } + // #nosec G115 latestUnconfirmedNonceGauge.Update(int64(unconfirmedNonce)) // We use unconfirmedNonce here to replace-by-fee transactions that aren't in a block, // excluding those that are in an unconfirmed block. If a reorg occurs, we'll continue @@ -1154,7 +1159,9 @@ func (p *DataPoster) Start(ctxIn context.Context) { confirmedNonce := unconfirmedNonce - 1 confirmedMeta, err := p.queue.Get(ctx, confirmedNonce) if err == nil && confirmedMeta != nil { + // #nosec G115 totalQueueWeightGauge.Update(int64(arbmath.SaturatingUSub(latestCumulativeWeight, confirmedMeta.CumulativeWeight()))) + // #nosec G115 totalQueueLengthGauge.Update(int64(arbmath.SaturatingUSub(latestNonce, confirmedNonce))) } else { log.Error("Failed to fetch latest confirmed tx from queue", "confirmedNonce", confirmedNonce, "err", err, "confirmedMeta", confirmedMeta) diff --git a/arbnode/dataposter/dbstorage/storage.go b/arbnode/dataposter/dbstorage/storage.go index 97055193a64..37ebfa50991 100644 --- a/arbnode/dataposter/dbstorage/storage.go +++ b/arbnode/dataposter/dbstorage/storage.go @@ -42,7 +42,7 @@ func (s *Storage) FetchContents(_ context.Context, startingIndex uint64, maxResu var res []*storage.QueuedTransaction it := s.db.NewIterator([]byte(""), idxToKey(startingIndex)) defer it.Release() - for i := 0; i < int(maxResults); i++ { + for i := uint64(0); i < maxResults; i++ { if !it.Next() { break } diff --git a/arbnode/dataposter/slice/slicestorage.go b/arbnode/dataposter/slice/slicestorage.go index 69de7564a32..8685ed6f54b 100644 --- a/arbnode/dataposter/slice/slicestorage.go +++ b/arbnode/dataposter/slice/slicestorage.go @@ -89,8 +89,8 @@ func (s *Storage) Put(_ context.Context, index uint64, prev, new *storage.Queued } s.queue = append(s.queue, newEnc) } else if index >= s.firstNonce { - queueIdx := int(index - s.firstNonce) - if queueIdx > len(s.queue) { + queueIdx := index - s.firstNonce + if queueIdx > uint64(len(s.queue)) { return fmt.Errorf("attempted to set out-of-bounds index %v in queue starting at %v of length %v", index, s.firstNonce, len(s.queue)) } prevEnc, err := s.encDec().Encode(prev) diff --git a/arbnode/dataposter/storage/time.go b/arbnode/dataposter/storage/time.go index aa15f291702..82f8a3dbf55 100644 --- a/arbnode/dataposter/storage/time.go +++ b/arbnode/dataposter/storage/time.go @@ -34,11 +34,13 @@ func (b *RlpTime) DecodeRLP(s *rlp.Stream) error { if err != nil { return err } + // #nosec G115 *b = RlpTime(time.Unix(int64(enc.Seconds), int64(enc.Nanos))) return nil } func (b RlpTime) EncodeRLP(w io.Writer) error { + // #nosec G115 return rlp.Encode(w, rlpTimeEncoding{ Seconds: uint64(time.Time(b).Unix()), Nanos: uint64(time.Time(b).Nanosecond()), diff --git a/arbnode/dataposter/storage_test.go b/arbnode/dataposter/storage_test.go index e2aa321e0df..8934d92b456 100644 --- a/arbnode/dataposter/storage_test.go +++ b/arbnode/dataposter/storage_test.go @@ -362,6 +362,7 @@ func TestLength(t *testing.T) { if err != nil { t.Fatalf("Length() unexpected error: %v", err) } + // #nosec G115 if want := arbmath.MaxInt(0, 20-int(tc.pruneFrom)); got != want { t.Errorf("Length() = %d want %d", got, want) } diff --git a/arbnode/inbox_reader.go b/arbnode/inbox_reader.go index 77a0b6e7a29..fd050b5f67b 100644 --- a/arbnode/inbox_reader.go +++ b/arbnode/inbox_reader.go @@ -437,8 +437,8 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } delayedMessages, err := r.delayedBridge.LookupMessagesInRange(ctx, from, to, func(batchNum uint64) ([]byte, error) { if len(sequencerBatches) > 0 && batchNum >= sequencerBatches[0].SequenceNumber { - idx := int(batchNum - sequencerBatches[0].SequenceNumber) - if idx < len(sequencerBatches) { + idx := batchNum - sequencerBatches[0].SequenceNumber + if idx < uint64(len(sequencerBatches)) { return sequencerBatches[idx].Serialize(ctx, r.l1Reader.Client()) } log.Warn("missing mentioned batch in L1 message lookup", "batch", batchNum) diff --git a/arbnode/inbox_tracker.go b/arbnode/inbox_tracker.go index 23b81bde627..fe4149c80eb 100644 --- a/arbnode/inbox_tracker.go +++ b/arbnode/inbox_tracker.go @@ -804,6 +804,7 @@ func (t *InboxTracker) AddSequencerBatches(ctx context.Context, client arbutil.L if len(messages) > 0 { latestTimestamp = messages[len(messages)-1].Message.Header.Timestamp } + // #nosec G115 log.Info( "InboxTracker", "sequencerBatchCount", pos, @@ -811,7 +812,9 @@ func (t *InboxTracker) AddSequencerBatches(ctx context.Context, client arbutil.L "l1Block", latestL1Block, "l1Timestamp", time.Unix(int64(latestTimestamp), 0), ) + // #nosec G115 inboxLatestBatchGauge.Update(int64(pos)) + // #nosec G115 inboxLatestBatchMessageGauge.Update(int64(newMessageCount)) if t.validator != nil { diff --git a/arbnode/node.go b/arbnode/node.go index c66598618f9..93b58e800f9 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -515,6 +515,7 @@ func createNodeImpl( if err != nil { return nil, err } + // #nosec G115 sequencerInbox, err := NewSequencerInbox(l1client, deployInfo.SequencerInbox, int64(deployInfo.DeployedAt)) if err != nil { return nil, err @@ -639,6 +640,7 @@ func createNodeImpl( tmpAddress := common.HexToAddress(config.Staker.ContractWalletAddress) existingWalletAddress = &tmpAddress } + // #nosec G115 wallet, err = validatorwallet.NewContract(dp, existingWalletAddress, deployInfo.ValidatorWalletCreator, deployInfo.Rollup, l1Reader, txOptsValidator, int64(deployInfo.DeployedAt), func(common.Address) {}, getExtraGas) if err != nil { return nil, err diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index 90e7feddc65..a5bab8342f9 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -840,6 +840,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil // Active broadcast reorg and L1 messages at or before start of broadcast messages // Or no active broadcast reorg and broadcast messages start before or immediately after last L1 message if messagesAfterPos >= broadcastStartPos { + // #nosec G115 broadcastSliceIndex := int(messagesAfterPos - broadcastStartPos) messagesOldLen := len(messages) if broadcastSliceIndex < len(s.broadcasterQueuedMessages) { diff --git a/arbos/addressSet/addressSet_test.go b/arbos/addressSet/addressSet_test.go index 7d06c74f0bb..d32e07a546d 100644 --- a/arbos/addressSet/addressSet_test.go +++ b/arbos/addressSet/addressSet_test.go @@ -316,6 +316,7 @@ func checkIfRectifyMappingWorks(t *testing.T, aset *AddressSet, owners []common. Fail(t, "RectifyMapping did not fix the mismatch") } + // #nosec G115 if clearList && int(size(t, aset)) != index+1 { Fail(t, "RectifyMapping did not fix the mismatch") } diff --git a/arbos/addressTable/addressTable.go b/arbos/addressTable/addressTable.go index 3fbb7b3782e..566c71b6895 100644 --- a/arbos/addressTable/addressTable.go +++ b/arbos/addressTable/addressTable.go @@ -118,6 +118,7 @@ func (atab *AddressTable) Decompress(buf []byte) (common.Address, uint64, error) if !exists { return common.Address{}, 0, errors.New("invalid index in compressed address") } + // #nosec G115 numBytesRead := uint64(rd.Size() - int64(rd.Len())) return addr, numBytesRead, nil } diff --git a/arbos/arbosState/initialization_test.go b/arbos/arbosState/initialization_test.go index 34802392fe7..b0fe1d0dac4 100644 --- a/arbos/arbosState/initialization_test.go +++ b/arbos/arbosState/initialization_test.go @@ -109,6 +109,7 @@ func pseudorandomAccountInitInfoForTesting(prand *testhelpers.PseudoRandomDataSo } func pseudorandomHashHashMapForTesting(prand *testhelpers.PseudoRandomDataSource, maxItems uint64) map[common.Hash]common.Hash { + // #nosec G115 size := int(prand.GetUint64() % maxItems) ret := make(map[common.Hash]common.Hash) for i := 0; i < size; i++ { diff --git a/arbos/l1pricing/l1pricing.go b/arbos/l1pricing/l1pricing.go index 9e00eeb581a..392bf36d377 100644 --- a/arbos/l1pricing/l1pricing.go +++ b/arbos/l1pricing/l1pricing.go @@ -509,7 +509,7 @@ func (ps *L1PricingState) getPosterUnitsWithoutCache(tx *types.Transaction, post return 0 } - l1Bytes, err := byteCountAfterBrotliLevel(txBytes, int(brotliCompressionLevel)) + l1Bytes, err := byteCountAfterBrotliLevel(txBytes, brotliCompressionLevel) if err != nil { panic(fmt.Sprintf("failed to compress tx: %v", err)) } @@ -594,7 +594,7 @@ func (ps *L1PricingState) PosterDataCost(message *core.Message, poster common.Ad return am.BigMulByUint(pricePerUnit, units), units } -func byteCountAfterBrotliLevel(input []byte, level int) (uint64, error) { +func byteCountAfterBrotliLevel(input []byte, level uint64) (uint64, error) { compressed, err := arbcompress.CompressLevel(input, level) if err != nil { return 0, err diff --git a/arbos/l1pricing_test.go b/arbos/l1pricing_test.go index 6e2b1b7eec3..1cda4b3d82b 100644 --- a/arbos/l1pricing_test.go +++ b/arbos/l1pricing_test.go @@ -100,7 +100,7 @@ func expectedResultsForL1Test(input *l1PricingTest) *l1TestExpectedResults { availableFunds = availableFundsCap } } - fundsWantedForRewards := big.NewInt(int64(input.unitReward * input.unitsPerSecond)) + fundsWantedForRewards := new(big.Int).SetUint64(input.unitReward * input.unitsPerSecond) unitsAllocated := arbmath.UintToBig(input.unitsPerSecond) if arbmath.BigLessThan(availableFunds, fundsWantedForRewards) { ret.rewardRecipientBalance = availableFunds @@ -111,7 +111,7 @@ func expectedResultsForL1Test(input *l1PricingTest) *l1TestExpectedResults { uncappedAvailableFunds = arbmath.BigSub(uncappedAvailableFunds, ret.rewardRecipientBalance) ret.unitsRemaining = (3 * input.unitsPerSecond) - unitsAllocated.Uint64() - maxCollectable := big.NewInt(int64(input.fundsSpent)) + maxCollectable := new(big.Int).SetUint64(input.fundsSpent) if arbmath.BigLessThan(availableFunds, maxCollectable) { maxCollectable = availableFunds } @@ -170,7 +170,7 @@ func _testL1PricingFundsDue(t *testing.T, testParams *l1PricingTest, expectedRes Require(t, err) // create some fake collection - balanceAdded := big.NewInt(int64(testParams.fundsCollectedPerSecond * 3)) + balanceAdded := new(big.Int).SetUint64(testParams.fundsCollectedPerSecond * 3) unitsAdded := testParams.unitsPerSecond * 3 evm.StateDB.AddBalance(l1pricing.L1PricerFundsPoolAddress, uint256.MustFromBig(balanceAdded)) err = l1p.SetL1FeesAvailable(balanceAdded) diff --git a/arbos/l2pricing/l2pricing_test.go b/arbos/l2pricing/l2pricing_test.go index 57759d7f829..aa1e785f70a 100644 --- a/arbos/l2pricing/l2pricing_test.go +++ b/arbos/l2pricing/l2pricing_test.go @@ -40,6 +40,7 @@ func TestPricingModelExp(t *testing.T) { // show that running at the speed limit with a full pool is a steady-state colors.PrintBlue("full pool & speed limit") for seconds := 0; seconds < 4; seconds++ { + // #nosec G115 fakeBlockUpdate(t, pricing, int64(seconds)*int64(limit), uint64(seconds)) if getPrice(t, pricing) != minPrice { Fail(t, "price changed when it shouldn't have") @@ -50,6 +51,7 @@ func TestPricingModelExp(t *testing.T) { // note that for large enough spans of time the price will rise a miniscule amount due to the pool's avg colors.PrintBlue("pool target & speed limit") for seconds := 0; seconds < 4; seconds++ { + // #nosec G115 fakeBlockUpdate(t, pricing, int64(seconds)*int64(limit), uint64(seconds)) if getPrice(t, pricing) != minPrice { Fail(t, "price changed when it shouldn't have") @@ -59,6 +61,7 @@ func TestPricingModelExp(t *testing.T) { // show that running over the speed limit escalates the price before the pool drains colors.PrintBlue("exceeding the speed limit") for { + // #nosec G115 fakeBlockUpdate(t, pricing, 8*int64(limit), 1) newPrice := getPrice(t, pricing) if newPrice < price { diff --git a/arbos/l2pricing/model.go b/arbos/l2pricing/model.go index 131af2c2cf6..476effa8aa8 100644 --- a/arbos/l2pricing/model.go +++ b/arbos/l2pricing/model.go @@ -30,22 +30,26 @@ func (ps *L2PricingState) AddToGasPool(gas int64) error { return err } // pay off some of the backlog with the added gas, stopping at 0 - backlog = arbmath.SaturatingUCast[uint64](arbmath.SaturatingSub(int64(backlog), gas)) + if gas > 0 { + backlog = arbmath.SaturatingUSub(backlog, uint64(gas)) + } else { + backlog = arbmath.SaturatingUAdd(backlog, uint64(-gas)) + } return ps.SetGasBacklog(backlog) } // UpdatePricingModel updates the pricing model with info from the last block func (ps *L2PricingState) UpdatePricingModel(l2BaseFee *big.Int, timePassed uint64, debug bool) { speedLimit, _ := ps.SpeedLimitPerSecond() - _ = ps.AddToGasPool(int64(timePassed * speedLimit)) + _ = ps.AddToGasPool(arbmath.SaturatingCast[int64](arbmath.SaturatingUMul(timePassed, speedLimit))) inertia, _ := ps.PricingInertia() tolerance, _ := ps.BacklogTolerance() backlog, _ := ps.GasBacklog() minBaseFee, _ := ps.MinBaseFeeWei() baseFee := minBaseFee if backlog > tolerance*speedLimit { - excess := int64(backlog - tolerance*speedLimit) - exponentBips := arbmath.NaturalToBips(excess) / arbmath.Bips(inertia*speedLimit) + excess := arbmath.SaturatingCast[int64](backlog - tolerance*speedLimit) + exponentBips := arbmath.NaturalToBips(excess) / arbmath.SaturatingCast[arbmath.Bips](inertia*speedLimit) baseFee = arbmath.BigMulByBips(minBaseFee, arbmath.ApproxExpBasisPoints(exponentBips, 4)) } _ = ps.SetBaseFeeWei(baseFee) diff --git a/arbos/storage/storage.go b/arbos/storage/storage.go index 6e6c976644e..352726778d0 100644 --- a/arbos/storage/storage.go +++ b/arbos/storage/storage.go @@ -156,11 +156,6 @@ func (s *Storage) GetUint64ByUint64(key uint64) (uint64, error) { return s.GetUint64(util.UintToHash(key)) } -func (s *Storage) GetUint32(key common.Hash) (uint32, error) { - value, err := s.Get(key) - return uint32(value.Big().Uint64()), err -} - func (s *Storage) Set(key common.Hash, value common.Hash) error { if s.burner.ReadOnly() { log.Error("Read-only burner attempted to mutate state", "key", key, "value", value) @@ -420,6 +415,7 @@ func (sbu *StorageBackedInt64) Get() (int64, error) { if !raw.Big().IsUint64() { panic("invalid value found in StorageBackedInt64 storage") } + // #nosec G115 return int64(raw.Big().Uint64()), err // see implementation note above } @@ -477,6 +473,7 @@ func (sbu *StorageBackedUint16) Get() (uint16, error) { if !big.IsUint64() || big.Uint64() > math.MaxUint16 { panic("expected uint16 compatible value in storage") } + // #nosec G115 return uint16(big.Uint64()), err } @@ -517,6 +514,7 @@ func (sbu *StorageBackedUint32) Get() (uint32, error) { if !big.IsUint64() || big.Uint64() > math.MaxUint32 { panic("expected uint32 compatible value in storage") } + // #nosec G115 return uint32(big.Uint64()), err } diff --git a/arbstate/inbox.go b/arbstate/inbox.go index 753ca19cd6c..b58a7420b75 100644 --- a/arbstate/inbox.go +++ b/arbstate/inbox.go @@ -246,7 +246,7 @@ func (r *inboxMultiplexer) IsCachedSegementLast() bool { if r.delayedMessagesRead < seqMsg.afterDelayedMessages { return false } - for segmentNum := int(r.cachedSegmentNum) + 1; segmentNum < len(seqMsg.segments); segmentNum++ { + for segmentNum := r.cachedSegmentNum + 1; segmentNum < uint64(len(seqMsg.segments)); segmentNum++ { segment := seqMsg.segments[segmentNum] if len(segment) == 0 { continue @@ -276,7 +276,7 @@ func (r *inboxMultiplexer) getNextMsg() (*arbostypes.MessageWithMetadata, error) if segmentNum >= uint64(len(seqMsg.segments)) { break } - segment = seqMsg.segments[int(segmentNum)] + segment = seqMsg.segments[segmentNum] if len(segment) == 0 { segmentNum++ continue @@ -322,7 +322,7 @@ func (r *inboxMultiplexer) getNextMsg() (*arbostypes.MessageWithMetadata, error) log.Warn("reading virtual delayed message segment", "delayedMessagesRead", r.delayedMessagesRead, "afterDelayedMessages", seqMsg.afterDelayedMessages) segment = []byte{BatchSegmentKindDelayedMessages} } else { - segment = seqMsg.segments[int(segmentNum)] + segment = seqMsg.segments[segmentNum] } if len(segment) == 0 { log.Error("empty sequencer message segment", "sequence", r.cachedSegmentNum, "segmentNum", segmentNum) diff --git a/arbutil/block_message_relation.go b/arbutil/block_message_relation.go index a69f9079ee1..e164cf2619a 100644 --- a/arbutil/block_message_relation.go +++ b/arbutil/block_message_relation.go @@ -15,5 +15,6 @@ func SignedBlockNumberToMessageCount(blockNumber int64, genesisBlockNumber uint6 } func MessageCountToBlockNumber(messageCount MessageIndex, genesisBlockNumber uint64) int64 { + // #nosec G115 return int64(uint64(messageCount)+genesisBlockNumber) - 1 } diff --git a/arbutil/correspondingl1blocknumber.go b/arbutil/correspondingl1blocknumber.go index 05323ed1832..d654e471e25 100644 --- a/arbutil/correspondingl1blocknumber.go +++ b/arbutil/correspondingl1blocknumber.go @@ -20,6 +20,7 @@ func ParentHeaderToL1BlockNumber(header *types.Header) uint64 { } func CorrespondingL1BlockNumber(ctx context.Context, client L1Interface, parentBlockNumber uint64) (uint64, error) { + // #nosec G115 header, err := client.HeaderByNumber(ctx, big.NewInt(int64(parentBlockNumber))) if err != nil { return 0, fmt.Errorf("error getting L1 block number %d header : %w", parentBlockNumber, err) diff --git a/blocks_reexecutor/blocks_reexecutor.go b/blocks_reexecutor/blocks_reexecutor.go index 1e4a06fe904..f7cc0d8c720 100644 --- a/blocks_reexecutor/blocks_reexecutor.go +++ b/blocks_reexecutor/blocks_reexecutor.go @@ -102,7 +102,8 @@ func New(c *Config, blockchain *core.BlockChain, fatalErrChan chan error) *Block if rng > end-start { rng = end - start } - start += uint64(rand.Intn(int(end - start - rng + 1))) + // #nosec G115 + start += uint64(rand.Int63n(int64(end - start - rng + 1))) end = start + rng } // Inclusive of block reexecution [start, end] diff --git a/broadcaster/backlog/backlog.go b/broadcaster/backlog/backlog.go index f6501105c2e..b7b935fb7a2 100644 --- a/broadcaster/backlog/backlog.go +++ b/broadcaster/backlog/backlog.go @@ -97,6 +97,7 @@ func (b *backlog) Append(bm *m.BroadcastMessage) error { if err != nil { log.Warn("error calculating backlogSizeInBytes", "err", err) } else { + // #nosec G115 backlogSizeInBytesGauge.Update(int64(size)) } } @@ -108,6 +109,7 @@ func (b *backlog) Append(bm *m.BroadcastMessage) error { segment = newBacklogSegment() b.head.Store(segment) b.tail.Store(segment) + // #nosec G115 confirmedSequenceNumberGauge.Update(int64(msg.SequenceNumber)) } @@ -143,9 +145,11 @@ func (b *backlog) Append(bm *m.BroadcastMessage) error { } lookupByIndex.Store(uint64(msg.SequenceNumber), segment) b.messageCount.Add(1) + // #nosec G115 backlogSizeInBytesGauge.Inc(int64(msg.Size())) } + // #nosec G115 backlogSizeGauge.Update(int64(b.Count())) return nil } @@ -174,7 +178,7 @@ func (b *backlog) Get(start, end uint64) (*m.BroadcastMessage, error) { } bm := &m.BroadcastMessage{Version: 1} - required := int(end-start) + 1 + required := end - start + 1 for { segMsgs, err := segment.Get(arbmath.MaxInt(start, segment.Start()), arbmath.MinInt(end, segment.End())) if err != nil { @@ -183,7 +187,7 @@ func (b *backlog) Get(start, end uint64) (*m.BroadcastMessage, error) { bm.Messages = append(bm.Messages, segMsgs...) segment = segment.Next() - if len(bm.Messages) == required { + if uint64(len(bm.Messages)) == required { break } else if segment == nil { return nil, errOutOfBounds @@ -213,6 +217,7 @@ func (b *backlog) delete(confirmed uint64) { return } + // #nosec G115 confirmedSequenceNumberGauge.Update(int64(confirmed)) // find the segment containing the confirmed message diff --git a/broadcaster/backlog/backlog_test.go b/broadcaster/backlog/backlog_test.go index ee712de9ede..d74389f692b 100644 --- a/broadcaster/backlog/backlog_test.go +++ b/broadcaster/backlog/backlog_test.go @@ -33,8 +33,8 @@ func validateBacklog(t *testing.T, b *backlog, count, start, end uint64, lookupK } } - expLen := len(lookupKeys) - actualLen := int(b.Count()) + expLen := uint64(len(lookupKeys)) + actualLen := b.Count() if expLen != actualLen { t.Errorf("expected length of lookupByIndex map (%d) does not equal actual length (%d)", expLen, actualLen) } diff --git a/broadcaster/broadcaster.go b/broadcaster/broadcaster.go index ba95f2d8af7..397698635af 100644 --- a/broadcaster/broadcaster.go +++ b/broadcaster/broadcaster.go @@ -145,6 +145,7 @@ func (b *Broadcaster) ListenerAddr() net.Addr { } func (b *Broadcaster) GetCachedMessageCount() int { + // #nosec G115 return int(b.backlog.Count()) } diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index a052c146d1e..39f204980d3 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -371,6 +371,7 @@ func mainImpl() int { if err != nil { log.Crit("error getting rollup addresses config", "err", err) } + // #nosec G115 addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) @@ -582,7 +583,7 @@ func mainImpl() int { l1TransactionOptsBatchPoster, dataSigner, fatalErrChan, - big.NewInt(int64(nodeConfig.ParentChain.ID)), + new(big.Int).SetUint64(nodeConfig.ParentChain.ID), blobReader, ) if err != nil { diff --git a/cmd/staterecovery/staterecovery.go b/cmd/staterecovery/staterecovery.go index bb014774142..5486ba3726c 100644 --- a/cmd/staterecovery/staterecovery.go +++ b/cmd/staterecovery/staterecovery.go @@ -60,6 +60,7 @@ func RecreateMissingStates(chainDb ethdb.Database, bc *core.BlockChain, cacheCon break } if time.Since(logged) > 1*time.Minute { + // #nosec G115 log.Info("Recreating missing states", "block", current, "target", target, "remaining", int64(target)-int64(current), "elapsed", time.Since(start), "recreated", recreated) logged = time.Now() } diff --git a/das/aggregator.go b/das/aggregator.go index d944f8d48ab..e8972447ad6 100644 --- a/das/aggregator.go +++ b/das/aggregator.go @@ -166,6 +166,7 @@ type storeResponse struct { // If Store gets not enough successful responses by the time its context is canceled // (eg via TimeoutWrapper) then it also returns an error. func (a *Aggregator) Store(ctx context.Context, message []byte, timeout uint64) (*daprovider.DataAvailabilityCertificate, error) { + // #nosec G115 log.Trace("das.Aggregator.Store", "message", pretty.FirstFewBytes(message), "timeout", time.Unix(int64(timeout), 0)) allBackendsSucceeded := false diff --git a/das/dasRpcClient.go b/das/dasRpcClient.go index ca2ee8e7d47..635696bdab2 100644 --- a/das/dasRpcClient.go +++ b/das/dasRpcClient.go @@ -138,6 +138,7 @@ func (c *DASRPCClient) sendChunk(ctx context.Context, batchId, i uint64, chunk [ } func (c *DASRPCClient) legacyStore(ctx context.Context, message []byte, timeout uint64) (*daprovider.DataAvailabilityCertificate, error) { + // #nosec G115 log.Trace("das.DASRPCClient.Store(...)", "message", pretty.FirstFewBytes(message), "timeout", time.Unix(int64(timeout), 0), "this", *c) reqSig, err := applyDasSigner(c.signer, message, timeout) diff --git a/das/dasRpcServer.go b/das/dasRpcServer.go index 9e6228ca5dc..d14766cc7e9 100644 --- a/das/dasRpcServer.go +++ b/das/dasRpcServer.go @@ -108,6 +108,7 @@ type StoreResult struct { } func (s *DASRPCServer) Store(ctx context.Context, message hexutil.Bytes, timeout hexutil.Uint64, sig hexutil.Bytes) (*StoreResult, error) { + // #nosec G115 log.Trace("dasRpc.DASRPCServer.Store", "message", pretty.FirstFewBytes(message), "message length", len(message), "timeout", time.Unix(int64(timeout), 0), "sig", pretty.FirstFewBytes(sig), "this", s) rpcStoreRequestGauge.Inc(1) start := time.Now() @@ -277,6 +278,7 @@ func (s *DASRPCServer) StartChunkedStore(ctx context.Context, timestamp, nChunks } // Prevent replay of old messages + // #nosec G115 if time.Since(time.Unix(int64(timestamp), 0)).Abs() > time.Minute { return nil, errors.New("too much time has elapsed since request was signed") } diff --git a/das/dastree/dastree.go b/das/dastree/dastree.go index d873f0568de..2bcbccaae3b 100644 --- a/das/dastree/dastree.go +++ b/das/dastree/dastree.go @@ -61,12 +61,13 @@ func RecordHash(record func(bytes32, []byte, arbutil.PreimageType), preimage ... return arbmath.FlipBit(keccord(prepend(LeafByte, keccord([]byte{}).Bytes())), 0) } - length := uint32(len(unrolled)) + length := len(unrolled) leaves := []node{} - for bin := uint32(0); bin < length; bin += BinSize { + for bin := 0; bin < length; bin += BinSize { end := arbmath.MinInt(bin+BinSize, length) hash := keccord(prepend(LeafByte, keccord(unrolled[bin:end]).Bytes())) - leaves = append(leaves, node{hash, end - bin}) + // #nosec G115 + leaves = append(leaves, node{hash, uint32(end - bin)}) } layer := leaves @@ -186,7 +187,9 @@ func Content(root bytes32, oracle func(bytes32) ([]byte, error)) ([]byte, error) leaves = append(leaves, leaf) case NodeByte: count := binary.BigEndian.Uint32(data[64:]) - power := uint32(arbmath.NextOrCurrentPowerOf2(uint64(count))) + power := arbmath.NextOrCurrentPowerOf2(uint64(count)) + // #nosec G115 + halfPower := uint32(power / 2) if place.size != count { return nil, fmt.Errorf("invalid size data: %v vs %v for %v", count, place.size, data) @@ -194,11 +197,11 @@ func Content(root bytes32, oracle func(bytes32) ([]byte, error)) ([]byte, error) prior := node{ hash: common.BytesToHash(data[:32]), - size: power / 2, + size: halfPower, } after := node{ hash: common.BytesToHash(data[32:64]), - size: count - power/2, + size: count - halfPower, } // we want to expand leftward so we reverse their order diff --git a/das/db_storage_service.go b/das/db_storage_service.go index e3b6183c37b..1d9e5348d46 100644 --- a/das/db_storage_service.go +++ b/das/db_storage_service.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "math" "os" "path/filepath" "time" @@ -172,7 +173,8 @@ func (dbs *DBStorageService) Put(ctx context.Context, data []byte, timeout uint6 return dbs.db.Update(func(txn *badger.Txn) error { e := badger.NewEntry(dastree.HashBytes(data), data) - if dbs.discardAfterTimeout { + if dbs.discardAfterTimeout && timeout <= math.MaxInt64 { + // #nosec G115 e = e.WithTTL(time.Until(time.Unix(int64(timeout), 0))) } return txn.SetEntry(e) diff --git a/das/local_file_storage_service.go b/das/local_file_storage_service.go index 65ca6fe15cf..ce86786718d 100644 --- a/das/local_file_storage_service.go +++ b/das/local_file_storage_service.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "math" "os" "path" "path/filepath" @@ -133,6 +134,10 @@ func (s *LocalFileStorageService) GetByHash(ctx context.Context, key common.Hash func (s *LocalFileStorageService) Put(ctx context.Context, data []byte, expiry uint64) error { logPut("das.LocalFileStorageService.Store", data, expiry, s) + if expiry > math.MaxInt64 { + return fmt.Errorf("request expiry time (%v) exceeds max int64", expiry) + } + // #nosec G115 expiryTime := time.Unix(int64(expiry), 0) currentTimePlusRetention := time.Now().Add(s.config.MaxRetention) if expiryTime.After(currentTimePlusRetention) { @@ -182,6 +187,7 @@ func (s *LocalFileStorageService) Put(ctx context.Context, data []byte, expiry u // new flat layout files, set their modification time accordingly. if s.enableLegacyLayout { tv := syscall.Timeval{ + // #nosec G115 Sec: int64(expiry - uint64(s.legacyLayout.retention.Seconds())), Usec: 0, } diff --git a/das/local_file_storage_service_test.go b/das/local_file_storage_service_test.go index cc27e293e3c..01b999f3563 100644 --- a/das/local_file_storage_service_test.go +++ b/das/local_file_storage_service_test.go @@ -99,6 +99,7 @@ func TestMigrationNoExpiry(t *testing.T) { getByHashAndCheck(t, s, "a", "b", "c", "d") // Can still iterate by timestamp even if expiry disabled + // #nosec G115 countTimestampEntries(t, &s.layout, time.Unix(int64(now+11), 0), 4) } diff --git a/das/s3_storage_service.go b/das/s3_storage_service.go index a1de200c52c..f9baa96547b 100644 --- a/das/s3_storage_service.go +++ b/das/s3_storage_service.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "io" + "math" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -110,7 +111,8 @@ func (s3s *S3StorageService) Put(ctx context.Context, value []byte, timeout uint Bucket: aws.String(s3s.bucket), Key: aws.String(s3s.objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))), Body: bytes.NewReader(value)} - if !s3s.discardAfterTimeout { + if !s3s.discardAfterTimeout && timeout <= math.MaxInt64 { + // #nosec G115 expires := time.Unix(int64(timeout), 0) putObjectInput.Expires = &expires } diff --git a/das/sign_after_store_das_writer.go b/das/sign_after_store_das_writer.go index 0e31d30ae97..40b03847d86 100644 --- a/das/sign_after_store_das_writer.go +++ b/das/sign_after_store_das_writer.go @@ -105,6 +105,7 @@ func NewSignAfterStoreDASWriter(ctx context.Context, config DataAvailabilityConf } func (d *SignAfterStoreDASWriter) Store(ctx context.Context, message []byte, timeout uint64) (c *daprovider.DataAvailabilityCertificate, err error) { + // #nosec G115 log.Trace("das.SignAfterStoreDASWriter.Store", "message", pretty.FirstFewBytes(message), "timeout", time.Unix(int64(timeout), 0), "this", d) c = &daprovider.DataAvailabilityCertificate{ Timeout: timeout, diff --git a/das/simple_das_reader_aggregator.go b/das/simple_das_reader_aggregator.go index dc6147a7e48..f45c56afe0e 100644 --- a/das/simple_das_reader_aggregator.go +++ b/das/simple_das_reader_aggregator.go @@ -50,8 +50,8 @@ var DefaultRestfulClientAggregatorConfig = RestfulClientAggregatorConfig{ } type SimpleExploreExploitStrategyConfig struct { - ExploreIterations int `koanf:"explore-iterations"` - ExploitIterations int `koanf:"exploit-iterations"` + ExploreIterations uint32 `koanf:"explore-iterations"` + ExploitIterations uint32 `koanf:"exploit-iterations"` } var DefaultSimpleExploreExploitStrategyConfig = SimpleExploreExploitStrategyConfig{ @@ -73,8 +73,8 @@ func RestfulClientAggregatorConfigAddOptions(prefix string, f *flag.FlagSet) { } func SimpleExploreExploitStrategyConfigAddOptions(prefix string, f *flag.FlagSet) { - f.Int(prefix+".explore-iterations", DefaultSimpleExploreExploitStrategyConfig.ExploreIterations, "number of consecutive GetByHash calls to the aggregator where each call will cause it to randomly select from REST endpoints until one returns successfully, before switching to exploit mode") - f.Int(prefix+".exploit-iterations", DefaultSimpleExploreExploitStrategyConfig.ExploitIterations, "number of consecutive GetByHash calls to the aggregator where each call will cause it to select from REST endpoints in order of best latency and success rate, before switching to explore mode") + f.Uint32(prefix+".explore-iterations", DefaultSimpleExploreExploitStrategyConfig.ExploreIterations, "number of consecutive GetByHash calls to the aggregator where each call will cause it to randomly select from REST endpoints until one returns successfully, before switching to exploit mode") + f.Uint32(prefix+".exploit-iterations", DefaultSimpleExploreExploitStrategyConfig.ExploitIterations, "number of consecutive GetByHash calls to the aggregator where each call will cause it to select from REST endpoints in order of best latency and success rate, before switching to explore mode") } func NewRestfulClientAggregator(ctx context.Context, config *RestfulClientAggregatorConfig) (*SimpleDASReaderAggregator, error) { @@ -120,8 +120,8 @@ func NewRestfulClientAggregator(ctx context.Context, config *RestfulClientAggreg switch strings.ToLower(config.Strategy) { case "simple-explore-exploit": a.strategy = &simpleExploreExploitStrategy{ - exploreIterations: uint32(config.SimpleExploreExploitStrategy.ExploreIterations), - exploitIterations: uint32(config.SimpleExploreExploitStrategy.ExploitIterations), + exploreIterations: config.SimpleExploreExploitStrategy.ExploreIterations, + exploitIterations: config.SimpleExploreExploitStrategy.ExploitIterations, } case "testing-sequential": a.strategy = &testingSequentialStrategy{} diff --git a/das/util.go b/das/util.go index de266c433f3..114e075e797 100644 --- a/das/util.go +++ b/das/util.go @@ -13,11 +13,13 @@ import ( func logPut(store string, data []byte, timeout uint64, reader daprovider.DASReader, more ...interface{}) { if len(more) == 0 { + // #nosec G115 log.Trace( store, "message", pretty.FirstFewBytes(data), "timeout", time.Unix(int64(timeout), 0), "this", reader, ) } else { + // #nosec G115 log.Trace( store, "message", pretty.FirstFewBytes(data), "timeout", time.Unix(int64(timeout), 0), "this", reader, more, diff --git a/execution/gethexec/api.go b/execution/gethexec/api.go index c19072ae771..2bff8026c27 100644 --- a/execution/gethexec/api.go +++ b/execution/gethexec/api.go @@ -78,6 +78,7 @@ func (api *ArbDebugAPI) evenlySpaceBlocks(start, end rpc.BlockNumber) (uint64, u end, _ = api.blockchain.ClipToPostNitroGenesis(end) blocks := end.Int64() - start.Int64() + 1 + // #nosec G115 bound := int64(api.blockRangeBound) step := int64(1) if blocks > bound { diff --git a/execution/gethexec/sequencer.go b/execution/gethexec/sequencer.go index 90e30820627..819cd105000 100644 --- a/execution/gethexec/sequencer.go +++ b/execution/gethexec/sequencer.go @@ -887,6 +887,7 @@ func (s *Sequencer) createBlock(ctx context.Context) (returnValue bool) { for _, queueItem := range queueItems { s.txRetryQueue.Push(queueItem) } + // #nosec G115 log.Error( "cannot sequence: unknown L1 block or L1 timestamp too far from local clock time", "l1Block", l1Block, @@ -1037,10 +1038,14 @@ func (s *Sequencer) updateExpectedSurplus(ctx context.Context) (int64, error) { if err != nil { return 0, fmt.Errorf("error encountered getting l1 pricing surplus while updating expectedSurplus: %w", err) } + // #nosec G115 backlogL1GasCharged := int64(s.execEngine.backlogL1GasCharged()) + // #nosec G115 backlogCallDataUnits := int64(s.execEngine.backlogCallDataUnits()) + // #nosec G115 expectedSurplus := int64(surplus) + backlogL1GasCharged - backlogCallDataUnits*int64(l1GasPrice) // update metrics + // #nosec G115 l1GasPriceGauge.Update(int64(l1GasPrice)) callDataUnitsBacklogGauge.Update(backlogCallDataUnits) unusedL1GasChargeGauge.Update(backlogL1GasCharged) diff --git a/execution/gethexec/tx_pre_checker.go b/execution/gethexec/tx_pre_checker.go index 191331b48a7..e0ae330148d 100644 --- a/execution/gethexec/tx_pre_checker.go +++ b/execution/gethexec/tx_pre_checker.go @@ -161,6 +161,7 @@ func PreCheckTx(bc *core.BlockChain, chainConfig *params.ChainConfig, header *ty oldHeader := header blocksTraversed := uint(0) // find a block that's old enough + // #nosec G115 for now-int64(oldHeader.Time) < config.RequiredStateAge && (config.RequiredStateMaxBlocks <= 0 || blocksTraversed < config.RequiredStateMaxBlocks) && oldHeader.Number.Uint64() > 0 { diff --git a/execution/nodeInterface/NodeInterface.go b/execution/nodeInterface/NodeInterface.go index 9179a527184..45fcebcdfaf 100644 --- a/execution/nodeInterface/NodeInterface.go +++ b/execution/nodeInterface/NodeInterface.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "math" "math/big" "sort" @@ -234,6 +235,7 @@ func (n NodeInterface) ConstructOutboxProof(c ctx, evm mech, size, leaf uint64) } balanced := size == arbmath.NextPowerOf2(size)/2 + // #nosec G115 treeLevels := int(arbmath.Log2ceil(size)) // the # of levels in the tree proofLevels := treeLevels - 1 // the # of levels where a hash is needed (all but root) walkLevels := treeLevels // the # of levels we need to consider when building walks @@ -297,6 +299,7 @@ func (n NodeInterface) ConstructOutboxProof(c ctx, evm mech, size, leaf uint64) mid := (lo + hi) / 2 + // #nosec G115 block, err := n.backend.BlockByNumber(n.context, rpc.BlockNumber(mid)) if err != nil { searchErr = err @@ -643,6 +646,10 @@ func (n NodeInterface) LegacyLookupMessageBatchProof(c ctx, evm mech, batchNum h // L2BlockRangeForL1 fetches the L1 block number of a given l2 block number. // c ctx and evm mech arguments are not used but supplied to match the precompile function type in NodeInterface contract func (n NodeInterface) BlockL1Num(c ctx, evm mech, l2BlockNum uint64) (uint64, error) { + if l2BlockNum > math.MaxInt64 { + return 0, fmt.Errorf("requested l2 block number %d out of range for int64", l2BlockNum) + } + // #nosec G115 blockHeader, err := n.backend.HeaderByNumber(n.context, rpc.BlockNumber(l2BlockNum)) if err != nil { return 0, err diff --git a/precompiles/ArbAddressTable.go b/precompiles/ArbAddressTable.go index 05f2275fd7d..102fd55c3b6 100644 --- a/precompiles/ArbAddressTable.go +++ b/precompiles/ArbAddressTable.go @@ -33,7 +33,7 @@ func (con ArbAddressTable) Decompress(c ctx, evm mech, buf []uint8, offset huge) return addr{}, nil, errors.New("invalid offset in ArbAddressTable.Decompress") } result, nbytes, err := c.State.AddressTable().Decompress(buf[ioffset:]) - return result, big.NewInt(int64(nbytes)), err + return result, new(big.Int).SetUint64(nbytes), err } // Lookup the index of an address in the table @@ -45,7 +45,7 @@ func (con ArbAddressTable) Lookup(c ctx, evm mech, addr addr) (huge, error) { if !exists { return nil, errors.New("address does not exist in AddressTable") } - return big.NewInt(int64(result)), nil + return new(big.Int).SetUint64(result), nil } // LookupIndex for an address in the table by index @@ -66,11 +66,11 @@ func (con ArbAddressTable) LookupIndex(c ctx, evm mech, index huge) (addr, error // Register adds an account to the table, shrinking its compressed representation func (con ArbAddressTable) Register(c ctx, evm mech, addr addr) (huge, error) { slot, err := c.State.AddressTable().Register(addr) - return big.NewInt(int64(slot)), err + return new(big.Int).SetUint64(slot), err } // Size gets the number of addresses in the table func (con ArbAddressTable) Size(c ctx, evm mech) (huge, error) { size, err := c.State.AddressTable().Size() - return big.NewInt(int64(size)), err + return new(big.Int).SetUint64(size), err } diff --git a/precompiles/ArbRetryableTx.go b/precompiles/ArbRetryableTx.go index d508d757521..93e80236039 100644 --- a/precompiles/ArbRetryableTx.go +++ b/precompiles/ArbRetryableTx.go @@ -149,7 +149,7 @@ func (con ArbRetryableTx) GetTimeout(c ctx, evm mech, ticketId bytes32) (huge, e if err != nil { return nil, err } - return big.NewInt(int64(timeout)), nil + return new(big.Int).SetUint64(timeout), nil } // Keepalive adds one lifetime period to the ticket's expiry @@ -176,8 +176,9 @@ func (con ArbRetryableTx) Keepalive(c ctx, evm mech, ticketId bytes32) (huge, er return big.NewInt(0), err } - err = con.LifetimeExtended(c, evm, ticketId, big.NewInt(int64(newTimeout))) - return big.NewInt(int64(newTimeout)), err + bigNewTimeout := new(big.Int).SetUint64(newTimeout) + err = con.LifetimeExtended(c, evm, ticketId, bigNewTimeout) + return bigNewTimeout, err } // GetBeneficiary gets the beneficiary of the ticket diff --git a/precompiles/ArbSys.go b/precompiles/ArbSys.go index 13f56d3b8e5..d55067a09c2 100644 --- a/precompiles/ArbSys.go +++ b/precompiles/ArbSys.go @@ -162,7 +162,7 @@ func (con *ArbSys) SendTxToL1(c ctx, evm mech, value huge, destination addr, cal } } - leafNum := big.NewInt(int64(size - 1)) + leafNum := new(big.Int).SetUint64(size - 1) var blockTime big.Int blockTime.SetUint64(evm.Context.Time) @@ -199,7 +199,7 @@ func (con ArbSys) SendMerkleTreeState(c ctx, evm mech) (huge, bytes32, []bytes32 for i, par := range rawPartials { partials[i] = par } - return big.NewInt(int64(size)), rootHash, partials, nil + return new(big.Int).SetUint64(size), rootHash, partials, nil } // WithdrawEth send paid eth to the destination on L1 diff --git a/relay/relay_stress_test.go b/relay/relay_stress_test.go index 9a8875a4295..9d5c4150566 100644 --- a/relay/relay_stress_test.go +++ b/relay/relay_stress_test.go @@ -160,7 +160,7 @@ func largeBacklogRelayTestImpl(t *testing.T, numClients, backlogSize, l2MsgSize connected++ } } - if int32(connected) != int32(numClients) { + if connected != numClients { t.Fail() } log.Info("number of clients connected", "expected", numClients, "got", connected) diff --git a/staker/block_challenge_backend.go b/staker/block_challenge_backend.go index 42351789ba0..0dd89865bd0 100644 --- a/staker/block_challenge_backend.go +++ b/staker/block_challenge_backend.go @@ -219,6 +219,6 @@ func (b *BlockChallengeBackend) IssueExecChallenge( }, machineStatuses, globalStateHashes, - big.NewInt(int64(numsteps)), + new(big.Int).SetUint64(numsteps), ) } diff --git a/staker/block_validator.go b/staker/block_validator.go index 8f5724beac4..2239952b37f 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -315,6 +315,7 @@ func NewBlockValidator( func atomicStorePos(addr *atomic.Uint64, val arbutil.MessageIndex, metr metrics.Gauge) { addr.Store(uint64(val)) + // #nosec G115 metr.Update(int64(val)) } @@ -573,6 +574,7 @@ func (v *BlockValidator) createNextValidationEntry(ctx context.Context) (bool, e v.nextCreateBatch = batch v.nextCreateBatchBlockHash = batchBlockHash v.nextCreateBatchMsgCount = count + // #nosec G115 validatorMsgCountCurrentBatch.Update(int64(count)) v.nextCreateBatchReread = false } @@ -723,6 +725,7 @@ func (v *BlockValidator) iterativeValidationPrint(ctx context.Context) time.Dura if err != nil { printedCount = -1 } else { + // #nosec G115 printedCount = int64(batchMsgs) + int64(validated.GlobalState.PosInBatch) } log.Info("validated execution", "messageCount", printedCount, "globalstate", validated.GlobalState, "WasmRoots", validated.WasmRoots) @@ -992,8 +995,10 @@ func (v *BlockValidator) UpdateLatestStaked(count arbutil.MessageIndex, globalSt if v.recordSentA.Load() < countUint64 { v.recordSentA.Store(countUint64) } + // #nosec G115 v.validatedA.Store(countUint64) v.valLoopPos = count + // #nosec G115 validatorMsgCountValidatedGauge.Update(int64(countUint64)) err = v.writeLastValidated(globalState, nil) // we don't know which wasm roots were validated if err != nil { @@ -1058,6 +1063,7 @@ func (v *BlockValidator) Reorg(ctx context.Context, count arbutil.MessageIndex) } if v.validatedA.Load() > countUint64 { v.validatedA.Store(countUint64) + // #nosec G115 validatorMsgCountValidatedGauge.Update(int64(countUint64)) err := v.writeLastValidated(v.nextCreateStartGS, nil) // we don't know which wasm roots were validated if err != nil { @@ -1249,6 +1255,7 @@ func (v *BlockValidator) checkValidatedGSCaughtUp() (bool, error) { atomicStorePos(&v.createdA, count, validatorMsgCountCreatedGauge) atomicStorePos(&v.recordSentA, count, validatorMsgCountRecordSentGauge) atomicStorePos(&v.validatedA, count, validatorMsgCountValidatedGauge) + // #nosec G115 validatorMsgCountValidatedGauge.Update(int64(count)) v.chainCaughtUp = true return true, nil diff --git a/staker/challenge-cache/cache.go b/staker/challenge-cache/cache.go index ed4fad6450e..5dca2764e89 100644 --- a/staker/challenge-cache/cache.go +++ b/staker/challenge-cache/cache.go @@ -187,12 +187,12 @@ func (c *Cache) Prune(ctx context.Context, messageNumber uint64) error { if info.IsDir() { matches := pattern.FindStringSubmatch(info.Name()) if len(matches) > 1 { - dirNameMessageNum, err := strconv.Atoi(matches[1]) + dirNameMessageNum, err := strconv.ParseUint(matches[1], 10, 64) if err != nil { return err } // Collect the directory path if the message number is <= the specified value. - if dirNameMessageNum <= int(messageNumber) { + if dirNameMessageNum <= messageNumber { pathsToDelete = append(pathsToDelete, path) } } diff --git a/staker/challenge_manager.go b/staker/challenge_manager.go index b1421d7e419..ef431d3c797 100644 --- a/staker/challenge_manager.go +++ b/staker/challenge_manager.go @@ -294,7 +294,7 @@ func (m *ChallengeManager) bisect(ctx context.Context, backend ChallengeBackend, if newChallengeLength < bisectionDegree { bisectionDegree = newChallengeLength } - newSegments := make([][32]byte, int(bisectionDegree+1)) + newSegments := make([][32]byte, bisectionDegree+1) position := startSegmentPosition normalSegmentLength := newChallengeLength / bisectionDegree for i := range newSegments { diff --git a/staker/challenge_test.go b/staker/challenge_test.go index 4534b04a257..33f1644c637 100644 --- a/staker/challenge_test.go +++ b/staker/challenge_test.go @@ -77,7 +77,7 @@ func CreateChallenge( resultReceiverAddr, maxInboxMessage, [2][32]byte{startHashBytes, endHashBytes}, - big.NewInt(int64(endMachineSteps)), + new(big.Int).SetUint64(endMachineSteps), asserter, challenger, big.NewInt(100), diff --git a/staker/l1_validator.go b/staker/l1_validator.go index dd9673ee0bf..6ea9fd8ded0 100644 --- a/staker/l1_validator.go +++ b/staker/l1_validator.go @@ -247,6 +247,7 @@ func (v *L1Validator) generateNodeAction( startStateProposedParentChain, err, ) } + // #nosec G115 startStateProposedTime := time.Unix(int64(startStateProposedHeader.Time), 0) v.txStreamer.PauseReorgs() @@ -375,6 +376,7 @@ func (v *L1Validator) generateNodeAction( return nil, false, fmt.Errorf("error getting rollup minimum assertion period: %w", err) } + // #nosec G115 timeSinceProposed := big.NewInt(int64(l1BlockNumber) - int64(startStateProposedL1)) if timeSinceProposed.Cmp(minAssertionPeriod) < 0 { // Too soon to assert diff --git a/staker/rollup_watcher.go b/staker/rollup_watcher.go index b35bebd1c62..5ef28a49dc4 100644 --- a/staker/rollup_watcher.go +++ b/staker/rollup_watcher.go @@ -196,7 +196,7 @@ func (r *RollupWatcher) LookupNodeChildren(ctx context.Context, nodeNum uint64, if logQueryRangeSize == 0 { query.ToBlock = toBlock } else { - query.ToBlock = new(big.Int).Add(fromBlock, big.NewInt(int64(logQueryRangeSize))) + query.ToBlock = new(big.Int).Add(fromBlock, new(big.Int).SetUint64(logQueryRangeSize)) } if query.ToBlock.Cmp(toBlock) > 0 { query.ToBlock = toBlock diff --git a/staker/staker.go b/staker/staker.go index c54e74be37b..6e93d273119 100644 --- a/staker/staker.go +++ b/staker/staker.go @@ -352,6 +352,7 @@ func (s *Staker) Initialize(ctx context.Context) error { if err != nil { return err } + // #nosec G115 stakerLatestStakedNodeGauge.Update(int64(latestStaked)) if latestStaked == 0 { return nil @@ -570,6 +571,7 @@ func (s *Staker) Start(ctxIn context.Context) { if err != nil && ctx.Err() == nil { log.Error("staker: error checking latest staked", "err", err) } + // #nosec G115 stakerLatestStakedNodeGauge.Update(int64(staked)) if stakedGlobalState != nil { for _, notifier := range s.stakedNotifiers { @@ -585,6 +587,7 @@ func (s *Staker) Start(ctxIn context.Context) { log.Error("staker: error checking latest confirmed", "err", err) } } + // #nosec G115 stakerLatestConfirmedNodeGauge.Update(int64(confirmed)) if confirmedGlobalState != nil { for _, notifier := range s.confirmedNotifiers { @@ -726,6 +729,7 @@ func (s *Staker) Act(ctx context.Context) (*types.Transaction, error) { if err != nil { return nil, fmt.Errorf("error getting latest staked node of own wallet %v: %w", walletAddressOrZero, err) } + // #nosec G115 stakerLatestStakedNodeGauge.Update(int64(latestStakedNodeNum)) if rawInfo != nil { rawInfo.LatestStakedNode = latestStakedNodeNum diff --git a/system_tests/block_validator_test.go b/system_tests/block_validator_test.go index bd0a1f33362..eef6c29b7a3 100644 --- a/system_tests/block_validator_test.go +++ b/system_tests/block_validator_test.go @@ -259,6 +259,7 @@ func testBlockValidatorSimple(t *testing.T, opts Options) { Require(t, err) // up to 3 extra references: awaiting validation, recently valid, lastValidatedHeader largestRefCount := lastBlockNow.NumberU64() - lastBlock.NumberU64() + 3 + // #nosec G115 if finalRefCount < 0 || finalRefCount > int64(largestRefCount) { Fatal(t, "unexpected refcount:", finalRefCount) } diff --git a/system_tests/forwarder_test.go b/system_tests/forwarder_test.go index 9fe419593ee..f87283432e8 100644 --- a/system_tests/forwarder_test.go +++ b/system_tests/forwarder_test.go @@ -246,6 +246,7 @@ func TestRedisForwarder(t *testing.T) { for i := range seqClients { userA := user("A", i) builder.L2Info.GenerateAccount(userA) + // #nosec G115 tx := builder.L2Info.PrepareTx("Owner", userA, builder.L2Info.TransferGas, big.NewInt(1e12+int64(builder.L2Info.TransferGas)*builder.L2Info.GasPrice.Int64()), nil) err := fallbackClient.SendTransaction(ctx, tx) Require(t, err) diff --git a/system_tests/initialization_test.go b/system_tests/initialization_test.go index f0797404a94..17e020e6abb 100644 --- a/system_tests/initialization_test.go +++ b/system_tests/initialization_test.go @@ -21,6 +21,7 @@ func InitOneContract(prand *testhelpers.PseudoRandomDataSource) (*statetransfer. storageMap := make(map[common.Hash]common.Hash) code := []byte{0x60, 0x0} // PUSH1 0 sum := big.NewInt(0) + // #nosec G115 numCells := int(prand.GetUint64() % 1000) for i := 0; i < numCells; i++ { storageAddr := prand.GetHash() diff --git a/system_tests/outbox_test.go b/system_tests/outbox_test.go index 739d756a310..c68df6ea229 100644 --- a/system_tests/outbox_test.go +++ b/system_tests/outbox_test.go @@ -146,6 +146,7 @@ func TestOutboxProofs(t *testing.T) { treeSize := root.size balanced := treeSize == arbmath.NextPowerOf2(treeSize)/2 + // #nosec G115 treeLevels := int(arbmath.Log2ceil(treeSize)) // the # of levels in the tree proofLevels := treeLevels - 1 // the # of levels where a hash is needed (all but root) walkLevels := treeLevels // the # of levels we need to consider when building walks diff --git a/system_tests/program_recursive_test.go b/system_tests/program_recursive_test.go index dbf527a2931..e928f9f3aa4 100644 --- a/system_tests/program_recursive_test.go +++ b/system_tests/program_recursive_test.go @@ -154,6 +154,7 @@ func testProgramResursiveCalls(t *testing.T, tests [][]multiCallRecurse, jit boo // execute transactions blockNum := uint64(0) for { + // #nosec G115 item := int(rander.GetUint64()/4) % len(tests) blockNum = testProgramRecursiveCall(t, builder, slotVals, rander, tests[item]) tests[item] = tests[len(tests)-1] diff --git a/system_tests/program_test.go b/system_tests/program_test.go index e171f2a4442..ed640809dbf 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -582,6 +582,7 @@ func testCalls(t *testing.T, jit bool) { for i := 0; i < 2; i++ { inner := nest(level - 1) + // #nosec G115 args = append(args, arbmath.Uint32ToBytes(uint32(len(inner)))...) args = append(args, inner...) } @@ -637,6 +638,7 @@ func testCalls(t *testing.T, jit bool) { colors.PrintBlue("Calling the ArbosTest precompile (Rust => precompile)") testPrecompile := func(gas uint64) uint64 { // Call the burnArbGas() precompile from Rust + // #nosec G115 burn := pack(burnArbGas(big.NewInt(int64(gas)))) args := argsForMulticall(vm.CALL, types.ArbosTestAddress, nil, burn) tx := l2info.PrepareTxTo("Owner", &callsAddr, 1e9, nil, args) @@ -650,6 +652,7 @@ func testCalls(t *testing.T, jit bool) { large := testPrecompile(largeGas) if !arbmath.Within(large-small, largeGas-smallGas, 2) { + // #nosec G115 ratio := float64(int64(large)-int64(small)) / float64(int64(largeGas)-int64(smallGas)) Fatal(t, "inconsistent burns", large, small, largeGas, smallGas, ratio) } @@ -1527,6 +1530,7 @@ func readWasmFile(t *testing.T, file string) ([]byte, []byte) { Require(t, err) // chose a random dictionary for testing, but keep the same files consistent + // #nosec G115 randDict := arbcompress.Dictionary((len(file) + len(t.Name())) % 2) wasmSource, err := programs.Wat2Wasm(source) @@ -1597,6 +1601,7 @@ func argsForMulticall(opcode vm.OpCode, address common.Address, value *big.Int, if opcode == vm.CALL { length += 32 } + // #nosec G115 args = append(args, arbmath.Uint32ToBytes(uint32(length))...) args = append(args, kinds[opcode]) if opcode == vm.CALL { diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 09d53669ee0..cd3904ca060 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -132,6 +132,7 @@ func TestRecreateStateForRPCNoDepthLimit(t *testing.T) { func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + // #nosec G115 depthGasLimit := int64(256 * util.NormalizeL2GasForL1GasInitial(800_000, params.GWei)) execConfig := ExecConfigDefaultTest() execConfig.RPC.MaxRecreateStateDepth = depthGasLimit @@ -407,6 +408,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig gas = 0 blocks = 0 } else { + // #nosec G115 if int(i) >= int(lastBlock)-int(cacheConfig.BlockCount) { // skipping nonexistence check - the state might have been saved on node shutdown continue @@ -471,6 +473,7 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { for _, skipGas := range skipGasValues { for _, skipBlocks := range skipBlockValues[:len(skipBlockValues)-2] { cacheConfig.MaxAmountOfGasToSkipStateSaving = skipGas + // #nosec G115 cacheConfig.MaxNumberOfBlocksToSkipStateSaving = uint32(skipBlocks) testSkippingSavingStateAndRecreatingAfterRestart(t, &cacheConfig, 100) } @@ -495,6 +498,7 @@ func TestGettingStateForRPCFullNode(t *testing.T) { if header == nil { Fatal(t, "failed to get current block header") } + // #nosec G115 state, _, err := api.StateAndHeaderByNumber(ctx, rpc.BlockNumber(header.Number.Uint64())) Require(t, err) addr := builder.L2Info.GetAddress("User2") @@ -505,6 +509,7 @@ func TestGettingStateForRPCFullNode(t *testing.T) { Fatal(t, "User2 address does not exist in the state") } // Get the state again to avoid caching + // #nosec G115 state, _, err = api.StateAndHeaderByNumber(ctx, rpc.BlockNumber(header.Number.Uint64())) Require(t, err) @@ -542,6 +547,7 @@ func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { if header == nil { Fatal(t, "failed to get current block header") } + // #nosec G115 state, _, err := api.StateAndHeaderByNumber(ctx, rpc.BlockNumber(header.Number.Uint64())) Require(t, err) addr := builder.L2Info.GetAddress("User2") @@ -552,6 +558,7 @@ func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { Fatal(t, "User2 address does not exist in the state") } // Get the state again to avoid caching + // #nosec G115 state, _, err = api.StateAndHeaderByNumber(ctx, rpc.BlockNumber(header.Number.Uint64())) Require(t, err) diff --git a/system_tests/seq_nonce_test.go b/system_tests/seq_nonce_test.go index 72629e19787..c099563e296 100644 --- a/system_tests/seq_nonce_test.go +++ b/system_tests/seq_nonce_test.go @@ -111,6 +111,7 @@ func TestSequencerNonceTooHighQueueFull(t *testing.T) { } for wait := 9; wait >= 0; wait-- { + // #nosec G115 got := int(completed.Load()) expected := count - builder.execConfig.Sequencer.NonceFailureCacheSize if got == expected { diff --git a/system_tests/seqinbox_test.go b/system_tests/seqinbox_test.go index 4dc8f4a6647..6babe5833f0 100644 --- a/system_tests/seqinbox_test.go +++ b/system_tests/seqinbox_test.go @@ -229,6 +229,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { reorgTargetNumber := blockStates[reorgTo].l1BlockNumber currentHeader, err := builder.L1.Client.HeaderByNumber(ctx, nil) Require(t, err) + // #nosec G115 if currentHeader.Number.Int64()-int64(reorgTargetNumber) < 65 { Fatal(t, "Less than 65 blocks of difference between current block", currentHeader.Number, "and target", reorgTargetNumber) } @@ -346,7 +347,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { BridgeAddr: builder.L1Info.GetAddress("Bridge"), DataPosterAddr: seqOpts.From, GasRefunderAddr: gasRefunderAddr, - SequencerInboxAccs: len(blockStates), + SequencerInboxAccs: uint64(len(blockStates)), AfterDelayedMessagesRead: 1, }) if diff := diffAccessList(accessed, *wantAL); diff != "" { @@ -374,6 +375,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { t.Fatalf("BalanceAt(%v) unexpected error: %v", seqOpts.From, err) } txCost := txRes.EffectiveGasPrice.Uint64() * txRes.GasUsed + // #nosec G115 if diff := before.Int64() - after.Int64(); diff >= int64(txCost) { t.Errorf("Transaction: %v was not refunded, balance diff: %v, cost: %v", tx.Hash(), diff, txCost) } @@ -424,11 +426,13 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { } for _, state := range blockStates { + // #nosec G115 block, err := l2Backend.APIBackend().BlockByNumber(ctx, rpc.BlockNumber(state.l2BlockNumber)) Require(t, err) if block == nil { Fatal(t, "missing state block", state.l2BlockNumber) } + // #nosec G115 stateDb, _, err := l2Backend.APIBackend().StateAndHeaderByNumber(ctx, rpc.BlockNumber(state.l2BlockNumber)) Require(t, err) for acct, expectedBalance := range state.balances { diff --git a/system_tests/snap_sync_test.go b/system_tests/snap_sync_test.go index a04d9f5bf39..7462b5f5f01 100644 --- a/system_tests/snap_sync_test.go +++ b/system_tests/snap_sync_test.go @@ -92,8 +92,10 @@ func TestSnapSync(t *testing.T) { waitForBlockToCatchupToMessageCount(ctx, t, nodeC.Client, finalMessageCount) // Fetching message count - 1 instead on the latest block number as the latest block number might not be // present in the snap sync node since it does not have the sequencer feed. + // #nosec G115 header, err := builder.L2.Client.HeaderByNumber(ctx, big.NewInt(int64(finalMessageCount)-1)) Require(t, err) + // #nosec G115 headerNodeC, err := nodeC.Client.HeaderByNumber(ctx, big.NewInt(int64(finalMessageCount)-1)) Require(t, err) // Once the node is synced up, check if the block hash is the same for the last block diff --git a/system_tests/twonodeslong_test.go b/system_tests/twonodeslong_test.go index 83cd975dd80..60707b83fb9 100644 --- a/system_tests/twonodeslong_test.go +++ b/system_tests/twonodeslong_test.go @@ -63,6 +63,7 @@ func testTwoNodesLong(t *testing.T, dasModeStr string) { builder.L2Info.GenerateAccount("ErrorTxSender") builder.L2.SendWaitTestTransactions(t, []*types.Transaction{ + // #nosec G115 builder.L2Info.PrepareTx("Faucet", "ErrorTxSender", builder.L2Info.TransferGas, big.NewInt(l2pricing.InitialBaseFeeWei*int64(builder.L2Info.TransferGas)), nil), }) diff --git a/system_tests/unsupported_txtypes_test.go b/system_tests/unsupported_txtypes_test.go index 4c3c8661c86..a228cb2454c 100644 --- a/system_tests/unsupported_txtypes_test.go +++ b/system_tests/unsupported_txtypes_test.go @@ -112,8 +112,8 @@ func TestBlobAndInternalTxsAsDelayedMsgReject(t *testing.T) { blocknum, err := builder.L2.Client.BlockNumber(ctx) Require(t, err) - for i := int64(0); i <= int64(blocknum); i++ { - block, err := builder.L2.Client.BlockByNumber(ctx, big.NewInt(i)) + for i := uint64(0); i <= blocknum; i++ { + block, err := builder.L2.Client.BlockByNumber(ctx, new(big.Int).SetUint64(i)) Require(t, err) for _, tx := range block.Transactions() { if _, ok := txAcceptStatus[tx.Hash()]; ok { diff --git a/util/arbmath/bips.go b/util/arbmath/bips.go index 8b7c47d82bc..646dad3a92d 100644 --- a/util/arbmath/bips.go +++ b/util/arbmath/bips.go @@ -20,7 +20,7 @@ func PercentToBips(percentage int64) Bips { } func BigToBips(natural *big.Int) Bips { - return Bips(natural.Uint64()) + return Bips(natural.Int64()) } func BigMulByBips(value *big.Int, bips Bips) *big.Int { @@ -51,5 +51,5 @@ func (bips Bips) Uint64() uint64 { func BigDivToBips(dividend, divisor *big.Int) Bips { value := BigMulByInt(dividend, int64(OneInBips)) value.Div(value, divisor) - return Bips(BigToUintSaturating(value)) + return Bips(BigToIntSaturating(value)) } diff --git a/util/arbmath/math.go b/util/arbmath/math.go index 62af1e26e01..e5bed67f6df 100644 --- a/util/arbmath/math.go +++ b/util/arbmath/math.go @@ -117,6 +117,18 @@ func BigToUintSaturating(value *big.Int) uint64 { return value.Uint64() } +// BigToUintSaturating casts a huge to an int, saturating if out of bounds +func BigToIntSaturating(value *big.Int) int64 { + if !value.IsInt64() { + if value.Sign() < 0 { + return math.MinInt64 + } else { + return math.MaxInt64 + } + } + return value.Int64() +} + // BigToUintOrPanic casts a huge to a uint, panicking if out of bounds func BigToUintOrPanic(value *big.Int) uint64 { if value.Sign() < 0 { @@ -260,10 +272,12 @@ func BigFloatMulByUint(multiplicand *big.Float, multiplier uint64) *big.Float { } func MaxSignedValue[T Signed]() T { + // #nosec G115 return T((uint64(1) << (8*unsafe.Sizeof(T(0)) - 1)) - 1) } func MinSignedValue[T Signed]() T { + // #nosec G115 return T(uint64(1) << ((8 * unsafe.Sizeof(T(0))) - 1)) } diff --git a/util/arbmath/math_test.go b/util/arbmath/math_test.go index 1be60dc58b1..528666dc19e 100644 --- a/util/arbmath/math_test.go +++ b/util/arbmath/math_test.go @@ -35,6 +35,7 @@ func TestMath(t *testing.T) { input := rand.Uint64() / 256 approx := ApproxSquareRoot(input) correct := math.Sqrt(float64(input)) + // #nosec G115 diff := int(approx) - int(correct) if diff < -1 || diff > 1 { Fail(t, "sqrt approximation off by too much", diff, input, approx, correct) @@ -46,6 +47,7 @@ func TestMath(t *testing.T) { input := uint64(i) approx := ApproxSquareRoot(input) correct := math.Sqrt(float64(input)) + // #nosec G115 diff := int(approx) - int(correct) if diff < 0 || diff > 1 { Fail(t, "sqrt approximation off by too much", diff, input, approx, correct) @@ -57,6 +59,7 @@ func TestMath(t *testing.T) { input := uint64(1 << i) approx := ApproxSquareRoot(input) correct := math.Sqrt(float64(input)) + // #nosec G115 diff := int(approx) - int(correct) if diff != 0 { Fail(t, "incorrect", "2^", i, diff, approx, correct) diff --git a/util/arbmath/uint24.go b/util/arbmath/uint24.go index 818f871a23e..a0c5aa27b79 100644 --- a/util/arbmath/uint24.go +++ b/util/arbmath/uint24.go @@ -9,10 +9,10 @@ import ( "math/big" ) -const MaxUint24 = 1<<24 - 1 // 16777215 - type Uint24 uint32 +const MaxUint24 = 1<<24 - 1 // 16777215 + func (value Uint24) ToBig() *big.Int { return UintToBig(uint64(value)) } @@ -26,8 +26,9 @@ func (value Uint24) ToUint64() uint64 { } func IntToUint24[T uint32 | uint64](value T) (Uint24, error) { + // #nosec G115 if value > T(MaxUint24) { - return Uint24(MaxUint24), errors.New("value out of range") + return MaxUint24, errors.New("value out of range") } return Uint24(value), nil } @@ -40,6 +41,7 @@ func BigToUint24OrPanic(value *big.Int) Uint24 { if !value.IsUint64() || value.Uint64() > MaxUint24 { panic("big.Int value exceeds the max Uint24") } + // #nosec G115 return Uint24(value.Uint64()) } diff --git a/util/headerreader/header_reader.go b/util/headerreader/header_reader.go index 074d24338e8..c8041dc8710 100644 --- a/util/headerreader/header_reader.go +++ b/util/headerreader/header_reader.go @@ -340,6 +340,7 @@ func (s *HeaderReader) logIfHeaderIsOld() { if storedHeader == nil { return } + // #nosec G115 l1Timetamp := time.Unix(int64(storedHeader.Time), 0) headerTime := time.Since(l1Timetamp) if headerTime >= s.config().OldHeaderTimeout { diff --git a/util/merkletree/merkleTree.go b/util/merkletree/merkleTree.go index 1b15d51d98c..fffa9bcabc9 100644 --- a/util/merkletree/merkleTree.go +++ b/util/merkletree/merkleTree.go @@ -43,8 +43,8 @@ func NewLevelAndLeaf(level, leaf uint64) LevelAndLeaf { func (place LevelAndLeaf) ToBigInt() *big.Int { return new(big.Int).Add( - new(big.Int).Lsh(big.NewInt(int64(place.Level)), 192), - big.NewInt(int64(place.Leaf)), + new(big.Int).Lsh(new(big.Int).SetUint64(place.Level), 192), + new(big.Int).SetUint64(place.Leaf), ) } diff --git a/util/rpcclient/rpcclient.go b/util/rpcclient/rpcclient.go index be5825a28db..a35d4b6665a 100644 --- a/util/rpcclient/rpcclient.go +++ b/util/rpcclient/rpcclient.go @@ -101,7 +101,7 @@ func (c *RpcClient) Close() { } type limitedMarshal struct { - limit int + limit uint value any } @@ -113,16 +113,18 @@ func (m limitedMarshal) String() string { } else { str = string(marshalled) } - if m.limit == 0 || len(str) <= m.limit { + // #nosec G115 + limit := int(m.limit) + if m.limit <= 0 || len(str) <= limit { return str } prefix := str[:m.limit/2-1] - postfix := str[len(str)-m.limit/2+1:] + postfix := str[len(str)-limit/2+1:] return fmt.Sprintf("%v..%v", prefix, postfix) } type limitedArgumentsMarshal struct { - limit int + limit uint args []any } @@ -162,9 +164,9 @@ func (c *RpcClient) CallContext(ctx_in context.Context, result interface{}, meth return errors.New("not connected") } logId := c.logId.Add(1) - log.Trace("sending RPC request", "method", method, "logId", logId, "args", limitedArgumentsMarshal{int(c.config().ArgLogLimit), args}) + log.Trace("sending RPC request", "method", method, "logId", logId, "args", limitedArgumentsMarshal{c.config().ArgLogLimit, args}) var err error - for i := 0; i < int(c.config().Retries)+1; i++ { + for i := uint(0); i < c.config().Retries+1; i++ { retryDelay := c.config().RetryDelay if i > 0 && retryDelay > 0 { select { @@ -188,7 +190,7 @@ func (c *RpcClient) CallContext(ctx_in context.Context, result interface{}, meth cancelCtx() logger := log.Trace - limit := int(c.config().ArgLogLimit) + limit := c.config().ArgLogLimit if err != nil && !IsAlreadyKnownError(err) { logger = log.Info } diff --git a/util/sharedmetrics/sharedmetrics.go b/util/sharedmetrics/sharedmetrics.go index 377eef53522..9b4b3609bc7 100644 --- a/util/sharedmetrics/sharedmetrics.go +++ b/util/sharedmetrics/sharedmetrics.go @@ -11,8 +11,10 @@ var ( ) func UpdateSequenceNumberGauge(sequenceNumber arbutil.MessageIndex) { + // #nosec G115 latestSequenceNumberGauge.Update(int64(sequenceNumber)) } func UpdateSequenceNumberInBlockGauge(sequenceNumber arbutil.MessageIndex) { + // #nosec G115 sequenceNumberInBlockGauge.Update(int64(sequenceNumber)) } diff --git a/util/testhelpers/testhelpers.go b/util/testhelpers/testhelpers.go index b1b08708e75..d681b422bf8 100644 --- a/util/testhelpers/testhelpers.go +++ b/util/testhelpers/testhelpers.go @@ -65,6 +65,7 @@ func RandomCallValue(limit int64) *big.Int { // Computes a psuedo-random uint64 on the interval [min, max] func RandomUint32(min, max uint32) uint32 { + //#nosec G115 return uint32(RandomUint64(uint64(min), uint64(max))) } diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 80cff666754..00bd992f46f 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -102,6 +102,7 @@ func (c *ValidationClient) Start(ctx context.Context) error { } else { log.Info("connected to validation server", "name", name, "room", room) } + // #nosec G115 c.room.Store(int32(room)) c.wasmModuleRoots = moduleRoots c.name = name diff --git a/validator/server_arb/execution_run_test.go b/validator/server_arb/execution_run_test.go index bdc1eefc4da..479db58515f 100644 --- a/validator/server_arb/execution_run_test.go +++ b/validator/server_arb/execution_run_test.go @@ -194,7 +194,7 @@ func Test_machineHashesWithStep(t *testing.T) { Batch: 1, PosInBatch: mm.totalSteps - 1, })) - if len(hashes) >= int(maxIterations) { + if uint64(len(hashes)) >= maxIterations { t.Fatal("Wanted fewer hashes than the max iterations") } for i := range hashes { diff --git a/validator/server_arb/machine_cache.go b/validator/server_arb/machine_cache.go index 23fcdef6d6f..55ef61cf11d 100644 --- a/validator/server_arb/machine_cache.go +++ b/validator/server_arb/machine_cache.go @@ -239,6 +239,7 @@ func (c *MachineCache) getClosestMachine(stepCount uint64) (int, MachineInterfac if c.machineStepInterval == 0 || stepsFromStart > c.machineStepInterval*uint64(len(c.machines)-1) { index = len(c.machines) - 1 } else { + // #nosec G115 index = int(stepsFromStart / c.machineStepInterval) } return index, c.machines[index] diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index 844a988d282..eb530703031 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -179,7 +179,10 @@ func (v *ArbitratorSpawner) execute( } steps += count } + + // #nosec G115 arbitratorValidationSteps.Update(int64(mach.GetStepCount())) + if mach.IsErrored() { log.Error("machine entered errored state during attempted validation", "block", entry.Id) return validator.GoGlobalState{}, errors.New("machine entered errored state during attempted validation") diff --git a/validator/server_jit/jit_machine.go b/validator/server_jit/jit_machine.go index 23a75bba833..e7753748aba 100644 --- a/validator/server_jit/jit_machine.go +++ b/validator/server_jit/jit_machine.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "math" "net" "os" "os/exec" @@ -125,6 +126,13 @@ func (machine *JitMachine) prove( writeUint32 := func(data uint32) error { return writeExact(arbmath.Uint32ToBytes(data)) } + writeIntAsUint32 := func(data int) error { + if data < 0 || data > math.MaxUint32 { + return fmt.Errorf("attempted to write out-of-bounds int %v as uint32", data) + } + // #nosec G115 + return writeUint32(uint32(data)) + } writeUint64 := func(data uint64) error { return writeExact(arbmath.UintToBytes(data)) } @@ -192,14 +200,14 @@ func (machine *JitMachine) prove( // send known preimages preimageTypes := entry.Preimages - if err := writeUint32(uint32(len(preimageTypes))); err != nil { + if err := writeIntAsUint32(len(preimageTypes)); err != nil { return state, err } for ty, preimages := range preimageTypes { if err := writeUint8(uint8(ty)); err != nil { return state, err } - if err := writeUint32(uint32(len(preimages))); err != nil { + if err := writeIntAsUint32(len(preimages)); err != nil { return state, err } for hash, preimage := range preimages { @@ -224,7 +232,7 @@ func (machine *JitMachine) prove( } } - if err := writeUint32(uint32(len(userWasms))); err != nil { + if err := writeIntAsUint32(len(userWasms)); err != nil { return state, err } for moduleHash, program := range userWasms { @@ -301,6 +309,7 @@ func (machine *JitMachine) prove( if memoryUsed > uint64(machine.wasmMemoryUsageLimit) { log.Warn("memory used by jit wasm exceeds the wasm memory usage limit", "limit", machine.wasmMemoryUsageLimit, "memoryUsed", memoryUsed) } + // #nosec G115 jitWasmMemoryUsage.Update(int64(memoryUsed)) return state, nil default: diff --git a/wavmio/stub.go b/wavmio/stub.go index 7fd29e20626..1395fb42350 100644 --- a/wavmio/stub.go +++ b/wavmio/stub.go @@ -60,13 +60,13 @@ func parsePreimageBytes(path string) { if read != len(lenBuf) { panic(fmt.Sprintf("missing bytes reading len got %d", read)) } - fieldSize := int(binary.LittleEndian.Uint64(lenBuf)) + fieldSize := binary.LittleEndian.Uint64(lenBuf) dataBuf := make([]byte, fieldSize) read, err = file.Read(dataBuf) if err != nil { panic(err) } - if read != fieldSize { + if uint64(read) != fieldSize { panic("missing bytes reading data") } hash := crypto.Keccak256Hash(dataBuf) @@ -125,7 +125,7 @@ func ReadInboxMessage(msgNum uint64) []byte { } func ReadDelayedInboxMessage(seqNum uint64) []byte { - if seqNum < delayedMsgFirstPos || (int(seqNum-delayedMsgFirstPos) > len(delayedMsgs)) { + if seqNum < delayedMsgFirstPos || (seqNum-delayedMsgFirstPos > uint64(len(delayedMsgs))) { panic(fmt.Sprintf("trying to read bad delayed msg %d", seqNum)) } return delayedMsgs[seqNum-delayedMsgFirstPos] diff --git a/wsbroadcastserver/clientconnection.go b/wsbroadcastserver/clientconnection.go index 16a8f64daf7..00ae0f0dcfc 100644 --- a/wsbroadcastserver/clientconnection.go +++ b/wsbroadcastserver/clientconnection.go @@ -135,6 +135,7 @@ func (cc *ClientConnection) writeBacklog(ctx context.Context, segment backlog.Ba msgs := prevSegment.Messages() if isFirstSegment && prevSegment.Contains(uint64(cc.requestedSeqNum)) { + // #nosec G115 requestedIdx := int(cc.requestedSeqNum) - int(prevSegment.Start()) // This might be false if messages were added after we fetched the segment's messages if len(msgs) >= requestedIdx { From 9443c1904566ee1b7b7b6eeb949a3cd01a967b0a Mon Sep 17 00:00:00 2001 From: Tristan Wilson Date: Wed, 21 Aug 2024 20:00:17 +0200 Subject: [PATCH 16/95] Fix inverted expiry logic for DAS S3 backends Also force a resync of all mirrors since if they were using S3, they may be missing data in S3. --- das/s3_storage_service.go | 2 +- das/syncing_fallback_storage.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/das/s3_storage_service.go b/das/s3_storage_service.go index a1de200c52c..a289b0d46fc 100644 --- a/das/s3_storage_service.go +++ b/das/s3_storage_service.go @@ -110,7 +110,7 @@ func (s3s *S3StorageService) Put(ctx context.Context, value []byte, timeout uint Bucket: aws.String(s3s.bucket), Key: aws.String(s3s.objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))), Body: bytes.NewReader(value)} - if !s3s.discardAfterTimeout { + if s3s.discardAfterTimeout { expires := time.Unix(int64(timeout), 0) putObjectInput.Expires = &expires } diff --git a/das/syncing_fallback_storage.go b/das/syncing_fallback_storage.go index ee0ddc6b5fc..8af46b7fc58 100644 --- a/das/syncing_fallback_storage.go +++ b/das/syncing_fallback_storage.go @@ -105,9 +105,9 @@ type l1SyncService struct { lastBatchAcc common.Hash } -// The original syncing process had a bug, so the file was renamed to cause any mirrors -// in the wild to re-sync from their configured starting block number. -const nextBlockNoFilename = "nextBlockNumberV2" +// The filename has been updated when we have discovered bugs that may have impacted +// syncing, to cause mirrors to re-sync. +const nextBlockNoFilename = "nextBlockNumberV3" func readSyncStateOrDefault(syncDir string, dflt uint64) uint64 { if syncDir == "" { From b4fa9bbd97abaa2b83d8499cc769fff629566f39 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 22 Aug 2024 09:39:41 -0500 Subject: [PATCH 17/95] Lower a couple of info logs down to debug --- arbnode/batch_poster.go | 2 +- execution/gethexec/executionengine.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 71239efdbbc..b7eee5cc476 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -1439,7 +1439,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) b.building.muxBackend.delayedInboxStart = batchPosition.DelayedMessageCount b.building.muxBackend.SetPositionWithinMessage(0) simMux := arbstate.NewInboxMultiplexer(b.building.muxBackend, batchPosition.DelayedMessageCount, dapReaders, daprovider.KeysetValidate) - log.Info("Begin checking the correctness of batch against inbox multiplexer", "startMsgSeqNum", batchPosition.MessageCount, "endMsgSeqNum", b.building.msgCount-1) + log.Debug("Begin checking the correctness of batch against inbox multiplexer", "startMsgSeqNum", batchPosition.MessageCount, "endMsgSeqNum", b.building.msgCount-1) for i := batchPosition.MessageCount; i < b.building.msgCount; i++ { msg, err := simMux.Pop(ctx) if err != nil { diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 806355b2c6d..19d77fc38f9 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -138,7 +138,7 @@ func (s *ExecutionEngine) MarkFeedStart(to arbutil.MessageIndex) { defer s.cachedL1PriceData.mutex.Unlock() if to < s.cachedL1PriceData.startOfL1PriceDataCache { - log.Info("trying to trim older cache which doesnt exist anymore") + log.Debug("trying to trim older L1 price data cache which doesnt exist anymore") } else if to >= s.cachedL1PriceData.endOfL1PriceDataCache { s.cachedL1PriceData.startOfL1PriceDataCache = 0 s.cachedL1PriceData.endOfL1PriceDataCache = 0 From 5f40b1c58653120c151d329b7a3fc91818e7d6ec Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 13:46:44 -0300 Subject: [PATCH 18/95] Creates wallet contract through Staker's DataPoster --- cmd/nitro/nitro.go | 2 +- staker/validatorwallet/contract.go | 90 ++++++++++++++++++++++-------- system_tests/fast_confirm_test.go | 8 +-- system_tests/staker_test.go | 4 +- 4 files changed, 74 insertions(+), 30 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 39f204980d3..8c26b95a927 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -372,7 +372,7 @@ func mainImpl() int { log.Crit("error getting rollup addresses config", "err", err) } // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, nil) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 77b403b6692..89d4fef11d2 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -26,8 +26,11 @@ import ( "github.com/offchainlabs/nitro/util/headerreader" ) -var validatorABI abi.ABI -var walletCreatedID common.Hash +var ( + validatorABI abi.ABI + validatorWalletCreatorABI abi.ABI + walletCreatedID common.Hash +) func init() { parsedValidator, err := abi.JSON(strings.NewReader(rollupgen.ValidatorWalletABI)) @@ -40,6 +43,7 @@ func init() { if err != nil { panic(err) } + validatorWalletCreatorABI = parsedValidatorWalletCreator walletCreatedID = parsedValidatorWalletCreator.Events["WalletCreated"].ID } @@ -179,6 +183,37 @@ func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } +// Exposed for tests +func (v *Contract) CreateWalletContract( + ctx context.Context, + validatorWalletFactoryAddr common.Address, +) (*types.Transaction, error) { + var initialExecutorAllowedDests []common.Address + txData, err := validatorWalletCreatorABI.Pack("createWallet", initialExecutorAllowedDests) + if err != nil { + return nil, err + } + + auth, err := v.getAuth(ctx, nil) + if err != nil { + return nil, err + } + + gas, err := gasForTxData( + ctx, + v.l1Reader, + auth, + &validatorWalletFactoryAddr, + txData, + v.getExtraGas, + ) + if err != nil { + return nil, fmt.Errorf("getting gas for tx data: %w", err) + } + + return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) +} + func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { if v.con != nil { return nil @@ -194,7 +229,7 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err if err != nil { return err } - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) if err != nil { return err } @@ -295,25 +330,29 @@ func (v *Contract) ExecuteTransactions(ctx context.Context, builder *txbuilder.B return arbTx, nil } -func (v *Contract) estimateGas(ctx context.Context, value *big.Int, data []byte) (uint64, error) { - h, err := v.l1Reader.LastHeader(ctx) +func gasForTxData(ctx context.Context, l1Reader *headerreader.HeaderReader, auth *bind.TransactOpts, to *common.Address, data []byte, getExtraGas func() uint64) (uint64, error) { + if auth.GasLimit != 0 { + return auth.GasLimit, nil + } + + h, err := l1Reader.LastHeader(ctx) if err != nil { return 0, fmt.Errorf("getting the last header: %w", err) } gasFeeCap := new(big.Int).Mul(h.BaseFee, big.NewInt(2)) gasFeeCap = arbmath.BigMax(gasFeeCap, arbmath.FloatToBig(params.GWei)) - gasTipCap, err := v.l1Reader.Client().SuggestGasTipCap(ctx) + gasTipCap, err := l1Reader.Client().SuggestGasTipCap(ctx) if err != nil { return 0, fmt.Errorf("getting suggested gas tip cap: %w", err) } gasFeeCap.Add(gasFeeCap, gasTipCap) - g, err := v.l1Reader.Client().EstimateGas( + g, err := l1Reader.Client().EstimateGas( ctx, ethereum.CallMsg{ - From: v.auth.From, - To: v.Address(), - Value: value, + From: auth.From, + To: to, + Value: auth.Value, Data: data, GasFeeCap: gasFeeCap, GasTipCap: gasTipCap, @@ -322,7 +361,11 @@ func (v *Contract) estimateGas(ctx context.Context, value *big.Int, data []byte) if err != nil { return 0, fmt.Errorf("estimating gas: %w", err) } - return g + v.getExtraGas(), nil + return g + getExtraGas(), nil +} + +func (v *Contract) gasForTxData(ctx context.Context, auth *bind.TransactOpts, data []byte) (uint64, error) { + return gasForTxData(ctx, v.l1Reader, auth, v.Address(), data, v.getExtraGas) } func (v *Contract) TimeoutChallenges(ctx context.Context, challenges []uint64) (*types.Transaction, error) { @@ -341,14 +384,6 @@ func (v *Contract) TimeoutChallenges(ctx context.Context, challenges []uint64) ( return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } -// gasForTxData returns auth.GasLimit if it's nonzero, otherwise returns estimate. -func (v *Contract) gasForTxData(ctx context.Context, auth *bind.TransactOpts, data []byte) (uint64, error) { - if auth.GasLimit != 0 { - return auth.GasLimit, nil - } - return v.estimateGas(ctx, auth.Value, data) -} - func (v *Contract) L1Client() arbutil.L1Interface { return v.l1Reader.Client() } @@ -407,6 +442,7 @@ func GetValidatorWalletContract( transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, + createWalletFunc func(context.Context, common.Address) (*types.Transaction, error), ) (*common.Address, error) { client := l1Reader.Client() @@ -443,10 +479,18 @@ func GetValidatorWalletContract( return nil, nil } - var initialExecutorAllowedDests []common.Address - tx, err := walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) - if err != nil { - return nil, err + var tx *types.Transaction + if createWalletFunc == nil { + var initialExecutorAllowedDests []common.Address + tx, err = walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) + if err != nil { + return nil, err + } + } else { + tx, err = createWalletFunc(ctx, validatorWalletFactoryAddr) + if err != nil { + return nil, err + } } receipt, err := l1Reader.WaitForTxApproval(ctx, tx) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 4a679e50772..79b77524a28 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -79,10 +79,10 @@ func TestFastConfirmation(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "Validator", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("Validator", ctx) - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -278,10 +278,10 @@ func TestFastConfirmationWithSafe(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 03c9fd36283..13530adfd56 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -132,10 +132,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 26bd0bd59e0ed1d0948c197937fd601809bf5342 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:23:46 -0300 Subject: [PATCH 19/95] Staker tests creates wallet contract through Staker's DataPoster --- system_tests/staker_test.go | 42 +++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 13530adfd56..ea34eb8a966 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -37,6 +37,7 @@ import ( "github.com/offchainlabs/nitro/util" "github.com/offchainlabs/nitro/util/arbmath" "github.com/offchainlabs/nitro/util/colors" + "github.com/offchainlabs/nitro/util/testhelpers" "github.com/offchainlabs/nitro/validator/valnode" ) @@ -57,7 +58,8 @@ func makeBackgroundTxs(ctx context.Context, builder *NodeBuilder) error { } func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) { - t.Parallel() + logHandler := testhelpers.InitTestLog(t, log.LvlTrace) + ctx, cancelCtx := context.WithCancel(context.Background()) defer cancelCtx() srv := externalsignertest.NewServer(t) @@ -132,15 +134,6 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - if valWalletAddrA == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2nodeA.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -149,16 +142,9 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address}, []bool{true, true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) @@ -190,6 +176,22 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } + + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address}, []bool{true, true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig @@ -464,6 +466,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) if !stakerBWasStaked { Fatal(t, "staker B was never staked") } + + if logHandler.WasLogged("data poster expected next transaction to have nonce \\d+ but was requested to post transaction with nonce \\d+") { + Fatal(t, "Staker's DataPoster inferred nonce incorrectly") + } } func TestStakersCooperative(t *testing.T) { From c3ee555bc6c3679aed7c1347fc8c771d3d02caca Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:24:02 -0300 Subject: [PATCH 20/95] TestGetValidatorWalletContractWithoutDataPoster --- system_tests/staker_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index ea34eb8a966..3fdf82ca744 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -475,3 +475,29 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) func TestStakersCooperative(t *testing.T) { stakerTestImpl(t, false, false) } + +func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { + t.Parallel() + + ctx, cancelCtx := context.WithCancel(context.Background()) + defer cancelCtx() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + cleanup := builder.Build(t) + defer cleanup() + + balance := big.NewInt(params.Ether) + balance.Mul(balance, big.NewInt(100)) + builder.L1Info.GenerateAccount("ValidatorA") + builder.L1.TransferBalance(t, "Faucet", "ValidatorA", balance, builder.L1Info) + l1auth := builder.L1Info.GetDefaultTransactOpts("ValidatorA", ctx) + + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } +} From 2fde94e651c991df1e4a3efe858d758bd784e4e4 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 15:52:10 -0300 Subject: [PATCH 21/95] Adds comment related to v.CreateWalletContract --- staker/validatorwallet/contract.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 89d4fef11d2..495e796cd25 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -229,6 +229,10 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err if err != nil { return err } + + // By passing v.CreateWalletContract as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. + // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. + // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) if err != nil { return err From 04e3f12507ceff95812d0b72901d84ed643954a1 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 12 Aug 2024 16:16:12 -0300 Subject: [PATCH 22/95] Fast confirmation test creates wallet contract through Staker's DataPoster --- system_tests/fast_confirm_test.go | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 79b77524a28..ae624be1e9b 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -79,15 +79,6 @@ func TestFastConfirmation(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "Validator", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("Validator", ctx) - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) - Require(t, err) - valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, nil) - Require(t, err) - if valWalletAddr == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2node.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -96,27 +87,13 @@ func TestFastConfirmation(t *testing.T) { rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddr, srv.Address}, []bool{true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) - setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", valWalletAddr) - Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) - Require(t, err, "unable to set anytrust fast confirmer") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - valConfig := staker.TestL1ValidatorConfig valConfig.EnableFastConfirmation = true parentChainID, err := builder.L1.Client.ChainID(ctx) @@ -138,6 +115,29 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + Require(t, err) + valWalletAddr := *valWalletAddrPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + Require(t, err) + if valWalletAddr == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) + } + + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddr, srv.Address}, []bool{true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + + setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", valWalletAddr) + Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2node.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) + Require(t, err, "unable to set anytrust fast confirmer") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig From fa3bf04edabd7d07b5986aaab73e5df4d030104b Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Thu, 22 Aug 2024 16:16:41 -0300 Subject: [PATCH 23/95] Fix linter warning --- system_tests/stylus_tracer_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index f7c83c73f67..3b95f38d218 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -167,6 +167,7 @@ func TestStylusTracer(t *testing.T) { } func intToBe32(v int) []byte { + // #nosec G115 return binary.BigEndian.AppendUint32(nil, uint32(v)) } From b933b74725e2dd34df1f472ab22fbdfcff277698 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 22 Aug 2024 22:19:16 +0200 Subject: [PATCH 24/95] pass targets to rawdb.WrapDatabaseWithWasm, change rawdb.Target to ethdb.WasmTarget --- arbnode/inbox_test.go | 4 +- arbos/programs/native.go | 11 ++--- arbos/programs/wasmstorehelper.go | 3 +- cmd/nitro/init.go | 6 +-- cmd/nitro/init_test.go | 4 ++ cmd/nitro/nitro.go | 2 +- execution/gethexec/executionengine.go | 35 ++++++++++------ execution/gethexec/node.go | 50 +++++++++++++++++++++-- go-ethereum | 2 +- staker/block_validator.go | 2 +- staker/challenge_manager.go | 3 +- staker/stateless_block_validator.go | 5 +-- system_tests/common_test.go | 47 +++++++++++---------- system_tests/forwarder_test.go | 4 +- system_tests/recreatestate_rpc_test.go | 18 ++++---- system_tests/retryable_test.go | 2 +- system_tests/validation_mock_test.go | 6 +-- validator/client/redis/producer.go | 9 ++-- validator/client/validation_client.go | 13 +++--- validator/interface.go | 4 +- validator/server_api/json.go | 8 ++-- validator/server_arb/validator_spawner.go | 5 ++- validator/server_jit/spawner.go | 5 ++- validator/validation_entry.go | 4 +- validator/valnode/validation_api.go | 4 +- 25 files changed, 161 insertions(+), 95 deletions(-) diff --git a/arbnode/inbox_test.go b/arbnode/inbox_test.go index 1c46c593b99..d579b7c2784 100644 --- a/arbnode/inbox_test.go +++ b/arbnode/inbox_test.go @@ -72,7 +72,9 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (* if err != nil { Fail(t, err) } - if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, &gethexec.DefaultStylusTargetConfig); err != nil { + stylusTargetConfig := &gethexec.DefaultStylusTargetConfig + Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets) + if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, stylusTargetConfig); err != nil { Fail(t, err) } execSeq := &execClientWrapper{execEngine, t} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index fd3dec25a04..36e200af4df 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbos/burn" "github.com/offchainlabs/nitro/arbos/util" @@ -71,7 +72,7 @@ func activateProgramInternal( version uint16, debug bool, gasLeft *uint64, -) (*activationInfo, map[rawdb.Target][]byte, error) { +) (*activationInfo, map[ethdb.WasmTarget][]byte, error) { output := &rustBytes{} moduleHash := &bytes32{} stylusData := &C.StylusData{} @@ -111,7 +112,7 @@ func activateProgramInternal( if status_asm != 0 { return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) } - asmMap := map[rawdb.Target][]byte{ + asmMap := map[ethdb.WasmTarget][]byte{ rawdb.TargetWavm: module, target: asm, } @@ -171,7 +172,7 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c } asm, exists := asmMap[localTarget] if !exists { - var availableTargets []rawdb.Target + var availableTargets []ethdb.WasmTarget for target := range asmMap { availableTargets = append(availableTargets, target) } @@ -203,7 +204,7 @@ func callProgram( } if db, ok := db.(*state.StateDB); ok { - targets := []rawdb.Target{ + targets := []ethdb.WasmTarget{ rawdb.TargetWavm, rawdb.LocalTarget(), } @@ -291,7 +292,7 @@ func ResizeWasmLruCache(size uint32) { const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2" -func SetTarget(name rawdb.Target, description string, native bool) error { +func SetTarget(name ethdb.WasmTarget, description string, native bool) error { output := &rustBytes{} status := userStatus(C.stylus_target_set( goSlice([]byte(name)), diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 4f82d802828..95daaf544ab 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -44,7 +45,7 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash } // If already in wasm store then return early - _, err = statedb.TryGetActivatedAsmMap([]rawdb.Target{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) + _, err = statedb.TryGetActivatedAsmMap([]ethdb.WasmTarget{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) if err == nil { return nil } diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index d9ae0df3b04..14d9a522030 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -475,7 +475,7 @@ func validateOrUpgradeWasmStoreSchemaVersion(db ethdb.Database) error { return nil } -func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { +func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { if chainConfig := gethexec.TryReadStoredChainConfig(readOnlyDb); chainConfig != nil { @@ -500,7 +500,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := dbutil.UnfinishedConversionCheck(wasmDb); err != nil { return nil, nil, fmt.Errorf("wasm unfinished database conversion check error: %w", err) } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err @@ -617,7 +617,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := validateOrUpgradeWasmStoreSchemaVersion(wasmDb); err != nil { return nil, nil, err } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index b2773ed8613..af2242af783 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -377,6 +377,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -393,6 +394,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -410,6 +412,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -545,6 +548,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + &gethexec.DefaultStylusTargetConfig, &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index a052c146d1e..052755d8a34 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -481,7 +481,7 @@ func mainImpl() int { } } - chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Persistent, l1Client, rollupAddrs) + chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Execution.StylusTarget, &nodeConfig.Persistent, l1Client, rollupAddrs) if l2BlockChain != nil { deferFuncs = append(deferFuncs, func() { l2BlockChain.Stop() }) } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 806355b2c6d..4e0217f527e 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -154,20 +154,31 @@ func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusT if rustCacheSize != 0 { programs.ResizeWasmLruCache(rustCacheSize) } - var effectiveStylusTarget string - target := rawdb.LocalTarget() - switch target { - case rawdb.TargetArm64: - effectiveStylusTarget = targetConfig.Arm64 - case rawdb.TargetAmd64: - effectiveStylusTarget = targetConfig.Amd64 - case rawdb.TargetHost: - effectiveStylusTarget = targetConfig.Host + + localTarget := rawdb.LocalTarget() + targets := targetConfig.WasmTargets() + var nativeSet bool + for _, target := range targets { + var effectiveStylusTarget string + switch target { + case rawdb.TargetArm64: + effectiveStylusTarget = targetConfig.Arm64 + case rawdb.TargetAmd64: + effectiveStylusTarget = targetConfig.Amd64 + case rawdb.TargetHost: + effectiveStylusTarget = targetConfig.Host + } + isNative := target == localTarget + err := programs.SetTarget(target, effectiveStylusTarget, isNative) + if err != nil { + return fmt.Errorf("Failed to set stylus target: %w", err) + } + nativeSet = nativeSet || isNative } - err := programs.SetTarget(target, effectiveStylusTarget, true) - if err != nil { - return fmt.Errorf("Failed to set stylus target: %w", err) + if !nativeSet { + return fmt.Errorf("local target %v missing in list of archs %v", localTarget, targets) } + return nil } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index b864332e83d..f519cce6ed7 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/arbitrum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/filters" @@ -29,21 +30,61 @@ import ( ) type StylusTargetConfig struct { - Arm64 string `koanf:"arm64"` - Amd64 string `koanf:"amd64"` - Host string `koanf:"host"` + Arm64 string `koanf:"arm64"` + Amd64 string `koanf:"amd64"` + Host string `koanf:"host"` + Archs []string `koanf:"archs"` + + wasmTargets []ethdb.WasmTarget +} + +func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { + return c.wasmTargets +} + +func (c *StylusTargetConfig) Validate() error { + localTarget := rawdb.LocalTarget() + var nativeFound bool + targets := make([]ethdb.WasmTarget, 0, len(c.Archs)) + for _, arch := range c.Archs { + target := ethdb.WasmTarget(arch) + if !rawdb.IsSupportedWasmTarget(target) { + return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) + } + var alreadyInList bool + for _, t := range targets { + if target == t { + alreadyInList = true + break + } + } + if alreadyInList { + continue + } + if target == localTarget { + nativeFound = true + } + targets = append(targets, target) + } + if !nativeFound { + return fmt.Errorf("native target not found in archs list, native target: %v, archs: %v", localTarget, c.Archs) + } + c.wasmTargets = targets + return nil } var DefaultStylusTargetConfig = StylusTargetConfig{ Arm64: programs.DefaultTargetDescriptionArm, Amd64: programs.DefaultTargetDescriptionX86, Host: "", + Archs: []string{string(rawdb.TargetWavm), string(rawdb.LocalTarget())}, } func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux") f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux") f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86") + f.StringSlice(prefix+".archs", DefaultStylusTargetConfig.Archs, fmt.Sprintf("Comma separated architectures list to compile stylus program to and cache in wasm store (available targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) } type Config struct { @@ -82,6 +123,9 @@ func (c *Config) Validate() error { if c.forwardingTarget != "" && c.Sequencer.Enable { return errors.New("ForwardingTarget set and sequencer enabled") } + if err := c.StylusTarget.Validate(); err != nil { + return err + } return nil } diff --git a/go-ethereum b/go-ethereum index 575062fad7f..cb03c12f7f0 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 575062fad7ff4db9d7c235f49472f658be29e2fe +Subproject commit cb03c12f7f02d47344d2d089eef748e92b99c95e diff --git a/staker/block_validator.go b/staker/block_validator.go index 8f5724beac4..5cfce84c991 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -499,7 +499,7 @@ func (v *BlockValidator) sendRecord(s *validationStatus) error { //nolint:gosec func (v *BlockValidator) writeToFile(validationEntry *validationEntry, moduleRoot common.Hash) error { - input, err := validationEntry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := validationEntry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return err } diff --git a/staker/challenge_manager.go b/staker/challenge_manager.go index b1421d7e419..978a876fd0f 100644 --- a/staker/challenge_manager.go +++ b/staker/challenge_manager.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" @@ -468,7 +469,7 @@ func (m *ChallengeManager) createExecutionBackend(ctx context.Context, step uint if err != nil { return fmt.Errorf("error creating validation entry for challenge %v msg %v for execution challenge: %w", m.challengeIndex, initialCount, err) } - input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := entry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return fmt.Errorf("error getting validation entry input of challenge %v msg %v: %w", m.challengeIndex, initialCount, err) } diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index d5eeb8eb694..f54ec8b58c1 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -12,7 +12,6 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -134,7 +133,7 @@ type validationEntry struct { DelayedMsg []byte } -func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.ValidationInput, error) { +func (e *validationEntry) ToInput(stylusArchs []ethdb.WasmTarget) (*validator.ValidationInput, error) { if e.Stage != Ready { return nil, errors.New("cannot create input from non-ready entry") } @@ -143,7 +142,7 @@ func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.Valida HasDelayedMsg: e.HasDelayedMsg, DelayedMsgNr: e.DelayedMsgNr, Preimages: e.Preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte, len(e.UserWasms)), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte, len(e.UserWasms)), BatchInfo: e.BatchInfo, DelayedMsg: e.DelayedMsg, StartState: e.Start, diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 6e7375a921d..392f6de8059 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -197,7 +197,7 @@ var TestSequencerConfig = gethexec.SequencerConfig{ EnableProfiling: false, } -func ExecConfigDefaultNonSequencerTest() *gethexec.Config { +func ExecConfigDefaultNonSequencerTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.ParentChainReader = headerreader.TestConfig @@ -206,12 +206,12 @@ func ExecConfigDefaultNonSequencerTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } -func ExecConfigDefaultTest() *gethexec.Config { +func ExecConfigDefaultTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.Sequencer = TestSequencerConfig @@ -219,7 +219,7 @@ func ExecConfigDefaultTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } @@ -272,7 +272,7 @@ func (b *NodeBuilder) DefaultConfig(t *testing.T, withL1 bool) *NodeBuilder { b.l2StackConfig = testhelpers.CreateStackConfigForTest(b.dataDir) cp := valnode.TestValidationConfig b.valnodeConfig = &cp - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) return b } @@ -310,7 +310,7 @@ func (b *NodeBuilder) CheckConfig(t *testing.T) { b.nodeConfig = arbnode.ConfigDefaultL1Test() } if b.execConfig == nil { - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) } if b.L1Info == nil { b.L1Info = NewL1TestInfo(t) @@ -346,7 +346,7 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { var l2arbDb ethdb.Database var l2blockchain *core.BlockChain _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createL2BlockChainWithStackConfig( - t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) var sequencerTxOptsPtr *bind.TransactOpts var dataSigner signature.DataSignerFunc @@ -409,7 +409,7 @@ func (b *NodeBuilder) BuildL2(t *testing.T) func() { var arbDb ethdb.Database var blockchain *core.BlockChain b.L2Info, b.L2.Stack, chainDb, arbDb, blockchain = createL2BlockChain( - t, b.L2Info, b.dataDir, b.chainConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.execConfig) Require(t, b.execConfig.Validate()) execConfig := b.execConfig @@ -460,7 +460,7 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { } b.L2.cleanup() - l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) execConfigFetcher := func() *gethexec.Config { return b.execConfig } execNode, err := gethexec.CreateExecutionNode(b.ctx, stack, chainDb, blockchain, nil, execConfigFetcher) @@ -1047,30 +1047,32 @@ func DeployOnTestL1( } func createL2BlockChain( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, cacheConfig) + return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) } func createL2BlockChainWithStackConfig( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { if l2info == nil { l2info = NewArbTestInfo(t, chainConfig.ChainID) } - var stack *node.Node - var err error if stackConfig == nil { stackConfig = testhelpers.CreateStackConfigForTest(dataDir) } - stack, err = node.New(stackConfig) + if execConfig == nil { + execConfig = ExecConfigDefaultTest(t) + } + + stack, err := node.New(stackConfig) Require(t, err) chainData, err := stack.OpenDatabaseWithExtraOptions("l2chaindata", 0, 0, "l2chaindata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")) Require(t, err) wasmData, err := stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) arbDb, err := stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1085,11 +1087,8 @@ func createL2BlockChainWithStackConfig( SerializedChainConfig: serializedChainConfig, } } - var coreCacheConfig *core.CacheConfig - if cacheConfig != nil { - coreCacheConfig = gethexec.DefaultCacheConfigFor(stack, cacheConfig) - } - blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &execConfig.Caching) + blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) return l2info, stack, chainDb, arbDb, blockchain @@ -1149,7 +1148,7 @@ func Create2ndNodeWithConfig( nodeConfig = arbnode.ConfigDefaultL1NonSequencerTest() } if execConfig == nil { - execConfig = ExecConfigDefaultNonSequencerTest() + execConfig = ExecConfigDefaultNonSequencerTest(t) } feedErrChan := make(chan error, 10) l1rpcClient := l1stack.Attach() @@ -1165,7 +1164,7 @@ func Create2ndNodeWithConfig( Require(t, err) wasmData, err := l2stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0) + l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) l2arbDb, err := l2stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1179,7 +1178,7 @@ func Create2ndNodeWithConfig( chainConfig := firstExec.ArbInterface.BlockChain().Config() coreCacheConfig := gethexec.DefaultCacheConfigFor(l2stack, &execConfig.Caching) - l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) diff --git a/system_tests/forwarder_test.go b/system_tests/forwarder_test.go index 9fe419593ee..1b083d2089d 100644 --- a/system_tests/forwarder_test.go +++ b/system_tests/forwarder_test.go @@ -38,7 +38,7 @@ func TestStaticForwarder(t *testing.T) { clientA := builder.L2.Client nodeConfigB := arbnode.ConfigDefaultL1Test() - execConfigB := ExecConfigDefaultTest() + execConfigB := ExecConfigDefaultTest(t) execConfigB.Sequencer.Enable = false nodeConfigB.Sequencer = false nodeConfigB.DelayedSequencer.Enable = false @@ -109,7 +109,7 @@ func createForwardingNode(t *testing.T, builder *NodeBuilder, ipcPath string, re nodeConfig.Sequencer = false nodeConfig.DelayedSequencer.Enable = false nodeConfig.BatchPoster.Enable = false - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Sequencer.Enable = false execConfig.Forwarder.RedisUrl = redisUrl execConfig.ForwardingTarget = fallbackPath diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 09d53669ee0..d5a6311cb45 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -95,7 +95,7 @@ func removeStatesFromDb(t *testing.T, bc *core.BlockChain, db ethdb.Database, fr func TestRecreateStateForRPCNoDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -133,7 +133,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() depthGasLimit := int64(256 * util.NormalizeL2GasForL1GasInitial(800_000, params.GWei)) - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = depthGasLimit execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -170,7 +170,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { func TestRecreateStateForRPCDepthLimitExceeded(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = int64(200) execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -207,7 +207,7 @@ func TestRecreateStateForRPCMissingBlockParent(t *testing.T) { var headerCacheLimit uint64 = 512 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -255,7 +255,7 @@ func TestRecreateStateForRPCBeyondGenesis(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -293,7 +293,7 @@ func TestRecreateStateForRPCBlockNotFoundWhileRecreating(t *testing.T) { var blockCacheLimit uint64 = 256 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -341,7 +341,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = maxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -480,7 +480,7 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { func TestGettingStateForRPCFullNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.SnapshotCache = 0 // disable snapshots execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are execConfig.Sequencer.MaxBlockSpeed = 0 @@ -522,7 +522,7 @@ func TestGettingStateForRPCFullNode(t *testing.T) { func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.Archive = true // For now Archive node should use HashScheme execConfig.Caching.StateScheme = rawdb.HashScheme diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 106dfc6d463..aa9fbfd72ea 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -1042,7 +1042,7 @@ func elevateL2Basefee(t *testing.T, ctx context.Context, builder *NodeBuilder) { _, err = precompilesgen.NewArbosTest(common.HexToAddress("0x69"), builder.L2.Client) Require(t, err, "failed to deploy ArbosTest") - burnAmount := ExecConfigDefaultTest().RPC.RPCGasCap + burnAmount := ExecConfigDefaultTest(t).RPC.RPCGasCap burnTarget := uint64(5 * l2pricing.InitialSpeedLimitPerSecondV6 * l2pricing.InitialBacklogTolerance) for i := uint64(0); i < (burnTarget+burnAmount)/burnAmount; i++ { burnArbGas := arbosTestAbi.Methods["burnArbGas"] diff --git a/system_tests/validation_mock_test.go b/system_tests/validation_mock_test.go index 88421e4c4b5..2739c7545e6 100644 --- a/system_tests/validation_mock_test.go +++ b/system_tests/validation_mock_test.go @@ -8,9 +8,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" @@ -61,8 +61,8 @@ func (s *mockSpawner) WasmModuleRoots() ([]common.Hash, error) { return mockWasmModuleRoots, nil } -func (s *mockSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{"mock"} +func (s *mockSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{"mock"} } func (s *mockSpawner) Launch(entry *validator.ValidationInput, moduleRoot common.Hash) validator.ValidationRun { diff --git a/validator/client/redis/producer.go b/validator/client/redis/producer.go index f98c246d0e6..adc2f34af57 100644 --- a/validator/client/redis/producer.go +++ b/validator/client/redis/producer.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/pubsub" @@ -35,7 +36,7 @@ func (c ValidationClientConfig) Enabled() bool { func (c ValidationClientConfig) Validate() error { for _, arch := range c.StylusArchs { - if !rawdb.Target(arch).IsValid() { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(arch)) { return fmt.Errorf("Invalid stylus arch: %v", arch) } } @@ -162,10 +163,10 @@ func (c *ValidationClient) Name() string { return c.config.Name } -func (c *ValidationClient) StylusArchs() []rawdb.Target { - stylusArchs := make([]rawdb.Target, 0, len(c.config.StylusArchs)) +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { + stylusArchs := make([]ethdb.WasmTarget, 0, len(c.config.StylusArchs)) for _, arch := range c.config.StylusArchs { - stylusArchs = append(stylusArchs, rawdb.Target(arch)) + stylusArchs = append(stylusArchs, ethdb.WasmTarget(arch)) } return stylusArchs } diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 80cff666754..9dd7780fbc2 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" @@ -31,7 +32,7 @@ type ValidationClient struct { stopwaiter.StopWaiter client *rpcclient.RpcClient name string - stylusArchs []rawdb.Target + stylusArchs []ethdb.WasmTarget room atomic.Int32 wasmModuleRoots []common.Hash } @@ -40,7 +41,7 @@ func NewValidationClient(config rpcclient.ClientConfigFetcher, stack *node.Node) return &ValidationClient{ client: rpcclient.NewRpcClient(config, stack), name: "not started", - stylusArchs: []rawdb.Target{"not started"}, + stylusArchs: []ethdb.WasmTarget{"not started"}, } } @@ -67,14 +68,14 @@ func (c *ValidationClient) Start(ctx context.Context) error { if len(name) == 0 { return errors.New("couldn't read name from server") } - var stylusArchs []rawdb.Target + var stylusArchs []ethdb.WasmTarget if err := c.client.CallContext(ctx, &stylusArchs, server_api.Namespace+"_stylusArchs"); err != nil { var rpcError rpc.Error ok := errors.As(err, &rpcError) if !ok || rpcError.ErrorCode() != -32601 { return fmt.Errorf("could not read stylus arch from server: %w", err) } - stylusArchs = []rawdb.Target{rawdb.Target("pre-stylus")} // invalid, will fail if trying to validate block with stylus + stylusArchs = []ethdb.WasmTarget{ethdb.WasmTarget("pre-stylus")} // invalid, will fail if trying to validate block with stylus } else { if len(stylusArchs) == 0 { return fmt.Errorf("could not read stylus archs from validation server") @@ -117,11 +118,11 @@ func (c *ValidationClient) WasmModuleRoots() ([]common.Hash, error) { return nil, errors.New("not started") } -func (c *ValidationClient) StylusArchs() []rawdb.Target { +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { if c.Started() { return c.stylusArchs } - return []rawdb.Target{"not started"} + return []ethdb.WasmTarget{"not started"} } func (c *ValidationClient) Stop() { diff --git a/validator/interface.go b/validator/interface.go index 81b40ae5cfc..af086291372 100644 --- a/validator/interface.go +++ b/validator/interface.go @@ -4,7 +4,7 @@ import ( "context" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/containers" ) @@ -14,7 +14,7 @@ type ValidationSpawner interface { Start(context.Context) error Stop() Name() string - StylusArchs() []rawdb.Target + StylusArchs() []ethdb.WasmTarget Room() int } diff --git a/validator/server_api/json.go b/validator/server_api/json.go index dbe2bb1fee6..6fe936e17d6 100644 --- a/validator/server_api/json.go +++ b/validator/server_api/json.go @@ -11,7 +11,7 @@ import ( "os" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbcompress" "github.com/offchainlabs/nitro/arbutil" @@ -64,7 +64,7 @@ type InputJSON struct { BatchInfo []BatchInfoJson DelayedMsgB64 string StartState validator.GoGlobalState - UserWasms map[rawdb.Target]map[common.Hash]string + UserWasms map[ethdb.WasmTarget]map[common.Hash]string DebugChain bool } @@ -96,7 +96,7 @@ func ValidationInputToJson(entry *validator.ValidationInput) *InputJSON { DelayedMsgB64: base64.StdEncoding.EncodeToString(entry.DelayedMsg), StartState: entry.StartState, PreimagesB64: jsonPreimagesMap, - UserWasms: make(map[rawdb.Target]map[common.Hash]string), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash]string), DebugChain: entry.DebugChain, } for _, binfo := range entry.BatchInfo { @@ -128,7 +128,7 @@ func ValidationInputFromJson(entry *InputJSON) (*validator.ValidationInput, erro DelayedMsgNr: entry.DelayedMsgNr, StartState: entry.StartState, Preimages: preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte), DebugChain: entry.DebugChain, } delayed, err := base64.StdEncoding.DecodeString(entry.DelayedMsgB64) diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index 844a988d282..96bc5d619d0 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/pflag" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/stopwaiter" @@ -89,8 +90,8 @@ func (s *ArbitratorSpawner) WasmModuleRoots() ([]common.Hash, error) { return s.locator.ModuleRoots(), nil } -func (s *ArbitratorSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.TargetWavm} +func (s *ArbitratorSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.TargetWavm} } func (s *ArbitratorSpawner) Name() string { diff --git a/validator/server_jit/spawner.go b/validator/server_jit/spawner.go index 92b50b17cba..d77317d2184 100644 --- a/validator/server_jit/spawner.go +++ b/validator/server_jit/spawner.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -72,8 +73,8 @@ func (v *JitSpawner) WasmModuleRoots() ([]common.Hash, error) { return v.locator.ModuleRoots(), nil } -func (v *JitSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.LocalTarget()} +func (v *JitSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.LocalTarget()} } func (v *JitSpawner) execute( diff --git a/validator/validation_entry.go b/validator/validation_entry.go index 2c357659ad2..d340993fa2e 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -2,7 +2,7 @@ package validator import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" ) @@ -17,7 +17,7 @@ type ValidationInput struct { HasDelayedMsg bool DelayedMsgNr uint64 Preimages map[arbutil.PreimageType]map[common.Hash][]byte - UserWasms map[rawdb.Target]map[common.Hash][]byte + UserWasms map[ethdb.WasmTarget]map[common.Hash][]byte BatchInfo []BatchInfo DelayedMsg []byte StartState GoGlobalState diff --git a/validator/valnode/validation_api.go b/validator/valnode/validation_api.go index a79ac7fa55e..a10d931dfc2 100644 --- a/validator/valnode/validation_api.go +++ b/validator/valnode/validation_api.go @@ -12,7 +12,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -45,7 +45,7 @@ func (a *ValidationServerAPI) WasmModuleRoots() ([]common.Hash, error) { return a.spawner.WasmModuleRoots() } -func (a *ValidationServerAPI) StylusArchs() ([]rawdb.Target, error) { +func (a *ValidationServerAPI) StylusArchs() ([]ethdb.WasmTarget, error) { return a.spawner.StylusArchs(), nil } From eefd7294777775f69689c316ca97c4b9d1338172 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 00:52:30 +0200 Subject: [PATCH 25/95] compile to targets in WasmTargets list --- arbos/programs/native.go | 61 +++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 36e200af4df..98abf9443ca 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -100,25 +100,50 @@ func activateProgramInternal( } return nil, nil, err } - target := rawdb.LocalTarget() - status_asm := C.stylus_compile( - goSlice(wasm), - u16(version), - cbool(debug), - goSlice([]byte(target)), - output, - ) - asm := output.intoBytes() - if status_asm != 0 { - return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) + targets := db.Database().WasmTargets() + type result struct { + target ethdb.WasmTarget + asm []byte + err error } - asmMap := map[ethdb.WasmTarget][]byte{ - rawdb.TargetWavm: module, - target: asm, + results := make(chan result, len(targets)) + for _, target := range targets { + if target == rawdb.TargetWavm { + results <- result{target, module, nil} + } else { + target := target + go func() { + output := &rustBytes{} + status_asm := C.stylus_compile( + goSlice(wasm), + u16(version), + cbool(debug), + goSlice([]byte(target)), + output, + ) + asm := output.intoBytes() + if status_asm != 0 { + results <- result{target, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))} + return + } + results <- result{target, asm, nil} + }() + } + } + asmMap := make(map[ethdb.WasmTarget][]byte, len(targets)) + for range targets { + res := <-results + if res.err != nil { + err = errors.Join(res.err, err) + } else { + asmMap[res.target] = res.asm + } + } + if err != nil { + return nil, nil, fmt.Errorf("compilation failed for one or more targets: %w", err) } hash := moduleHash.toHash() - info := &activationInfo{ moduleHash: hash, initGas: uint16(stylusData.init_cost), @@ -204,11 +229,7 @@ func callProgram( } if db, ok := db.(*state.StateDB); ok { - targets := []ethdb.WasmTarget{ - rawdb.TargetWavm, - rawdb.LocalTarget(), - } - db.RecordProgram(targets, moduleHash) + db.RecordProgram(moduleHash) } evmApi := newApi(interpreter, tracingInfo, scope, memoryModel) From 2832703599b4749c32e9af9760b4d3ffdacbba9c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 01:21:02 +0200 Subject: [PATCH 26/95] use wasm targets list in SaveActiveProgramToWasmStore --- arbos/programs/wasmstorehelper.go | 4 ++-- go-ethereum | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 95daaf544ab..434820dd9c9 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -12,7 +12,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -44,8 +43,9 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash return err } + targets := statedb.Database().WasmTargets() // If already in wasm store then return early - _, err = statedb.TryGetActivatedAsmMap([]ethdb.WasmTarget{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) + _, err = statedb.TryGetActivatedAsmMap(targets, moduleHash) if err == nil { return nil } diff --git a/go-ethereum b/go-ethereum index cb03c12f7f0..c7d753f2c4d 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit cb03c12f7f02d47344d2d089eef748e92b99c95e +Subproject commit c7d753f2c4d5eb41c94e2e2499219a72fbf241a0 From 10f9b159550b59d97304cd935db25b076adf6443 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 01:48:47 +0200 Subject: [PATCH 27/95] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index c7d753f2c4d..7fc4c646075 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit c7d753f2c4d5eb41c94e2e2499219a72fbf241a0 +Subproject commit 7fc4c646075eb9b6e26374886ea6fe9463b106aa From 8bfc9e680ad20ef8e5d4d292167af3c73a6149e1 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 23 Aug 2024 17:21:21 +0530 Subject: [PATCH 28/95] Allow force rebuilding of local wasm store --- cmd/conf/init.go | 14 +++++++++++--- cmd/nitro/init.go | 16 ++++++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index d88bcdd241d..f3606916936 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -30,7 +30,7 @@ type InitConfig struct { PruneThreads int `koanf:"prune-threads"` PruneTrieCleanCache int `koanf:"prune-trie-clean-cache"` RecreateMissingStateFrom uint64 `koanf:"recreate-missing-state-from"` - RebuildLocalWasm bool `koanf:"rebuild-local-wasm"` + RebuildLocalWasm string `koanf:"rebuild-local-wasm"` ReorgToBatch int64 `koanf:"reorg-to-batch"` ReorgToMessageBatch int64 `koanf:"reorg-to-message-batch"` ReorgToBlockBatch int64 `koanf:"reorg-to-block-batch"` @@ -56,7 +56,7 @@ var InitConfigDefault = InitConfig{ PruneThreads: runtime.NumCPU(), PruneTrieCleanCache: 600, RecreateMissingStateFrom: 0, // 0 = disabled - RebuildLocalWasm: true, + RebuildLocalWasm: "auto", ReorgToBatch: -1, ReorgToMessageBatch: -1, ReorgToBlockBatch: -1, @@ -82,10 +82,14 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Int(prefix+".prune-threads", InitConfigDefault.PruneThreads, "the number of threads to use when pruning") f.Int(prefix+".prune-trie-clean-cache", InitConfigDefault.PruneTrieCleanCache, "amount of memory in megabytes to cache unchanged state trie nodes with when traversing state database during pruning") f.Uint64(prefix+".recreate-missing-state-from", InitConfigDefault.RecreateMissingStateFrom, "block number to start recreating missing states from (0 = disabled)") - f.Bool(prefix+".rebuild-local-wasm", InitConfigDefault.RebuildLocalWasm, "rebuild local wasm database on boot if needed (otherwise-will be done lazily)") f.Int64(prefix+".reorg-to-batch", InitConfigDefault.ReorgToBatch, "rolls back the blockchain to a specified batch number") f.Int64(prefix+".reorg-to-message-batch", InitConfigDefault.ReorgToMessageBatch, "rolls back the blockchain to the first batch at or before a given message index") f.Int64(prefix+".reorg-to-block-batch", InitConfigDefault.ReorgToBlockBatch, "rolls back the blockchain to the first batch at or before a given block number") + f.String(prefix+".rebuild-local-wasm", InitConfigDefault.RebuildLocalWasm, "rebuild local wasm database on boot if needed (otherwise-will be done lazily). Three modes are supported \n"+ + "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild,\n"+ + "\"force\"- force rebuilding which would commence rebuilding despite the status of previous attempts,\n"+ + "\"false\"- do not rebuild on startup", + ) } func (c *InitConfig) Validate() error { @@ -110,6 +114,10 @@ func (c *InitConfig) Validate() error { } } } + c.RebuildLocalWasm = strings.ToLower(c.RebuildLocalWasm) + if c.RebuildLocalWasm != "auto" && c.RebuildLocalWasm != "force" && c.RebuildLocalWasm != "false" { + return fmt.Errorf("invalid value of rebuild-local-wasm, want: auto or force or false, got: %s", c.RebuildLocalWasm) + } return nil } diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index d9ae0df3b04..fc3439a56dc 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -532,13 +532,21 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err = gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, gethexec.RebuildingDone); err != nil { return nil, nil, fmt.Errorf("unable to set rebuilding status of wasm store to done: %w", err) } - } else if config.Init.RebuildLocalWasm { - position, err := gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingPositionKey) - if err != nil { - log.Info("Unable to get codehash position in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it and starting rebuilding", "err", err) + } else if config.Init.RebuildLocalWasm != "false" { + var position common.Hash + if config.Init.RebuildLocalWasm == "force" { + log.Info("Commencing force rebuilding of wasm store by setting codehash position in rebuilding to beginning") if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) } + } else { + position, err = gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingPositionKey) + if err != nil { + log.Info("Unable to get codehash position in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it and starting rebuilding", "err", err) + if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { + return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) + } + } } if position != gethexec.RebuildingDone { startBlockHash, err := gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingStartBlockHashKey) From 803384395ad64c116ec543f459888e059fa9e95a Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 23 Aug 2024 19:36:24 +0530 Subject: [PATCH 29/95] fix G115 lint errors --- system_tests/contract_tx_test.go | 1 + util/blobs/blobs.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/system_tests/contract_tx_test.go b/system_tests/contract_tx_test.go index 7d66e516b4d..c1ef840c434 100644 --- a/system_tests/contract_tx_test.go +++ b/system_tests/contract_tx_test.go @@ -51,6 +51,7 @@ func TestContractTxDeploy(t *testing.T) { 0xF3, // RETURN } var requestId common.Hash + // #nosec G115 requestId[0] = uint8(stateNonce) contractTx := &types.ArbitrumContractTx{ ChainId: params.ArbitrumDevTestChainConfig().ChainID, diff --git a/util/blobs/blobs.go b/util/blobs/blobs.go index 405c776bad7..f5914edd2e3 100644 --- a/util/blobs/blobs.go +++ b/util/blobs/blobs.go @@ -41,6 +41,7 @@ func fillBlobBits(blob []byte, data []byte) ([]byte, error) { accBits += 8 data = data[1:] } + // #nosec G115 blob[fieldElement*32] = uint8(acc & ((1 << spareBlobBits) - 1)) accBits -= spareBlobBits if accBits < 0 { @@ -88,6 +89,7 @@ func DecodeBlobs(blobs []kzg4844.Blob) ([]byte, error) { acc |= uint16(blob[fieldIndex*32]) << accBits accBits += spareBlobBits if accBits >= 8 { + // #nosec G115 rlpData = append(rlpData, uint8(acc)) acc >>= 8 accBits -= 8 From 17319ad2d1bb00d0c63a22693844652f6b739b1e Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 16:18:01 +0200 Subject: [PATCH 30/95] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 7fc4c646075..0a376604c9c 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 7fc4c646075eb9b6e26374886ea6fe9463b106aa +Subproject commit 0a376604c9ce0c588de0d943184380acd7e25db4 From cdb43d77e5ef976c6ad3e09ac63d07034c85bfcf Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 17:57:18 +0200 Subject: [PATCH 31/95] update validation client stylus archs sanity check --- validator/client/validation_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 36316a83288..3b18ad18513 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -81,7 +81,7 @@ func (c *ValidationClient) Start(ctx context.Context) error { return fmt.Errorf("could not read stylus archs from validation server") } for _, stylusArch := range stylusArchs { - if stylusArch != rawdb.TargetWavm && stylusArch != rawdb.LocalTarget() && stylusArch != "mock" { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(stylusArch)) && stylusArch != "mock" { return fmt.Errorf("unsupported stylus architecture: %v", stylusArch) } } From 65b58b69fc644cf4036422107d0683126c3791fd Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 23 Aug 2024 11:18:58 -0500 Subject: [PATCH 32/95] Fix new integer cast overflow lints --- system_tests/contract_tx_test.go | 1 + util/blobs/blobs.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/system_tests/contract_tx_test.go b/system_tests/contract_tx_test.go index 7d66e516b4d..c1ef840c434 100644 --- a/system_tests/contract_tx_test.go +++ b/system_tests/contract_tx_test.go @@ -51,6 +51,7 @@ func TestContractTxDeploy(t *testing.T) { 0xF3, // RETURN } var requestId common.Hash + // #nosec G115 requestId[0] = uint8(stateNonce) contractTx := &types.ArbitrumContractTx{ ChainId: params.ArbitrumDevTestChainConfig().ChainID, diff --git a/util/blobs/blobs.go b/util/blobs/blobs.go index 405c776bad7..f5914edd2e3 100644 --- a/util/blobs/blobs.go +++ b/util/blobs/blobs.go @@ -41,6 +41,7 @@ func fillBlobBits(blob []byte, data []byte) ([]byte, error) { accBits += 8 data = data[1:] } + // #nosec G115 blob[fieldElement*32] = uint8(acc & ((1 << spareBlobBits) - 1)) accBits -= spareBlobBits if accBits < 0 { @@ -88,6 +89,7 @@ func DecodeBlobs(blobs []kzg4844.Blob) ([]byte, error) { acc |= uint16(blob[fieldIndex*32]) << accBits accBits += spareBlobBits if accBits >= 8 { + // #nosec G115 rlpData = append(rlpData, uint8(acc)) acc >>= 8 accBits -= 8 From 71ab928cb42b66247ac9fdc3acd82280255de712 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 23 Aug 2024 19:27:15 +0200 Subject: [PATCH 33/95] change archs list option to extra-archs that must include wavm and is appended with local target if not in list --- execution/gethexec/node.go | 47 ++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index f519cce6ed7..bc29e9b7f23 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -30,10 +30,10 @@ import ( ) type StylusTargetConfig struct { - Arm64 string `koanf:"arm64"` - Amd64 string `koanf:"amd64"` - Host string `koanf:"host"` - Archs []string `koanf:"archs"` + Arm64 string `koanf:"arm64"` + Amd64 string `koanf:"amd64"` + Host string `koanf:"host"` + ExtraArchs []string `koanf:"extra-archs"` wasmTargets []ethdb.WasmTarget } @@ -44,47 +44,40 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { func (c *StylusTargetConfig) Validate() error { localTarget := rawdb.LocalTarget() - var nativeFound bool - targets := make([]ethdb.WasmTarget, 0, len(c.Archs)) - for _, arch := range c.Archs { + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)) + targetsSet := make(map[ethdb.WasmTarget]struct{}, len(c.ExtraArchs)+1) + for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - var alreadyInList bool - for _, t := range targets { - if target == t { - alreadyInList = true - break - } - } - if alreadyInList { - continue + if _, duplicate := targetsSet[target]; !duplicate { + targets = append(targets, target) + targetsSet[target] = struct{}{} } - if target == localTarget { - nativeFound = true - } - targets = append(targets, target) } - if !nativeFound { - return fmt.Errorf("native target not found in archs list, native target: %v, archs: %v", localTarget, c.Archs) + if _, has := targetsSet[rawdb.TargetWavm]; !has { + return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) + } + if _, has := targetsSet[localTarget]; !has { + targets = append(targets, localTarget) } c.wasmTargets = targets return nil } var DefaultStylusTargetConfig = StylusTargetConfig{ - Arm64: programs.DefaultTargetDescriptionArm, - Amd64: programs.DefaultTargetDescriptionX86, - Host: "", - Archs: []string{string(rawdb.TargetWavm), string(rawdb.LocalTarget())}, + Arm64: programs.DefaultTargetDescriptionArm, + Amd64: programs.DefaultTargetDescriptionX86, + Host: "", + ExtraArchs: []string{string(rawdb.TargetWavm)}, } func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux") f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux") f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86") - f.StringSlice(prefix+".archs", DefaultStylusTargetConfig.Archs, fmt.Sprintf("Comma separated architectures list to compile stylus program to and cache in wasm store (available targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) + f.StringSlice(prefix+".extra-archs", DefaultStylusTargetConfig.ExtraArchs, fmt.Sprintf("Comma separated list of extra architectures to cross-compile stylus program to and cache in wasm store (additionally to local target). Currently must include at least %s. (supported targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) } type Config struct { From 29df43a287833aa9541fa12db853bc6e3b07fd5f Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Fri, 23 Aug 2024 12:06:22 -0300 Subject: [PATCH 34/95] Remove wasm database after downloading snapshot By default, remove the wasm DB because it contains native executables. Hence, it is a security concern when downloading a snapshot from a untrusted source. --- cmd/conf/init.go | 3 +++ cmd/nitro/init.go | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index d88bcdd241d..242f43dc82e 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -22,6 +22,7 @@ type InitConfig struct { DevInitAddress string `koanf:"dev-init-address"` DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` Empty bool `koanf:"empty"` + ExtractWasm bool `koanf:"extract-wasm"` AccountsPerSync uint `koanf:"accounts-per-sync"` ImportFile string `koanf:"import-file"` ThenQuit bool `koanf:"then-quit"` @@ -48,6 +49,7 @@ var InitConfigDefault = InitConfig{ DevInitAddress: "", DevInitBlockNum: 0, Empty: false, + ExtractWasm: false, ImportFile: "", AccountsPerSync: 100000, ThenQuit: false, @@ -74,6 +76,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".dev-init-address", InitConfigDefault.DevInitAddress, "Address of dev-account. Leave empty to use the dev-wallet.") f.Uint64(prefix+".dev-init-blocknum", InitConfigDefault.DevInitBlockNum, "Number of preinit blocks. Must exist in ancient database.") f.Bool(prefix+".empty", InitConfigDefault.Empty, "init with empty state") + f.Bool(prefix+".extract-wasm", InitConfigDefault.ExtractWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") f.Bool(prefix+".then-quit", InitConfigDefault.ThenQuit, "quit after init is done") f.String(prefix+".import-file", InitConfigDefault.ImportFile, "path for json data to import") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index d9ae0df3b04..b5cab75cf61 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -602,6 +602,17 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err != nil { return nil, nil, fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) } + wasmDb := path.Join(stack.InstanceDir(), "wasm") + if dirExists(wasmDb) && !config.Init.ExtractWasm { + // By default, remove the wasm DB because it contains native executables. + // Hence, it is a security concern when downloading a snapshot from a + // untrusted source. + err := os.RemoveAll(wasmDb) + if err != nil { + return nil, nil, fmt.Errorf("failed to remove extracted wasm database: %w", err) + } + log.Debug("init: removed wasm database") + } } var initDataReader statetransfer.InitDataReader = nil From a84a88803db272922160d620d1d4df9bdd550064 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 23 Aug 2024 14:53:37 -0300 Subject: [PATCH 35/95] Don't recurse submodules in submodule-pin-check github action --- .github/workflows/submodule-pin-check.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index e459bad34d3..b8b97a27c5b 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -14,8 +14,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: recursive + submodules: true - name: Check all submodules are ancestors of origin/HEAD or configured branch run: ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh - From f4da3d42d2f7166f4acddcb5d79e8cc7d4c0f05c Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 23 Aug 2024 15:07:58 -0300 Subject: [PATCH 36/95] Workflow "Merge Checks" creates "Submodule Pin Check" through github API --- .github/workflows/submodule-pin-check.yml | 32 +++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index b8b97a27c5b..12a9656e846 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,10 +1,13 @@ -name: Submodule Pin Check +name: Merge Checks on: pull_request: branches: [ master ] types: [synchronize, opened, reopened] +permissions: + statuses: write + jobs: submodule-pin-check: name: Submodule Pin Check @@ -17,4 +20,29 @@ jobs: submodules: true - name: Check all submodules are ancestors of origin/HEAD or configured branch - run: ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh + run: | + status_state="pending" + if ${{ github.workspace }}/.github/workflows/submodule-pin-check.sh; then + status_state="success" + else + resp="$(curl -sSL --fail-with-body \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/commits/${{ github.event.pull_request.head.sha }}/statuses")" + + if ! jq -e '.[] | select(.context == "Submodule Pin Check")' > /dev/null <<< "$resp"; then + # Submodule pin check is failling and no status exists + # Keep it without a status to keep the green checkmark appearing + # Otherwise, the commit and PR's CI will appear to be indefinitely pending + # Merging will still be blocked until the required status appears + exit 0 + fi + fi + curl -sSL --fail-with-body \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/statuses/${{ github.event.pull_request.head.sha }}" \ + -d '{"context":"Submodule Pin Check","state":"'"$status_state"'"}' From 966ec11ac96d892c56d7f4dcffb6b4dadd2e7abb Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 11:13:45 -0300 Subject: [PATCH 37/95] Rename conf init.extract-wasm to init.import-wasm --- cmd/conf/init.go | 6 +++--- cmd/nitro/init.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index d0ca592e429..dddf3c0c28c 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -22,7 +22,7 @@ type InitConfig struct { DevInitAddress string `koanf:"dev-init-address"` DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` Empty bool `koanf:"empty"` - ExtractWasm bool `koanf:"extract-wasm"` + ImportWasm bool `koanf:"import-wasm"` AccountsPerSync uint `koanf:"accounts-per-sync"` ImportFile string `koanf:"import-file"` ThenQuit bool `koanf:"then-quit"` @@ -49,7 +49,7 @@ var InitConfigDefault = InitConfig{ DevInitAddress: "", DevInitBlockNum: 0, Empty: false, - ExtractWasm: false, + ImportWasm: false, ImportFile: "", AccountsPerSync: 100000, ThenQuit: false, @@ -76,7 +76,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".dev-init-address", InitConfigDefault.DevInitAddress, "Address of dev-account. Leave empty to use the dev-wallet.") f.Uint64(prefix+".dev-init-blocknum", InitConfigDefault.DevInitBlockNum, "Number of preinit blocks. Must exist in ancient database.") f.Bool(prefix+".empty", InitConfigDefault.Empty, "init with empty state") - f.Bool(prefix+".extract-wasm", InitConfigDefault.ExtractWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") + f.Bool(prefix+".import-wasm", InitConfigDefault.ImportWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") f.Bool(prefix+".then-quit", InitConfigDefault.ThenQuit, "quit after init is done") f.String(prefix+".import-file", InitConfigDefault.ImportFile, "path for json data to import") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index aaef5798f56..55325f1b934 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -611,7 +611,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return nil, nil, fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) } wasmDb := path.Join(stack.InstanceDir(), "wasm") - if dirExists(wasmDb) && !config.Init.ExtractWasm { + if dirExists(wasmDb) && !config.Init.ImportWasm { // By default, remove the wasm DB because it contains native executables. // Hence, it is a security concern when downloading a snapshot from a // untrusted source. From a3bf49a54e5dddba0e36f80162f7e2d9115e4e44 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 11:14:52 -0300 Subject: [PATCH 38/95] Improve init.import-wasm description --- cmd/conf/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index dddf3c0c28c..f01d99f8b7d 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -76,7 +76,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".dev-init-address", InitConfigDefault.DevInitAddress, "Address of dev-account. Leave empty to use the dev-wallet.") f.Uint64(prefix+".dev-init-blocknum", InitConfigDefault.DevInitBlockNum, "Number of preinit blocks. Must exist in ancient database.") f.Bool(prefix+".empty", InitConfigDefault.Empty, "init with empty state") - f.Bool(prefix+".import-wasm", InitConfigDefault.ImportWasm, "if set, extract the wasm directory when downloading a database (this can be security concern because it contains native executables)") + f.Bool(prefix+".import-wasm", InitConfigDefault.ImportWasm, "if set, import the wasm directory when downloading a database (contains executable code - only use with highly trusted source)") f.Bool(prefix+".then-quit", InitConfigDefault.ThenQuit, "quit after init is done") f.String(prefix+".import-file", InitConfigDefault.ImportFile, "path for json data to import") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") From 57249c83fc7ad215a874d56c5e7200b0ae77cefb Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 11:27:35 -0300 Subject: [PATCH 39/95] Remove WASM database even if the extraction fails --- cmd/nitro/init.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 55325f1b934..318c134eb12 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -608,18 +608,22 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo log.Info("extracting downloaded init archive", "size", fmt.Sprintf("%dMB", stat.Size()/1024/1024)) err = extract.Archive(context.Background(), reader, stack.InstanceDir(), nil) if err != nil { - return nil, nil, fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) + err = fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) } wasmDb := path.Join(stack.InstanceDir(), "wasm") if dirExists(wasmDb) && !config.Init.ImportWasm { // By default, remove the wasm DB because it contains native executables. // Hence, it is a security concern when downloading a snapshot from a // untrusted source. - err := os.RemoveAll(wasmDb) - if err != nil { - return nil, nil, fmt.Errorf("failed to remove extracted wasm database: %w", err) + removeErr := os.RemoveAll(wasmDb) + if removeErr != nil { + err = errors.Join(err, fmt.Errorf("failed to remove extracted wasm database: %w", removeErr)) + } else { + log.Debug("init: removed wasm database") } - log.Debug("init: removed wasm database") + } + if err != nil { + return nil, nil, err } } From 304f2c1127a4def9d292b617b3b52a437e9891b2 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 26 Aug 2024 16:53:34 +0200 Subject: [PATCH 40/95] add cross compilation to progam test --- system_tests/common_test.go | 5 ++ system_tests/program_test.go | 92 +++++++++++++++++++++++++++++++++--- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 392f6de8059..457dae09101 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -293,6 +293,11 @@ func (b *NodeBuilder) WithWasmRootDir(wasmRootDir string) *NodeBuilder { return b } +func (b *NodeBuilder) WithExtraArchs(targets []string) *NodeBuilder { + b.execConfig.StylusTarget.ExtraArchs = targets + return b +} + func (b *NodeBuilder) Build(t *testing.T) func() { b.CheckConfig(t) if b.withL1 { diff --git a/system_tests/program_test.go b/system_tests/program_test.go index ed640809dbf..8c4ca2e4386 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -45,18 +45,31 @@ import ( var oneEth = arbmath.UintToBig(1e18) +var allWasmTargets = []string{string(rawdb.TargetWavm), string(rawdb.TargetArm64), string(rawdb.TargetAmd64), string(rawdb.TargetHost)} + func TestProgramKeccak(t *testing.T) { t.Parallel() - keccakTest(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + keccakTest(t, true) + }) + + t.Run("WithAllWasmTargets", func(t *testing.T) { + keccakTest(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func keccakTest(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func keccakTest(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2client := builder.L2.Client defer cleanup() programAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + wasm, _ := readWasmFile(t, rustFile("keccak")) otherAddressSameCode := deployContract(t, ctx, auth, l2client, wasm) arbWasm, err := pgen.NewArbWasm(types.ArbWasmAddress, l2client) @@ -68,6 +81,7 @@ func keccakTest(t *testing.T, jit bool) { Fatal(t, "activate should have failed with ProgramUpToDate", err) } }) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) if programAddress == otherAddressSameCode { Fatal(t, "expected to deploy at two separate program addresses") @@ -141,11 +155,18 @@ func keccakTest(t *testing.T, jit bool) { func TestProgramActivateTwice(t *testing.T) { t.Parallel() - testActivateTwice(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + testActivateTwice(t, true) + }) + t.Run("WithAllWasmTargets", func(t *testing.T) { + testActivateTwice(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func testActivateTwice(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -171,6 +192,10 @@ func testActivateTwice(t *testing.T, jit bool) { colors.PrintBlue("keccak program B deployed to ", keccakB) multiAddr := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + preimage := []byte("it's time to du-du-du-du d-d-d-d-d-d-d de-duplicate") keccakArgs := []byte{0x01} // keccak the preimage once @@ -194,6 +219,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Calling the contract pre-activation should fail. checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // mechanisms for creating calldata activateProgram, _ := util.NewCallParser(pgen.ArbWasmABI, "activateProgram") @@ -216,6 +242,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Ensure the revert also reverted keccak's activation checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // Activate keccak program A, then call into B, which should succeed due to being the same codehash args = argsForMulticall(vm.CALL, types.ArbWasmAddress, oneEth, pack(activateProgram(keccakA))) @@ -223,6 +250,7 @@ func testActivateTwice(t *testing.T, jit bool) { tx = l2info.PrepareTxTo("Owner", &multiAddr, 1e9, oneEth, args) ensure(tx, l2client.SendTransaction(ctx, tx)) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 2) validateBlocks(t, 7, jit, builder) } @@ -1834,7 +1862,9 @@ func createMapFromDb(db ethdb.KeyValueStore) (map[string][]byte, error) { } func TestWasmStoreRebuilding(t *testing.T) { - builder, auth, cleanup := setupProgramTest(t, true) + builder, auth, cleanup := setupProgramTest(t, true, func(b *NodeBuilder) { + b.WithExtraArchs(allWasmTargets) + }) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -1871,6 +1901,7 @@ func TestWasmStoreRebuilding(t *testing.T) { storeMap, err := createMapFromDb(wasmDb) Require(t, err) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // close nodeB cleanupB() @@ -1926,5 +1957,52 @@ func TestWasmStoreRebuilding(t *testing.T) { } } + checkWasmStoreContent(t, wasmDbAfterRebuild, builder.execConfig.StylusTarget.ExtraArchs, 1) cleanupB() } + +func readModuleHashes(t *testing.T, wasmDb ethdb.KeyValueStore) []common.Hash { + modulesSet := make(map[common.Hash]struct{}) + asmPrefix := []byte{0x00, 'w'} + it := wasmDb.NewIterator(asmPrefix, nil) + defer it.Release() + for it.Next() { + key := it.Key() + if len(key) != rawdb.WasmKeyLen { + t.Fatalf("unexpected activated module key length, len: %d, key: %v", len(key), key) + } + moduleHash := key[rawdb.WasmPrefixLen:] + if len(moduleHash) != common.HashLength { + t.Fatalf("Invalid moduleHash length in key: %v, moduleHash: %v", key, moduleHash) + } + modulesSet[common.BytesToHash(moduleHash)] = struct{}{} + } + modules := make([]common.Hash, 0, len(modulesSet)) + for module := range modulesSet { + modules = append(modules, module) + } + return modules +} + +func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []string, numModules int) { + modules := readModuleHashes(t, wasmDb) + if len(modules) != numModules { + t.Fatalf("Unexpected number of module hashes found in wasm store, want: %d, have: %d", numModules, len(modules)) + } + for _, module := range modules { + for _, target := range targets { + wasmTarget := ethdb.WasmTarget(target) + if !rawdb.IsSupportedWasmTarget(wasmTarget) { + t.Fatalf("internal test error - unsupported target passed to checkWasmStoreContent: %v", target) + } + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Failed to read activated asm for target: %v, module: %v", target, module) + } + }() + _ = rawdb.ReadActivatedAsm(wasmDb, wasmTarget, module) + }() + } + } +} From 4a29e0d21be963f17d3cfbcdf105799096a76a26 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 12:02:16 -0300 Subject: [PATCH 41/95] Move stack struct to util/containers pkg --- execution/gethexec/stylus_tracer.go | 14 +++++++------- system_tests/stylus_tracer_test.go | 8 ++++---- util/{stack => containers}/stack.go | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) rename util/{stack => containers}/stack.go (97%) diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index 16c43d71d86..d1246989f0a 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -14,7 +14,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" - "github.com/offchainlabs/nitro/util/stack" + "github.com/offchainlabs/nitro/util/containers" ) func init() { @@ -24,8 +24,8 @@ func init() { // stylusTracer captures Stylus HostIOs and returns them in a structured format to be used in Cargo // Stylus Replay. type stylusTracer struct { - open *stack.Stack[HostioTraceInfo] - stack *stack.Stack[*stack.Stack[HostioTraceInfo]] + open *containers.Stack[HostioTraceInfo] + stack *containers.Stack[*containers.Stack[HostioTraceInfo]] interrupt atomic.Bool reason error } @@ -55,7 +55,7 @@ type HostioTraceInfo struct { Address *common.Address `json:"address,omitempty"` // For *call HostIOs, the steps performed by the called contract. - Steps *stack.Stack[HostioTraceInfo] `json:"steps,omitempty"` + Steps *containers.Stack[HostioTraceInfo] `json:"steps,omitempty"` } // nestsHostios contains the hostios with nested calls. @@ -67,8 +67,8 @@ var nestsHostios = map[string]bool{ func newStylusTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { return &stylusTracer{ - open: stack.NewStack[HostioTraceInfo](), - stack: stack.NewStack[*stack.Stack[HostioTraceInfo]](), + open: containers.NewStack[HostioTraceInfo](), + stack: containers.NewStack[*containers.Stack[HostioTraceInfo]](), }, nil } @@ -126,7 +126,7 @@ func (t *stylusTracer) CaptureEnter(typ vm.OpCode, from common.Address, to commo name = "evm_self_destruct" } - inner := stack.NewStack[HostioTraceInfo]() + inner := containers.NewStack[HostioTraceInfo]() info := HostioTraceInfo{ Name: name, Address: &to, diff --git a/system_tests/stylus_tracer_test.go b/system_tests/stylus_tracer_test.go index 3b95f38d218..7fda39f04e8 100644 --- a/system_tests/stylus_tracer_test.go +++ b/system_tests/stylus_tracer_test.go @@ -12,7 +12,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/offchainlabs/nitro/execution/gethexec" "github.com/offchainlabs/nitro/solgen/go/mocksgen" - "github.com/offchainlabs/nitro/util/stack" + "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/testhelpers" ) @@ -89,7 +89,7 @@ func TestStylusTracer(t *testing.T) { Args: append(stylusMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), Outs: common.Hex2Bytes("0000000000"), Address: &stylusMulticall, - Steps: (*stack.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ + Steps: (*containers.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ {Name: "user_entrypoint", Args: intToBe32(1)}, {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, {Name: "read_args", Outs: []byte{0x00}}, @@ -117,7 +117,7 @@ func TestStylusTracer(t *testing.T) { Args: append(evmMulticall.Bytes(), common.Hex2Bytes("ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000")...), Outs: common.Hex2Bytes("0000000000"), Address: &evmMulticall, - Steps: stack.NewStack[gethexec.HostioTraceInfo](), + Steps: containers.NewStack[gethexec.HostioTraceInfo](), }, {Name: "storage_flush_cache", Args: []byte{0x00}}, {Name: "write_result"}, @@ -133,7 +133,7 @@ func TestStylusTracer(t *testing.T) { { Name: "evm_call_contract", Address: &stylusMulticall, - Steps: (*stack.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ + Steps: (*containers.Stack[gethexec.HostioTraceInfo])(&[]gethexec.HostioTraceInfo{ {Name: "user_entrypoint", Args: intToBe32(1)}, {Name: "pay_for_memory_grow", Args: []byte{0x00, 0x01}}, {Name: "read_args", Outs: []byte{0x00}}, diff --git a/util/stack/stack.go b/util/containers/stack.go similarity index 97% rename from util/stack/stack.go rename to util/containers/stack.go index 1b7ac3f9d94..97fcd44a6d9 100644 --- a/util/stack/stack.go +++ b/util/containers/stack.go @@ -1,7 +1,7 @@ // Copyright 2024, Offchain Labs, Inc. // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE -package stack +package containers import ( "fmt" From 2e5a6eadc03a562fa32e96781fda77ab430b55a0 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 26 Aug 2024 12:22:25 -0300 Subject: [PATCH 42/95] Renames job name related to checking submodule pin --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 12a9656e846..870f5208770 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -10,7 +10,7 @@ permissions: jobs: submodule-pin-check: - name: Submodule Pin Check + name: Check Submodule Pin runs-on: ubuntu-latest steps: - name: Checkout From 82b48d414ec4fd9369d1ca0fa74c62d6108a700d Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 26 Aug 2024 17:35:30 +0200 Subject: [PATCH 43/95] populate stylus target cache before rebuilding wasm store --- cmd/nitro/init.go | 2 +- execution/gethexec/executionengine.go | 15 +++++++++++---- execution/gethexec/wasmstorerebuilder.go | 7 ++++++- system_tests/program_test.go | 3 ++- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index fc3439a56dc..5573bd6ab27 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -558,7 +558,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo startBlockHash = latestBlock.Hash() } log.Info("Starting or continuing rebuilding of wasm store", "codeHash", position, "startBlockHash", startBlockHash) - if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, l2BlockChain, position, startBlockHash); err != nil { + if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, &config.Execution.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { return nil, nil, fmt.Errorf("error rebuilding of wasm store: %w", err) } } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 19d77fc38f9..7eb7920d8ad 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -150,10 +150,7 @@ func (s *ExecutionEngine) MarkFeedStart(to arbutil.MessageIndex) { } } -func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusTargetConfig) error { - if rustCacheSize != 0 { - programs.ResizeWasmLruCache(rustCacheSize) - } +func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { var effectiveStylusTarget string target := rawdb.LocalTarget() switch target { @@ -171,6 +168,16 @@ func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusT return nil } +func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusTargetConfig) error { + if rustCacheSize != 0 { + programs.ResizeWasmLruCache(rustCacheSize) + } + if err := populateStylusTargetCache(targetConfig); err != nil { + return err + } + return nil +} + func (s *ExecutionEngine) SetRecorder(recorder *BlockRecorder) { if s.Started() { panic("trying to set recorder after start") diff --git a/execution/gethexec/wasmstorerebuilder.go b/execution/gethexec/wasmstorerebuilder.go index dcbee45a3f9..698ba3ec8a4 100644 --- a/execution/gethexec/wasmstorerebuilder.go +++ b/execution/gethexec/wasmstorerebuilder.go @@ -59,9 +59,14 @@ func WriteToKeyValueStore[T any](store ethdb.KeyValueStore, key []byte, val T) e // It also stores a special value that is only set once when rebuilding commenced in RebuildingStartBlockHashKey as the block // time of the latest block when rebuilding was first called, this is used to avoid recomputing of assembly and module of // contracts that were created after rebuilding commenced since they would anyway already be added during sync. -func RebuildWasmStore(ctx context.Context, wasmStore ethdb.KeyValueStore, chainDb ethdb.Database, maxRecreateStateDepth int64, l2Blockchain *core.BlockChain, position, rebuildingStartBlockHash common.Hash) error { +func RebuildWasmStore(ctx context.Context, wasmStore ethdb.KeyValueStore, chainDb ethdb.Database, maxRecreateStateDepth int64, targetConfig *StylusTargetConfig, l2Blockchain *core.BlockChain, position, rebuildingStartBlockHash common.Hash) error { var err error var stateDb *state.StateDB + + if err := populateStylusTargetCache(targetConfig); err != nil { + return fmt.Errorf("error populating stylus target cache: %w", err) + } + latestHeader := l2Blockchain.CurrentBlock() // Attempt to get state at the start block when rebuilding commenced, if not available (in case of non-archival nodes) use latest state rebuildingStartHeader := l2Blockchain.GetHeaderByHash(rebuildingStartBlockHash) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index ed640809dbf..c89a13e2050 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -1896,7 +1896,8 @@ func TestWasmStoreRebuilding(t *testing.T) { // Start rebuilding and wait for it to finish log.Info("starting rebuilding of wasm store") - Require(t, gethexec.RebuildWasmStore(ctx, wasmDbAfterDelete, nodeB.ExecNode.ChainDB, nodeB.ExecNode.ConfigFetcher().RPC.MaxRecreateStateDepth, bc, common.Hash{}, bc.CurrentBlock().Hash())) + execConfig := nodeB.ExecNode.ConfigFetcher() + Require(t, gethexec.RebuildWasmStore(ctx, wasmDbAfterDelete, nodeB.ExecNode.ChainDB, execConfig.RPC.MaxRecreateStateDepth, &execConfig.StylusTarget, bc, common.Hash{}, bc.CurrentBlock().Hash())) wasmDbAfterRebuild := nodeB.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() From 4d16436406e3381612d9849d07b68fd498f02b62 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Mon, 26 Aug 2024 17:42:05 +0200 Subject: [PATCH 44/95] add error description --- execution/gethexec/executionengine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 7eb7920d8ad..991c94540ed 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -173,7 +173,7 @@ func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusT programs.ResizeWasmLruCache(rustCacheSize) } if err := populateStylusTargetCache(targetConfig); err != nil { - return err + return fmt.Errorf("error populating stylus target cache: %w", err) } return nil } From 5e178f4c149ad6c3d8a825b7bf0956ef4f2194d8 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Mon, 26 Aug 2024 12:58:23 -0300 Subject: [PATCH 45/95] Add sanity checks to stylus tracer --- execution/gethexec/stylus_tracer.go | 17 ++++++++++++++++- util/containers/stack.go | 13 ++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/execution/gethexec/stylus_tracer.go b/execution/gethexec/stylus_tracer.go index d1246989f0a..4c18bb2ebe2 100644 --- a/execution/gethexec/stylus_tracer.go +++ b/execution/gethexec/stylus_tracer.go @@ -5,6 +5,7 @@ package gethexec import ( "encoding/json" + "errors" "fmt" "math/big" "strings" @@ -14,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/util/containers" ) @@ -152,9 +154,22 @@ func (t *stylusTracer) GetResult() (json.RawMessage, error) { if t.reason != nil { return nil, t.reason } + + var internalErr error if t.open == nil { - return nil, fmt.Errorf("trace is nil") + internalErr = errors.Join(internalErr, fmt.Errorf("tracer.open is nil")) + } + if t.stack == nil { + internalErr = errors.Join(internalErr, fmt.Errorf("tracer.stack is nil")) + } + if !t.stack.Empty() { + internalErr = errors.Join(internalErr, fmt.Errorf("tracer.stack should be empty, but has %d values", t.stack.Len())) } + if internalErr != nil { + log.Error("stylusTracer: internal error when generating a trace", "error", internalErr) + return nil, fmt.Errorf("internal error: %w", internalErr) + } + msg, err := json.Marshal(t.open) if err != nil { return nil, err diff --git a/util/containers/stack.go b/util/containers/stack.go index 97fcd44a6d9..ea7f31013b5 100644 --- a/util/containers/stack.go +++ b/util/containers/stack.go @@ -28,7 +28,7 @@ func (s *Stack[T]) Pop() (T, error) { var zeroVal T return zeroVal, fmt.Errorf("trying to pop nil stack") } - if len(*s) == 0 { + if s.Empty() { var zeroVal T return zeroVal, fmt.Errorf("trying to pop empty stack") } @@ -37,3 +37,14 @@ func (s *Stack[T]) Pop() (T, error) { *s = (*s)[:i] return val, nil } + +func (s *Stack[T]) Empty() bool { + return s == nil || len(*s) == 0 +} + +func (s *Stack[T]) Len() int { + if s == nil { + return 0 + } + return len(*s) +} From 18de0ad594cf3882f3d2c6a85a497080f3816f1b Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Mon, 26 Aug 2024 18:32:29 -0600 Subject: [PATCH 46/95] update default x86 architecture --- arbos/programs/native.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index fd3dec25a04..9316e8f230e 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -289,7 +289,7 @@ func ResizeWasmLruCache(size uint32) { } const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" -const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2" +const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2+lzcnt+bmi" func SetTarget(name rawdb.Target, description string, native bool) error { output := &rustBytes{} From 9e9bcc43cfe6b2d0bf0c3710e416130ac9321328 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Mon, 26 Aug 2024 19:17:02 -0600 Subject: [PATCH 47/95] testcompile: test loaded compile with default target --- arbos/programs/cgo_test.go | 8 ++++++++ arbos/programs/testcompile.go | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/arbos/programs/cgo_test.go b/arbos/programs/cgo_test.go index c0e146d98d7..e16c362ef8d 100644 --- a/arbos/programs/cgo_test.go +++ b/arbos/programs/cgo_test.go @@ -40,5 +40,13 @@ func TestCompileArch(t *testing.T) { if err != nil { t.Fatal(err) } + err = resetNativeTarget() + if err != nil { + t.Fatal(err) + } + err = testCompileLoad() + if err != nil { + t.Fatal(err) + } } } diff --git a/arbos/programs/testcompile.go b/arbos/programs/testcompile.go index a16bae52c0c..1daf470620e 100644 --- a/arbos/programs/testcompile.go +++ b/arbos/programs/testcompile.go @@ -178,6 +178,28 @@ func testCompileArch(store bool) error { return nil } +func resetNativeTarget() error { + output := &rustBytes{} + + _, err := fmt.Print("resetting native target\n") + if err != nil { + return err + } + + localCompileName := []byte("local") + + status := C.stylus_target_set(goSlice(localCompileName), + goSlice([]byte{}), + output, + cbool(true)) + + if status != 0 { + return fmt.Errorf("failed setting compilation target arm: %v", string(output.intoBytes())) + } + + return nil +} + func testCompileLoad() error { filePath := "../../target/testdata/host.bin" localTarget := rawdb.LocalTarget() From ea0cbc672d82802e91be407b595463d6d9415f8c Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 27 Aug 2024 15:05:59 +0200 Subject: [PATCH 48/95] update geth pin --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 0a376604c9c..4ca4b4d7340 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 0a376604c9ce0c588de0d943184380acd7e25db4 +Subproject commit 4ca4b4d734021df09cb42a796ca8f263d47383f9 From af6320388b2bf9928251115d7edfafb33cbcf81b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 27 Aug 2024 14:03:04 -0500 Subject: [PATCH 49/95] Return false if nil is passed to IsNotExistError --- util/dbutil/dbutil.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/util/dbutil/dbutil.go b/util/dbutil/dbutil.go index ca0f5aaaebe..6573c5742cd 100644 --- a/util/dbutil/dbutil.go +++ b/util/dbutil/dbutil.go @@ -6,7 +6,7 @@ package dbutil import ( "errors" "fmt" - "os" + "io/fs" "regexp" "github.com/cockroachdb/pebble" @@ -22,13 +22,15 @@ func IsErrNotFound(err error) bool { var pebbleNotExistErrorRegex = regexp.MustCompile("pebble: database .* does not exist") func isPebbleNotExistError(err error) bool { - return pebbleNotExistErrorRegex.MatchString(err.Error()) + return err != nil && pebbleNotExistErrorRegex.MatchString(err.Error()) } func isLeveldbNotExistError(err error) bool { - return os.IsNotExist(err) + return errors.Is(err, fs.ErrNotExist) } +// IsNotExistError returns true if the error is a "database not found" error. +// It must return false if err is nil. func IsNotExistError(err error) bool { return isLeveldbNotExistError(err) || isPebbleNotExistError(err) } From a32669038ec7d6009d3614973f8b779890aa13c4 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 27 Aug 2024 14:05:11 -0500 Subject: [PATCH 50/95] Add test case --- util/dbutil/dbutil_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util/dbutil/dbutil_test.go b/util/dbutil/dbutil_test.go index b28f8a2c230..b303bb56b6e 100644 --- a/util/dbutil/dbutil_test.go +++ b/util/dbutil/dbutil_test.go @@ -28,6 +28,9 @@ func testIsNotExistError(t *testing.T, dbEngine string, isNotExist func(error) b if isNotExist(err) { t.Fatalf("Classified other error as not exist, err: %v", err) } + if isNotExist(nil) { + t.Fatal("Classified nil as not exist") + } } func TestIsNotExistError(t *testing.T) { From eb1fd9e1bd366b89431e29b3ddcc05586f90d0b4 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Tue, 27 Aug 2024 17:55:09 -0300 Subject: [PATCH 51/95] Do not extract wasm db instead of deleting it Use the extract.Renamer function to avoid extracting the wasm DB when the init.import-wasm flag is set to false. --- cmd/nitro/init.go | 57 ++++++++++--------- cmd/nitro/init_test.go | 121 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 27 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 9a1230651b0..53048d34861 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -24,7 +24,7 @@ import ( "time" "github.com/cavaliergopher/grab/v3" - extract "github.com/codeclysm/extract/v3" + "github.com/codeclysm/extract/v3" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -405,6 +405,34 @@ func databaseIsEmpty(db ethdb.Database) bool { return !it.Next() } +func extractSnapshot(archive string, location string, importWasm bool) error { + isWasmDB := regexp.MustCompile("^(./)?wasm") + reader, err := os.Open(archive) + if err != nil { + return fmt.Errorf("couln't open init '%v' archive: %w", archive, err) + } + stat, err := reader.Stat() + if err != nil { + return err + } + log.Info("extracting downloaded init archive", "size", fmt.Sprintf("%dMB", stat.Size()/1024/1024)) + var rename extract.Renamer + if !importWasm { + rename = func(path string) string { + path = strings.ToLower(path) + if isWasmDB.MatchString(path) { + return "" // do not extract wasm files + } + return path + } + } + err = extract.Archive(context.Background(), reader, location, rename) + if err != nil { + return fmt.Errorf("couln't extract init archive '%v' err: %w", archive, err) + } + return nil +} + // removes all entries with keys prefixed with prefixes and of length used in initial version of wasm store schema func purgeVersion0WasmStoreEntries(db ethdb.Database) error { prefixes, keyLength := rawdb.DeprecatedPrefixesV0() @@ -597,32 +625,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo } if initFile != "" { - reader, err := os.Open(initFile) - if err != nil { - return nil, nil, fmt.Errorf("couln't open init '%v' archive: %w", initFile, err) - } - stat, err := reader.Stat() - if err != nil { - return nil, nil, err - } - log.Info("extracting downloaded init archive", "size", fmt.Sprintf("%dMB", stat.Size()/1024/1024)) - err = extract.Archive(context.Background(), reader, stack.InstanceDir(), nil) - if err != nil { - err = fmt.Errorf("couln't extract init archive '%v' err:%w", initFile, err) - } - wasmDb := path.Join(stack.InstanceDir(), "wasm") - if dirExists(wasmDb) && !config.Init.ImportWasm { - // By default, remove the wasm DB because it contains native executables. - // Hence, it is a security concern when downloading a snapshot from a - // untrusted source. - removeErr := os.RemoveAll(wasmDb) - if removeErr != nil { - err = errors.Join(err, fmt.Errorf("failed to remove extracted wasm database: %w", removeErr)) - } else { - log.Debug("init: removed wasm database") - } - } - if err != nil { + if err := extractSnapshot(initFile, stack.InstanceDir(), config.Init.ImportWasm); err != nil { return nil, nil, err } } diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index b2773ed8613..9282839f75c 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -4,18 +4,21 @@ package main import ( + "archive/tar" "bytes" "context" "crypto/sha256" "encoding/hex" "errors" "fmt" + "io" "math/big" "net" "net/http" "os" "path" "path/filepath" + "slices" "strings" "testing" "time" @@ -24,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" + "github.com/google/go-cmp/cmp" "github.com/offchainlabs/nitro/arbnode" "github.com/offchainlabs/nitro/cmd/chaininfo" "github.com/offchainlabs/nitro/cmd/conf" @@ -554,3 +558,120 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { err = chainDb.Close() Require(t, err) } + +func TestExtractSnapshot(t *testing.T) { + testCases := []struct { + name string + archiveFiles []string + importWasm bool + wantFiles []string + }{ + { + name: "extractAll", + importWasm: true, + archiveFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "l2chaindata/ancients/000001.ldb", + "nodes/000001.ldb", + "wasm/000001.ldb", + }, + wantFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "l2chaindata/ancients/000001.ldb", + "nodes/000001.ldb", + "wasm/000001.ldb", + }, + }, + { + name: "extractAllButWasm", + importWasm: false, + archiveFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "nodes/000001.ldb", + "wasm/000001.ldb", + }, + wantFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "nodes/000001.ldb", + }, + }, + { + name: "extractAllButWasmWithPrefixDot", + importWasm: false, + archiveFiles: []string{ + "./arbitrumdata/000001.ldb", + "./l2chaindata/000001.ldb", + "./nodes/000001.ldb", + "./wasm/000001.ldb", + }, + wantFiles: []string{ + "arbitrumdata/000001.ldb", + "l2chaindata/000001.ldb", + "nodes/000001.ldb", + }, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + // Create archive with dummy files + archiveDir := t.TempDir() + archivePath := path.Join(archiveDir, "archive.tar") + { + // Create context to close the file handlers + archiveFile, err := os.Create(archivePath) + Require(t, err) + defer archiveFile.Close() + tarWriter := tar.NewWriter(archiveFile) + defer tarWriter.Close() + for _, relativePath := range testCase.archiveFiles { + filePath := path.Join(archiveDir, relativePath) + dir := filepath.Dir(filePath) + const dirPerm = 0700 + err := os.MkdirAll(dir, dirPerm) + Require(t, err) + const filePerm = 0600 + err = os.WriteFile(filePath, []byte{0xbe, 0xef}, filePerm) + Require(t, err) + file, err := os.Open(filePath) + Require(t, err) + info, err := file.Stat() + Require(t, err) + header, err := tar.FileInfoHeader(info, "") + Require(t, err) + header.Name = relativePath + err = tarWriter.WriteHeader(header) + Require(t, err) + _, err = io.Copy(tarWriter, file) + Require(t, err) + } + } + + // Extract archive and compare contents + targetDir := t.TempDir() + err := extractSnapshot(archivePath, targetDir, testCase.importWasm) + Require(t, err, "failed to extract snapshot") + gotFiles := []string{} + err = filepath.WalkDir(targetDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + gotFiles = append(gotFiles, path) + } + return nil + }) + Require(t, err) + slices.Sort(gotFiles) + for i, f := range testCase.wantFiles { + testCase.wantFiles[i] = path.Join(targetDir, f) + } + if diff := cmp.Diff(gotFiles, testCase.wantFiles); diff != "" { + t.Fatal("extracted files don't match", diff) + } + }) + } +} From 8ffca7e70304cede8048afa981b442b14677d3bb Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Tue, 27 Aug 2024 17:59:27 -0300 Subject: [PATCH 52/95] Do not change case when extracting DB --- cmd/nitro/init.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 53048d34861..435c73c997e 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -419,8 +419,7 @@ func extractSnapshot(archive string, location string, importWasm bool) error { var rename extract.Renamer if !importWasm { rename = func(path string) string { - path = strings.ToLower(path) - if isWasmDB.MatchString(path) { + if isWasmDB.MatchString(strings.ToLower(path)) { return "" // do not extract wasm files } return path From 7b4834facf7e1c5834ea630551b9908a15183321 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 13:24:35 +0200 Subject: [PATCH 53/95] pass WasmTargets to RecordProgram call --- arbos/programs/native.go | 4 ++-- go-ethereum | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 98abf9443ca..08431b12096 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -228,8 +228,8 @@ func callProgram( panic("missing asm") } - if db, ok := db.(*state.StateDB); ok { - db.RecordProgram(moduleHash) + if stateDb, ok := db.(*state.StateDB); ok { + stateDb.RecordProgram(db.Database().WasmTargets(), moduleHash) } evmApi := newApi(interpreter, tracingInfo, scope, memoryModel) diff --git a/go-ethereum b/go-ethereum index 4ca4b4d7340..81114dde8a2 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 4ca4b4d734021df09cb42a796ca8f263d47383f9 +Subproject commit 81114dde8a26bae90c188605c4a36d5919a4a265 From 99b34b301af91888b32b3395b58aad7eb0ad46b9 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 13:39:09 +0200 Subject: [PATCH 54/95] init test: call Validate on StylusTargetConfig --- cmd/nitro/init_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index af2242af783..eadc6234c0d 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -350,6 +350,12 @@ func TestEmptyDatabaseDir(t *testing.T) { } } +func defaultStylusTargetConfigForTest(t *testing.T) *gethexec.StylusTargetConfig { + targetConfig := gethexec.DefaultStylusTargetConfig + Require(t, targetConfig.Validate()) + return &targetConfig +} + func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { t.Parallel() @@ -377,7 +383,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -394,7 +400,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -412,7 +418,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -548,7 +554,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), - &gethexec.DefaultStylusTargetConfig, + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, From bf735f466ce13e25006ce2f87f6362692df51659 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:15:46 +0200 Subject: [PATCH 55/95] don't call programs.SetTarget for TargetWavm --- execution/gethexec/executionengine.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 192408794bf..56b52ed986e 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -157,6 +157,9 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { for _, target := range targets { var effectiveStylusTarget string switch target { + case rawdb.TargetWavm: + // skip wavm target + continue case rawdb.TargetArm64: effectiveStylusTarget = targetConfig.Arm64 case rawdb.TargetAmd64: From 29327a8be174e5c582caef4fbe8d4fd55839688e Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:16:57 +0200 Subject: [PATCH 56/95] refactor StylusTargetConfig.Validate --- execution/gethexec/node.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index bc29e9b7f23..01464123fdd 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -44,22 +44,24 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { func (c *StylusTargetConfig) Validate() error { localTarget := rawdb.LocalTarget() - targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)) - targetsSet := make(map[ethdb.WasmTarget]struct{}, len(c.ExtraArchs)+1) + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) + targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs)) for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) + if targetsSet[target] { + // skip duplicate + continue + } if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - if _, duplicate := targetsSet[target]; !duplicate { - targets = append(targets, target) - targetsSet[target] = struct{}{} - } + targets = append(targets, target) + targetsSet[target] = true } - if _, has := targetsSet[rawdb.TargetWavm]; !has { + if !targetsSet[rawdb.TargetWavm] { return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) } - if _, has := targetsSet[localTarget]; !has { + if !targetsSet[localTarget] { targets = append(targets, localTarget) } c.wasmTargets = targets From aba94150dbb0a1b4c6533762284c83948a299e89 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 28 Aug 2024 14:21:21 +0200 Subject: [PATCH 57/95] error for unsupported stylus target in populateStylusTargetCache --- execution/gethexec/executionengine.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 56b52ed986e..8594d5867df 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -166,11 +166,13 @@ func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { effectiveStylusTarget = targetConfig.Amd64 case rawdb.TargetHost: effectiveStylusTarget = targetConfig.Host + default: + return fmt.Errorf("unsupported stylus target: %v", target) } isNative := target == localTarget err := programs.SetTarget(target, effectiveStylusTarget, isNative) if err != nil { - return fmt.Errorf("Failed to set stylus target: %w", err) + return fmt.Errorf("failed to set stylus target: %w", err) } nativeSet = nativeSet || isNative } From a69cb63efc66e86bd5f5bce998fdc66998e536ae Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 11:43:11 -0300 Subject: [PATCH 58/95] Replace regex with filepath functions --- cmd/nitro/init.go | 15 +++++++++++++-- cmd/nitro/init_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 435c73c997e..e0709bb3c94 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -405,8 +405,19 @@ func databaseIsEmpty(db ethdb.Database) bool { return !it.Next() } +func isWasmDb(path string) bool { + path = filepath.Clean(strings.ToLower(path)) + parts := strings.Split(path, string(filepath.Separator)) + if len(parts) >= 1 && parts[0] == "wasm" { + return true + } + if len(parts) >= 2 && parts[0] == "" && parts[1] == "wasm" { // Cover "/wasm" case + return true + } + return false +} + func extractSnapshot(archive string, location string, importWasm bool) error { - isWasmDB := regexp.MustCompile("^(./)?wasm") reader, err := os.Open(archive) if err != nil { return fmt.Errorf("couln't open init '%v' archive: %w", archive, err) @@ -419,7 +430,7 @@ func extractSnapshot(archive string, location string, importWasm bool) error { var rename extract.Renamer if !importWasm { rename = func(path string) string { - if isWasmDB.MatchString(strings.ToLower(path)) { + if isWasmDb(path) { return "" // do not extract wasm files } return path diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 9282839f75c..7b9153a9cf7 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -675,3 +675,35 @@ func TestExtractSnapshot(t *testing.T) { }) } } + +func TestIsWasmDb(t *testing.T) { + testCases := []struct { + path string + want bool + }{ + {"wasm", true}, + {"wasm/", true}, + {"wasm/something", true}, + {"/wasm", true}, + {"./wasm", true}, + {"././wasm", true}, + {"/./wasm", true}, + {"WASM", true}, + {"wAsM", true}, + {"nitro/../wasm", true}, + {"/nitro/../wasm", true}, + {".//nitro/.//../wasm", true}, + {"not-wasm", false}, + {"l2chaindata/example@@", false}, + {"somedir/wasm", false}, + } + for _, testCase := range testCases { + name := fmt.Sprintf("%q", testCase.path) + t.Run(name, func(t *testing.T) { + got := isWasmDb(testCase.path) + if testCase.want != got { + t.Fatalf("want %v, but got %v", testCase.want, got) + } + }) + } +} From 2e9f4a5a1e77c0c63da526d89874f1225550fab0 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 12:07:46 -0300 Subject: [PATCH 59/95] Disallow any wasm prefix --- cmd/nitro/init.go | 4 ++-- cmd/nitro/init_test.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index e0709bb3c94..92398c92075 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -408,10 +408,10 @@ func databaseIsEmpty(db ethdb.Database) bool { func isWasmDb(path string) bool { path = filepath.Clean(strings.ToLower(path)) parts := strings.Split(path, string(filepath.Separator)) - if len(parts) >= 1 && parts[0] == "wasm" { + if len(parts) >= 1 && strings.HasPrefix(parts[0], "wasm") { return true } - if len(parts) >= 2 && parts[0] == "" && parts[1] == "wasm" { // Cover "/wasm" case + if len(parts) >= 2 && parts[0] == "" && strings.HasPrefix(parts[1], "wasm") { // Cover "/wasm" case return true } return false diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 7b9153a9cf7..50d88c9b402 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -693,6 +693,9 @@ func TestIsWasmDb(t *testing.T) { {"nitro/../wasm", true}, {"/nitro/../wasm", true}, {".//nitro/.//../wasm", true}, + {"wasm-something", true}, + {"wasm\x00something", true}, + {"wasm\x00test/example", true}, {"not-wasm", false}, {"l2chaindata/example@@", false}, {"somedir/wasm", false}, From 3f9461d132899134e527bd7fdaf84cd8efbdf773 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 28 Aug 2024 22:40:17 +0530 Subject: [PATCH 60/95] Support rebuilding of wasm store on init --- cmd/nitro/init.go | 92 ++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 5573bd6ab27..db9f3bc0fc5 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -475,6 +475,51 @@ func validateOrUpgradeWasmStoreSchemaVersion(db ethdb.Database) error { return nil } +func rebuildLocalWasm(ctx context.Context, config *NodeConfig, l2BlockChain *core.BlockChain, chainConfig *params.ChainConfig, chainDb, wasmDb ethdb.Database, rebuildMode string) (ethdb.Database, *core.BlockChain, error) { + var err error + latestBlock := l2BlockChain.CurrentBlock() + if latestBlock == nil || latestBlock.Number.Uint64() <= chainConfig.ArbitrumChainParams.GenesisBlockNum || + types.DeserializeHeaderExtraInformation(latestBlock).ArbOSFormatVersion < params.ArbosVersion_Stylus { + // If there is only genesis block or no blocks in the blockchain, set Rebuilding of wasm store to Done + // If Stylus upgrade hasn't yet happened, skipping rebuilding of wasm store + log.Info("Setting rebuilding of wasm store to done") + if err = gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, gethexec.RebuildingDone); err != nil { + return nil, nil, fmt.Errorf("unable to set rebuilding status of wasm store to done: %w", err) + } + } else if rebuildMode != "false" { + var position common.Hash + if rebuildMode == "force" { + log.Info("Commencing force rebuilding of wasm store by setting codehash position in rebuilding to beginning") + if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { + return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) + } + } else { + position, err = gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingPositionKey) + if err != nil { + log.Info("Unable to get codehash position in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it and starting rebuilding", "err", err) + if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { + return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) + } + } + } + if position != gethexec.RebuildingDone { + startBlockHash, err := gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingStartBlockHashKey) + if err != nil { + log.Info("Unable to get start block hash in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it to latest block hash", "err", err) + if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingStartBlockHashKey, latestBlock.Hash()); err != nil { + return nil, nil, fmt.Errorf("unable to initialize start block hash in rebuilding of wasm store to latest block hash: %w", err) + } + startBlockHash = latestBlock.Hash() + } + log.Info("Starting or continuing rebuilding of wasm store", "codeHash", position, "startBlockHash", startBlockHash) + if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, &config.Execution.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { + return nil, nil, fmt.Errorf("error rebuilding of wasm store: %w", err) + } + } + } + return chainDb, l2BlockChain, nil +} + func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { @@ -523,47 +568,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, fmt.Errorf("failed to recreate missing states: %w", err) } } - latestBlock := l2BlockChain.CurrentBlock() - if latestBlock == nil || latestBlock.Number.Uint64() <= chainConfig.ArbitrumChainParams.GenesisBlockNum || - types.DeserializeHeaderExtraInformation(latestBlock).ArbOSFormatVersion < params.ArbosVersion_Stylus { - // If there is only genesis block or no blocks in the blockchain, set Rebuilding of wasm store to Done - // If Stylus upgrade hasn't yet happened, skipping rebuilding of wasm store - log.Info("Setting rebuilding of wasm store to done") - if err = gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, gethexec.RebuildingDone); err != nil { - return nil, nil, fmt.Errorf("unable to set rebuilding status of wasm store to done: %w", err) - } - } else if config.Init.RebuildLocalWasm != "false" { - var position common.Hash - if config.Init.RebuildLocalWasm == "force" { - log.Info("Commencing force rebuilding of wasm store by setting codehash position in rebuilding to beginning") - if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { - return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) - } - } else { - position, err = gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingPositionKey) - if err != nil { - log.Info("Unable to get codehash position in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it and starting rebuilding", "err", err) - if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingPositionKey, common.Hash{}); err != nil { - return nil, nil, fmt.Errorf("unable to initialize codehash position in rebuilding of wasm store to beginning: %w", err) - } - } - } - if position != gethexec.RebuildingDone { - startBlockHash, err := gethexec.ReadFromKeyValueStore[common.Hash](wasmDb, gethexec.RebuildingStartBlockHashKey) - if err != nil { - log.Info("Unable to get start block hash in rebuilding of wasm store, its possible it isnt initialized yet, so initializing it to latest block hash", "err", err) - if err := gethexec.WriteToKeyValueStore(wasmDb, gethexec.RebuildingStartBlockHashKey, latestBlock.Hash()); err != nil { - return nil, nil, fmt.Errorf("unable to initialize start block hash in rebuilding of wasm store to latest block hash: %w", err) - } - startBlockHash = latestBlock.Hash() - } - log.Info("Starting or continuing rebuilding of wasm store", "codeHash", position, "startBlockHash", startBlockHash) - if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, &config.Execution.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { - return nil, nil, fmt.Errorf("error rebuilding of wasm store: %w", err) - } - } - } - return chainDb, l2BlockChain, nil + return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, config.Init.RebuildLocalWasm) } readOnlyDb.Close() } else if !dbutil.IsNotExistError(err) { @@ -795,6 +800,11 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, err } + if config.Init.RebuildLocalWasm != "false" { + // In order to rebuild wasm store correctly we have to use force option + return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, "force") + } + return chainDb, l2BlockChain, nil } From 2499675af22d1c760a59dbed4b21cc6cd3821977 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 28 Aug 2024 22:46:31 +0530 Subject: [PATCH 61/95] update flag description --- cmd/conf/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index f3606916936..208969fce28 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -86,7 +86,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Int64(prefix+".reorg-to-message-batch", InitConfigDefault.ReorgToMessageBatch, "rolls back the blockchain to the first batch at or before a given message index") f.Int64(prefix+".reorg-to-block-batch", InitConfigDefault.ReorgToBlockBatch, "rolls back the blockchain to the first batch at or before a given block number") f.String(prefix+".rebuild-local-wasm", InitConfigDefault.RebuildLocalWasm, "rebuild local wasm database on boot if needed (otherwise-will be done lazily). Three modes are supported \n"+ - "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild,\n"+ + "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild. Is equivalent to force when starting out with --init.force or downloading db snapshot via init,\n"+ "\"force\"- force rebuilding which would commence rebuilding despite the status of previous attempts,\n"+ "\"false\"- do not rebuild on startup", ) From caa421013a96664a5f71653bde8a392165a78421 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 15:19:26 -0300 Subject: [PATCH 62/95] Revert "Disallow any wasm prefix" This reverts commit 2e9f4a5a1e77c0c63da526d89874f1225550fab0. --- cmd/nitro/init.go | 4 ++-- cmd/nitro/init_test.go | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 92398c92075..e0709bb3c94 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -408,10 +408,10 @@ func databaseIsEmpty(db ethdb.Database) bool { func isWasmDb(path string) bool { path = filepath.Clean(strings.ToLower(path)) parts := strings.Split(path, string(filepath.Separator)) - if len(parts) >= 1 && strings.HasPrefix(parts[0], "wasm") { + if len(parts) >= 1 && parts[0] == "wasm" { return true } - if len(parts) >= 2 && parts[0] == "" && strings.HasPrefix(parts[1], "wasm") { // Cover "/wasm" case + if len(parts) >= 2 && parts[0] == "" && parts[1] == "wasm" { // Cover "/wasm" case return true } return false diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 50d88c9b402..7b9153a9cf7 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -693,9 +693,6 @@ func TestIsWasmDb(t *testing.T) { {"nitro/../wasm", true}, {"/nitro/../wasm", true}, {".//nitro/.//../wasm", true}, - {"wasm-something", true}, - {"wasm\x00something", true}, - {"wasm\x00test/example", true}, {"not-wasm", false}, {"l2chaindata/example@@", false}, {"somedir/wasm", false}, From 9644416f86ac09478f99b6c56fe3784229318abf Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 18:38:06 -0300 Subject: [PATCH 63/95] Improve out of gas warning log For the setTrieSlots and setTransientBytes32 HostIOs, when there is an out-of-gas error, Nitro emits a warning line with `Caused by:\n \x02`. This commit improves this error message by printing the debug representation of the API status code and the warning becomes: `Caused by:\n OutOfGas`. This commit also solves a potential panic when the API does not return the status code. --- arbitrator/arbutil/src/evm/req.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/arbitrator/arbutil/src/evm/req.rs b/arbitrator/arbutil/src/evm/req.rs index b1c8d999727..f76d1407f1b 100644 --- a/arbitrator/arbutil/src/evm/req.rs +++ b/arbitrator/arbutil/src/evm/req.rs @@ -140,8 +140,13 @@ impl> EvmApi for EvmApiRequestor { } let (res, _, cost) = self.request(EvmApiMethod::SetTrieSlots, data); - if res[0] != EvmApiStatus::Success.into() { - bail!("{}", String::from_utf8_or_hex(res)); + let status = res + .first() + .copied() + .map(EvmApiStatus::from) + .unwrap_or(EvmApiStatus::Failure); + if status != EvmApiStatus::Success { + bail!("{:?}", status); } Ok(cost) } @@ -156,8 +161,13 @@ impl> EvmApi for EvmApiRequestor { data.extend(key); data.extend(value); let (res, ..) = self.request(EvmApiMethod::SetTransientBytes32, data); - if res[0] != EvmApiStatus::Success.into() { - bail!("{}", String::from_utf8_or_hex(res)); + let status = res + .first() + .copied() + .map(EvmApiStatus::from) + .unwrap_or(EvmApiStatus::Failure); + if status != EvmApiStatus::Success { + bail!("{:?}", status); } Ok(()) } From 03dd3abfd7fb27f5be4bfffa6f0cb8e9deadd8ef Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 22:20:57 -0300 Subject: [PATCH 64/95] Remove unused import --- arbitrator/arbutil/src/evm/req.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/arbitrator/arbutil/src/evm/req.rs b/arbitrator/arbutil/src/evm/req.rs index f76d1407f1b..287db357f32 100644 --- a/arbitrator/arbutil/src/evm/req.rs +++ b/arbitrator/arbutil/src/evm/req.rs @@ -7,7 +7,6 @@ use crate::{ storage::{StorageCache, StorageWord}, user::UserOutcomeKind, }, - format::Utf8OrHex, pricing::EVM_API_INK, Bytes20, Bytes32, }; From f6efc1b491a39e0eb52e62dc7dc01cda829bd168 Mon Sep 17 00:00:00 2001 From: Gabriel de Quadros Ligneul Date: Wed, 28 Aug 2024 22:25:41 -0300 Subject: [PATCH 65/95] Add comment about case-insensitive file systems --- cmd/nitro/init.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index e0709bb3c94..b338e0a49ef 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -406,7 +406,8 @@ func databaseIsEmpty(db ethdb.Database) bool { } func isWasmDb(path string) bool { - path = filepath.Clean(strings.ToLower(path)) + path = strings.ToLower(path) // lowers the path to handle case-insensitive file systems + path = filepath.Clean(path) parts := strings.Split(path, string(filepath.Separator)) if len(parts) >= 1 && parts[0] == "wasm" { return true From 9a326e281871d3c29dd5a796972298c87e917235 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 28 Aug 2024 22:04:10 -0500 Subject: [PATCH 66/95] Disable preimage validation in JIT --- arbitrator/jit/src/wavmio.rs | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/arbitrator/jit/src/wavmio.rs b/arbitrator/jit/src/wavmio.rs index 062d18d8e92..0ca666d3b20 100644 --- a/arbitrator/jit/src/wavmio.rs +++ b/arbitrator/jit/src/wavmio.rs @@ -8,8 +8,6 @@ use crate::{ }; use arbutil::{Color, PreimageType}; use caller_env::{GuestPtr, MemAccess}; -use sha2::Sha256; -use sha3::{Digest, Keccak256}; use std::{ io, io::{BufReader, BufWriter, ErrorKind}, @@ -170,19 +168,25 @@ pub fn resolve_preimage_impl( error!("Missing requested preimage for hash {hash_hex} in {name}") }; - // Check if preimage rehashes to the provided hash. Exclude blob preimages - let calculated_hash: [u8; 32] = match preimage_type { - PreimageType::Keccak256 => Keccak256::digest(preimage).into(), - PreimageType::Sha2_256 => Sha256::digest(preimage).into(), - PreimageType::EthVersionedHash => *hash, - }; - if calculated_hash != *hash { - error!( - "Calculated hash {} of preimage {} does not match provided hash {}", - hex::encode(calculated_hash), - hex::encode(preimage), - hex::encode(*hash) - ); + #[cfg(debug_assertions)] + { + use sha2::Sha256; + use sha3::{Digest, Keccak256}; + + // Check if preimage rehashes to the provided hash. Exclude blob preimages + let calculated_hash: [u8; 32] = match preimage_type { + PreimageType::Keccak256 => Keccak256::digest(preimage).into(), + PreimageType::Sha2_256 => Sha256::digest(preimage).into(), + PreimageType::EthVersionedHash => *hash, + }; + if calculated_hash != *hash { + error!( + "Calculated hash {} of preimage {} does not match provided hash {}", + hex::encode(calculated_hash), + hex::encode(preimage), + hex::encode(*hash) + ); + } } if offset % 32 != 0 { From 972508da88fb9797c3151cbdc23349b096a098f5 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 29 Aug 2024 09:44:34 +0530 Subject: [PATCH 67/95] address PR comments --- cmd/nitro/init.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index db9f3bc0fc5..6ae3c9125d2 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -475,10 +475,10 @@ func validateOrUpgradeWasmStoreSchemaVersion(db ethdb.Database) error { return nil } -func rebuildLocalWasm(ctx context.Context, config *NodeConfig, l2BlockChain *core.BlockChain, chainConfig *params.ChainConfig, chainDb, wasmDb ethdb.Database, rebuildMode string) (ethdb.Database, *core.BlockChain, error) { +func rebuildLocalWasm(ctx context.Context, config *gethexec.Config, l2BlockChain *core.BlockChain, chainDb, wasmDb ethdb.Database, rebuildMode string) (ethdb.Database, *core.BlockChain, error) { var err error latestBlock := l2BlockChain.CurrentBlock() - if latestBlock == nil || latestBlock.Number.Uint64() <= chainConfig.ArbitrumChainParams.GenesisBlockNum || + if latestBlock == nil || latestBlock.Number.Uint64() <= l2BlockChain.Config().ArbitrumChainParams.GenesisBlockNum || types.DeserializeHeaderExtraInformation(latestBlock).ArbOSFormatVersion < params.ArbosVersion_Stylus { // If there is only genesis block or no blocks in the blockchain, set Rebuilding of wasm store to Done // If Stylus upgrade hasn't yet happened, skipping rebuilding of wasm store @@ -512,7 +512,7 @@ func rebuildLocalWasm(ctx context.Context, config *NodeConfig, l2BlockChain *cor startBlockHash = latestBlock.Hash() } log.Info("Starting or continuing rebuilding of wasm store", "codeHash", position, "startBlockHash", startBlockHash) - if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.Execution.RPC.MaxRecreateStateDepth, &config.Execution.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { + if err := gethexec.RebuildWasmStore(ctx, wasmDb, chainDb, config.RPC.MaxRecreateStateDepth, &config.StylusTarget, l2BlockChain, position, startBlockHash); err != nil { return nil, nil, fmt.Errorf("error rebuilding of wasm store: %w", err) } } @@ -568,7 +568,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, fmt.Errorf("failed to recreate missing states: %w", err) } } - return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, config.Init.RebuildLocalWasm) + return rebuildLocalWasm(ctx, &config.Execution, l2BlockChain, chainDb, wasmDb, config.Init.RebuildLocalWasm) } readOnlyDb.Close() } else if !dbutil.IsNotExistError(err) { @@ -800,12 +800,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, err } - if config.Init.RebuildLocalWasm != "false" { - // In order to rebuild wasm store correctly we have to use force option - return rebuildLocalWasm(ctx, config, l2BlockChain, chainConfig, chainDb, wasmDb, "force") - } - - return chainDb, l2BlockChain, nil + return rebuildLocalWasm(ctx, &config.Execution, l2BlockChain, chainDb, wasmDb, config.Init.RebuildLocalWasm) } func testTxIndexUpdated(chainDb ethdb.Database, lastBlock uint64) bool { From a710d0c4a9203af55e9a43b4687647e3d1444a55 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Thu, 29 Aug 2024 09:46:29 +0530 Subject: [PATCH 68/95] undo flag description --- cmd/conf/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index 208969fce28..f3606916936 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -86,7 +86,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Int64(prefix+".reorg-to-message-batch", InitConfigDefault.ReorgToMessageBatch, "rolls back the blockchain to the first batch at or before a given message index") f.Int64(prefix+".reorg-to-block-batch", InitConfigDefault.ReorgToBlockBatch, "rolls back the blockchain to the first batch at or before a given block number") f.String(prefix+".rebuild-local-wasm", InitConfigDefault.RebuildLocalWasm, "rebuild local wasm database on boot if needed (otherwise-will be done lazily). Three modes are supported \n"+ - "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild. Is equivalent to force when starting out with --init.force or downloading db snapshot via init,\n"+ + "\"auto\"- (enabled by default) if any previous rebuilding attempt was successful then rebuilding is disabled else continues to rebuild,\n"+ "\"force\"- force rebuilding which would commence rebuilding despite the status of previous attempts,\n"+ "\"false\"- do not rebuild on startup", ) From 138bcb7c5c3c5ab4427be249d48430afbc0a5699 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 29 Aug 2024 10:47:20 -0500 Subject: [PATCH 69/95] Add safety check that min basefee must be positive for gas estimation --- precompiles/ArbOwner.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/precompiles/ArbOwner.go b/precompiles/ArbOwner.go index 066fc0a4c41..8b87445e0e5 100644 --- a/precompiles/ArbOwner.go +++ b/precompiles/ArbOwner.go @@ -69,6 +69,9 @@ func (con ArbOwner) SetL2BaseFee(c ctx, evm mech, priceInWei huge) error { // SetMinimumL2BaseFee sets the minimum base fee needed for a transaction to succeed func (con ArbOwner) SetMinimumL2BaseFee(c ctx, evm mech, priceInWei huge) error { + if c.txProcessor.MsgIsNonMutating() && priceInWei.Sign() == 0 { + return errors.New("minimum base fee must be nonzero") + } return c.State.L2PricingState().SetMinBaseFeeWei(priceInWei) } From 7a4fe007e5e3edb198130265ad167dc39d531073 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 29 Aug 2024 21:21:50 +0200 Subject: [PATCH 70/95] simplify StylusTargetConfig.Validate --- execution/gethexec/node.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 01464123fdd..21c2b4beced 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -43,26 +43,21 @@ func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { } func (c *StylusTargetConfig) Validate() error { - localTarget := rawdb.LocalTarget() - targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs)) for _, arch := range c.ExtraArchs { target := ethdb.WasmTarget(arch) - if targetsSet[target] { - // skip duplicate - continue - } if !rawdb.IsSupportedWasmTarget(target) { return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) } - targets = append(targets, target) targetsSet[target] = true } if !targetsSet[rawdb.TargetWavm] { return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) } - if !targetsSet[localTarget] { - targets = append(targets, localTarget) + targetsSet[rawdb.LocalTarget()] = true + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) + for target := range targetsSet { + targets = append(targets, target) } c.wasmTargets = targets return nil From 01c91c72db803410631f51b8614cd69021c8e786 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 10:11:46 -0300 Subject: [PATCH 71/95] Use pull_request_target instead of pull_request in submodule pin check --- .github/workflows/submodule-pin-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submodule-pin-check.yml b/.github/workflows/submodule-pin-check.yml index 870f5208770..f045f71f68c 100644 --- a/.github/workflows/submodule-pin-check.yml +++ b/.github/workflows/submodule-pin-check.yml @@ -1,7 +1,7 @@ name: Merge Checks on: - pull_request: + pull_request_target: branches: [ master ] types: [synchronize, opened, reopened] From ae67f4e575ac90b97119429ebf2dccc5fcc0ef17 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Fri, 30 Aug 2024 12:11:40 -0600 Subject: [PATCH 72/95] redis consimer: workers and beffers --- validator/valnode/redis/consumer.go | 69 ++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index fb7db1e8702..8458ec021af 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -3,6 +3,7 @@ package redis import ( "context" "fmt" + "runtime" "time" "github.com/ethereum/go-ethereum/common" @@ -22,8 +23,9 @@ type ValidationServer struct { spawner validator.ValidationSpawner // consumers stores moduleRoot to consumer mapping. - consumers map[common.Hash]*pubsub.Consumer[*validator.ValidationInput, validator.GoGlobalState] - streamTimeout time.Duration + consumers map[common.Hash]*pubsub.Consumer[*validator.ValidationInput, validator.GoGlobalState] + + config *ValidationServerConfig } func NewValidationServer(cfg *ValidationServerConfig, spawner validator.ValidationSpawner) (*ValidationServer, error) { @@ -44,9 +46,9 @@ func NewValidationServer(cfg *ValidationServerConfig, spawner validator.Validati consumers[mr] = c } return &ValidationServer{ - consumers: consumers, - spawner: spawner, - streamTimeout: cfg.StreamTimeout, + consumers: consumers, + spawner: spawner, + config: cfg, }, nil } @@ -54,6 +56,19 @@ func (s *ValidationServer) Start(ctx_in context.Context) { s.StopWaiter.Start(ctx_in, s) // Channel that all consumers use to indicate their readiness. readyStreams := make(chan struct{}, len(s.consumers)) + type workUnit struct { + req *pubsub.Message[*validator.ValidationInput] + moduleRoot common.Hash + } + workers := s.config.Workers + if workers == 0 { + workers = runtime.NumCPU() + } + bufferSize := 0 + if s.config.BufferReads { + bufferSize = workers + } + workQueue := make(chan workUnit, bufferSize) for moduleRoot, c := range s.consumers { c := c moduleRoot := moduleRoot @@ -93,17 +108,11 @@ func (s *ValidationServer) Start(ctx_in context.Context) { // There's nothing in the queue. return time.Second } - valRun := s.spawner.Launch(req.Value, moduleRoot) - res, err := valRun.Await(ctx) - if err != nil { - log.Error("Error validating", "request value", req.Value, "error", err) - return 0 - } - if err := c.SetResult(ctx, req.ID, res); err != nil { - log.Error("Error setting result for request", "id", req.ID, "result", res, "error", err) - return 0 + select { + case <-ctx.Done(): + case workQueue <- workUnit{req, moduleRoot}: } - return time.Second + return 0 }) }) } @@ -113,7 +122,7 @@ func (s *ValidationServer) Start(ctx_in context.Context) { case <-readyStreams: log.Trace("At least one stream is ready") return // Don't block Start if at least one of the stream is ready. - case <-time.After(s.streamTimeout): + case <-time.After(s.config.StreamTimeout): log.Error("Waiting for redis streams timed out") case <-ctx.Done(): log.Info("Context done while waiting redis streams to be ready, failed to start") @@ -121,6 +130,26 @@ func (s *ValidationServer) Start(ctx_in context.Context) { } } }) + for i := 0; i < workers; i++ { + s.StopWaiter.LaunchThread(func(ctx context.Context) { + for { + var work workUnit + select { + case <-ctx.Done(): + return + case work = <-workQueue: + } + valRun := s.spawner.Launch(work.req.Value, work.moduleRoot) + res, err := valRun.Await(ctx) + if err != nil { + log.Error("Error validating", "request value", work.req.Value, "error", err) + } + if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { + log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } + } + }) + } } type ValidationServerConfig struct { @@ -131,6 +160,8 @@ type ValidationServerConfig struct { // Timeout on polling for existence of each redis stream. StreamTimeout time.Duration `koanf:"stream-timeout"` StreamPrefix string `koanf:"stream-prefix"` + Workers int `koanf:"workers"` + BufferReads bool `koanf:"buffer-reads"` } var DefaultValidationServerConfig = ValidationServerConfig{ @@ -139,6 +170,8 @@ var DefaultValidationServerConfig = ValidationServerConfig{ ConsumerConfig: pubsub.DefaultConsumerConfig, ModuleRoots: []string{}, StreamTimeout: 10 * time.Minute, + Workers: 1, + BufferReads: true, } var TestValidationServerConfig = ValidationServerConfig{ @@ -147,6 +180,8 @@ var TestValidationServerConfig = ValidationServerConfig{ ConsumerConfig: pubsub.TestConsumerConfig, ModuleRoots: []string{}, StreamTimeout: time.Minute, + Workers: 1, + BufferReads: true, } func ValidationServerConfigAddOptions(prefix string, f *pflag.FlagSet) { @@ -155,6 +190,8 @@ func ValidationServerConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".redis-url", DefaultValidationServerConfig.RedisURL, "url of redis server") f.String(prefix+".stream-prefix", DefaultValidationServerConfig.StreamPrefix, "prefix for stream name") f.Duration(prefix+".stream-timeout", DefaultValidationServerConfig.StreamTimeout, "Timeout on polling for existence of redis streams") + f.Int(prefix+".workers", DefaultValidationServerConfig.Workers, "number of validation threads (0 to use number of CPUs)") + f.Bool(prefix+".buffer-reads", DefaultValidationServerConfig.BufferReads, "buffer reads (read next while working)") } func (cfg *ValidationServerConfig) Enabled() bool { From d7ee2eed4b5e07c609d95d6cba60e625d9df67ae Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Fri, 30 Aug 2024 15:41:57 -0600 Subject: [PATCH 73/95] redis consumer: use tokens for buffer, default 0 --- validator/valnode/redis/consumer.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 8458ec021af..74564665336 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -64,11 +64,15 @@ func (s *ValidationServer) Start(ctx_in context.Context) { if workers == 0 { workers = runtime.NumCPU() } - bufferSize := 0 + workQueue := make(chan workUnit, workers) + tokensCount := workers if s.config.BufferReads { - bufferSize = workers + tokensCount += workers + } + requestTokenQueue := make(chan struct{}, tokensCount) + for i := 0; i < tokensCount; i++ { + requestTokenQueue <- struct{}{} } - workQueue := make(chan workUnit, bufferSize) for moduleRoot, c := range s.consumers { c := c moduleRoot := moduleRoot @@ -99,6 +103,11 @@ func (s *ValidationServer) Start(ctx_in context.Context) { case <-ready: // Wait until the stream exists and start consuming iteratively. } s.StopWaiter.CallIteratively(func(ctx context.Context) time.Duration { + select { + case <-ctx.Done(): + return 0 + case <-requestTokenQueue: + } req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) @@ -147,6 +156,11 @@ func (s *ValidationServer) Start(ctx_in context.Context) { if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) } + select { + case <-ctx.Done(): + return + case requestTokenQueue <- struct{}{}: + } } }) } @@ -170,7 +184,7 @@ var DefaultValidationServerConfig = ValidationServerConfig{ ConsumerConfig: pubsub.DefaultConsumerConfig, ModuleRoots: []string{}, StreamTimeout: 10 * time.Minute, - Workers: 1, + Workers: 0, BufferReads: true, } From 0d08e481f039c778ff0f9a7c60f38167eadb29f2 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 20:46:02 -0300 Subject: [PATCH 74/95] Also uses DataPoster when only creating validator wallet contract and exiting --- arbnode/node.go | 23 ++++++++++ cmd/nitro/nitro.go | 15 ++++++- staker/validatorwallet/contract.go | 72 +++++++++++++++--------------- system_tests/fast_confirm_test.go | 54 +++++++++++----------- system_tests/staker_test.go | 25 ++++++++--- 5 files changed, 119 insertions(+), 70 deletions(-) diff --git a/arbnode/node.go b/arbnode/node.go index 93b58e800f9..e13f92f30f3 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -339,6 +339,29 @@ func checkArbDbSchemaVersion(arbDb ethdb.Database) error { return nil } +func DataposterOnlyUsedToCreateValidatorWalletContract( + ctx context.Context, + l1Reader *headerreader.HeaderReader, + transactOpts *bind.TransactOpts, + cfg *dataposter.DataPosterConfig, + parentChainID *big.Int, +) (*dataposter.DataPoster, error) { + return dataposter.NewDataPoster(ctx, + &dataposter.DataPosterOpts{ + HeaderReader: l1Reader, + Auth: transactOpts, + Config: func() *dataposter.DataPosterConfig { + cfg.UseNoOpStorage = true + return cfg + }, + MetadataRetriever: func(ctx context.Context, blockNum *big.Int) ([]byte, error) { + return nil, nil + }, + ParentChainID: parentChainID, + }, + ) +} + func StakerDataposter( ctx context.Context, db ethdb.Database, l1Reader *headerreader.HeaderReader, transactOpts *bind.TransactOpts, cfgFetcher ConfigFetcher, syncMonitor *SyncMonitor, diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 8c26b95a927..bad48f430ca 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -371,8 +371,21 @@ func mainImpl() int { if err != nil { log.Crit("error getting rollup addresses config", "err", err) } + + dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( + ctx, + l1Reader, + l1TransactionOptsBatchPoster, + &nodeConfig.Node.Staker.DataPoster, + new(big.Int).SetUint64(nodeConfig.ParentChain.ID), + ) + if err != nil { + log.Crit("error creating data poster to create validator wallet contract", "err", err) + } + getExtraGas := func() uint64 { return nodeConfig.Node.Staker.ExtraGas } + // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, nil) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, dataPoster, getExtraGas) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 495e796cd25..3db35d8232c 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -155,16 +155,19 @@ func (v *Contract) From() common.Address { } // nil value == 0 value -func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { - newAuth := *v.auth - newAuth.Context = ctx - newAuth.Value = value - nonce, err := v.L1Client().NonceAt(ctx, v.auth.From, nil) +func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { + auth.Context = ctx + auth.Value = value + nonce, err := l1Reader.Client().NonceAt(ctx, auth.From, nil) if err != nil { return nil, err } - newAuth.Nonce = new(big.Int).SetUint64(nonce) - return &newAuth, nil + auth.Nonce = new(big.Int).SetUint64(nonce) + return &auth, nil +} + +func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { + return getAuthWithUpdatedNonce(ctx, v.l1Reader, *v.auth, value) } func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction, gasRefunder common.Address) (*types.Transaction, error) { @@ -183,9 +186,12 @@ func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), *v.Address(), data, gas, auth.Value) } -// Exposed for tests -func (v *Contract) CreateWalletContract( +func createWalletContract( ctx context.Context, + l1Reader *headerreader.HeaderReader, + auth *bind.TransactOpts, + dataPoster *dataposter.DataPoster, + getExtraGas func() uint64, validatorWalletFactoryAddr common.Address, ) (*types.Transaction, error) { var initialExecutorAllowedDests []common.Address @@ -194,24 +200,19 @@ func (v *Contract) CreateWalletContract( return nil, err } - auth, err := v.getAuth(ctx, nil) - if err != nil { - return nil, err - } - gas, err := gasForTxData( ctx, - v.l1Reader, + l1Reader, auth, &validatorWalletFactoryAddr, txData, - v.getExtraGas, + getExtraGas, ) if err != nil { return nil, fmt.Errorf("getting gas for tx data: %w", err) } - return v.dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) + return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) } func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { @@ -225,15 +226,10 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err return nil } if v.address.Load() == nil { - auth, err := v.getAuth(ctx, nil) - if err != nil { - return err - } - - // By passing v.CreateWalletContract as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. + // By passing v.dataPoster as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, auth, v.l1Reader, createIfMissing, v.CreateWalletContract) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.auth, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) if err != nil { return err } @@ -439,6 +435,11 @@ func (b *Contract) DataPoster() *dataposter.DataPoster { return b.dataPoster } +// Exported for testing +func (b *Contract) GetExtraGas() func() uint64 { + return b.getExtraGas +} + func GetValidatorWalletContract( ctx context.Context, validatorWalletFactoryAddr common.Address, @@ -446,7 +447,8 @@ func GetValidatorWalletContract( transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, - createWalletFunc func(context.Context, common.Address) (*types.Transaction, error), + dataPoster *dataposter.DataPoster, + getExtraGas func() uint64, ) (*common.Address, error) { client := l1Reader.Client() @@ -483,18 +485,14 @@ func GetValidatorWalletContract( return nil, nil } - var tx *types.Transaction - if createWalletFunc == nil { - var initialExecutorAllowedDests []common.Address - tx, err = walletCreator.CreateWallet(transactAuth, initialExecutorAllowedDests) - if err != nil { - return nil, err - } - } else { - tx, err = createWalletFunc(ctx, validatorWalletFactoryAddr) - if err != nil { - return nil, err - } + transactAuth, err = getAuthWithUpdatedNonce(ctx, l1Reader, *transactAuth, nil) + if err != nil { + return nil, err + } + + tx, err := createWalletContract(ctx, l1Reader, transactAuth, dataPoster, getExtraGas, validatorWalletFactoryAddr) + if err != nil { + return nil, err } receipt, err := l1Reader.WaitForTxApproval(ctx, tx) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index ae624be1e9b..45988274186 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -115,10 +115,10 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.CreateWalletContract) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -278,15 +278,6 @@ func TestFastConfirmationWithSafe(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorB", balance, builder.L1Info) l1authB := builder.L1Info.GetDefaultTransactOpts("ValidatorB", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, nil) - Require(t, err) - if valWalletAddrA == *valWalletAddrCheck { - Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) - } - rollup, err := rollupgen.NewRollupAdminLogic(l2nodeA.DeployInfo.Rollup, builder.L1.Client) Require(t, err) @@ -295,28 +286,13 @@ func TestFastConfirmationWithSafe(t *testing.T) { rollupABI, err := abi.JSON(strings.NewReader(rollupgen.RollupAdminLogicABI)) Require(t, err, "unable to parse rollup ABI") - safeAddress := deploySafe(t, builder.L1, builder.L1.Client, deployAuth, []common.Address{valWalletAddrA, srv.Address}) - setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address, safeAddress}, []bool{true, true, true, true}) - Require(t, err, "unable to generate setValidator calldata") - tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) - Require(t, err, "unable to set validators") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - setMinAssertPeriodCalldata, err := rollupABI.Pack("setMinimumAssertionPeriod", big.NewInt(1)) Require(t, err, "unable to generate setMinimumAssertionPeriod calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) + tx, err := upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setMinAssertPeriodCalldata) Require(t, err, "unable to set minimum assertion period") _, err = builder.L1.EnsureTxSucceeded(tx) Require(t, err) - setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", safeAddress) - Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") - tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) - Require(t, err, "unable to set anytrust fast confirmer") - _, err = builder.L1.EnsureTxSucceeded(tx) - Require(t, err) - valConfigA := staker.TestL1ValidatorConfig valConfigA.EnableFastConfirmation = true @@ -339,6 +315,30 @@ func TestFastConfirmationWithSafe(t *testing.T) { Require(t, err) valConfigA.Strategy = "MakeNodes" + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + Require(t, err) + valWalletAddrA := *valWalletAddrAPtr + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + Require(t, err) + if valWalletAddrA == *valWalletAddrCheck { + Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) + } + + safeAddress := deploySafe(t, builder.L1, builder.L1.Client, deployAuth, []common.Address{valWalletAddrA, srv.Address}) + setValidatorCalldata, err := rollupABI.Pack("setValidator", []common.Address{valWalletAddrA, l1authB.From, srv.Address, safeAddress}, []bool{true, true, true, true}) + Require(t, err, "unable to generate setValidator calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setValidatorCalldata) + Require(t, err, "unable to set validators") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + + setAnyTrustFastConfirmerCalldata, err := rollupABI.Pack("setAnyTrustFastConfirmer", safeAddress) + Require(t, err, "unable to generate setAnyTrustFastConfirmer calldata") + tx, err = upgradeExecutor.ExecuteCall(&deployAuth, l2nodeA.DeployInfo.Rollup, setAnyTrustFastConfirmerCalldata) + Require(t, err, "unable to set anytrust fast confirmer") + _, err = builder.L1.EnsureTxSucceeded(tx) + Require(t, err) + _, valStack := createTestValidationNode(t, ctx, &valnode.TestValidationConfig) blockValidatorConfig := staker.TestBlockValidatorConfig diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index 3fdf82ca744..e61dcedc2b4 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -176,10 +176,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.CreateWalletContract) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) @@ -476,7 +476,7 @@ func TestStakersCooperative(t *testing.T) { stakerTestImpl(t, false, false) } -func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { +func TestGetValidatorWalletContractWithDataposterOnlyUsedToCreateValidatorWalletContract(t *testing.T) { t.Parallel() ctx, cancelCtx := context.WithCancel(context.Background()) @@ -492,10 +492,25 @@ func TestGetValidatorWalletContractWithoutDataPoster(t *testing.T) { builder.L1.TransferBalance(t, "Faucet", "ValidatorA", balance, builder.L1Info) l1auth := builder.L1Info.GetDefaultTransactOpts("ValidatorA", ctx) - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + parentChainID, err := builder.L1.Client.ChainID(ctx) + Require(t, err) + + dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( + ctx, + builder.L2.ConsensusNode.L1Reader, + &l1auth, + &builder.nodeConfig.Staker.DataPoster, + parentChainID, + ) + if err != nil { + log.Crit("error creating data poster to create validator wallet contract", "err", err) + } + getExtraGas := func() uint64 { return builder.nodeConfig.Staker.ExtraGas } + + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, nil) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 6eae487866df45f46db437141e30254c0eefa4c6 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 20:47:15 -0300 Subject: [PATCH 75/95] Creates validator wallet contract with transaction with value set to zero --- staker/validatorwallet/contract.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 3db35d8232c..8a522b44a3d 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -212,7 +212,7 @@ func createWalletContract( return nil, fmt.Errorf("getting gas for tx data: %w", err) } - return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, auth.Value) + return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, common.Big0) } func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) error { From caadac62a7be06a2296a0c1c0db0330e5745d800 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Fri, 30 Aug 2024 21:05:48 -0300 Subject: [PATCH 76/95] Improves error message when getting gas to create validator wallet --- staker/validatorwallet/contract.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 8a522b44a3d..1910db1d637 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -209,7 +209,7 @@ func createWalletContract( getExtraGas, ) if err != nil { - return nil, fmt.Errorf("getting gas for tx data: %w", err) + return nil, fmt.Errorf("getting gas for tx data when creating validator wallet, validatorWalletFactory=%v: %w", validatorWalletFactoryAddr, err) } return dataPoster.PostSimpleTransaction(ctx, auth.Nonce.Uint64(), validatorWalletFactoryAddr, txData, gas, common.Big0) From 716bfea74964a6f2e51a2a7f119a1d9f9d314149 Mon Sep 17 00:00:00 2001 From: Pepper Lebeck-Jobe Date: Mon, 2 Sep 2024 12:23:03 +0200 Subject: [PATCH 77/95] Add Mac OSX environment variables The unfortunate thing is that it would still be good for developers to set these in their shell environment so that if they do end up running any "go" commands directly from the CLI, they won't be bombarded with all the linker warnings. But, at least this makes the default experience of using the make files a little nicer. --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index e46bdbbe612..0a71d64f12c 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,14 @@ ifneq ($(origin GOLANG_LDFLAGS),undefined) GOLANG_PARAMS = -ldflags="-extldflags '-ldl' $(GOLANG_LDFLAGS)" endif +UNAME_S := $(shell uname -s) + +# In Mac OSX, there are a lot of warnings emitted if these environment variables aren't set. +ifeq ($(UNAME_S), Darwin) + export MACOSX_DEPLOYMENT_TARGET := $(shell sw_vers -productVersion) + export CGO_LDFLAGS := -Wl,-no_warn_duplicate_libraries +endif + precompile_names = AddressTable Aggregator BLS Debug FunctionTable GasInfo Info osTest Owner RetryableTx Statistics Sys precompiles = $(patsubst %,./solgen/generated/%.go, $(precompile_names)) From 33337b8cd074c4a2ddf38dbcbe574e9712d1123f Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 10:39:38 -0300 Subject: [PATCH 78/95] Renames getAuthWithUpdatedNonce to getAuthWithUpdatedNonceFromL1 --- staker/validatorwallet/contract.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 1910db1d637..83dedcad841 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -155,7 +155,7 @@ func (v *Contract) From() common.Address { } // nil value == 0 value -func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { +func getAuthWithUpdatedNonceFromL1(ctx context.Context, l1Reader *headerreader.HeaderReader, auth bind.TransactOpts, value *big.Int) (*bind.TransactOpts, error) { auth.Context = ctx auth.Value = value nonce, err := l1Reader.Client().NonceAt(ctx, auth.From, nil) @@ -167,7 +167,7 @@ func getAuthWithUpdatedNonce(ctx context.Context, l1Reader *headerreader.HeaderR } func (v *Contract) getAuth(ctx context.Context, value *big.Int) (*bind.TransactOpts, error) { - return getAuthWithUpdatedNonce(ctx, v.l1Reader, *v.auth, value) + return getAuthWithUpdatedNonceFromL1(ctx, v.l1Reader, *v.auth, value) } func (v *Contract) executeTransaction(ctx context.Context, tx *types.Transaction, gasRefunder common.Address) (*types.Transaction, error) { @@ -485,7 +485,7 @@ func GetValidatorWalletContract( return nil, nil } - transactAuth, err = getAuthWithUpdatedNonce(ctx, l1Reader, *transactAuth, nil) + transactAuth, err = getAuthWithUpdatedNonceFromL1(ctx, l1Reader, *transactAuth, nil) if err != nil { return nil, err } From 9150808093cbfbc2fb98c9bb44e34b3b4ad927a7 Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 16:12:49 -0300 Subject: [PATCH 79/95] Don't modify config inside config fetcher --- arbnode/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbnode/node.go b/arbnode/node.go index e13f92f30f3..a9da4ea24bb 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -346,12 +346,12 @@ func DataposterOnlyUsedToCreateValidatorWalletContract( cfg *dataposter.DataPosterConfig, parentChainID *big.Int, ) (*dataposter.DataPoster, error) { + cfg.UseNoOpStorage = true return dataposter.NewDataPoster(ctx, &dataposter.DataPosterOpts{ HeaderReader: l1Reader, Auth: transactOpts, Config: func() *dataposter.DataPosterConfig { - cfg.UseNoOpStorage = true return cfg }, MetadataRetriever: func(ctx context.Context, blockNum *big.Int) ([]byte, error) { From 1bc34f20ff8f21a7c860dc059393309fd11d1d00 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 2 Sep 2024 14:23:02 -0500 Subject: [PATCH 80/95] Close opened files explicitly instead of relying on the GC --- arbnode/resourcemanager/resource_management.go | 2 ++ cmd/nitro/init.go | 1 + 2 files changed, 3 insertions(+) diff --git a/arbnode/resourcemanager/resource_management.go b/arbnode/resourcemanager/resource_management.go index aba823cc252..249b6894436 100644 --- a/arbnode/resourcemanager/resource_management.go +++ b/arbnode/resourcemanager/resource_management.go @@ -256,6 +256,7 @@ func readIntFromFile(fileName string) (int, error) { if err != nil { return 0, err } + defer file.Close() var limit int if _, err = fmt.Fscanf(file, "%d", &limit); err != nil { @@ -269,6 +270,7 @@ func readFromMemStats(fileName string, re *regexp.Regexp) (int, error) { if err != nil { return 0, err } + defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 26cd698efc8..51d895c0697 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -423,6 +423,7 @@ func extractSnapshot(archive string, location string, importWasm bool) error { if err != nil { return fmt.Errorf("couln't open init '%v' archive: %w", archive, err) } + defer reader.Close() stat, err := reader.Stat() if err != nil { return err From 0bf0b9accf2ca8b6e6ef81d2ac6f9fa67bd6bdaa Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 18:07:15 -0300 Subject: [PATCH 81/95] Uses auth from data poster in GetValidatorWalletContract --- cmd/nitro/nitro.go | 2 +- staker/validatorwallet/contract.go | 4 ++-- system_tests/fast_confirm_test.go | 8 ++++---- system_tests/staker_test.go | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index bad48f430ca..0cf81c45e9d 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -385,7 +385,7 @@ func mainImpl() int { getExtraGas := func() uint64 { return nodeConfig.Node.Staker.ExtraGas } // #nosec G115 - addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1TransactionOptsValidator, l1Reader, true, dataPoster, getExtraGas) + addr, err := validatorwallet.GetValidatorWalletContract(ctx, deployInfo.ValidatorWalletCreator, int64(deployInfo.DeployedAt), l1Reader, true, dataPoster, getExtraGas) if err != nil { log.Crit("error creating validator wallet contract", "error", err, "address", l1TransactionOptsValidator.From.Hex()) } diff --git a/staker/validatorwallet/contract.go b/staker/validatorwallet/contract.go index 83dedcad841..6346029c3a3 100644 --- a/staker/validatorwallet/contract.go +++ b/staker/validatorwallet/contract.go @@ -229,7 +229,7 @@ func (v *Contract) populateWallet(ctx context.Context, createIfMissing bool) err // By passing v.dataPoster as a parameter to GetValidatorWalletContract we force to create a validator wallet through the Staker's DataPoster object. // DataPoster keeps in its internal state information related to the transactions sent through it, which is used to infer the expected nonce in a transaction for example. // If a transaction is sent using the Staker's DataPoster key, but not through the Staker's DataPoster object, DataPoster's internal state will be outdated, which can compromise the expected nonce inference. - addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.auth, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) + addr, err := GetValidatorWalletContract(ctx, v.walletFactoryAddr, v.rollupFromBlock, v.l1Reader, createIfMissing, v.dataPoster, v.getExtraGas) if err != nil { return err } @@ -444,13 +444,13 @@ func GetValidatorWalletContract( ctx context.Context, validatorWalletFactoryAddr common.Address, fromBlock int64, - transactAuth *bind.TransactOpts, l1Reader *headerreader.HeaderReader, createIfMissing bool, dataPoster *dataposter.DataPoster, getExtraGas func() uint64, ) (*common.Address, error) { client := l1Reader.Client() + transactAuth := dataPoster.Auth() // TODO: If we just save a mapping in the wallet creator we won't need log search walletCreator, err := rollupgen.NewValidatorWalletCreator(validatorWalletFactoryAddr, client) diff --git a/system_tests/fast_confirm_test.go b/system_tests/fast_confirm_test.go index 45988274186..dae2699b9f7 100644 --- a/system_tests/fast_confirm_test.go +++ b/system_tests/fast_confirm_test.go @@ -115,10 +115,10 @@ func TestFastConfirmation(t *testing.T) { Require(t, err) valConfig.Strategy = "MakeNodes" - valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) + valWalletAddrPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) valWalletAddr := *valWalletAddrPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, &l1auth, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2node.DeployInfo.ValidatorWalletCreator, 0, l2node.L1Reader, true, valWallet.DataPoster(), valWallet.GetExtraGas()) Require(t, err) if valWalletAddr == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddr.String(), "vs", valWalletAddrCheck.String()) @@ -315,10 +315,10 @@ func TestFastConfirmationWithSafe(t *testing.T) { Require(t, err) valConfigA.Strategy = "MakeNodes" - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) diff --git a/system_tests/staker_test.go b/system_tests/staker_test.go index e61dcedc2b4..67ce2605290 100644 --- a/system_tests/staker_test.go +++ b/system_tests/staker_test.go @@ -176,10 +176,10 @@ func stakerTestImpl(t *testing.T, faultyStaker bool, honestStakerInactive bool) valConfigA.Strategy = "MakeNodes" } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, &l1authA, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, l2nodeA.DeployInfo.ValidatorWalletCreator, 0, l2nodeA.L1Reader, true, valWalletA.DataPoster(), valWalletA.GetExtraGas()) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) @@ -507,10 +507,10 @@ func TestGetValidatorWalletContractWithDataposterOnlyUsedToCreateValidatorWallet } getExtraGas := func() uint64 { return builder.nodeConfig.Staker.ExtraGas } - valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) + valWalletAddrAPtr, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) valWalletAddrA := *valWalletAddrAPtr - valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, &l1auth, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) + valWalletAddrCheck, err := validatorwallet.GetValidatorWalletContract(ctx, builder.L2.ConsensusNode.DeployInfo.ValidatorWalletCreator, 0, builder.L2.ConsensusNode.L1Reader, true, dataPoster, getExtraGas) Require(t, err) if valWalletAddrA == *valWalletAddrCheck { Require(t, err, "didn't cache validator wallet address", valWalletAddrA.String(), "vs", valWalletAddrCheck.String()) From 857073870e455354ba13aa99c6bf06b81ba846ba Mon Sep 17 00:00:00 2001 From: Diego Ximenes Date: Mon, 2 Sep 2024 18:08:05 -0300 Subject: [PATCH 82/95] Uses validator auth instead of batch poster auth when creating data poster to create validator wallet contract --- cmd/nitro/nitro.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 0cf81c45e9d..d4b2a87a304 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -375,7 +375,7 @@ func mainImpl() int { dataPoster, err := arbnode.DataposterOnlyUsedToCreateValidatorWalletContract( ctx, l1Reader, - l1TransactionOptsBatchPoster, + l1TransactionOptsValidator, &nodeConfig.Node.Staker.DataPoster, new(big.Int).SetUint64(nodeConfig.ParentChain.ID), ) From 8e4e8f9dafcb40057ab59516e2dbc7604ef6e39d Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 3 Sep 2024 10:34:17 -0500 Subject: [PATCH 83/95] Use a macro to define native WASM imports --- arbitrator/stylus/src/host.rs | 88 +++++++++++---- arbitrator/stylus/src/native.rs | 186 +++++++++----------------------- 2 files changed, 118 insertions(+), 156 deletions(-) diff --git a/arbitrator/stylus/src/host.rs b/arbitrator/stylus/src/host.rs index 1afc1b4e51e..ca48da35db6 100644 --- a/arbitrator/stylus/src/host.rs +++ b/arbitrator/stylus/src/host.rs @@ -441,26 +441,76 @@ pub(crate) fn pay_for_memory_grow>( hostio!(env, pay_for_memory_grow(pages)) } -pub(crate) fn console_log_text>( - mut env: WasmEnvMut, - ptr: GuestPtr, - len: u32, -) -> MaybeEscape { - hostio!(env, console_log_text(ptr, len)) -} +pub(crate) mod console { + use super::*; -pub(crate) fn console_log, T: Into>( - mut env: WasmEnvMut, - value: T, -) -> MaybeEscape { - hostio!(env, console_log(value)) -} + pub(crate) fn log_txt>( + mut env: WasmEnvMut, + ptr: GuestPtr, + len: u32, + ) -> MaybeEscape { + hostio!(env, console_log_text(ptr, len)) + } -pub(crate) fn console_tee, T: Into + Copy>( - mut env: WasmEnvMut, - value: T, -) -> Result { - hostio!(env, console_tee(value)) + pub(crate) fn log_i32>( + mut env: WasmEnvMut, + value: u32, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_i32>( + mut env: WasmEnvMut, + value: u32, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_i64>( + mut env: WasmEnvMut, + value: u64, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_i64>( + mut env: WasmEnvMut, + value: u64, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_f32>( + mut env: WasmEnvMut, + value: f32, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_f32>( + mut env: WasmEnvMut, + value: f32, + ) -> Result { + hostio!(env, console_tee(value)) + } + + pub(crate) fn log_f64>( + mut env: WasmEnvMut, + value: f64, + ) -> MaybeEscape { + hostio!(env, console_log(value)) + } + + pub(crate) fn tee_f64>( + mut env: WasmEnvMut, + value: f64, + ) -> Result { + hostio!(env, console_tee(value)) + } } -pub(crate) fn null_host>(_: WasmEnvMut) {} +pub(crate) mod debug { + use super::*; + + pub(crate) fn null_host>(_: WasmEnvMut) {} +} diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs index cc1d191fe2b..7a82314fbca 100644 --- a/arbitrator/stylus/src/native.rs +++ b/arbitrator/stylus/src/native.rs @@ -33,7 +33,7 @@ use std::{ ops::{Deref, DerefMut}, }; use wasmer::{ - imports, AsStoreMut, Function, FunctionEnv, Instance, Memory, Module, Pages, Store, Target, + AsStoreMut, Function, FunctionEnv, Imports, Instance, Memory, Module, Pages, Store, Target, TypedFunction, Value, WasmTypeList, }; use wasmer_vm::VMExtern; @@ -151,68 +151,58 @@ impl> NativeInstance { fn from_module(module: Module, mut store: Store, env: WasmEnv) -> Result { let debug_funcs = env.compile.debug.debug_funcs; let func_env = FunctionEnv::new(&mut store, env); + let mut imports = Imports::new(); macro_rules! func { - ($func:expr) => { - Function::new_typed_with_env(&mut store, &func_env, $func) + ($rust_mod:path, $func:ident) => {{ + use $rust_mod as rust_mod; + Function::new_typed_with_env(&mut store, &func_env, rust_mod::$func) + }}; + } + macro_rules! define_imports { + ($($wasm_mod:literal => $rust_mod:path { $( $import:ident ),* $(,)? },)* $(,)?) => { + $( + $( + define_imports!(@@imports $wasm_mod, func!($rust_mod, $import), $import, "arbitrator_forward__"); + )* + )* + }; + (@@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { + define_imports!(@imports $wasm_mod, $func, $import, $($p),*, ""); + }; + (@imports $wasm_mod:literal, $func:expr, $import:ident, $($p:expr),*) => { + $( + imports.define($wasm_mod, concat!($p, stringify!($import)), $func); + )* }; } - let mut imports = imports! { - "vm_hooks" => { - "read_args" => func!(host::read_args), - "write_result" => func!(host::write_result), - "exit_early" => func!(host::exit_early), - "storage_load_bytes32" => func!(host::storage_load_bytes32), - "storage_cache_bytes32" => func!(host::storage_cache_bytes32), - "storage_flush_cache" => func!(host::storage_flush_cache), - "transient_load_bytes32" => func!(host::transient_load_bytes32), - "transient_store_bytes32" => func!(host::transient_store_bytes32), - "call_contract" => func!(host::call_contract), - "delegate_call_contract" => func!(host::delegate_call_contract), - "static_call_contract" => func!(host::static_call_contract), - "create1" => func!(host::create1), - "create2" => func!(host::create2), - "read_return_data" => func!(host::read_return_data), - "return_data_size" => func!(host::return_data_size), - "emit_log" => func!(host::emit_log), - "account_balance" => func!(host::account_balance), - "account_code" => func!(host::account_code), - "account_codehash" => func!(host::account_codehash), - "account_code_size" => func!(host::account_code_size), - "evm_gas_left" => func!(host::evm_gas_left), - "evm_ink_left" => func!(host::evm_ink_left), - "block_basefee" => func!(host::block_basefee), - "chainid" => func!(host::chainid), - "block_coinbase" => func!(host::block_coinbase), - "block_gas_limit" => func!(host::block_gas_limit), - "block_number" => func!(host::block_number), - "block_timestamp" => func!(host::block_timestamp), - "contract_address" => func!(host::contract_address), - "math_div" => func!(host::math_div), - "math_mod" => func!(host::math_mod), - "math_pow" => func!(host::math_pow), - "math_add_mod" => func!(host::math_add_mod), - "math_mul_mod" => func!(host::math_mul_mod), - "msg_reentrant" => func!(host::msg_reentrant), - "msg_sender" => func!(host::msg_sender), - "msg_value" => func!(host::msg_value), - "tx_gas_price" => func!(host::tx_gas_price), - "tx_ink_price" => func!(host::tx_ink_price), - "tx_origin" => func!(host::tx_origin), - "pay_for_memory_grow" => func!(host::pay_for_memory_grow), - "native_keccak256" => func!(host::native_keccak256), + define_imports!( + "vm_hooks" => host { + read_args, write_result, exit_early, + storage_load_bytes32, storage_cache_bytes32, storage_flush_cache, transient_load_bytes32, transient_store_bytes32, + call_contract, delegate_call_contract, static_call_contract, create1, create2, read_return_data, return_data_size, + emit_log, + account_balance, account_code, account_codehash, account_code_size, + evm_gas_left, evm_ink_left, + block_basefee, chainid, block_coinbase, block_gas_limit, block_number, block_timestamp, + contract_address, + math_div, math_mod, math_pow, math_add_mod, math_mul_mod, + msg_reentrant, msg_sender, msg_value, + tx_gas_price, tx_ink_price, tx_origin, + pay_for_memory_grow, + native_keccak256, }, - }; + ); if debug_funcs { - imports.define("console", "log_txt", func!(host::console_log_text)); - imports.define("console", "log_i32", func!(host::console_log::)); - imports.define("console", "log_i64", func!(host::console_log::)); - imports.define("console", "log_f32", func!(host::console_log::)); - imports.define("console", "log_f64", func!(host::console_log::)); - imports.define("console", "tee_i32", func!(host::console_tee::)); - imports.define("console", "tee_i64", func!(host::console_tee::)); - imports.define("console", "tee_f32", func!(host::console_tee::)); - imports.define("console", "tee_f64", func!(host::console_tee::)); - imports.define("debug", "null_host", func!(host::null_host)); + define_imports!( + "console" => host::console { + log_txt, + log_i32, log_i64, log_f32, log_f64, + tee_i32, tee_i64, tee_f32, tee_f64, + }, + "debug" => host::debug { + null_host, + }, + ); } let instance = Instance::new(&mut store, &module, &imports)?; let exports = &instance.exports; @@ -351,86 +341,8 @@ impl> StartlessMachine for NativeInstance { } pub fn module(wasm: &[u8], compile: CompileConfig, target: Target) -> Result> { - let mut store = compile.store(target); + let store = compile.store(target); let module = Module::new(&store, wasm)?; - macro_rules! stub { - (u8 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u8 { panic!("incomplete import") }) - }; - (u32 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u32 { panic!("incomplete import") }) - }; - (u64 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> u64 { panic!("incomplete import") }) - }; - (f32 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> f32 { panic!("incomplete import") }) - }; - (f64 <- $($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> f64 { panic!("incomplete import") }) - }; - ($($types:tt)+) => { - Function::new_typed(&mut store, $($types)+ -> () { panic!("incomplete import") }) - }; - } - let mut imports = imports! { - "vm_hooks" => { - "read_args" => stub!(|_: u32|), - "write_result" => stub!(|_: u32, _: u32|), - "exit_early" => stub!(|_: u32|), - "storage_load_bytes32" => stub!(|_: u32, _: u32|), - "storage_cache_bytes32" => stub!(|_: u32, _: u32|), - "storage_flush_cache" => stub!(|_: u32|), - "transient_load_bytes32" => stub!(|_: u32, _: u32|), - "transient_store_bytes32" => stub!(|_: u32, _: u32|), - "call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u32, _: u64, _: u32|), - "delegate_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), - "static_call_contract" => stub!(u8 <- |_: u32, _: u32, _: u32, _: u64, _: u32|), - "create1" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32|), - "create2" => stub!(|_: u32, _: u32, _: u32, _: u32, _: u32, _: u32|), - "read_return_data" => stub!(u32 <- |_: u32, _: u32, _: u32|), - "return_data_size" => stub!(u32 <- ||), - "emit_log" => stub!(|_: u32, _: u32, _: u32|), - "account_balance" => stub!(|_: u32, _: u32|), - "account_code" => stub!(u32 <- |_: u32, _: u32, _: u32, _: u32|), - "account_codehash" => stub!(|_: u32, _: u32|), - "account_code_size" => stub!(u32 <- |_: u32|), - "evm_gas_left" => stub!(u64 <- ||), - "evm_ink_left" => stub!(u64 <- ||), - "block_basefee" => stub!(|_: u32|), - "chainid" => stub!(u64 <- ||), - "block_coinbase" => stub!(|_: u32|), - "block_gas_limit" => stub!(u64 <- ||), - "block_number" => stub!(u64 <- ||), - "block_timestamp" => stub!(u64 <- ||), - "contract_address" => stub!(|_: u32|), - "math_div" => stub!(|_: u32, _: u32|), - "math_mod" => stub!(|_: u32, _: u32|), - "math_pow" => stub!(|_: u32, _: u32|), - "math_add_mod" => stub!(|_: u32, _: u32, _: u32|), - "math_mul_mod" => stub!(|_: u32, _: u32, _: u32|), - "msg_reentrant" => stub!(u32 <- ||), - "msg_sender" => stub!(|_: u32|), - "msg_value" => stub!(|_: u32|), - "tx_gas_price" => stub!(|_: u32|), - "tx_ink_price" => stub!(u32 <- ||), - "tx_origin" => stub!(|_: u32|), - "pay_for_memory_grow" => stub!(|_: u16|), - "native_keccak256" => stub!(|_: u32, _: u32, _: u32|), - }, - }; - if compile.debug.debug_funcs { - imports.define("console", "log_txt", stub!(|_: u32, _: u32|)); - imports.define("console", "log_i32", stub!(|_: u32|)); - imports.define("console", "log_i64", stub!(|_: u64|)); - imports.define("console", "log_f32", stub!(|_: f32|)); - imports.define("console", "log_f64", stub!(|_: f64|)); - imports.define("console", "tee_i32", stub!(u32 <- |_: u32|)); - imports.define("console", "tee_i64", stub!(u64 <- |_: u64|)); - imports.define("console", "tee_f32", stub!(f32 <- |_: f32|)); - imports.define("console", "tee_f64", stub!(f64 <- |_: f64|)); - imports.define("debug", "null_host", stub!(||)); - } let module = module.serialize()?; Ok(module.to_vec()) From b913d6b53779489593fe4289d4a2335c86ce2b51 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 3 Sep 2024 10:51:48 -0500 Subject: [PATCH 84/95] Remove unused import --- arbitrator/stylus/src/host.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/arbitrator/stylus/src/host.rs b/arbitrator/stylus/src/host.rs index ca48da35db6..fbe5657c5f4 100644 --- a/arbitrator/stylus/src/host.rs +++ b/arbitrator/stylus/src/host.rs @@ -13,7 +13,6 @@ use arbutil::{ }; use caller_env::GuestPtr; use eyre::Result; -use prover::value::Value; use std::{ fmt::Display, mem::{self, MaybeUninit}, From d39f362da5c86964a68992b984a732ad618bf16c Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 4 Sep 2024 14:58:22 -0600 Subject: [PATCH 85/95] redis consumer fix --- validator/valnode/redis/consumer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 74564665336..6432741eba3 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -111,10 +111,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) + requestTokenQueue <- struct{}{} return 0 } if req == nil { - // There's nothing in the queue. + // There's nothing in the queue + requestTokenQueue <- struct{}{} return time.Second } select { From 8d61504e109a89b12112ddbd4be94eb2753b0802 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 4 Sep 2024 15:00:53 -0600 Subject: [PATCH 86/95] redis consumer: dont set result if error --- validator/valnode/redis/consumer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 6432741eba3..2b025600cc0 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -154,9 +154,10 @@ func (s *ValidationServer) Start(ctx_in context.Context) { res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) - } - if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { - log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } else { + if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { + log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } } select { case <-ctx.Done(): From 8f946b3d46c4db46b03389c3502ac4ddcae323e5 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 5 Sep 2024 07:48:00 -0600 Subject: [PATCH 87/95] block_validator: introduce recording-iter-limit --- staker/block_validator.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/staker/block_validator.go b/staker/block_validator.go index e6773a08541..7a7efca8460 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -106,6 +106,7 @@ type BlockValidatorConfig struct { ValidationServerConfigs []rpcclient.ClientConfig `koanf:"validation-server-configs"` ValidationPoll time.Duration `koanf:"validation-poll" reload:"hot"` PrerecordedBlocks uint64 `koanf:"prerecorded-blocks" reload:"hot"` + RecordingIterLimit uint64 `koanf:"recording-iter-limit"` ForwardBlocks uint64 `koanf:"forward-blocks" reload:"hot"` CurrentModuleRoot string `koanf:"current-module-root"` // TODO(magic) requires reinitialization on hot reload PendingUpgradeModuleRoot string `koanf:"pending-upgrade-module-root"` // TODO(magic) requires StatelessBlockValidator recreation on hot reload @@ -174,6 +175,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (small footprint)") f.Uint64(prefix+".prerecorded-blocks", DefaultBlockValidatorConfig.PrerecordedBlocks, "record that many blocks ahead of validation (larger footprint)") f.String(prefix+".current-module-root", DefaultBlockValidatorConfig.CurrentModuleRoot, "current wasm module root ('current' read from chain, 'latest' from machines/latest dir, or provide hash)") + f.Uint64(prefix+".recording-iter-limit", DefaultBlockValidatorConfig.RecordingIterLimit, "limit on block recordings sent per iteration") f.String(prefix+".pending-upgrade-module-root", DefaultBlockValidatorConfig.PendingUpgradeModuleRoot, "pending upgrade wasm module root to additionally validate (hash, 'latest' or empty)") f.Bool(prefix+".failure-is-fatal", DefaultBlockValidatorConfig.FailureIsFatal, "failing a validation is treated as a fatal error") BlockValidatorDangerousConfigAddOptions(prefix+".dangerous", f) @@ -197,6 +199,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ FailureIsFatal: true, Dangerous: DefaultBlockValidatorDangerousConfig, MemoryFreeLimit: "default", + RecordingIterLimit: 20, } var TestBlockValidatorConfig = BlockValidatorConfig{ @@ -207,6 +210,7 @@ var TestBlockValidatorConfig = BlockValidatorConfig{ ValidationPoll: 100 * time.Millisecond, ForwardBlocks: 128, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), + RecordingIterLimit: 20, CurrentModuleRoot: "latest", PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, @@ -652,6 +656,10 @@ func (v *BlockValidator) sendNextRecordRequests(ctx context.Context) (bool, erro if recordUntil < pos { return false, nil } + recordUntilLimit := pos + arbutil.MessageIndex(v.config().RecordingIterLimit) + if recordUntil > recordUntilLimit { + recordUntil = recordUntilLimit + } log.Trace("preparing to record", "pos", pos, "until", recordUntil) // prepare could take a long time so we do it without a lock err := v.recorder.PrepareForRecord(ctx, pos, recordUntil) From 0d5293d24de3c995609c063bdc8743622c6d0762 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 6 Sep 2024 13:43:11 +0530 Subject: [PATCH 88/95] Fix ArbWASM precompile minInitGas() method from reverting --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index f7894d3a6d4..a5c2d54561c 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce +Subproject commit a5c2d54561c2547e05d6118d6f072882c5fc7a9d From 19ef7eaf78ceb07ad6eb5ddf3a7df39b8e920f95 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 6 Sep 2024 09:56:11 -0500 Subject: [PATCH 89/95] Revert "Fix ArbWASM precompile minInitGas() method from reverting" --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index a5c2d54561c..f7894d3a6d4 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit a5c2d54561c2547e05d6118d6f072882c5fc7a9d +Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce From 077213c4593ade8d5afac8eb54db5dd467cd033b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 6 Sep 2024 11:39:09 -0500 Subject: [PATCH 90/95] Bump contracts to latest develop --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index f7894d3a6d4..23fc7962829 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce +Subproject commit 23fc79628292aa5d604d449fed48937ae7faeb2f From bf7b6ac85058feab3e42f9f85edeb5618935b02b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Sun, 8 Sep 2024 11:13:31 -0500 Subject: [PATCH 91/95] Resolve parent chain block by hash when resolving delayed messages --- arbnode/delayed.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arbnode/delayed.go b/arbnode/delayed.go index c166aa2b906..082f0ecf9db 100644 --- a/arbnode/delayed.go +++ b/arbnode/delayed.go @@ -215,7 +215,7 @@ func (b *DelayedBridge) logsToDeliveredMessages(ctx context.Context, logs []type } messages := make([]*DelayedInboxMessage, 0, len(logs)) - var lastParentChainBlockNumber uint64 + var lastParentChainBlockHash common.Hash var lastL1BlockNumber uint64 for _, parsedLog := range parsedLogs { msgKey := common.BigToHash(parsedLog.MessageIndex) @@ -228,17 +228,17 @@ func (b *DelayedBridge) logsToDeliveredMessages(ctx context.Context, logs []type } requestId := common.BigToHash(parsedLog.MessageIndex) - parentChainBlockNumber := parsedLog.Raw.BlockNumber + parentChainBlockHash := parsedLog.Raw.BlockHash var l1BlockNumber uint64 - if lastParentChainBlockNumber == parentChainBlockNumber && lastParentChainBlockNumber > 0 { + if lastParentChainBlockHash == parentChainBlockHash && lastParentChainBlockHash != (common.Hash{}) { l1BlockNumber = lastL1BlockNumber } else { - var err error - l1BlockNumber, err = arbutil.CorrespondingL1BlockNumber(ctx, b.client, parentChainBlockNumber) + parentChainHeader, err := b.client.HeaderByHash(ctx, parentChainBlockHash) if err != nil { return nil, err } - lastParentChainBlockNumber = parentChainBlockNumber + l1BlockNumber = arbutil.ParentHeaderToL1BlockNumber(parentChainHeader) + lastParentChainBlockHash = parentChainBlockHash lastL1BlockNumber = l1BlockNumber } msg := &DelayedInboxMessage{ From 8b9f4ae319d11630af5a5ff53ff770071f26691b Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 9 Sep 2024 12:09:46 -0500 Subject: [PATCH 92/95] Improve block validator test stability --- system_tests/block_validator_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/system_tests/block_validator_test.go b/system_tests/block_validator_test.go index eef6c29b7a3..cca5572f4e7 100644 --- a/system_tests/block_validator_test.go +++ b/system_tests/block_validator_test.go @@ -63,7 +63,6 @@ func testBlockValidatorSimple(t *testing.T, opts Options) { var delayEvery int if opts.workloadLoops > 1 { - l1NodeConfigA.BatchPoster.MaxDelay = time.Millisecond * 500 delayEvery = opts.workloadLoops / 3 } From 01e1c4af5b4dcd3f05c364bccc1f29e409ef25e9 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 9 Sep 2024 16:04:34 -0500 Subject: [PATCH 93/95] Fix new G115 lints in golangci-lint v1.61.0 --- arbnode/batch_poster.go | 13 +++++++--- arbnode/dataposter/data_poster.go | 10 ++++---- arbnode/dataposter/dataposter_test.go | 4 +-- arbnode/dataposter/dbstorage/storage.go | 4 +-- arbnode/dataposter/storage_test.go | 17 +++++++++--- arbnode/delayed_sequencer.go | 2 ++ arbnode/inbox_reader.go | 1 + arbnode/transaction_streamer.go | 17 +++++++----- arbos/activate_test.go | 1 + arbos/addressSet/addressSet.go | 1 + arbos/addressTable/addressTable.go | 1 + arbos/arbosState/initialization_test.go | 1 + arbos/arbosState/initialize.go | 4 +-- arbos/l1pricing_test.go | 2 ++ arbos/merkleAccumulator/merkleAccumulator.go | 1 + arbos/programs/data_pricer.go | 4 +-- arbos/retryable_test.go | 3 +++ arbos/retryables/retryable.go | 4 ++- arbos/storage/storage.go | 9 ++++--- arbutil/block_message_relation.go | 1 + blocks_reexecutor/blocks_reexecutor.go | 1 + broadcastclient/broadcastclient_test.go | 2 ++ broadcaster/broadcaster.go | 1 + broadcaster/message/message.go | 1 + cmd/datool/datool.go | 3 +++ cmd/nitro/init.go | 1 + cmd/nitro/nitro.go | 4 +++ cmd/pruning/pruning.go | 1 + das/aggregator.go | 1 + das/dasRpcClient.go | 1 + das/dasRpcServer.go | 4 +-- das/das_test.go | 2 ++ das/db_storage_service.go | 1 + das/factory.go | 8 +----- das/fallback_storage_service.go | 1 + das/local_file_storage_service.go | 1 + das/local_file_storage_service_test.go | 13 ++++++++++ das/redis_storage_service_test.go | 1 + das/redundant_storage_test.go | 1 + das/restful_server_test.go | 1 + das/rpc_aggregator.go | 2 +- das/s3_storage_service_test.go | 1 + das/simple_das_reader_aggregator_test.go | 4 +++ das/store_signing_test.go | 1 + das/syncing_fallback_storage.go | 1 + execution/gethexec/api.go | 4 +++ execution/gethexec/blockchain.go | 1 + execution/gethexec/sequencer.go | 4 +-- execution/nodeInterface/NodeInterface.go | 3 +++ precompiles/precompile.go | 3 +++ precompiles/precompile_test.go | 2 ++ relay/relay_stress_test.go | 1 + staker/challenge-cache/cache_test.go | 15 ++++++----- staker/challenge_manager.go | 1 + system_tests/bloom_test.go | 6 +++++ system_tests/common_test.go | 2 +- system_tests/estimation_test.go | 2 +- system_tests/nodeinterface_test.go | 2 ++ system_tests/outbox_test.go | 3 +++ system_tests/recreatestate_rpc_test.go | 5 ++++ system_tests/seqfeed_test.go | 4 +-- system_tests/seqinbox_test.go | 3 +++ system_tests/stylus_trace_test.go | 1 + util/arbmath/bips.go | 27 ++++++++++++++------ util/arbmath/math.go | 7 +++-- util/arbmath/math_test.go | 1 + util/headerreader/blob_client.go | 1 + util/merkletree/merkleEventProof_test.go | 1 + validator/server_arb/machine_cache.go | 11 ++++---- validator/server_jit/jit_machine.go | 1 + wavmio/stub.go | 13 +++++----- 71 files changed, 207 insertions(+), 75 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 6b4b95f8e00..44b360e76e1 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -168,7 +168,7 @@ type BatchPosterConfig struct { L1BlockBound string `koanf:"l1-block-bound" reload:"hot"` L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"` UseAccessLists bool `koanf:"use-access-lists" reload:"hot"` - GasEstimateBaseFeeMultipleBips arbmath.Bips `koanf:"gas-estimate-base-fee-multiple-bips"` + GasEstimateBaseFeeMultipleBips arbmath.UBips `koanf:"gas-estimate-base-fee-multiple-bips"` Dangerous BatchPosterDangerousConfig `koanf:"dangerous"` ReorgResistanceMargin time.Duration `koanf:"reorg-resistance-margin" reload:"hot"` CheckBatchCorrectness bool `koanf:"check-batch-correctness"` @@ -253,7 +253,7 @@ var DefaultBatchPosterConfig = BatchPosterConfig{ L1BlockBoundBypass: time.Hour, UseAccessLists: true, RedisLock: redislock.DefaultCfg, - GasEstimateBaseFeeMultipleBips: arbmath.OneInBips * 3 / 2, + GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, ReorgResistanceMargin: 10 * time.Minute, CheckBatchCorrectness: true, } @@ -285,7 +285,7 @@ var TestBatchPosterConfig = BatchPosterConfig{ L1BlockBound: "", L1BlockBoundBypass: time.Hour, UseAccessLists: true, - GasEstimateBaseFeeMultipleBips: arbmath.OneInBips * 3 / 2, + GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, CheckBatchCorrectness: true, } @@ -1035,7 +1035,7 @@ func (b *BatchPoster) estimateGas(ctx context.Context, sequencerMessage []byte, if err != nil { return 0, err } - maxFeePerGas := arbmath.BigMulByBips(latestHeader.BaseFee, config.GasEstimateBaseFeeMultipleBips) + maxFeePerGas := arbmath.BigMulByUBips(latestHeader.BaseFee, config.GasEstimateBaseFeeMultipleBips) if useNormalEstimation { _, realBlobHashes, err := blobs.ComputeCommitmentsAndHashes(realBlobs) if err != nil { @@ -1250,7 +1250,9 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) l1BoundMinTimestamp = arbmath.SaturatingUSub(latestHeader.Time, arbmath.BigToUintSaturating(maxTimeVariationDelaySeconds)) if config.L1BlockBoundBypass > 0 { + // #nosec G115 blockNumberWithPadding := arbmath.SaturatingUAdd(latestBlockNumber, uint64(config.L1BlockBoundBypass/ethPosBlockTime)) + // #nosec G115 timestampWithPadding := arbmath.SaturatingUAdd(latestHeader.Time, uint64(config.L1BlockBoundBypass/time.Second)) l1BoundMinBlockNumberWithBypass = arbmath.SaturatingUSub(blockNumberWithPadding, arbmath.BigToUintSaturating(maxTimeVariationDelayBlocks)) l1BoundMinTimestampWithBypass = arbmath.SaturatingUSub(timestampWithPadding, arbmath.BigToUintSaturating(maxTimeVariationDelaySeconds)) @@ -1316,7 +1318,9 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) if hasL1Bound && config.ReorgResistanceMargin > 0 { firstMsgBlockNumber := firstMsg.Message.Header.BlockNumber firstMsgTimeStamp := firstMsg.Message.Header.Timestamp + // #nosec G115 batchNearL1BoundMinBlockNumber := firstMsgBlockNumber <= arbmath.SaturatingUAdd(l1BoundMinBlockNumber, uint64(config.ReorgResistanceMargin/ethPosBlockTime)) + // #nosec G115 batchNearL1BoundMinTimestamp := firstMsgTimeStamp <= arbmath.SaturatingUAdd(l1BoundMinTimestamp, uint64(config.ReorgResistanceMargin/time.Second)) if batchNearL1BoundMinTimestamp || batchNearL1BoundMinBlockNumber { log.Error( @@ -1361,6 +1365,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) batchPosterDAFailureCounter.Inc(1) return false, fmt.Errorf("%w: nonce changed from %d to %d while creating batch", storage.ErrStorageRace, nonce, gotNonce) } + // #nosec G115 sequencerMsg, err = b.dapWriter.Store(ctx, sequencerMsg, uint64(time.Now().Add(config.DASRetentionPeriod).Unix()), config.DisableDapFallbackStoreDataOnChain) if err != nil { batchPosterDAFailureCounter.Inc(1) diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index 6a483929b26..31b9a983dbf 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -638,11 +638,11 @@ func (p *DataPoster) feeAndTipCaps(ctx context.Context, nonce uint64, gasLimit u if config.MaxFeeBidMultipleBips > 0 { // Limit the fee caps to be no greater than max(MaxFeeBidMultipleBips, minRbf) - maxNonBlobFee := arbmath.BigMulByBips(currentNonBlobFee, config.MaxFeeBidMultipleBips) + maxNonBlobFee := arbmath.BigMulByUBips(currentNonBlobFee, config.MaxFeeBidMultipleBips) if lastTx != nil { maxNonBlobFee = arbmath.BigMax(maxNonBlobFee, arbmath.BigMulByBips(lastTx.GasFeeCap(), minRbfIncrease)) } - maxBlobFee := arbmath.BigMulByBips(currentBlobFee, config.MaxFeeBidMultipleBips) + maxBlobFee := arbmath.BigMulByUBips(currentBlobFee, config.MaxFeeBidMultipleBips) if lastTx != nil && lastTx.BlobGasFeeCap() != nil { maxBlobFee = arbmath.BigMax(maxBlobFee, arbmath.BigMulByBips(lastTx.BlobGasFeeCap(), minRbfIncrease)) } @@ -1241,7 +1241,7 @@ type DataPosterConfig struct { MinBlobTxTipCapGwei float64 `koanf:"min-blob-tx-tip-cap-gwei" reload:"hot"` MaxTipCapGwei float64 `koanf:"max-tip-cap-gwei" reload:"hot"` MaxBlobTxTipCapGwei float64 `koanf:"max-blob-tx-tip-cap-gwei" reload:"hot"` - MaxFeeBidMultipleBips arbmath.Bips `koanf:"max-fee-bid-multiple-bips" reload:"hot"` + MaxFeeBidMultipleBips arbmath.UBips `koanf:"max-fee-bid-multiple-bips" reload:"hot"` NonceRbfSoftConfs uint64 `koanf:"nonce-rbf-soft-confs" reload:"hot"` AllocateMempoolBalance bool `koanf:"allocate-mempool-balance" reload:"hot"` UseDBStorage bool `koanf:"use-db-storage"` @@ -1345,7 +1345,7 @@ var DefaultDataPosterConfig = DataPosterConfig{ MinBlobTxTipCapGwei: 1, // default geth minimum, and relays aren't likely to accept lower values given propagation time MaxTipCapGwei: 1.2, MaxBlobTxTipCapGwei: 1, // lower than normal because 4844 rbf is a minimum of a 2x - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, NonceRbfSoftConfs: 1, AllocateMempoolBalance: true, UseDBStorage: true, @@ -1380,7 +1380,7 @@ var TestDataPosterConfig = DataPosterConfig{ MinBlobTxTipCapGwei: 1, MaxTipCapGwei: 5, MaxBlobTxTipCapGwei: 1, - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, NonceRbfSoftConfs: 1, AllocateMempoolBalance: true, UseDBStorage: false, diff --git a/arbnode/dataposter/dataposter_test.go b/arbnode/dataposter/dataposter_test.go index 7f2f61c07e3..d2c49427bef 100644 --- a/arbnode/dataposter/dataposter_test.go +++ b/arbnode/dataposter/dataposter_test.go @@ -204,7 +204,7 @@ func TestFeeAndTipCaps_EnoughBalance_NoBacklog_NoUnconfirmed_BlobTx(t *testing.T MinBlobTxTipCapGwei: 1, MaxTipCapGwei: 5, MaxBlobTxTipCapGwei: 10, - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, AllocateMempoolBalance: true, UrgencyGwei: 2., @@ -335,7 +335,7 @@ func TestFeeAndTipCaps_RBF_RisingBlobFee_FallingBaseFee(t *testing.T) { MinBlobTxTipCapGwei: 1, MaxTipCapGwei: 5, MaxBlobTxTipCapGwei: 10, - MaxFeeBidMultipleBips: arbmath.OneInBips * 10, + MaxFeeBidMultipleBips: arbmath.OneInUBips * 10, AllocateMempoolBalance: true, UrgencyGwei: 2., diff --git a/arbnode/dataposter/dbstorage/storage.go b/arbnode/dataposter/dbstorage/storage.go index 37ebfa50991..6a6cd3cfa43 100644 --- a/arbnode/dataposter/dbstorage/storage.go +++ b/arbnode/dataposter/dbstorage/storage.go @@ -95,11 +95,11 @@ func (s *Storage) PruneAll(ctx context.Context) error { if err != nil { return fmt.Errorf("pruning all keys: %w", err) } - until, err := strconv.Atoi(string(idx)) + until, err := strconv.ParseUint(string(idx), 10, 64) if err != nil { return fmt.Errorf("converting last item index bytes to integer: %w", err) } - return s.Prune(ctx, uint64(until+1)) + return s.Prune(ctx, until+1) } func (s *Storage) Prune(ctx context.Context, until uint64) error { diff --git a/arbnode/dataposter/storage_test.go b/arbnode/dataposter/storage_test.go index 8934d92b456..c6316caea7e 100644 --- a/arbnode/dataposter/storage_test.go +++ b/arbnode/dataposter/storage_test.go @@ -72,24 +72,29 @@ func newRedisStorage(ctx context.Context, t *testing.T, encF storage.EncoderDeco func valueOf(t *testing.T, i int) *storage.QueuedTransaction { t.Helper() + // #nosec G115 meta, err := rlp.EncodeToBytes(storage.BatchPosterPosition{DelayedMessageCount: uint64(i)}) if err != nil { t.Fatalf("Encoding batch poster position, error: %v", err) } return &storage.QueuedTransaction{ FullTx: types.NewTransaction( + // #nosec G115 uint64(i), common.Address{}, big.NewInt(int64(i)), + // #nosec G115 uint64(i), big.NewInt(int64(i)), []byte{byte(i)}), Meta: meta, DeprecatedData: types.DynamicFeeTx{ - ChainID: big.NewInt(int64(i)), - Nonce: uint64(i), - GasTipCap: big.NewInt(int64(i)), - GasFeeCap: big.NewInt(int64(i)), + ChainID: big.NewInt(int64(i)), + // #nosec G115 + Nonce: uint64(i), + GasTipCap: big.NewInt(int64(i)), + GasFeeCap: big.NewInt(int64(i)), + // #nosec G115 Gas: uint64(i), Value: big.NewInt(int64(i)), Data: []byte{byte(i % 8)}, @@ -113,6 +118,7 @@ func values(t *testing.T, from, to int) []*storage.QueuedTransaction { func initStorage(ctx context.Context, t *testing.T, s QueueStorage) QueueStorage { t.Helper() for i := 0; i < 20; i++ { + // #nosec G115 if err := s.Put(ctx, uint64(i), nil, valueOf(t, i)); err != nil { t.Fatalf("Error putting a key/value: %v", err) } @@ -153,6 +159,7 @@ func TestPruneAll(t *testing.T) { s := newLevelDBStorage(t, func() storage.EncoderDecoderInterface { return &storage.EncoderDecoder{} }) ctx := context.Background() for i := 0; i < 20; i++ { + // #nosec G115 if err := s.Put(ctx, uint64(i), nil, valueOf(t, i)); err != nil { t.Fatalf("Error putting a key/value: %v", err) } @@ -236,6 +243,7 @@ func TestLast(t *testing.T) { ctx := context.Background() for i := 0; i < cnt; i++ { val := valueOf(t, i) + // #nosec G115 if err := s.Put(ctx, uint64(i), nil, val); err != nil { t.Fatalf("Error putting a key/value: %v", err) } @@ -255,6 +263,7 @@ func TestLast(t *testing.T) { for i := 0; i < cnt-1; i++ { prev := valueOf(t, i) newVal := valueOf(t, cnt+i) + // #nosec G115 if err := s.Put(ctx, uint64(i), prev, newVal); err != nil { t.Fatalf("Error putting a key/value: %v, prev: %v, new: %v", err, prev, newVal) } diff --git a/arbnode/delayed_sequencer.go b/arbnode/delayed_sequencer.go index 4f18531a764..b29a66dd054 100644 --- a/arbnode/delayed_sequencer.go +++ b/arbnode/delayed_sequencer.go @@ -121,6 +121,7 @@ func (d *DelayedSequencer) sequenceWithoutLockout(ctx context.Context, lastBlock if currentNum < config.FinalizeDistance { return nil } + // #nosec G115 finalized = uint64(currentNum - config.FinalizeDistance) } @@ -189,6 +190,7 @@ func (d *DelayedSequencer) sequenceWithoutLockout(ctx context.Context, lastBlock return fmt.Errorf("inbox reader at delayed message %v db accumulator %v doesn't match delayed bridge accumulator %v at L1 block %v", pos-1, lastDelayedAcc, delayedBridgeAcc, finalized) } for i, msg := range messages { + // #nosec G115 err = d.exec.SequenceDelayedMessage(msg, startPos+uint64(i)) if err != nil { return err diff --git a/arbnode/inbox_reader.go b/arbnode/inbox_reader.go index fd050b5f67b..c596cfa9b0d 100644 --- a/arbnode/inbox_reader.go +++ b/arbnode/inbox_reader.go @@ -542,6 +542,7 @@ func (r *InboxReader) run(ctx context.Context, hadError bool) error { } else { from = arbmath.BigAddByUint(to, 1) } + // #nosec G115 haveMessages := uint64(len(delayedMessages) + len(sequencerBatches)) if haveMessages <= (config.TargetMessagesRead / 2) { blocksToFetch += (blocksToFetch + 4) / 5 diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index a5bab8342f9..24a0564b976 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -279,6 +279,7 @@ func (s *TransactionStreamer) reorg(batch ethdb.Batch, count arbutil.MessageInde return err } config := s.config() + // #nosec G115 maxResequenceMsgCount := count + arbutil.MessageIndex(config.MaxReorgResequenceDepth) if config.MaxReorgResequenceDepth >= 0 && maxResequenceMsgCount < targetMsgCount { log.Error( @@ -388,6 +389,7 @@ func (s *TransactionStreamer) reorg(batch ethdb.Batch, count arbutil.MessageInde } for i := 0; i < len(messagesResults); i++ { + // #nosec G115 pos := count + arbutil.MessageIndex(i) err = s.storeResult(pos, *messagesResults[i], batch) if err != nil { @@ -680,7 +682,7 @@ func (s *TransactionStreamer) AddMessagesAndEndBatch(pos arbutil.MessageIndex, m if err != nil { return err } - if dups == len(messages) { + if dups == uint64(len(messages)) { return endBatch(batch) } // cant keep reorg lock when catching insertionMutex. @@ -715,10 +717,10 @@ func (s *TransactionStreamer) countDuplicateMessages( pos arbutil.MessageIndex, messages []arbostypes.MessageWithMetadataAndBlockHash, batch *ethdb.Batch, -) (int, bool, *arbostypes.MessageWithMetadata, error) { - curMsg := 0 +) (uint64, bool, *arbostypes.MessageWithMetadata, error) { + var curMsg uint64 for { - if len(messages) == curMsg { + if uint64(len(messages)) == curMsg { break } key := dbKey(messagePrefix, uint64(pos)) @@ -818,7 +820,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil broadcastStartPos := arbutil.MessageIndex(s.broadcasterQueuedMessagesPos.Load()) if messagesAreConfirmed { - var duplicates int + var duplicates uint64 var err error duplicates, confirmedReorg, oldMsg, err = s.countDuplicateMessages(messageStartPos, messages, &batch) if err != nil { @@ -857,7 +859,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil var feedReorg bool if !hasNewConfirmedMessages { - var duplicates int + var duplicates uint64 var err error duplicates, feedReorg, oldMsg, err = s.countDuplicateMessages(messageStartPos, messages, nil) if err != nil { @@ -889,6 +891,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil // Validate delayed message counts of remaining messages for i, msg := range messages { + // #nosec G115 msgPos := messageStartPos + arbutil.MessageIndex(i) diff := msg.MessageWithMeta.DelayedMessagesRead - lastDelayedRead if diff != 0 && diff != 1 { @@ -924,6 +927,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil // Check if new messages were added at the end of cache, if they were, then dont remove those particular messages if len(s.broadcasterQueuedMessages) > cacheClearLen { s.broadcasterQueuedMessages = s.broadcasterQueuedMessages[cacheClearLen:] + // #nosec G115 s.broadcasterQueuedMessagesPos.Store(uint64(broadcastStartPos) + uint64(cacheClearLen)) } else { s.broadcasterQueuedMessages = s.broadcasterQueuedMessages[:0] @@ -1044,6 +1048,7 @@ func (s *TransactionStreamer) writeMessages(pos arbutil.MessageIndex, messages [ batch = s.db.NewBatch() } for i, msg := range messages { + // #nosec G115 err := s.writeMessage(pos+arbutil.MessageIndex(i), msg, batch) if err != nil { return err diff --git a/arbos/activate_test.go b/arbos/activate_test.go index 55440bb208b..a89a38639a9 100644 --- a/arbos/activate_test.go +++ b/arbos/activate_test.go @@ -20,6 +20,7 @@ func TestActivationDataFee(t *testing.T) { rand.Seed(time.Now().UTC().UnixNano()) state, _ := arbosState.NewArbosMemoryBackedArbOSState() pricer := state.Programs().DataPricer() + // #nosec G115 time := uint64(time.Now().Unix()) assert := func(cond bool) { diff --git a/arbos/addressSet/addressSet.go b/arbos/addressSet/addressSet.go index 1f09ff1440f..156f36e7e77 100644 --- a/arbos/addressSet/addressSet.go +++ b/arbos/addressSet/addressSet.go @@ -79,6 +79,7 @@ func (as *AddressSet) AllMembers(maxNumToReturn uint64) ([]common.Address, error } ret := make([]common.Address, size) for i := range ret { + // #nosec G115 sba := as.backingStorage.OpenStorageBackedAddress(uint64(i + 1)) ret[i], err = sba.Get() if err != nil { diff --git a/arbos/addressTable/addressTable.go b/arbos/addressTable/addressTable.go index 566c71b6895..6ae271060da 100644 --- a/arbos/addressTable/addressTable.go +++ b/arbos/addressTable/addressTable.go @@ -103,6 +103,7 @@ func (atab *AddressTable) Decompress(buf []byte) (common.Address, uint64, error) return common.Address{}, 0, err } if len(input) == 20 { + // #nosec G115 numBytesRead := uint64(rd.Size() - int64(rd.Len())) return common.BytesToAddress(input), numBytesRead, nil } else { diff --git a/arbos/arbosState/initialization_test.go b/arbos/arbosState/initialization_test.go index b0fe1d0dac4..5e605b8bd26 100644 --- a/arbos/arbosState/initialization_test.go +++ b/arbos/arbosState/initialization_test.go @@ -126,6 +126,7 @@ func checkAddressTable(arbState *ArbosState, addrTable []common.Address, t *test Fail(t) } for i, addr := range addrTable { + // #nosec G115 res, exists, err := atab.LookupIndex(uint64(i)) Require(t, err) if !exists { diff --git a/arbos/arbosState/initialize.go b/arbos/arbosState/initialize.go index 56fa579c153..427bdc30877 100644 --- a/arbos/arbosState/initialize.go +++ b/arbos/arbosState/initialize.go @@ -108,7 +108,7 @@ func InitializeArbosInDatabase(db ethdb.Database, cacheConfig *core.CacheConfig, if err != nil { return common.Hash{}, err } - for i := 0; addressReader.More(); i++ { + for i := uint64(0); addressReader.More(); i++ { addr, err := addressReader.GetNext() if err != nil { return common.Hash{}, err @@ -117,7 +117,7 @@ func InitializeArbosInDatabase(db ethdb.Database, cacheConfig *core.CacheConfig, if err != nil { return common.Hash{}, err } - if uint64(i) != slot { + if i != slot { return common.Hash{}, errors.New("address table slot mismatch") } } diff --git a/arbos/l1pricing_test.go b/arbos/l1pricing_test.go index 1cda4b3d82b..6f9e3ecb356 100644 --- a/arbos/l1pricing_test.go +++ b/arbos/l1pricing_test.go @@ -279,7 +279,9 @@ func _testL1PriceEquilibration(t *testing.T, initialL1BasefeeEstimate *big.Int, evm.StateDB, evm, 3, + // #nosec G115 uint64(10*(i+1)), + // #nosec G115 uint64(10*(i+1)+5), bpAddr, arbmath.BigMulByUint(equilibriumL1BasefeeEstimate, unitsToAdd), diff --git a/arbos/merkleAccumulator/merkleAccumulator.go b/arbos/merkleAccumulator/merkleAccumulator.go index 2e060c58400..e62303e5fd7 100644 --- a/arbos/merkleAccumulator/merkleAccumulator.go +++ b/arbos/merkleAccumulator/merkleAccumulator.go @@ -97,6 +97,7 @@ func (acc *MerkleAccumulator) GetPartials() ([]*common.Hash, error) { } partials := make([]*common.Hash, CalcNumPartials(size)) for i := range partials { + // #nosec G115 p, err := acc.getPartial(uint64(i)) if err != nil { return nil, err diff --git a/arbos/programs/data_pricer.go b/arbos/programs/data_pricer.go index ed7c98556db..d82aa81f045 100644 --- a/arbos/programs/data_pricer.go +++ b/arbos/programs/data_pricer.go @@ -83,8 +83,8 @@ func (p *DataPricer) UpdateModel(tempBytes uint32, time uint64) (*big.Int, error } exponent := arbmath.OneInBips * arbmath.Bips(demand) / arbmath.Bips(inertia) - multiplier := arbmath.ApproxExpBasisPoints(exponent, 12).Uint64() - costPerByte := arbmath.SaturatingUMul(uint64(minPrice), multiplier) / 10000 + multiplier := arbmath.ApproxExpBasisPoints(exponent, 12) + costPerByte := arbmath.UintSaturatingMulByBips(uint64(minPrice), multiplier) costInWei := arbmath.SaturatingUMul(costPerByte, uint64(tempBytes)) return arbmath.UintToBig(costInWei), nil } diff --git a/arbos/retryable_test.go b/arbos/retryable_test.go index ddb88348ddf..2eccaea6c2a 100644 --- a/arbos/retryable_test.go +++ b/arbos/retryable_test.go @@ -38,6 +38,7 @@ func TestRetryableLifecycle(t *testing.T) { retryableState := state.RetryableState() lifetime := uint64(retryables.RetryableLifetimeSeconds) + // #nosec G115 timestampAtCreation := uint64(rand.Int63n(1 << 16)) timeoutAtCreation := timestampAtCreation + lifetime currentTime := timeoutAtCreation @@ -57,6 +58,7 @@ func TestRetryableLifecycle(t *testing.T) { checkQueueSize := func(expected int, message string) { timeoutQueueSize, err := retryableState.TimeoutQueue.Size() Require(t, err) + // #nosec G115 if timeoutQueueSize != uint64(expected) { Fail(t, currentTime, message, timeoutQueueSize) } @@ -167,6 +169,7 @@ func TestRetryableCleanup(t *testing.T) { callvalue := big.NewInt(0) calldata := testhelpers.RandomizeSlice(make([]byte, rand.Intn(1<<12))) + // #nosec G115 timeout := uint64(rand.Int63n(1 << 16)) timestamp := 2 * timeout diff --git a/arbos/retryables/retryable.go b/arbos/retryables/retryable.go index e1cfe48bcf3..5938244782a 100644 --- a/arbos/retryables/retryable.go +++ b/arbos/retryables/retryable.go @@ -367,5 +367,7 @@ func RetryableEscrowAddress(ticketId common.Hash) common.Address { } func RetryableSubmissionFee(calldataLengthInBytes int, l1BaseFee *big.Int) *big.Int { - return arbmath.BigMulByUint(l1BaseFee, uint64(1400+6*calldataLengthInBytes)) + // This can't overflow because calldataLengthInBytes would need to be 3 exabytes + // #nosec G115 + return arbmath.BigMulByUint(l1BaseFee, 1400+6*uint64(calldataLengthInBytes)) } diff --git a/arbos/storage/storage.go b/arbos/storage/storage.go index 352726778d0..bc16491af03 100644 --- a/arbos/storage/storage.go +++ b/arbos/storage/storage.go @@ -322,11 +322,11 @@ func (s *Storage) Burner() burn.Burner { } func (s *Storage) Keccak(data ...[]byte) ([]byte, error) { - byteCount := 0 + var byteCount uint64 for _, part := range data { - byteCount += len(part) + byteCount += uint64(len(part)) } - cost := 30 + 6*arbmath.WordsForBytes(uint64(byteCount)) + cost := 30 + 6*arbmath.WordsForBytes(byteCount) if err := s.burner.Burn(cost); err != nil { return nil, err } @@ -420,6 +420,7 @@ func (sbu *StorageBackedInt64) Get() (int64, error) { } func (sbu *StorageBackedInt64) Set(value int64) error { + // #nosec G115 return sbu.StorageSlot.Set(util.UintToHash(uint64(value))) // see implementation note above } @@ -456,7 +457,7 @@ func (sbu *StorageBackedUBips) Get() (arbmath.UBips, error) { } func (sbu *StorageBackedUBips) Set(bips arbmath.UBips) error { - return sbu.backing.Set(bips.Uint64()) + return sbu.backing.Set(uint64(bips)) } type StorageBackedUint16 struct { diff --git a/arbutil/block_message_relation.go b/arbutil/block_message_relation.go index e164cf2619a..dcf4c86084e 100644 --- a/arbutil/block_message_relation.go +++ b/arbutil/block_message_relation.go @@ -11,6 +11,7 @@ func BlockNumberToMessageCount(blockNumber uint64, genesisBlockNumber uint64) Me // Block number must correspond to a message count, meaning it may not be less than -1 func SignedBlockNumberToMessageCount(blockNumber int64, genesisBlockNumber uint64) MessageIndex { + // #nosec G115 return MessageIndex(uint64(blockNumber+1) - genesisBlockNumber) } diff --git a/blocks_reexecutor/blocks_reexecutor.go b/blocks_reexecutor/blocks_reexecutor.go index f7cc0d8c720..b43999a7dbc 100644 --- a/blocks_reexecutor/blocks_reexecutor.go +++ b/blocks_reexecutor/blocks_reexecutor.go @@ -113,6 +113,7 @@ func New(c *Config, blockchain *core.BlockChain, fatalErrChan chan error) *Block } // Divide work equally among available threads when BlocksPerThread is zero if c.BlocksPerThread == 0 { + // #nosec G115 work := (end - start) / uint64(c.Room) if work > 0 { blocksPerThread = work diff --git a/broadcastclient/broadcastclient_test.go b/broadcastclient/broadcastclient_test.go index 44b48192abb..a499628cd5c 100644 --- a/broadcastclient/broadcastclient_test.go +++ b/broadcastclient/broadcastclient_test.go @@ -105,6 +105,7 @@ func testReceiveMessages(t *testing.T, clientCompression bool, serverCompression go func() { for i := 0; i < messageCount; i++ { + // #nosec G115 Require(t, b.BroadcastSingle(arbostypes.TestMessageWithMetadataAndRequestId, arbutil.MessageIndex(i), nil)) } }() @@ -156,6 +157,7 @@ func TestInvalidSignature(t *testing.T) { go func() { for i := 0; i < messageCount; i++ { + // #nosec G115 Require(t, b.BroadcastSingle(arbostypes.TestMessageWithMetadataAndRequestId, arbutil.MessageIndex(i), nil)) } }() diff --git a/broadcaster/broadcaster.go b/broadcaster/broadcaster.go index 397698635af..4fe8657bfa5 100644 --- a/broadcaster/broadcaster.go +++ b/broadcaster/broadcaster.go @@ -104,6 +104,7 @@ func (b *Broadcaster) BroadcastMessages( }() var feedMessages []*m.BroadcastFeedMessage for i, msg := range messagesWithBlockHash { + // #nosec G115 bfm, err := b.NewBroadcastFeedMessage(msg.MessageWithMeta, seq+arbutil.MessageIndex(i), msg.BlockHash) if err != nil { return err diff --git a/broadcaster/message/message.go b/broadcaster/message/message.go index aca95987540..1e26e6da5e6 100644 --- a/broadcaster/message/message.go +++ b/broadcaster/message/message.go @@ -41,6 +41,7 @@ type BroadcastFeedMessage struct { } func (m *BroadcastFeedMessage) Size() uint64 { + // #nosec G115 return uint64(len(m.Signature) + len(m.Message.Message.L2msg) + 160) } diff --git a/cmd/datool/datool.go b/cmd/datool/datool.go index ba60cbbd4d9..f791d8cbc4e 100644 --- a/cmd/datool/datool.go +++ b/cmd/datool/datool.go @@ -166,8 +166,10 @@ func startClientStore(args []string) error { if err != nil { return err } + // #nosec G115 cert, err = client.Store(ctx, message, uint64(time.Now().Add(config.DASRetentionPeriod).Unix())) } else if len(config.Message) > 0 { + // #nosec G115 cert, err = client.Store(ctx, []byte(config.Message), uint64(time.Now().Add(config.DASRetentionPeriod).Unix())) } else { return errors.New("--message or --random-message-size must be specified") @@ -363,6 +365,7 @@ func dumpKeyset(args []string) error { return err } + // #nosec G115 keysetHash, keysetBytes, err := das.KeysetHashFromServices(services, uint64(config.Keyset.AssumedHonest)) if err != nil { return err diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index a8463a7d210..fc59f2d231a 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -876,6 +876,7 @@ func testUpdateTxIndex(chainDb ethdb.Database, chainConfig *params.ChainConfig, localWg.Add(1) go func() { batch := chainDb.NewBatch() + // #nosec G115 for blockNum := uint64(thread); blockNum <= lastBlock; blockNum += uint64(threads) { blockHash := rawdb.ReadCanonicalHash(chainDb, blockNum) block := rawdb.ReadBlock(chainDb, blockHash, blockNum) diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 146a0049e72..e66d99b56e7 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -892,6 +892,7 @@ func ParseNode(ctx context.Context, args []string) (*NodeConfig, *genericconf.Wa l2ChainInfoIpfsDownloadPath := k.String("chain.info-ipfs-download-path") l2ChainInfoFiles := k.Strings("chain.info-files") l2ChainInfoJson := k.String("chain.info-json") + // #nosec G115 err = applyChainParameters(ctx, k, uint64(l2ChainId), l2ChainName, l2ChainInfoFiles, l2ChainInfoJson, l2ChainInfoIpfsUrl, l2ChainInfoIpfsDownloadPath) if err != nil { return nil, nil, err @@ -1037,13 +1038,16 @@ func applyChainParameters(ctx context.Context, k *koanf.Koanf, chainId uint64, c func initReorg(initConfig conf.InitConfig, chainConfig *params.ChainConfig, inboxTracker *arbnode.InboxTracker) error { var batchCount uint64 if initConfig.ReorgToBatch >= 0 { + // #nosec G115 batchCount = uint64(initConfig.ReorgToBatch) + 1 } else { var messageIndex arbutil.MessageIndex if initConfig.ReorgToMessageBatch >= 0 { + // #nosec G115 messageIndex = arbutil.MessageIndex(initConfig.ReorgToMessageBatch) } else if initConfig.ReorgToBlockBatch > 0 { genesis := chainConfig.ArbitrumChainParams.GenesisBlockNum + // #nosec G115 blockNum := uint64(initConfig.ReorgToBlockBatch) if blockNum < genesis { return fmt.Errorf("ReorgToBlockBatch %d before genesis %d", blockNum, genesis) diff --git a/cmd/pruning/pruning.go b/cmd/pruning/pruning.go index 096bb4b1aec..6fc7741478c 100644 --- a/cmd/pruning/pruning.go +++ b/cmd/pruning/pruning.go @@ -212,6 +212,7 @@ func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node } if meta.ParentChainBlock <= l1BlockNum { signedBlockNum := arbutil.MessageCountToBlockNumber(meta.MessageCount, genesisNum) + // #nosec G115 blockNum := uint64(signedBlockNum) l2Hash := rawdb.ReadCanonicalHash(chainDb, blockNum) l2Header := rawdb.ReadHeader(chainDb, l2Hash, blockNum) diff --git a/das/aggregator.go b/das/aggregator.go index e8972447ad6..e7460fa371a 100644 --- a/das/aggregator.go +++ b/das/aggregator.go @@ -130,6 +130,7 @@ func NewAggregatorWithSeqInboxCaller( seqInboxCaller *bridgegen.SequencerInboxCaller, ) (*Aggregator, error) { + // #nosec G115 keysetHash, keysetBytes, err := KeysetHashFromServices(services, uint64(config.RPCAggregator.AssumedHonest)) if err != nil { return nil, err diff --git a/das/dasRpcClient.go b/das/dasRpcClient.go index 635696bdab2..7d48ed796d8 100644 --- a/das/dasRpcClient.go +++ b/das/dasRpcClient.go @@ -58,6 +58,7 @@ func NewDASRPCClient(target string, signer signature.DataSignerFunc, maxStoreChu } func (c *DASRPCClient) Store(ctx context.Context, message []byte, timeout uint64) (*daprovider.DataAvailabilityCertificate, error) { + // #nosec G115 timestamp := uint64(time.Now().Unix()) nChunks := uint64(len(message)) / c.chunkSize lastChunkSize := uint64(len(message)) % c.chunkSize diff --git a/das/dasRpcServer.go b/das/dasRpcServer.go index d14766cc7e9..bb1be0384ed 100644 --- a/das/dasRpcServer.go +++ b/das/dasRpcServer.go @@ -153,7 +153,7 @@ type SendChunkResult struct { type batch struct { chunks [][]byte expectedChunks uint64 - seenChunks atomic.Int64 + seenChunks atomic.Uint64 expectedChunkSize, expectedSize uint64 timeout uint64 startTime time.Time @@ -248,7 +248,7 @@ func (b *batchBuilder) close(id uint64) ([]byte, uint64, time.Time, error) { return nil, 0, time.Time{}, fmt.Errorf("unknown batch(%d)", id) } - if batch.expectedChunks != uint64(batch.seenChunks.Load()) { + if batch.expectedChunks != batch.seenChunks.Load() { return nil, 0, time.Time{}, fmt.Errorf("incomplete batch(%d): got %d/%d chunks", id, batch.seenChunks.Load(), batch.expectedChunks) } diff --git a/das/das_test.go b/das/das_test.go index 179734c8b11..4971d454e54 100644 --- a/das/das_test.go +++ b/das/das_test.go @@ -55,6 +55,7 @@ func testDASStoreRetrieveMultipleInstances(t *testing.T, storageType string) { Require(t, err, "no das") var daReader DataAvailabilityServiceReader = storageService + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour * 24).Unix()) messageSaved := []byte("hello world") cert, err := daWriter.Store(firstCtx, messageSaved, timeout) @@ -146,6 +147,7 @@ func testDASMissingMessage(t *testing.T, storageType string) { var daReader DataAvailabilityServiceReader = storageService messageSaved := []byte("hello world") + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour * 24).Unix()) cert, err := daWriter.Store(ctx, messageSaved, timeout) Require(t, err, "Error storing message") diff --git a/das/db_storage_service.go b/das/db_storage_service.go index 1d9e5348d46..74bf12b927e 100644 --- a/das/db_storage_service.go +++ b/das/db_storage_service.go @@ -267,6 +267,7 @@ func (dbs *DBStorageService) String() string { func (dbs *DBStorageService) HealthCheck(ctx context.Context) error { testData := []byte("Test-Data") + // #nosec G115 err := dbs.Put(ctx, testData, uint64(time.Now().Add(time.Minute).Unix())) if err != nil { return err diff --git a/das/factory.go b/das/factory.go index 5742a394794..7f696912b3c 100644 --- a/das/factory.go +++ b/das/factory.go @@ -7,7 +7,6 @@ import ( "context" "errors" "fmt" - "math" "github.com/ethereum/go-ethereum/common" @@ -187,12 +186,7 @@ func CreateDAComponentsForDaserver( dasLifecycleManager.Register(restAgg) syncConf := &config.RestAggregator.SyncToStorage - var retentionPeriodSeconds uint64 - if uint64(syncConf.RetentionPeriod) == math.MaxUint64 { - retentionPeriodSeconds = math.MaxUint64 - } else { - retentionPeriodSeconds = uint64(syncConf.RetentionPeriod.Seconds()) - } + retentionPeriodSeconds := uint64(syncConf.RetentionPeriod.Seconds()) if syncConf.Eager { if l1Reader == nil || seqInboxAddress == nil { diff --git a/das/fallback_storage_service.go b/das/fallback_storage_service.go index 49f961da60d..0a451678d08 100644 --- a/das/fallback_storage_service.go +++ b/das/fallback_storage_service.go @@ -85,6 +85,7 @@ func (f *FallbackStorageService) GetByHash(ctx context.Context, key common.Hash) } if dastree.ValidHash(key, data) { putErr := f.StorageService.Put( + // #nosec G115 ctx, data, arbmath.SaturatingUAdd(uint64(time.Now().Unix()), f.backupRetentionSeconds), ) if putErr != nil && !f.ignoreRetentionWriteErrors { diff --git a/das/local_file_storage_service.go b/das/local_file_storage_service.go index ce86786718d..5e64c34b100 100644 --- a/das/local_file_storage_service.go +++ b/das/local_file_storage_service.go @@ -377,6 +377,7 @@ func migrate(fl *flatLayout, tl *trieLayout) error { return err } + // #nosec G115 expiryPath := tl.expiryPath(batch.key, uint64(batch.expiry.Unix())) if err = createHardLink(newPath, expiryPath); err != nil { return err diff --git a/das/local_file_storage_service_test.go b/das/local_file_storage_service_test.go index 01b999f3563..8a36664670e 100644 --- a/das/local_file_storage_service_test.go +++ b/das/local_file_storage_service_test.go @@ -78,6 +78,7 @@ func TestMigrationNoExpiry(t *testing.T) { Require(t, err) s.enableLegacyLayout = true + // #nosec G115 now := uint64(time.Now().Unix()) err = s.Put(ctx, []byte("a"), now+1) @@ -121,14 +122,19 @@ func TestMigrationExpiry(t *testing.T) { now := time.Now() // Use increments of expiry divisor in order to span multiple by-expiry-timestamp dirs + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(-2*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("b"), uint64(now.Add(-1*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("c"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("d"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("e"), uint64(now.Add(2*time.Second*expiryDivisor).Unix())) Require(t, err) @@ -171,19 +177,26 @@ func TestExpiryDuplicates(t *testing.T) { now := time.Now() // Use increments of expiry divisor in order to span multiple by-expiry-timestamp dirs + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(-2*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(-1*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("a"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("d"), uint64(now.Add(time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("e"), uint64(now.Add(2*time.Second*expiryDivisor).Unix())) Require(t, err) + // #nosec G115 err = s.Put(ctx, []byte("f"), uint64(now.Add(3*time.Second*expiryDivisor).Unix())) Require(t, err) // Put the same entry and expiry again, should have no effect + // #nosec G115 err = s.Put(ctx, []byte("f"), uint64(now.Add(3*time.Second*expiryDivisor).Unix())) Require(t, err) diff --git a/das/redis_storage_service_test.go b/das/redis_storage_service_test.go index 55f3ecd82c9..77d3e8cd0f5 100644 --- a/das/redis_storage_service_test.go +++ b/das/redis_storage_service_test.go @@ -16,6 +16,7 @@ import ( func TestRedisStorageService(t *testing.T) { ctx := context.Background() + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour).Unix()) baseStorageService := NewMemoryBackedStorageService(ctx) server, err := miniredis.Run() diff --git a/das/redundant_storage_test.go b/das/redundant_storage_test.go index b56f62ee248..11d3b58264e 100644 --- a/das/redundant_storage_test.go +++ b/das/redundant_storage_test.go @@ -17,6 +17,7 @@ const NumServices = 3 func TestRedundantStorageService(t *testing.T) { ctx := context.Background() + // #nosec G115 timeout := uint64(time.Now().Add(time.Hour).Unix()) services := []StorageService{} for i := 0; i < NumServices; i++ { diff --git a/das/restful_server_test.go b/das/restful_server_test.go index 1d3675749a6..e6982f9db5a 100644 --- a/das/restful_server_test.go +++ b/das/restful_server_test.go @@ -48,6 +48,7 @@ func TestRestfulClientServer(t *testing.T) { server, port, err := NewRestfulDasServerOnRandomPort(LocalServerAddressForTest, storage) Require(t, err) + // #nosec G115 err = storage.Put(ctx, data, uint64(time.Now().Add(time.Hour).Unix())) Require(t, err) diff --git a/das/rpc_aggregator.go b/das/rpc_aggregator.go index 24a470be5bf..1b3e2b8f447 100644 --- a/das/rpc_aggregator.go +++ b/das/rpc_aggregator.go @@ -119,7 +119,7 @@ func ParseServices(config AggregatorConfig, signer signature.DataSignerFunc) ([] return nil, err } - d, err := NewServiceDetails(service, *pubKey, 1< uint64(countsNum) && sectionNum > 1 { break } diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 457dae09101..7a78cee309e 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -671,7 +671,7 @@ func l2MessageBatchDataFromTxes(txes types.Transactions) ([]byte, error) { if err != nil { return nil, err } - binary.BigEndian.PutUint64(sizeBuf, uint64(len(txBytes)+1)) + binary.BigEndian.PutUint64(sizeBuf, uint64(len(txBytes))+1) l2Message = append(l2Message, sizeBuf...) l2Message = append(l2Message, arbos.L2MessageKind_SignedTx) l2Message = append(l2Message, txBytes...) diff --git a/system_tests/estimation_test.go b/system_tests/estimation_test.go index 284c709fad3..62857023429 100644 --- a/system_tests/estimation_test.go +++ b/system_tests/estimation_test.go @@ -214,7 +214,7 @@ func TestComponentEstimate(t *testing.T) { userBalance := big.NewInt(1e16) maxPriorityFeePerGas := big.NewInt(0) - maxFeePerGas := arbmath.BigMulByUfrac(l2BaseFee, 3, 2) + maxFeePerGas := arbmath.BigMulByUFrac(l2BaseFee, 3, 2) builder.L2Info.GenerateAccount("User") builder.L2.TransferBalance(t, "Owner", "User", userBalance, builder.L2Info) diff --git a/system_tests/nodeinterface_test.go b/system_tests/nodeinterface_test.go index 17bfb18892e..927dc1b6304 100644 --- a/system_tests/nodeinterface_test.go +++ b/system_tests/nodeinterface_test.go @@ -163,6 +163,7 @@ func TestGetL1Confirmations(t *testing.T) { numTransactions := 200 + // #nosec G115 if l1Confs >= uint64(numTransactions) { t.Fatalf("L1Confirmations for latest block %v is already %v (over %v)", genesisBlock.Number(), l1Confs, numTransactions) } @@ -175,6 +176,7 @@ func TestGetL1Confirmations(t *testing.T) { Require(t, err) // Allow a gap of 10 for asynchronicity, just in case + // #nosec G115 if l1Confs+10 < uint64(numTransactions) { t.Fatalf("L1Confirmations for latest block %v is only %v (did not hit expected %v)", genesisBlock.Number(), l1Confs, numTransactions) } diff --git a/system_tests/outbox_test.go b/system_tests/outbox_test.go index c68df6ea229..25c52396f93 100644 --- a/system_tests/outbox_test.go +++ b/system_tests/outbox_test.go @@ -175,6 +175,7 @@ func TestOutboxProofs(t *testing.T) { sibling := place ^ which position := merkletree.LevelAndLeaf{ + // #nosec G115 Level: uint64(level), Leaf: sibling, } @@ -201,6 +202,7 @@ func TestOutboxProofs(t *testing.T) { leaf := total - 1 // preceding it. We subtract 1 since we count from 0 partial := merkletree.LevelAndLeaf{ + // #nosec G115 Level: uint64(level), Leaf: leaf, } @@ -289,6 +291,7 @@ func TestOutboxProofs(t *testing.T) { step.Leaf += 1 << step.Level // we start on the min partial's zero-hash sibling known[step] = zero + // #nosec G115 for step.Level < uint64(treeLevels) { curr, ok := known[step] diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 7b09552e28a..4833d355361 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -362,12 +362,14 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig Require(t, err) l2info.GenerateAccount("User2") + // #nosec G115 for i := genesis; i < uint64(txCount)+genesis; i++ { tx := l2info.PrepareTx("Owner", "User2", l2info.TransferGas, common.Big1, nil) err := client.SendTransaction(ctx, tx) Require(t, err) receipt, err := EnsureTxSucceeded(ctx, client, tx) Require(t, err) + // #nosec G115 if have, want := receipt.BlockNumber.Uint64(), uint64(i)+1; have != want { Fatal(t, "internal test error - tx got included in unexpected block number, have:", have, "want:", want) } @@ -378,6 +380,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig Fatal(t, "missing current block") } lastBlock := currentHeader.Number.Uint64() + // #nosec G115 if want := genesis + uint64(txCount); lastBlock < want { Fatal(t, "internal test error - not enough blocks produced during preparation, want:", want, "have:", lastBlock) } @@ -391,6 +394,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig bc = builder.L2.ExecNode.Backend.ArbInterface().BlockChain() gas := skipGas blocks := skipBlocks + // #nosec G115 for i := genesis; i <= genesis+uint64(txCount); i++ { block := bc.GetBlockByNumber(i) if block == nil { @@ -423,6 +427,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig } } } + // #nosec G115 for i := genesis + 1; i <= genesis+uint64(txCount); i += i % 10 { _, err = client.BalanceAt(ctx, GetTestAddressForAccountName(t, "User2"), new(big.Int).SetUint64(i)) if err != nil { diff --git a/system_tests/seqfeed_test.go b/system_tests/seqfeed_test.go index 5e70fdf0982..21f07552250 100644 --- a/system_tests/seqfeed_test.go +++ b/system_tests/seqfeed_test.go @@ -164,12 +164,12 @@ func compareAllMsgResultsFromConsensusAndExecution( } var lastResult *execution.MessageResult - for msgCount := 1; arbutil.MessageIndex(msgCount) <= consensusMsgCount; msgCount++ { + for msgCount := arbutil.MessageIndex(1); msgCount <= consensusMsgCount; msgCount++ { pos := msgCount - 1 resultExec, err := testClient.ExecNode.ResultAtPos(arbutil.MessageIndex(pos)) Require(t, err) - resultConsensus, err := testClient.ConsensusNode.TxStreamer.ResultAtCount(arbutil.MessageIndex(msgCount)) + resultConsensus, err := testClient.ConsensusNode.TxStreamer.ResultAtCount(msgCount) Require(t, err) if !reflect.DeepEqual(resultExec, resultConsensus) { diff --git a/system_tests/seqinbox_test.go b/system_tests/seqinbox_test.go index 6babe5833f0..a9f66b0e2f9 100644 --- a/system_tests/seqinbox_test.go +++ b/system_tests/seqinbox_test.go @@ -265,6 +265,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { for j := 0; j < numMessages; j++ { sourceNum := rand.Int() % len(state.accounts) source := state.accounts[sourceNum] + // #nosec G115 amount := new(big.Int).SetUint64(uint64(rand.Int()) % state.balances[source].Uint64()) reserveAmount := new(big.Int).SetUint64(l2pricing.InitialBaseFeeWei * 100000000) if state.balances[source].Cmp(new(big.Int).Add(amount, reserveAmount)) < 0 { @@ -314,6 +315,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { for j := 0; ; j++ { haveNonce, err := builder.L1.Client.PendingNonceAt(ctx, seqOpts.From) Require(t, err) + // #nosec G115 if haveNonce == uint64(seqNonce) { break } @@ -380,6 +382,7 @@ func testSequencerInboxReaderImpl(t *testing.T, validator bool) { t.Errorf("Transaction: %v was not refunded, balance diff: %v, cost: %v", tx.Hash(), diff, txCost) } + // #nosec G115 state.l2BlockNumber += uint64(numMessages) state.l1BlockNumber = txRes.BlockNumber.Uint64() blockStates = append(blockStates, state) diff --git a/system_tests/stylus_trace_test.go b/system_tests/stylus_trace_test.go index cb303874d61..5c4463d9f77 100644 --- a/system_tests/stylus_trace_test.go +++ b/system_tests/stylus_trace_test.go @@ -76,6 +76,7 @@ func sendAndTraceTransaction( } func intToBytes(v int) []byte { + // #nosec G115 return binary.BigEndian.AppendUint64(nil, uint64(v)) } diff --git a/util/arbmath/bips.go b/util/arbmath/bips.go index 646dad3a92d..39b014f3ac8 100644 --- a/util/arbmath/bips.go +++ b/util/arbmath/bips.go @@ -27,24 +27,35 @@ func BigMulByBips(value *big.Int, bips Bips) *big.Int { return BigMulByFrac(value, int64(bips), int64(OneInBips)) } +func BigMulByUBips(value *big.Int, bips UBips) *big.Int { + return BigMulByUFrac(value, uint64(bips), uint64(OneInUBips)) +} + func IntMulByBips(value int64, bips Bips) int64 { return value * int64(bips) / int64(OneInBips) } +// UintMulByBips multiplies a uint value by a bips value +// bips must be positive and not cause an overflow func UintMulByBips(value uint64, bips Bips) uint64 { + // #nosec G115 return value * uint64(bips) / uint64(OneInBips) } -func SaturatingCastToBips(value uint64) Bips { - return Bips(SaturatingCast[int64](value)) -} - -func (bips UBips) Uint64() uint64 { - return uint64(bips) +// UintSaturatingMulByBips multiplies a uint value by a bips value, +// saturating at the maximum bips value (not the maximum uint64 result), +// then rounding down and returning a uint64. +// Returns 0 if bips is less than or equal to zero +func UintSaturatingMulByBips(value uint64, bips Bips) uint64 { + if bips <= 0 { + return 0 + } + // #nosec G115 + return SaturatingUMul(value, uint64(bips)) / uint64(OneInBips) } -func (bips Bips) Uint64() uint64 { - return uint64(bips) +func SaturatingCastToBips(value uint64) Bips { + return Bips(SaturatingCast[int64](value)) } // BigDivToBips returns dividend/divisor as bips, saturating if out of bounds diff --git a/util/arbmath/math.go b/util/arbmath/math.go index e5bed67f6df..07a9941b659 100644 --- a/util/arbmath/math.go +++ b/util/arbmath/math.go @@ -29,6 +29,7 @@ func NextOrCurrentPowerOf2(value uint64) uint64 { // Log2ceil the log2 of the int, rounded up func Log2ceil(value uint64) uint64 { + // #nosec G115 return uint64(64 - bits.LeadingZeros64(value)) } @@ -228,8 +229,8 @@ func BigMulByFrac(value *big.Int, numerator, denominator int64) *big.Int { return value } -// BigMulByUfrac multiply a huge by a rational whose components are non-negative -func BigMulByUfrac(value *big.Int, numerator, denominator uint64) *big.Int { +// BigMulByUFrac multiply a huge by a rational whose components are non-negative +func BigMulByUFrac(value *big.Int, numerator, denominator uint64) *big.Int { value = new(big.Int).Set(value) value.Mul(value, new(big.Int).SetUint64(numerator)) value.Div(value, new(big.Int).SetUint64(denominator)) @@ -407,6 +408,8 @@ func ApproxExpBasisPoints(value Bips, accuracy uint64) Bips { if negative { input = -value } + // This cast is safe because input is always positive + // #nosec G115 x := uint64(input) bips := uint64(OneInBips) diff --git a/util/arbmath/math_test.go b/util/arbmath/math_test.go index 528666dc19e..3660f3657e1 100644 --- a/util/arbmath/math_test.go +++ b/util/arbmath/math_test.go @@ -44,6 +44,7 @@ func TestMath(t *testing.T) { // try the first million sqrts for i := 0; i < 1000000; i++ { + // #nosec G115 input := uint64(i) approx := ApproxSquareRoot(input) correct := math.Sqrt(float64(input)) diff --git a/util/headerreader/blob_client.go b/util/headerreader/blob_client.go index 2b47a940c31..160323cf603 100644 --- a/util/headerreader/blob_client.go +++ b/util/headerreader/blob_client.go @@ -191,6 +191,7 @@ func (b *BlobClient) blobSidecars(ctx context.Context, slot uint64, versionedHas rawData, err := beaconRequest[json.RawMessage](b, ctx, fmt.Sprintf("/eth/v1/beacon/blob_sidecars/%d", slot)) if err != nil || len(rawData) == 0 { // blobs are pruned after 4096 epochs (1 epoch = 32 slots), we determine if the requested slot were to be pruned by a non-archive endpoint + // #nosec G115 roughAgeOfSlot := uint64(time.Now().Unix()) - (b.genesisTime + slot*b.secondsPerSlot) if roughAgeOfSlot > b.secondsPerSlot*32*4096 { return nil, fmt.Errorf("beacon client in blobSidecars got error or empty response fetching older blobs in slot: %d, an archive endpoint is required, please refer to https://docs.arbitrum.io/run-arbitrum-node/l1-ethereum-beacon-chain-rpc-providers, err: %w", slot, err) diff --git a/util/merkletree/merkleEventProof_test.go b/util/merkletree/merkleEventProof_test.go index b64cc88c2a9..6af84791904 100644 --- a/util/merkletree/merkleEventProof_test.go +++ b/util/merkletree/merkleEventProof_test.go @@ -22,6 +22,7 @@ func initializedMerkleAccumulatorForTesting() *merkleAccumulator.MerkleAccumulat func TestProofForNext(t *testing.T) { leaves := make([]common.Hash, 13) for i := range leaves { + // #nosec G115 leaves[i] = pseudorandomForTesting(uint64(i)) } diff --git a/validator/server_arb/machine_cache.go b/validator/server_arb/machine_cache.go index 55ef61cf11d..35f34062364 100644 --- a/validator/server_arb/machine_cache.go +++ b/validator/server_arb/machine_cache.go @@ -31,7 +31,7 @@ type MachineCache struct { } type MachineCacheConfig struct { - CachedChallengeMachines int `koanf:"cached-challenge-machines"` + CachedChallengeMachines uint64 `koanf:"cached-challenge-machines"` InitialSteps uint64 `koanf:"initial-steps"` } @@ -42,7 +42,7 @@ var DefaultMachineCacheConfig = MachineCacheConfig{ func MachineCacheConfigConfigAddOptions(prefix string, f *flag.FlagSet) { f.Uint64(prefix+".initial-steps", DefaultMachineCacheConfig.InitialSteps, "initial steps between machines") - f.Int(prefix+".cached-challenge-machines", DefaultMachineCacheConfig.CachedChallengeMachines, "how many machines to store in cache while working on a challenge (should be even)") + f.Uint64(prefix+".cached-challenge-machines", DefaultMachineCacheConfig.CachedChallengeMachines, "how many machines to store in cache while working on a challenge (should be even)") } // `initialMachine` won't be mutated by this function. @@ -140,7 +140,7 @@ func (c *MachineCache) unlockBuild(err error) { } func (c *MachineCache) setRangeLocked(ctx context.Context, start uint64, end uint64) error { - newInterval := (end - start) / uint64(c.config.CachedChallengeMachines) + newInterval := (end - start) / c.config.CachedChallengeMachines if newInterval == 0 { newInterval = 2 } @@ -150,7 +150,7 @@ func (c *MachineCache) setRangeLocked(ctx context.Context, start uint64, end uin if end >= c.finalMachineStep { end = c.finalMachineStep - newInterval/2 } - newInterval = (end - start) / uint64(c.config.CachedChallengeMachines) + newInterval = (end - start) / c.config.CachedChallengeMachines if newInterval == 0 { newInterval = 1 } @@ -212,7 +212,7 @@ func (c *MachineCache) populateCache(ctx context.Context) error { if nextMachine.GetStepCount()+c.machineStepInterval >= c.finalMachineStep { break } - if len(c.machines) >= c.config.CachedChallengeMachines { + if uint64(len(c.machines)) >= c.config.CachedChallengeMachines { break } nextMachine = nextMachine.CloneMachineInterface() @@ -236,6 +236,7 @@ func (c *MachineCache) getClosestMachine(stepCount uint64) (int, MachineInterfac } stepsFromStart := stepCount - c.firstMachineStep var index int + // #nosec G115 if c.machineStepInterval == 0 || stepsFromStart > c.machineStepInterval*uint64(len(c.machines)-1) { index = len(c.machines) - 1 } else { diff --git a/validator/server_jit/jit_machine.go b/validator/server_jit/jit_machine.go index e7753748aba..06c451bda18 100644 --- a/validator/server_jit/jit_machine.go +++ b/validator/server_jit/jit_machine.go @@ -306,6 +306,7 @@ func (machine *JitMachine) prove( if err != nil { return state, fmt.Errorf("failed to read memory usage from Jit machine: %w", err) } + // #nosec G115 if memoryUsed > uint64(machine.wasmMemoryUsageLimit) { log.Warn("memory used by jit wasm exceeds the wasm memory usage limit", "limit", machine.wasmMemoryUsageLimit, "memoryUsed", memoryUsed) } diff --git a/wavmio/stub.go b/wavmio/stub.go index 1395fb42350..0c82506ff39 100644 --- a/wavmio/stub.go +++ b/wavmio/stub.go @@ -66,6 +66,7 @@ func parsePreimageBytes(path string) { if err != nil { panic(err) } + // #nosec G115 if uint64(read) != fieldSize { panic("missing bytes reading data") } @@ -77,18 +78,18 @@ func parsePreimageBytes(path string) { func StubInit() { preimages = make(map[common.Hash][]byte) var delayedMsgPath arrayFlags - seqMsgPosFlag := flag.Int("inbox-position", 0, "position for sequencer inbox message") - posWithinMsgFlag := flag.Int("position-within-message", 0, "position inside sequencer inbox message") - delayedPositionFlag := flag.Int("delayed-inbox-position", 0, "position for first delayed inbox message") + seqMsgPosFlag := flag.Uint64("inbox-position", 0, "position for sequencer inbox message") + posWithinMsgFlag := flag.Uint64("position-within-message", 0, "position inside sequencer inbox message") + delayedPositionFlag := flag.Uint64("delayed-inbox-position", 0, "position for first delayed inbox message") lastBlockFlag := flag.String("last-block-hash", "0000000000000000000000000000000000000000000000000000000000000000", "lastBlockHash") flag.Var(&delayedMsgPath, "delayed-inbox", "delayed inbox messages (multiple values)") inboxPath := flag.String("inbox", "", "file to load sequencer message") preimagesPath := flag.String("preimages", "", "file to load preimages from") flag.Parse() - seqMsgPos = uint64(*seqMsgPosFlag) - posWithinMsg = uint64(*posWithinMsgFlag) - delayedMsgFirstPos = uint64(*delayedPositionFlag) + seqMsgPos = *seqMsgPosFlag + posWithinMsg = *posWithinMsgFlag + delayedMsgFirstPos = *delayedPositionFlag lastBlockHash = common.HexToHash(*lastBlockFlag) for _, path := range delayedMsgPath { msg, err := os.ReadFile(path) From f31392d0ddea81466f5510d01e4f3efa9b64d867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikola=20Bebi=C4=87=E2=80=AE?= Date: Thu, 12 Sep 2024 12:02:05 +0200 Subject: [PATCH 94/95] Update go-ethereum submodule --- go-ethereum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-ethereum b/go-ethereum index 96d2c1ca917..6b279c3114d 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 96d2c1ca9175041b8458a3ef51b62b9e29b92de5 +Subproject commit 6b279c3114d4caa2218a31b607d0b36526d47e9b From 15f20465dbe3fabc6902322b1a7bbb90e4c82e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikola=20Bebi=C4=87=E2=80=AE?= Date: Thu, 12 Sep 2024 12:13:02 +0200 Subject: [PATCH 95/95] upate --- execution/gethexec/executionengine.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 712394d451e..8594d5867df 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -192,16 +192,6 @@ func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusT return nil } -func (s *ExecutionEngine) Initialize(rustCacheSize uint32, targetConfig *StylusTargetConfig) error { - if rustCacheSize != 0 { - programs.ResizeWasmLruCache(rustCacheSize) - } - if err := populateStylusTargetCache(targetConfig); err != nil { - return fmt.Errorf("error populating stylus target cache: %w", err) - } - return nil -} - func (s *ExecutionEngine) SetRecorder(recorder *BlockRecorder) { if s.Started() { panic("trying to set recorder after start")