devel merged and conflict resolved

.github/ISSUE_TEMPLATE/bug.md

@@ -16,9 +16,9 @@ Commit hash:
 
 Erigon Command (with flags/config):
 
-Concensus Layer:
+Consensus Layer:
 
-Concensus Layer Command (with flags/config):
+Consensus Layer Command (with flags/config):
 
 Chain/Network: 
 

.github/workflows/test-integration.yml

@@ -29,6 +29,11 @@ jobs:
       - name: test-integration
         run: make test-integration
 
+      - name: Test erigon as a library
+        env:
+          GIT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: make test-erigon-ext GIT_COMMIT=$GIT_COMMIT
+
   #        name: history-v3-test-integration
   #        run: make test3-integration
 

Dockerfile

@@ -78,7 +78,7 @@ COPY --from=builder /app/build/bin/sentry /usr/local/bin/sentry
 COPY --from=builder /app/build/bin/state /usr/local/bin/state
 COPY --from=builder /app/build/bin/txpool /usr/local/bin/txpool
 COPY --from=builder /app/build/bin/verkle /usr/local/bin/verkle
-COPY --from=builder /app/build/bin/caplin-phase1 /usr/local/bin/caplin-phase1
+COPY --from=builder /app/build/bin/caplin /usr/local/bin/caplin
 COPY --from=builder /app/build/bin/caplin-regression /usr/local/bin/caplin-regression
 
 

Dockerfile.debian

@@ -1,5 +1,5 @@
 # syntax = docker/dockerfile:1.2
-FROM docker.io/library/golang:1.19-bullseye AS builder
+FROM docker.io/library/golang:1.20-bullseye AS builder
 
 RUN apt update
 RUN apt install -y build-essential git bash ca-certificates libstdc++6
@@ -17,7 +17,7 @@ RUN --mount=type=cache,target=/root/.cache \
     make all
 
 
-FROM docker.io/library/golang:1.19-alpine3.16 AS tools-builder
+FROM docker.io/library/golang:1.20-alpine3.17 AS tools-builder
 
 RUN apk --no-cache add build-base linux-headers git bash ca-certificates libstdc++
 WORKDIR /app
@@ -74,7 +74,7 @@ COPY --from=builder /app/build/bin/sentry /usr/local/bin/sentry
 COPY --from=builder /app/build/bin/state /usr/local/bin/state
 COPY --from=builder /app/build/bin/txpool /usr/local/bin/txpool
 COPY --from=builder /app/build/bin/verkle /usr/local/bin/verkle
-COPY --from=builder /app/build/bin/caplin-phase1 /usr/local/bin/caplin-phase1
+COPY --from=builder /app/build/bin/caplin /usr/local/bin/caplin
 COPY --from=builder /app/build/bin/caplin-regression /usr/local/bin/caplin-regression
 
 EXPOSE 8545 \

Makefile

@@ -1,4 +1,4 @@
-GO = go # if using docker, should not need to be installed/linked
+GO ?= go # if using docker, should not need to be installed/linked
 GOBIN = $(CURDIR)/build/bin
 UNAME = $(shell uname) # Supported: Darwin, Linux
 DOCKER := $(shell command -v docker 2> /dev/null)
@@ -24,8 +24,7 @@ CGO_CFLAGS += -DMDBX_FORCE_ASSERTIONS=0 # Enable MDBX's asserts by default in 'd
 #CGO_CFLAGS += -DMDBX_ENV_CHECKPID=0 # Erigon doesn't do fork() syscall
 CGO_CFLAGS += -O
 CGO_CFLAGS += -D__BLST_PORTABLE__
-CGO_CFLAGS += -Wno-unknown-warning-option -Wno-enum-int-mismatch -Wno-strict-prototypes
-#CGO_CFLAGS += -Wno-error=strict-prototypes # for Clang15, remove it when can https://github.com/ledgerwatch/erigon/issues/6113#issuecomment-1359526277
+CGO_CFLAGS += -Wno-unknown-warning-option -Wno-enum-int-mismatch -Wno-strict-prototypes -Wno-unused-but-set-variable
 
 # about netgo see: https://github.com/golang/go/issues/30310#issuecomment-471669125 and https://github.com/golang/go/issues/57757
 BUILD_TAGS = nosqlite,noboltdb
@@ -42,8 +41,8 @@ default: all
 
 ## go-version:                        print and verify go version
 go-version:
-	@if [ $(shell $(GO) version | cut -c 16-17) -lt 19 ]; then \
-		echo "minimum required Golang version is 1.19"; \
+	@if [ $(shell $(GO) version | cut -c 16-17) -lt 20 ]; then \
+		echo "minimum required Golang version is 1.20"; \
 		exit 1 ;\
 	fi
 
@@ -106,6 +105,7 @@ erigon: go-version erigon.cmd
 	@rm -f $(GOBIN)/tg # Remove old binary to prevent confusion where users still use it because of the scripts
 
 COMMANDS += devnet
+COMMANDS += capcli
 COMMANDS += downloader
 COMMANDS += hack
 COMMANDS += integration
@@ -119,7 +119,7 @@ COMMANDS += txpool
 COMMANDS += verkle
 COMMANDS += evm
 COMMANDS += sentinel
-COMMANDS += caplin-phase1
+COMMANDS += caplin
 COMMANDS += caplin-regression
 
 
@@ -140,22 +140,24 @@ db-tools:
 	rm -rf vendor
 	@echo "Run \"$(GOBIN)/mdbx_stat -h\" to get info about mdbx db file."
 
-## test:                              run unit tests with a 100s timeout
-test:
+test-erigon-lib:
 	@cd erigon-lib && $(MAKE) test
+
+test-erigon-ext:
+	@cd tests/erigon-ext-test && ./test.sh $(GIT_COMMIT)
+
+## test:                              run unit tests with a 100s timeout
+test: test-erigon-lib
 	$(GOTEST) --timeout 10m
 
-test3:
-	@cd erigon-lib && $(MAKE) test
+test3: test-erigon-lib
 	$(GOTEST) --timeout 10m -tags $(BUILD_TAGS),e3
 
 ## test-integration:                  run integration tests with a 30m timeout
-test-integration:
-	@cd erigon-lib && $(MAKE) test
+test-integration: test-erigon-lib
 	$(GOTEST) --timeout 240m -tags $(BUILD_TAGS),integration
 
-test3-integration:
-	@cd erigon-lib && $(MAKE) test
+test3-integration: test-erigon-lib
 	$(GOTEST) --timeout 240m -tags $(BUILD_TAGS),integration,e3
 
 ## lint-deps:                         install lint dependencies

README.md

@@ -55,16 +55,17 @@ System Requirements
 
 * Goerli Full node (see `--prune*` flags): 189GB on Beta, 114GB on Alpha (April 2022).
 
-* Gnosis Chain Archive: 370GB (January 2023).
+* Gnosis Chain Archive: 600GB (October 2023).
 
-* Polygon Mainnet Archive: 5TB. (April 2022). `--prune.*.older 15768000`: 5.1Tb (Sept 2023). Polygon Mumbai Archive: 1TB. (April 2022).
+* Polygon Mainnet Archive: 5TB. (April 2022). `--prune.*.older 15768000`: 5.1Tb (Sept 2023). Polygon Mumbai Archive:
+  1TB. (April 2022).
 
 SSD or NVMe. Do not recommend HDD - on HDD Erigon will always stay N blocks behind chain tip, but not fall behind.
 Bear in mind that SSD performance deteriorates when close to capacity.
 
 RAM: >=16GB, 64-bit architecture.
 
-[Golang version >= 1.19](https://golang.org/doc/install); GCC 10+ or Clang; On Linux: kernel > v4
+[Golang version >= 1.20](https://golang.org/doc/install); GCC 10+ or Clang; On Linux: kernel > v4
 
 <code>🔬 more details on disk storage [here](https://erigon.substack.com/p/disk-footprint-changes-in-new-erigon?s=r)
 and [here](https://ledgerwatch.github.io/turbo_geth_release.html#Disk-space).</code>
@@ -111,7 +112,8 @@ Running `make help` will list and describe the convenience commands available in
 ### Datadir structure
 
 - chaindata: recent blocks, state, recent state history. low-latency disk recommended.
-- snapshots: old blocks, old state history. can symlink/mount it to cheaper disk. mostly immutable. must have ~100gb free space (for merge recent files to bigger one).
+- snapshots: old blocks, old state history. can symlink/mount it to cheaper disk. mostly immutable. must have ~100gb
+  free space (for merge recent files to bigger one).
 - temp: can grow to ~100gb, but usually empty. can symlink/mount it to cheaper disk.
 - txpool: pending transactions. safe to remove.
 - nodes:  p2p peers. safe to remove.
@@ -141,7 +143,9 @@ int value specifying the highest output log level:
   LvlTrace = 5
 ```
 
-To set an output dir for logs to be collected on disk, please set `--log.dir.path` If you want to change the filename prodiced from `erigon` you should also set the `--log.dir.prefix` flag to an alternate name. The flag `--log.dir.verbosity` is
+To set an output dir for logs to be collected on disk, please set `--log.dir.path` If you want to change the filename
+produced from `erigon` you should also set the `--log.dir.prefix` flag to an alternate name. The
+flag `--log.dir.verbosity` is
 also available to control the verbosity of this logging, with the same int value as above, or the string value e.g. '
 debug' or 'info'. Default verbosity is 'debug' (4), for disk logging.
 
@@ -210,7 +214,7 @@ Windows users may run erigon in 3 possible ways:
   build on windows :
     * [Git](https://git-scm.com/downloads) for Windows must be installed. If you're cloning this repository is very
       likely you already have it
-    * [GO Programming Language](https://golang.org/dl/) must be installed. Minimum required version is 1.19
+    * [GO Programming Language](https://golang.org/dl/) must be installed. Minimum required version is 1.20
     * GNU CC Compiler at least version 10 (is highly suggested that you install `chocolatey` package manager - see
       following point)
     * If you need to build MDBX tools (i.e. `.\wmake.ps1 db-tools`)
@@ -530,13 +534,10 @@ Detailed explanation: [./docs/programmers_guide/db_faq.md](./docs/programmers_gu
 
 ### Default Ports and Firewalls
 
-
-
 #### `erigon` ports
 
-
 | Component | Port  | Protocol  | Purpose                     | Should Expose |
-| --------- | ----- | --------- | --------------------------- | ------------- |
+|-----------|-------|-----------|-----------------------------|---------------|
 | engine    | 9090  | TCP       | gRPC Server                 | Private       |
 | engine    | 42069 | TCP & UDP | Snap sync (Bittorrent)      | Public        |
 | engine    | 8551  | TCP       | Engine API (JWT auth)       | Private       |
@@ -545,42 +546,31 @@ Detailed explanation: [./docs/programmers_guide/db_faq.md](./docs/programmers_gu
 | sentry    | 9091  | TCP       | incoming gRPC Connections   | Private       |
 | rpcdaemon | 8545  | TCP       | HTTP & WebSockets & GraphQL | Private       |
 
-
-
-
 Typically, 30303 and 30304 are exposed to the internet to allow incoming peering connections. 9090 is exposed only
 internally for rpcdaemon or other connections, (e.g. rpcdaemon -> erigon).
 Port 8551 (JWT authenticated) is exposed only internally for [Engine API] JSON-RPC queries from the Consensus Layer
 node.
 
-
-
-
-
 #### `caplin` ports
+
 | Component | Port | Protocol | Purpose          | Should Expose |
-| --------- | ---- | -------- | ---------------- | ------------- |
+|-----------|------|----------|------------------|---------------|
 | sentinel  | 4000 | UDP      | Peering          | Public        |
 | sentinel  | 4001 | TCP      | Peering          | Public        |
-| sentinel  | 7777 | TCP      | gRPC Connections | Private       |
-
 
 If you are using `--internalcl` aka `caplin` as your consensus client, then also look at the chart above
 
-
 #### `shared` ports
 
-| Component | Port  | Protocol  | Purpose                     | Should Expose |
-| --------- | ----- | --------- | --------------------------- | ------------- |
-| all       | 6060 | TCP      | pprof            | Private       |
-| all       | 6060 | TCP      | metrics          | Private       |
-
+| Component | Port | Protocol | Purpose | Should Expose |
+|-----------|------|----------|---------|---------------|
+| all       | 6060 | TCP      | pprof   | Private       |
+| all       | 6060 | TCP      | metrics | Private       |
 
 Optional flags can be enabled that enable pprof or metrics (or both) - however, they both run on 6060 by default, so
 
 you'll have to change one if you want to run both at the same time. use `--help` with the binary for more info.
 
-
 #### `other` ports
 
 Reserved for future use: **gRPC ports**: `9092` consensus engine, `9093` snapshot downloader, `9094` TxPool
@@ -608,15 +598,16 @@ Device Benchmark Testing   RFC 2544
 RFC 922, Section 7
 ```
 
-Same in [IpTables syntax](https://ethereum.stackexchange.com/questions/6386/how-to-prevent-being-blacklisted-for-running-an-ethereum-client/13068#13068)
+Same
+in [IpTables syntax](https://ethereum.stackexchange.com/questions/6386/how-to-prevent-being-blacklisted-for-running-an-ethereum-client/13068#13068)
 
 ### How to get diagnostic for bug report?
 
 - Get stack trace: `kill -SIGUSR1 <pid>`, get trace and stop: `kill -6 <pid>`
 - Get CPU profiling: add `--pprof flag`
-    run `go tool pprof -png  http://127.0.0.1:6060/debug/pprof/profile\?seconds\=20 > cpu.png`
+  run `go tool pprof -png  http://127.0.0.1:6060/debug/pprof/profile\?seconds\=20 > cpu.png`
 - Get RAM profiling: add `--pprof flag`
-    run `go tool pprof -inuse_space -png  http://127.0.0.1:6060/debug/pprof/heap > mem.png`
+  run `go tool pprof -inuse_space -png  http://127.0.0.1:6060/debug/pprof/heap > mem.png`
 
 ### How to run local devnet?
 
@@ -630,6 +621,13 @@ UID/GID (1000).
 More details
 in [post](https://www.fullstaq.com/knowledge-hub/blogs/docker-and-the-host-filesystem-owner-matching-problem)
 
+### How to run public RPC api
+
+- `--txpool.nolocals=true`
+- don't add `admin` in `--http.api` list
+- to increase throughput may need
+  increase/decrease: `--db.read.concurrency`, `--rpc.batch.concurrency`, `--rpc.batch.limit`
+
 ### Run RaspberyPI
 
 https://github.com/mathMakesArt/Erigon-on-RPi-4
@@ -644,7 +642,8 @@ Getting in touch
 
 ### Erigon Discord Server
 
-The main discussions are happening on our Discord server. To get an invite, send an email to `bloxster [at] proton.me` with
+The main discussions are happening on our Discord server. To get an invite, send an email to `bloxster [at] proton.me`
+with
 your name, occupation, a brief explanation of why you want to join the Discord, and how you heard about Erigon.
 
 ### Reporting security issues/concerns
@@ -671,20 +670,22 @@ Next tools show correct memory usage of Erigon:
 
 - `vmmap -summary PID | grep -i "Physical footprint"`. Without `grep` you can see details
     - `section MALLOC ZONE column Resident Size` shows App memory usage, `section REGION TYPE column Resident Size`
-        shows OS pages cache size.
+      shows OS pages cache size.
 - `Prometheus` dashboard shows memory of Go app without OS pages cache (`make prometheus`, open in
-    browser `localhost:3000`, credentials `admin/admin`)
+  browser `localhost:3000`, credentials `admin/admin`)
 - `cat /proc/<PID>/smaps`
 
-    Erigon uses ~4Gb of RAM during genesis sync and ~1Gb during normal work. OS pages cache can utilize unlimited amount of
-    memory.
-
-    **Warning:** Multiple instances of Erigon on same machine will touch Disk concurrently, it impacts performance - one of
-    main Erigon optimisations: "reduce Disk random access".
-    "Blocks Execution stage" still does many random reads - this is reason why it's slowest stage. We do not recommend
-    running
-    multiple genesis syncs on same Disk. If genesis sync passed, then it's fine to run multiple Erigon instances on same
-    Disk.
+  Erigon uses ~4Gb of RAM during genesis sync and ~1Gb during normal work. OS pages cache can utilize unlimited amount
+  of
+  memory.
+
+  **Warning:** Multiple instances of Erigon on same machine will touch Disk concurrently, it impacts performance - one
+  of
+  main Erigon optimisations: "reduce Disk random access".
+  "Blocks Execution stage" still does many random reads - this is reason why it's slowest stage. We do not recommend
+  running
+  multiple genesis syncs on same Disk. If genesis sync passed, then it's fine to run multiple Erigon instances on same
+  Disk.
 
 ### Blocks Execution is slow on cloud-network-drives
 

accounts/abi/abi_test.go

@@ -1137,7 +1137,7 @@ func TestUnpackRevert(t *testing.T) {
 	}
 	for index, c := range cases {
 		t.Run(fmt.Sprintf("case %d", index), func(t *testing.T) {
-			got, err := UnpackRevert(common.Hex2Bytes(c.input))
+			got, err := UnpackRevert(libcommon.Hex2Bytes(c.input))
 			if c.expectErr != nil {
 				if err == nil {
 					t.Fatalf("Expected non-nil error")

accounts/abi/bind/backends/simulated_test.go

@@ -467,7 +467,7 @@ func TestSimulatedBackend_EstimateGas(t *testing.T) {
 			Gas:      0,
 			GasPrice: u256.Num0,
 			Value:    nil,
-			Data:     common.Hex2Bytes("d8b98391"),
+			Data:     libcommon.Hex2Bytes("d8b98391"),
 		}, 0, errors.New("execution reverted: revert reason"), "0x08c379a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000d72657665727420726561736f6e00000000000000000000000000000000000000"},
 
 		{"PureRevert", ethereum.CallMsg{
@@ -476,7 +476,7 @@ func TestSimulatedBackend_EstimateGas(t *testing.T) {
 			Gas:      0,
 			GasPrice: u256.Num0,
 			Value:    nil,
-			Data:     common.Hex2Bytes("aa8b1d30"),
+			Data:     libcommon.Hex2Bytes("aa8b1d30"),
 		}, 0, errors.New("execution reverted"), nil},
 
 		{"OOG", ethereum.CallMsg{
@@ -485,7 +485,7 @@ func TestSimulatedBackend_EstimateGas(t *testing.T) {
 			Gas:      100000,
 			GasPrice: u256.Num0,
 			Value:    nil,
-			Data:     common.Hex2Bytes("50f6fe34"),
+			Data:     libcommon.Hex2Bytes("50f6fe34"),
 		}, 0, errors.New("gas required exceeds allowance (100000)"), nil},
 
 		{"Assert", ethereum.CallMsg{
@@ -494,7 +494,7 @@ func TestSimulatedBackend_EstimateGas(t *testing.T) {
 			Gas:      100000,
 			GasPrice: u256.Num0,
 			Value:    nil,
-			Data:     common.Hex2Bytes("b9b046f9"),
+			Data:     libcommon.Hex2Bytes("b9b046f9"),
 		}, 0, errors.New("invalid opcode: INVALID"), nil},
 
 		{"Valid", ethereum.CallMsg{
@@ -503,7 +503,7 @@ func TestSimulatedBackend_EstimateGas(t *testing.T) {
 			Gas:      100000,
 			GasPrice: u256.Num0,
 			Value:    nil,
-			Data:     common.Hex2Bytes("e09fface"),
+			Data:     libcommon.Hex2Bytes("e09fface"),
 		}, 21275, nil, nil},
 	}
 	for _, c := range cases {

accounts/abi/bind/base_test.go

@@ -18,6 +18,7 @@ package bind_test
 
 import (
 	"context"
+	"github.com/ledgerwatch/erigon-lib/common/hexutil"
 	"math/big"
 	"reflect"
 	"strings"
@@ -28,7 +29,6 @@ import (
 	ethereum "github.com/ledgerwatch/erigon"
 	"github.com/ledgerwatch/erigon/accounts/abi"
 	"github.com/ledgerwatch/erigon/accounts/abi/bind"
-	"github.com/ledgerwatch/erigon/common/hexutil"
 	"github.com/ledgerwatch/erigon/core/types"
 	"github.com/ledgerwatch/erigon/crypto"
 	"github.com/ledgerwatch/erigon/rlp"