feat: merge with upstream bsc develop

.nancy-ignore

@@ -0,0 +1 @@
+CVE-2024-34478 # "CWE-754: Improper Check for Unusual or Exceptional Conditions." This vulnerability is BTC only, BSC does not have the issue.

CHANGELOG.md

@@ -1,4 +1,18 @@
 # Changelog
+## v1.4.7
+### FEATURE
+* [\#2439](https://github.com/bnb-chain/bsc/pull/2439) config: setup Mainnet Tycho(Cancun) hardfork date
+
+### IMPROVEMENT
+* [\#2396](https://github.com/bnb-chain/bsc/pull/2396) metrics: add blockInsertMgaspsGauge to trace mgasps
+* [\#2411](https://github.com/bnb-chain/bsc/pull/2411) build(deps): bump golang.org/x/net from 0.19.0 to 0.23.0
+* [\#2435](https://github.com/bnb-chain/bsc/pull/2435) txpool: limit max gas when mining is enabled
+* [\#2438](https://github.com/bnb-chain/bsc/pull/2438) fix: performance issue when load journal
+* [\#2440](https://github.com/bnb-chain/bsc/pull/2440) nancy: add files .nancy-ignore
+
+### BUGFIX
+NA
+
 ## v1.4.6
 ### FEATURE
 * [\#2227](https://github.com/bnb-chain/bsc/pull/2227) core: separated databases for block data

cmd/geth/config.go

@@ -193,14 +193,6 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 		v := ctx.Uint64(utils.OverrideVerkle.Name)
 		cfg.Eth.OverrideVerkle = &v
 	}
-	if ctx.IsSet(utils.OverrideFeynman.Name) {
-		v := ctx.Uint64(utils.OverrideFeynman.Name)
-		cfg.Eth.OverrideFeynman = &v
-	}
-	if ctx.IsSet(utils.OverrideFeynmanFix.Name) {
-		v := ctx.Uint64(utils.OverrideFeynmanFix.Name)
-		cfg.Eth.OverrideFeynmanFix = &v
-	}
 	if ctx.IsSet(utils.OverrideFullImmutabilityThreshold.Name) {
 		params.FullImmutabilityThreshold = ctx.Uint64(utils.OverrideFullImmutabilityThreshold.Name)
 		downloader.FullMaxForkAncestry = ctx.Uint64(utils.OverrideFullImmutabilityThreshold.Name)
@@ -211,6 +203,9 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	if ctx.IsSet(utils.OverrideDefaultExtraReserveForBlobRequests.Name) {
 		params.DefaultExtraReserveForBlobRequests = ctx.Uint64(utils.OverrideDefaultExtraReserveForBlobRequests.Name)
 	}
+	if ctx.IsSet(utils.OverrideBreatheBlockInterval.Name) {
+		params.BreatheBlockInterval = ctx.Uint64(utils.OverrideBreatheBlockInterval.Name)
+	}
 
 	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
 

cmd/geth/dbcmd.go

@@ -106,12 +106,12 @@ Remove blockchain and state databases`,
 	dbInspectTrieCmd = &cli.Command{
 		Action:    inspectTrie,
 		Name:      "inspect-trie",
-		ArgsUsage: "<blocknum> <jobnum>",
+		ArgsUsage: "<blocknum> <jobnum> <topn>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
 			utils.SyncModeFlag,
 		},
-		Usage:       "Inspect the MPT tree of the account and contract.",
+		Usage:       "Inspect the MPT tree of the account and contract. 'blocknum' can be latest/snapshot/number. 'topn' means output the top N storage tries info ranked by the total number of TrieNodes",
 		Description: `This commands iterates the entrie WorldState.`,
 	}
 	dbCheckStateContentCmd = &cli.Command{
@@ -386,6 +386,7 @@ func inspectTrie(ctx *cli.Context) error {
 		blockNumber  uint64
 		trieRootHash common.Hash
 		jobnum       uint64
+		topN         uint64
 	)
 
 	stack, _ := makeConfigNode(ctx)
@@ -411,12 +412,25 @@ func inspectTrie(ctx *cli.Context) error {
 
 		if ctx.NArg() == 1 {
 			jobnum = 1000
+			topN = 10
+		} else if ctx.NArg() == 2 {
+			var err error
+			jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64)
+			if err != nil {
+				return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err)
+			}
+			topN = 10
 		} else {
 			var err error
 			jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64)
 			if err != nil {
 				return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err)
 			}
+
+			topN, err = strconv.ParseUint(ctx.Args().Get(2), 10, 64)
+			if err != nil {
+				return fmt.Errorf("failed to Parse topn, Args[1]: %v, err: %v", ctx.Args().Get(1), err)
+			}
 		}
 
 		if blockNumber != math.MaxUint64 {
@@ -437,6 +451,7 @@ func inspectTrie(ctx *cli.Context) error {
 		if dbScheme == rawdb.PathScheme {
 			config = &triedb.Config{
 				PathDB: utils.PathDBConfigAddJournalFilePath(stack, pathdb.ReadOnly),
+				Cache:  0,
 			}
 		} else if dbScheme == rawdb.HashScheme {
 			config = triedb.HashDefaults
@@ -448,7 +463,7 @@ func inspectTrie(ctx *cli.Context) error {
 			fmt.Printf("fail to new trie tree, err: %v, rootHash: %v\n", err, trieRootHash.String())
 			return err
 		}
-		theInspect, err := trie.NewInspector(theTrie, triedb, trieRootHash, blockNumber, jobnum)
+		theInspect, err := trie.NewInspector(theTrie, triedb, trieRootHash, blockNumber, jobnum, int(topN))
 		if err != nil {
 			return err
 		}

cmd/geth/main.go

@@ -25,6 +25,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ethereum/go-ethereum/params"
+
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -72,11 +74,10 @@ var (
 		utils.RialtoHash,
 		utils.OverrideCancun,
 		utils.OverrideVerkle,
-		utils.OverrideFeynman,
-		utils.OverrideFeynmanFix,
 		utils.OverrideFullImmutabilityThreshold,
 		utils.OverrideMinBlocksForBlobRequests,
 		utils.OverrideDefaultExtraReserveForBlobRequests,
+		utils.OverrideBreatheBlockInterval,
 		utils.EnablePersonal,
 		utils.TxPoolLocalsFlag,
 		utils.TxPoolNoLocalsFlag,
@@ -456,6 +457,10 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon
 		// Set the gas price to the limits from the CLI and start mining
 		gasprice := flags.GlobalBig(ctx, utils.MinerGasPriceFlag.Name)
 		ethBackend.TxPool().SetGasTip(gasprice)
+		gasCeil := ethBackend.Miner().GasCeil()
+		if gasCeil > params.SystemTxsGas {
+			ethBackend.TxPool().SetMaxGas(gasCeil - params.SystemTxsGas)
+		}
 		if err := ethBackend.StartMining(); err != nil {
 			utils.Fatalf("Failed to start mining: %v", err)
 		}

cmd/geth/snapshot.go

@@ -409,7 +409,7 @@ func pruneBlock(ctx *cli.Context) error {
 	}
 
 	if _, err := os.Stat(newAncientPath); err == nil {
-		// No file lock found for old ancientDB but new ancientDB exsisted, indicating the geth was interrupted
+		// No file lock found for old ancientDB but new ancientDB existed, indicating the geth was interrupted
 		// after old ancientDB removal, this happened after backup successfully, so just rename the new ancientDB
 		if err := blockpruner.AncientDbReplacer(); err != nil {
 			log.Error("Failed to rename new ancient directory")

cmd/jsutils/check_blobtx.js

@@ -0,0 +1,51 @@
+import { ethers } from "ethers";
+import program from "commander";
+
+// depends on ethjs v6.11.0+ for 4844, https://github.com/ethers-io/ethers.js/releases/tag/v6.11.0
+// BSC testnet enabled 4844 on block: 39539137
+// Usage:
+//   nvm use 20
+//   node check_blobtx.js --rpc https://data-seed-prebsc-1-s1.binance.org:8545 --startNum 39539137
+//   node check_blobtx.js --rpc https://data-seed-prebsc-1-s1.binance.org:8545 --startNum 39539137 --endNum 40345994
+program.option("--rpc <Rpc>", "Rpc Server URL");
+program.option("--startNum <Num>", "start block", 0);
+program.option("--endNum <Num>", "end block", 0);
+program.parse(process.argv);
+
+const provider = new ethers.JsonRpcProvider(program.rpc);
+const main = async () => {
+    var startBlock = parseInt(program.startNum)
+    var endBlock = parseInt(program.endNum)
+    if (isNaN(endBlock) || isNaN(startBlock) || startBlock == 0) {
+        console.error("invalid input, --startNum", program.startNum, "--end", program.endNum)
+        return
+    }
+    // if --endNum is not specified, set it to the latest block number.
+    if (endBlock == 0) {
+        endBlock = await provider.getBlockNumber();
+    }
+    if (startBlock > endBlock) {
+        console.error("invalid input, startBlock:",startBlock, " endBlock:", endBlock);
+        return
+    }
+
+    for (let i = startBlock; i <= endBlock; i++) {
+        let blockData = await provider.getBlock(i);
+        console.log("startBlock:",startBlock, "endBlock:", endBlock, "curBlock", i, "blobGasUsed", blockData.blobGasUsed);
+        if (blockData.blobGasUsed == 0) {
+            continue
+        }
+        for  (let txIndex = 0; txIndex<= blockData.transactions.length - 1; txIndex++) {
+            let txHash = blockData.transactions[txIndex]
+            let txData =  await provider.getTransaction(txHash);
+            if (txData.type == 3) {
+                console.log("BlobTx in block:",i, " txIndex:", txIndex, " txHash:", txHash);
+            }
+        }
+    }
+};
+main().then(() => process.exit(0))
+    .catch((error) => {
+        console.error(error);
+        process.exit(1);
+    });

cmd/utils/flags.go

@@ -315,15 +315,6 @@ var (
 		Usage:    "Manually specify the Verkle fork timestamp, overriding the bundled setting",
 		Category: flags.EthCategory,
 	}
-	OverrideFeynman = &cli.Uint64Flag{
-		Name:     "override.feynman",
-		Usage:    "Manually specify the Feynman fork timestamp, overriding the bundled setting",
-		Category: flags.EthCategory,
-	}
-	OverrideFeynmanFix = &cli.Uint64Flag{
-		Name:  "override.feynmanfix",
-		Usage: "Manually specify the FeynmanFix fork timestamp, overriding the bundled setting",
-	}
 	OverrideFullImmutabilityThreshold = &cli.Uint64Flag{
 		Name:     "override.immutabilitythreshold",
 		Usage:    "It is the number of blocks after which a chain segment is considered immutable, only for testing purpose",
@@ -342,6 +333,12 @@ var (
 		Value:    params.DefaultExtraReserveForBlobRequests,
 		Category: flags.EthCategory,
 	}
+	OverrideBreatheBlockInterval = &cli.Uint64Flag{
+		Name:     "override.breatheblockinterval",
+		Usage:    "It changes the interval between breathe blocks, only for testing purpose",
+		Value:    params.BreatheBlockInterval,
+		Category: flags.EthCategory,
+	}
 	SyncModeFlag = &flags.TextMarshalerFlag{
 		Name:     "syncmode",
 		Usage:    `Blockchain sync mode ("snap" or "full")`,

consensus/parlia/feynmanfork.go

@@ -15,14 +15,13 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
-const SecondsPerDay uint64 = 86400
-
 // the params should be two blocks' time(timestamp)
 func sameDayInUTC(first, second uint64) bool {
-	return first/SecondsPerDay == second/SecondsPerDay
+	return first/params.BreatheBlockInterval == second/params.BreatheBlockInterval
 }
 
 func isBreatheBlock(lastBlockTime, blockTime uint64) bool {

core/block_validator.go

@@ -66,6 +66,31 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin
 	return validator
 }
 
+// ValidateListsInBody validates that UncleHash, WithdrawalsHash, and WithdrawalsHash correspond to the lists in the block body, respectively.
+func ValidateListsInBody(block *types.Block) error {
+	header := block.Header()
+	if hash := types.CalcUncleHash(block.Uncles()); hash != header.UncleHash {
+		return fmt.Errorf("uncle root hash mismatch (header value %x, calculated %x)", header.UncleHash, hash)
+	}
+	if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash {
+		return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
+	}
+	// Withdrawals are present after the Shanghai fork.
+	if header.WithdrawalsHash != nil {
+		// Withdrawals list must be present in body after Shanghai.
+		if block.Withdrawals() == nil {
+			return errors.New("missing withdrawals in block body")
+		}
+		if hash := types.DeriveSha(block.Withdrawals(), trie.NewStackTrie(nil)); hash != *header.WithdrawalsHash {
+			return fmt.Errorf("withdrawals root hash mismatch (header value %x, calculated %x)", *header.WithdrawalsHash, hash)
+		}
+	} else if block.Withdrawals() != nil { // Withdrawals turn into empty from nil when BlockBody has Sidecars
+		// Withdrawals are not allowed prior to shanghai fork
+		return errors.New("withdrawals present in block body")
+	}
+	return nil
+}
+
 // ValidateBody validates the given block's uncles and verifies the block
 // header's transaction and uncle roots. The headers are assumed to be already
 // validated at this point.
@@ -83,31 +108,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if err := v.engine.VerifyUncles(v.bc, block); err != nil {
 		return err
 	}
-	if hash := types.CalcUncleHash(block.Uncles()); hash != header.UncleHash {
-		return fmt.Errorf("uncle root hash mismatch (header value %x, calculated %x)", header.UncleHash, hash)
-	}
 
 	validateFuns := []func() error{
 		func() error {
-			if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash {
-				return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
-			}
-			return nil
+			return ValidateListsInBody(block)
 		},
 		func() error {
-			// Withdrawals are present after the Shanghai fork.
-			if header.WithdrawalsHash != nil {
-				// Withdrawals list must be present in body after Shanghai.
-				if block.Withdrawals() == nil {
-					return errors.New("missing withdrawals in block body")
-				}
-				if hash := types.DeriveSha(block.Withdrawals(), trie.NewStackTrie(nil)); hash != *header.WithdrawalsHash {
-					return fmt.Errorf("withdrawals root hash mismatch (header value %x, calculated %x)", *header.WithdrawalsHash, hash)
-				}
-			} else if block.Withdrawals() != nil { // Withdrawals turn into empty from nil when BlockBody has Sidecars
-				// Withdrawals are not allowed prior to shanghai fork
-				return errors.New("withdrawals present in block body")
-			}
 			// Blob transactions may be present after the Cancun fork.
 			var blobs int
 			for i, tx := range block.Transactions() {

core/blockchain.go

@@ -71,6 +71,8 @@ var (
 	justifiedBlockGauge = metrics.NewRegisteredGauge("chain/head/justified", nil)
 	finalizedBlockGauge = metrics.NewRegisteredGauge("chain/head/finalized", nil)
 
+	blockInsertMgaspsGauge = metrics.NewRegisteredGauge("chain/insert/mgasps", nil)
+
 	chainInfoGauge = metrics.NewRegisteredGaugeInfo("chain/info", nil)
 
 	accountReadTimer   = metrics.NewRegisteredTimer("chain/account/reads", nil)

core/blockchain_insert.go

@@ -58,11 +58,13 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn
 		end := chain[index]
 
 		// Assemble the log context and send it to the logger
+		mgasps := float64(st.usedGas) * 1000 / float64(elapsed)
 		context := []interface{}{
 			"number", end.Number(), "hash", end.Hash(), "miner", end.Coinbase(),
 			"blocks", st.processed, "txs", txs, "blobs", blobs, "mgas", float64(st.usedGas) / 1000000,
-			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
+			"elapsed", common.PrettyDuration(elapsed), "mgasps", mgasps,
 		}
+		blockInsertMgaspsGauge.Update(int64(mgasps))
 		if timestamp := time.Unix(int64(end.Time()), 0); time.Since(timestamp) > time.Minute {
 			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
 		}

core/blockchain_reader.go

@@ -511,3 +511,12 @@ func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscr
 func (bc *BlockChain) SubscribeFinalizedHeaderEvent(ch chan<- FinalizedHeaderEvent) event.Subscription {
 	return bc.scope.Track(bc.finalizedHeaderFeed.Subscribe(ch))
 }
+
+// AncientTail retrieves the tail the ancients blocks
+func (bc *BlockChain) AncientTail() (uint64, error) {
+	tail, err := bc.db.Tail()
+	if err != nil {
+		return 0, err
+	}
+	return tail, nil
+}

core/genesis.go

@@ -216,10 +216,8 @@ func (e *GenesisMismatchError) Error() string {
 // ChainOverrides contains the changes to chain config
 // Typically, these modifications involve hardforks that are not enabled on the BSC mainnet, intended for testing purposes.
 type ChainOverrides struct {
-	OverrideCancun     *uint64
-	OverrideVerkle     *uint64
-	OverrideFeynman    *uint64
-	OverrideFeynmanFix *uint64
+	OverrideCancun *uint64
+	OverrideVerkle *uint64
 }
 
 // SetupGenesisBlock writes or updates the genesis block in db.
@@ -251,12 +249,6 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g
 			if overrides != nil && overrides.OverrideVerkle != nil {
 				config.VerkleTime = overrides.OverrideVerkle
 			}
-			if overrides != nil && overrides.OverrideFeynman != nil {
-				config.FeynmanTime = overrides.OverrideFeynman
-			}
-			if overrides != nil && overrides.OverrideFeynmanFix != nil {
-				config.FeynmanFixTime = overrides.OverrideFeynmanFix
-			}
 		}
 	}
 	// Just commit the new block if there is no stored genesis block.

core/rawdb/accessors_chain.go

@@ -316,8 +316,8 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu
 	if count == 0 {
 		return rlpHeaders
 	}
-	// read remaining from ancients
-	data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 0)
+	// read remaining from ancients, cap at 2M
+	data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 2*1024*1024)
 	if err != nil {
 		log.Error("Failed to read headers from freezer", "err", err)
 		return rlpHeaders