Merge branch 'main' into rp/extract-minfee-on-upstream

.github/workflows/docker-build-publish.yml

@@ -21,7 +21,7 @@ jobs:
     permissions:
       contents: write
       packages: write
-    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.6
+    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.5.0
     with:
       dockerfile: docker/Dockerfile
       checkout_ref: ${{ github.event.inputs.ref }}
@@ -31,7 +31,7 @@ jobs:
     permissions:
       contents: write
       packages: write
-    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.6
+    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.5.0
     with:
       dockerfile: docker/txsim/Dockerfile
       packageName: txsim

.github/workflows/lint.yml

@@ -29,12 +29,12 @@ jobs:
 
   # hadolint lints the Dockerfile
   hadolint:
-    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.4.6
+    uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.5.0
     with:
       dockerfile: "docker/Dockerfile"
 
   yamllint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: celestiaorg/.github/.github/actions/yamllint@v0.4.6
+      - uses: celestiaorg/.github/.github/actions/yamllint@v0.5.0

.github/workflows/pr-review-requester.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   auto-request-review:
     name: Auto request reviews
-    uses: celestiaorg/.github/.github/workflows/reusable_housekeeping.yml@v0.4.6
+    uses: celestiaorg/.github/.github/workflows/reusable_housekeeping.yml@v0.5.0
     secrets: inherit
     # write access for issues and pull requests is needed because the called
     # workflow requires write access to issues and pull requests and the

Makefile

@@ -251,10 +251,10 @@ prebuilt-binary:
 ## check-bbr: Check if your system uses BBR congestion control algorithm. Only works on Linux.
 check-bbr:
 	@echo "Checking if BBR is enabled..."
-	@if [ "$(sysctl net.ipv4.tcp_congestion_control | awk '{print $3}')" != "bbr" ]; then \
-	    echo "WARNING: BBR is not enabled. Please enable BBR for optimal performance. Call make enable-bbr or see Usage section in the README."; \
+	@if [ "$$(sysctl net.ipv4.tcp_congestion_control | awk '{print $$3}')" != "bbr" ]; then \
+		echo "WARNING: BBR is not enabled. Please enable BBR for optimal performance. Call make enable-bbr or see Usage section in the README."; \
 	else \
-	    echo "BBR is enabled."; \
+		echo "BBR is enabled."; \
 	fi
 .PHONY: check-bbr
 
@@ -264,6 +264,7 @@ enable-bbr:
 	@if [ "$(sysctl net.ipv4.tcp_congestion_control | awk '{print $3}')" != "bbr" ]; then \
 	    echo "BBR is not enabled. Configuring BBR..."; \
 	    sudo modprobe tcp_bbr; \
+            echo tcp_bbr | sudo tee -a /etc/modules; \
 	    echo "net.core.default_qdisc=fq" | sudo tee -a /etc/sysctl.conf; \
 	    echo "net.ipv4.tcp_congestion_control=bbr" | sudo tee -a /etc/sysctl.conf; \
 	    sudo sysctl -p; \
@@ -273,8 +274,54 @@ enable-bbr:
 	fi
 .PHONY: enable-bbr
 
+## enable-mptcp: Enable mptcp over multiple ports (not interfaces). Only works on Linux Kernel 5.6 and above.
+enable-mptcp:
+	@echo "Configuring system to use mptcp..."
+	@sudo sysctl -w net.mptcp.enabled=1
+	@sudo sysctl -w net.mptcp.mptcp_path_manager=ndiffports
+	@sudo sysctl -w net.mptcp.mptcp_ndiffports=16
+	@echo "Making MPTCP settings persistent across reboots..."
+	@echo "net.mptcp.enabled=1" | sudo tee -a /etc/sysctl.conf
+	@echo "net.mptcp.mptcp_path_manager=ndiffports" | sudo tee -a /etc/sysctl.conf
+	@echo "net.mptcp.mptcp_ndiffports=16" | sudo tee -a /etc/sysctl.conf
+	@echo "MPTCP configuration complete and persistent!"
+
+.PHONY: enable-mptcp
+
+## disable-mptcp: Disables mptcp over multiple ports. Only works on Linux Kernel 5.6 and above.
+disable-mptcp:
+	@echo "Disabling MPTCP..."
+	@sudo sysctl -w net.mptcp.enabled=0
+	@sudo sysctl -w net.mptcp.mptcp_path_manager=default
+	@echo "Removing MPTCP settings from /etc/sysctl.conf..."
+	@sudo sed -i '/net.mptcp.enabled=1/d' /etc/sysctl.conf
+	@sudo sed -i '/net.mptcp.mptcp_path_manager=ndiffports/d' /etc/sysctl.conf
+	@sudo sed -i '/net.mptcp.mptcp_ndiffports=16/d' /etc/sysctl.conf
+	@echo "MPTCP configuration reverted!"
+
+.PHONY: disable-mptcp
+
+CONFIG_FILE ?= ${HOME}/.celestia-app/config/config.toml
+SEND_RECV_RATE ?= 10485760  # 10 MiB
+
+configure-v3:
+	@echo "Using config file at: $(CONFIG_FILE)"
+	@if [ "$$(uname)" = "Darwin" ]; then \
+		sed -i '' "s/^recv_rate = .*/recv_rate = $(SEND_RECV_RATE)/" $(CONFIG_FILE); \
+		sed -i '' "s/^send_rate = .*/send_rate = $(SEND_RECV_RATE)/" $(CONFIG_FILE); \
+		sed -i '' "s/ttl-num-blocks = .*/ttl-num-blocks = 12/" $(CONFIG_FILE); \
+	else \
+		sed -i "s/^recv_rate = .*/recv_rate = $(SEND_RECV_RATE)/" $(CONFIG_FILE); \
+		sed -i "s/^send_rate = .*/send_rate = $(SEND_RECV_RATE)/" $(CONFIG_FILE); \
+		sed -i "s/ttl-num-blocks = .*/ttl-num-blocks = 12/" $(CONFIG_FILE); \
+	fi
+
+.PHONY: configure-v3
+
+
 ## debug-version: Print the git tag and version.
 debug-version:
 	@echo "GIT_TAG: $(GIT_TAG)"
 	@echo "VERSION: $(VERSION)"
 .PHONY: debug-version
+

README.md

@@ -84,7 +84,7 @@ system's kernel. The result should contain `bbr`:
 sysctl net.ipv4.tcp_congestion_control
 ```
 
-If not, enable it on Linux by calling the `make use-bbr` or by running:
+If not, enable it on Linux by calling the `make enable-bbr` or by running:
 
 ```sh
 sudo modprobe tcp_bbr

app/default_overrides.go

@@ -274,6 +274,10 @@ func DefaultConsensusConfig() *tmcfg.Config {
 	cfg.TxIndex.Indexer = "null"
 	cfg.Storage.DiscardABCIResponses = true
 
+	const mebibyte = 1048576
+	cfg.P2P.SendRate = 10 * mebibyte
+	cfg.P2P.RecvRate = 10 * mebibyte
+
 	return cfg
 }
 

app/test/check_tx_test.go

@@ -189,11 +189,18 @@ func TestCheckTx(t *testing.T) {
 			checkType: abci.CheckTxType_New,
 			getTx: func() []byte {
 				signer := createSigner(t, kr, accs[10], encCfg.TxConfig, 11)
-				blob, err := share.NewV1Blob(share.RandomBlobNamespace(), []byte("data"), testnode.RandomAddress().(sdk.AccAddress))
+				blob, err := share.NewV1Blob(share.RandomBlobNamespace(), []byte("data"), signer.Account(accs[10]).Address())
 				require.NoError(t, err)
 				blobTx, _, err := signer.CreatePayForBlobs(accs[10], []*share.Blob{blob}, opts...)
 				require.NoError(t, err)
-				return blobTx
+				blob, err = share.NewV1Blob(share.RandomBlobNamespace(), []byte("data"), testnode.RandomAddress().(sdk.AccAddress))
+				require.NoError(t, err)
+				bTx, _, err := tx.UnmarshalBlobTx(blobTx)
+				require.NoError(t, err)
+				bTx.Blobs[0] = blob
+				blobTxBytes, err := tx.MarshalBlobTx(bTx.Tx, bTx.Blobs[0])
+				require.NoError(t, err)
+				return blobTxBytes
 			},
 			expectedABCICode: blobtypes.ErrInvalidBlobSigner.ABCICode(),
 		},

app/test/process_proposal_test.go

@@ -321,8 +321,9 @@ func TestProcessProposal(t *testing.T) {
 				falseAddr := testnode.RandomAddress().(sdk.AccAddress)
 				blob, err := share.NewV1Blob(ns1, data, falseAddr)
 				require.NoError(t, err)
-				msg, err := blobtypes.NewMsgPayForBlobs(addr.String(), appconsts.LatestVersion, blob)
+				msg, err := blobtypes.NewMsgPayForBlobs(falseAddr.String(), appconsts.LatestVersion, blob)
 				require.NoError(t, err)
+				msg.Signer = addr.String()
 
 				rawTx, err := signer.CreateTx([]sdk.Msg{msg}, user.SetGasLimit(100000), user.SetFee(100000))
 				require.NoError(t, err)

app/test/square_size_test.go

@@ -46,8 +46,10 @@ func (s *SquareSizeIntegrationTest) SetupSuite() {
 	t := s.T()
 	t.Log("setting up square size integration test")
 	s.ecfg = encoding.MakeConfig(app.ModuleEncodingRegisters...)
+
 	cfg := testnode.DefaultConfig().
-		WithModifiers(genesis.ImmediateProposals(s.ecfg.Codec))
+		WithModifiers(genesis.ImmediateProposals(s.ecfg.Codec)).
+		WithTimeoutCommit(time.Second)
 
 	cctx, rpcAddr, grpcAddr := testnode.NewNetwork(t, cfg)
 

app/test/upgrade_test.go

@@ -106,7 +106,7 @@ func TestAppUpgradeV3(t *testing.T) {
 
 	// brace yourselfs, this part may take a while
 	initialHeight := int64(4)
-	for height := initialHeight; height < initialHeight+appconsts.UpgradeHeightDelay(v2.Version); height++ {
+	for height := initialHeight; height < initialHeight+appconsts.UpgradeHeightDelay(testApp.GetChainID(), v2.Version); height++ {
 		appVersion := v2.Version
 		_ = testApp.BeginBlock(abci.RequestBeginBlock{
 			Header: tmproto.Header{
@@ -116,7 +116,7 @@ func TestAppUpgradeV3(t *testing.T) {
 		})
 
 		endBlockResp = testApp.EndBlock(abci.RequestEndBlock{
-			Height: 3 + appconsts.UpgradeHeightDelay(v2.Version),
+			Height: 3 + appconsts.UpgradeHeightDelay(testApp.GetChainID(), v2.Version),
 		})
 
 		require.Equal(t, appconsts.GetTimeoutCommit(appVersion), endBlockResp.Timeouts.TimeoutCommit)
@@ -141,7 +141,7 @@ func TestAppUpgradeV3(t *testing.T) {
 	_ = testApp.BeginBlock(abci.RequestBeginBlock{
 		Header: tmproto.Header{
 			ChainID: genesis.ChainID,
-			Height:  initialHeight + appconsts.UpgradeHeightDelay(v3.Version),
+			Height:  initialHeight + appconsts.UpgradeHeightDelay(testApp.GetChainID(), v3.Version),
 			Version: tmversion.Consensus{App: 3},
 		},
 	})
@@ -152,7 +152,7 @@ func TestAppUpgradeV3(t *testing.T) {
 	require.Equal(t, abci.CodeTypeOK, deliverTxResp.Code, deliverTxResp.Log)
 
 	respEndBlock := testApp.EndBlock(abci.
-		RequestEndBlock{Height: initialHeight + appconsts.UpgradeHeightDelay(v3.Version)})
+		RequestEndBlock{Height: initialHeight + appconsts.UpgradeHeightDelay(testApp.GetChainID(), v3.Version)})
 	require.Equal(t, appconsts.GetTimeoutCommit(v3.Version), respEndBlock.Timeouts.TimeoutCommit)
 	require.Equal(t, appconsts.GetTimeoutPropose(v3.Version), respEndBlock.Timeouts.TimeoutPropose)
 }

docs/release-notes/release-notes.md

@@ -6,12 +6,64 @@ This guide provides notes for major version releases. These notes may be helpful
 
 ### Node Operators (v3.0.0)
 
-- Consensus node operators must enable the BBR (Bottleneck Bandwidth and Round-trip propagation time) congestion control algorithm. See [#3774](https://github.com/celestiaorg/celestia-app/pull/3774).
-- Consensus node operators should manually configure their node's mempool `ttl-num-blocks = 12` in config.toml. An example command to do this:
+#### Enabling BBR and MCTCP
 
-  ```bash
-  sed -i 's/ttl-num-blocks = 5/ttl-num-blocks = 12/' ~/.celestia-app/config/config.toml
-  ```
+Consensus node operators must enable the BBR (Bottleneck Bandwidth and Round-trip propagation time) congestion control algorithm. See [#3774](https://github.com/celestiaorg/celestia-app/pull/3774).
+if using linux in docker, kubernetes, a vm or baremetal, this can be done by calling 
+
+```sh
+make enable-bbr 
+```
+
+command on the host machine.
+
+#### Configure Node for V3
+
+Consensus node operators should update several configurations for v3. This can be done by calling:
+
+```sh
+make configure-v3
+```
+
+If the config file is not in the default spot, it can be provided using:
+
+```sh
+make configure-v3 CONFIG_FILE=path/to/other/config.toml
+```
+
+**Alternatively**, the configurations can be changed manually. This involves updating the mempool TTLs and the send and the receive rates.
+
+- Configuring Bandwidth Settings
+  - update `recv_rate` and `send_rate` in your TOML config file to 10MiB (10485760).
+- Extend TTLs
+  - update `ttl-num-blocks` in your TOML config file to 12.
+
+#### Signaling Upgrades
+
+- Upgrades now use the `x/signal` module to coordinate the network to an upgrade height.
+
+The following command can be used, if you are a validator in the active set, to signal to upgrade to v3
+
+```bash
+celestia-appd tx signal signal 3 <plus transaction flags>
+```
+
+You can track the tally of signalling by validators using the following query
+
+```bash
+celestia-appd query signal tally 3
+```
+
+Once 5/6+ of the voting power have signalled, the upgrade will be ready. There is a hard coded delay between confirmation of the upgrade and execution to the new state machine. 
+
+To view the upcoming upgrade height use the following query:
+
+```bash
+celestia-appd query signal upgrade
+> An upgrade is pending to app version 3 at height 2348907.
+```
+
+For more information refer to the module [docs](../../x/signal/README.md)
 
 ### Library Consumers (v3.0.0)
 

go.mod

@@ -34,10 +34,10 @@ require (
 	github.com/tendermint/tm-db v0.6.7
 	golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e
 	google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38
-	google.golang.org/grpc v1.67.1
+	google.golang.org/grpc v1.68.0
 	google.golang.org/protobuf v1.35.1
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/apimachinery v0.31.1
+	k8s.io/apimachinery v0.31.2
 )
 
 require (
@@ -230,7 +230,7 @@ require (
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.27.0 // indirect
 	golang.org/x/net v0.29.0 // indirect
-	golang.org/x/oauth2 v0.22.0 // indirect
+	golang.org/x/oauth2 v0.23.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/sys v0.25.0 // indirect
 	golang.org/x/term v0.24.0 // indirect

go.sum

@@ -1544,8 +1544,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri
 golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
 golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
 golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A=
-golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
-golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
+golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1995,8 +1995,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu
 google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
 google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
 google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
-google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
-google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
+google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0=
+google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -2066,8 +2066,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI=
 k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI=
-k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U=
-k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw=
+k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
 k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50=
 k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=

pkg/appconsts/chain_ids.go

@@ -0,0 +1,5 @@
+package appconsts
+
+const (
+	ArabicaChainID = "arabica-11"
+)

pkg/appconsts/versioned_consts.go

@@ -79,7 +79,7 @@ func GetTimeoutCommit(v uint64) time.Duration {
 }
 
 // UpgradeHeightDelay returns the delay in blocks after a quorum has been reached that the chain should upgrade to the new version.
-func UpgradeHeightDelay(v uint64) int64 {
+func UpgradeHeightDelay(chainID string, v uint64) int64 {
 	if OverrideUpgradeHeightDelayStr != "" {
 		parsedValue, err := strconv.ParseInt(OverrideUpgradeHeightDelayStr, 10, 64)
 		if err != nil {
@@ -91,6 +91,12 @@ func UpgradeHeightDelay(v uint64) int64 {
 	case v1.Version:
 		return v1.UpgradeHeightDelay
 	case v2.Version:
+		// ONLY ON ARABICA: don't return the v2 value even when the app version is
+		// v2 on arabica. This is due to a bug that was shipped on arabica, where
+		// the next version was used.
+		if chainID == ArabicaChainID {
+			return v3.UpgradeHeightDelay
+		}
 		return v2.UpgradeHeightDelay
 	default:
 		return v3.UpgradeHeightDelay