Feature/go synchronizer (#53)

Co-authored-by: Shailu-s <[email protected]>
Co-authored-by: Shailendra Singh <[email protected]>

.gitignore

@@ -19,7 +19,7 @@ public.json
 *.wtns
 
 #.env
-.env
+sequencer/.env
 
 # etc
 .DS_Store

sequencer/common/account.go

@@ -26,7 +26,7 @@ type Account struct {
 }
 
 // AccountIdx represents the account Index in the MerkleTree
-type AccountIdx uint64
+type AccountIdx uint32
 
 const (
 	// NAccountLeafElems is the number of elements for a leaf in account tree
@@ -62,10 +62,10 @@ func (idx AccountIdx) Bytes() ([NLevelsAsBytes]byte, error) {
 	if idx > maxAccountIdxValue {
 		return [NLevelsAsBytes]byte{}, Wrap(ErrIdxOverflow)
 	}
-	var idxBytes [8]byte
-	binary.BigEndian.PutUint64(idxBytes[:], uint64(idx))
+	var idxBytes [4]byte
+	binary.BigEndian.PutUint32(idxBytes[:], uint32(idx))
 	var b [NLevelsAsBytes]byte
-	copy(b[:], idxBytes[8-NLevelsAsBytes:])
+	copy(b[:], idxBytes[4-NLevelsAsBytes:])
 	return b, nil
 }
 
@@ -75,9 +75,9 @@ func AccountIdxFromBytes(b []byte) (AccountIdx, error) {
 		return 0, Wrap(fmt.Errorf("can not parse Idx, bytes len %d, expected %d",
 			len(b), AccountIdxBytesLen))
 	}
-	var idxBytes [8]byte
-	copy(idxBytes[8-NLevelsAsBytes:], b[:])
-	idx := binary.BigEndian.Uint64(idxBytes[:])
+	var idxBytes [4]byte
+	copy(idxBytes[4-NLevelsAsBytes:], b[:])
+	idx := binary.BigEndian.Uint32(idxBytes[:])
 	return AccountIdx(idx), nil
 }
 
@@ -86,17 +86,17 @@ func (idx AccountIdx) BigInt() *big.Int {
 	return big.NewInt(int64(idx))
 }
 
-// IdxFromBytes returns Idx from a byte array
-func IdxFromBytes(b []byte) (AccountIdx, error) {
-	if len(b) != AccountIdxBytesLen {
-		return 0, Wrap(fmt.Errorf("can not parse Idx, bytes len %d, expected %d",
-			len(b), AccountIdxBytesLen))
-	}
-	var idxBytes [8]byte
-	copy(idxBytes[2:], b[:])
-	idx := binary.BigEndian.Uint64(idxBytes[:])
-	return AccountIdx(idx), nil
-}
+// // IdxFromBytes returns Idx from a byte array
+// func IdxFromBytes(b []byte) (AccountIdx, error) {
+// 	if len(b) != AccountIdxBytesLen {
+// 		return 0, Wrap(fmt.Errorf("can not parse Idx, bytes len %d, expected %d",
+// 			len(b), AccountIdxBytesLen))
+// 	}
+// 	var idxBytes [8]byte
+// 	copy(idxBytes[2:], b[:])
+// 	idx := binary.BigEndian.Uint32(idxBytes[:])
+// 	return AccountIdx(idx), nil
+// }
 
 // Bytes returns the bytes representing the Account, in a way that each BigInt
 // is represented by 32 bytes, in spite of the BigInt could be represented in

sequencer/common/errors.go

@@ -11,7 +11,7 @@ var ErrNotInFF = errors.New("BigInt not inside the Finite Field")
 var ErrNumOverflow = errors.New("Value overflows the type")
 
 // ErrIdxOverflow is used when a given nonce overflows the maximum capacity of the Idx (2**48-1)
-var ErrIdxOverflow = errors.New("idx overflow, max value: 2**48 -1")
+var ErrIdxOverflow = errors.New("idx overflow, max value: 2**24 -1")
 
 // ErrScoreOverflow is used when a given score overflows the maximum capacity of the Score (2**32-1)
 var ErrScoreOverflow = errors.New("Score overflow, max value: 2**32-1")

sequencer/common/l1tx.go

@@ -171,8 +171,8 @@ func (tx L1Tx) Tx() Tx {
 // [ 1 bits  ] empty (toBJJSign) // 1 byte
 // [ 8 bits  ] empty (userFee) // 1 byte
 // [ 40 bits ] empty (nonce) // 5 bytes
-// [ 48 bits ] toIdx // 6 bytes
-// [ 48 bits ] fromIdx // 6 bytes
+// [ 48 bits ] toIdx // 3 bytes
+// [ 48 bits ] fromIdx // 3 bytes
 // [ 16 bits ] chainId // 2 bytes
 // [ 32 bits ] empty (signatureConstant) // 4 bytes
 // Total bits compressed data:  225 bits // 29 bytes in *big.Int representation
@@ -213,7 +213,7 @@ func L1UserTxFromBytes(b []byte) (*L1Tx, error) {
 	pkCompB := b[20:52]
 	pkCompL := SwapEndianness(pkCompB)
 	copy(tx.FromBJJ[:], pkCompL)
-	fromIdx, err := IdxFromBytes(b[52:58])
+	fromIdx, err := AccountIdxFromBytes(b[52:55])
 	if err != nil {
 		return nil, Wrap(err)
 	}
@@ -226,7 +226,7 @@ func L1UserTxFromBytes(b []byte) (*L1Tx, error) {
 	if err != nil {
 		return nil, Wrap(err)
 	}
-	tx.ToIdx, err = IdxFromBytes(b[72:78])
+	tx.ToIdx, err = AccountIdxFromBytes(b[72:75])
 	if err != nil {
 		return nil, Wrap(err)
 	}
@@ -243,12 +243,12 @@ func L1TxFromDataAvailability(b []byte, nLevels uint32) (*L1Tx, error) {
 	amountBytes := b[idxLen*2 : idxLen*2+Float40BytesLength]
 
 	l1tx := L1Tx{}
-	fromIdx, err := IdxFromBytes(ethCommon.LeftPadBytes(fromIdxBytes, 6))
+	fromIdx, err := AccountIdxFromBytes(ethCommon.LeftPadBytes(fromIdxBytes, 3))
 	if err != nil {
 		return nil, Wrap(err)
 	}
 	l1tx.FromIdx = fromIdx
-	toIdx, err := IdxFromBytes(ethCommon.LeftPadBytes(toIdxBytes, 6))
+	toIdx, err := AccountIdxFromBytes(ethCommon.LeftPadBytes(toIdxBytes, 3))
 	if err != nil {
 		return nil, Wrap(err)
 	}

sequencer/common/l2tx.go

@@ -123,16 +123,16 @@ func L2TxFromBytesDataAvailability(b []byte, nLevels int) (*L2Tx, error) {
 	tx := &L2Tx{}
 	var err error
 
-	var paddedFromIdxBytes [6]byte
-	copy(paddedFromIdxBytes[6-idxLen:], b[0:idxLen])
-	tx.FromIdx, err = IdxFromBytes(paddedFromIdxBytes[:])
+	var paddedFromIdxBytes [3]byte
+	copy(paddedFromIdxBytes[3-idxLen:], b[0:idxLen])
+	tx.FromIdx, err = AccountIdxFromBytes(paddedFromIdxBytes[:])
 	if err != nil {
 		return nil, Wrap(err)
 	}
 
-	var paddedToIdxBytes [6]byte
-	copy(paddedToIdxBytes[6-idxLen:6], b[idxLen:idxLen*2])
-	tx.ToIdx, err = IdxFromBytes(paddedToIdxBytes[:])
+	var paddedToIdxBytes [3]byte
+	copy(paddedToIdxBytes[3-idxLen:3], b[idxLen:idxLen*2])
+	tx.ToIdx, err = AccountIdxFromBytes(paddedToIdxBytes[:])
 	if err != nil {
 		return nil, Wrap(err)
 	}

sequencer/common/token.go

@@ -25,4 +25,4 @@ func (t TokenID) Bytes() []byte {
 	var tokenIDBytes [4]byte
 	binary.BigEndian.PutUint32(tokenIDBytes[:], uint32(t))
 	return tokenIDBytes[:]
-}
+}

sequencer/common/vouch.go

@@ -38,16 +38,12 @@ func (idx VouchIdx) Bytes() ([2 * NLevelsAsBytes]byte, error) {
 }
 
 // GenerateVouchIdx
-func GenerateVouchIdx(fromIdx AccountIdx, toIdx AccountIdx) *big.Int {
-	// Create a new big.Int to hold the result
-	result := new(big.Int)
+func GenerateVouchIdx(fromIdx AccountIdx, toIdx AccountIdx) VouchIdx {
 
-	// Shift fromIdx left by 64 bits and add toIdx
-	result.SetUint64(uint64(fromIdx))
-	result.Lsh(result, 24)
-	result.Or(result, new(big.Int).SetUint64(uint64(toIdx)))
+	// Shift fromIdx left by 32 bits then concat with toIdx
+	vouchIdx := (uint64(fromIdx) << 32) | uint64(toIdx)
 
-	return result
+	return VouchIdx(vouchIdx)
 }
 
 func VouchIdxFromBytes(b []byte) (VouchIdx, error) {

sequencer/database/historydb/historydb.go

@@ -387,6 +387,9 @@ func (hdb *HistoryDB) AddL2Txs(l2txs []common.L2Tx) error {
 
 // addL2Txs inserts L2 txs to the DB. TokenID, USD and FeeUSD will be set automatically before storing the tx.
 func (hdb *HistoryDB) addL2Txs(d meddler.DB, l2txs []common.L2Tx) error {
+	if len(l2txs) == 0 {
+		return nil
+	}
 	txs := []txWrite{}
 	for i := 0; i < len(l2txs); i++ {
 		f := new(big.Float).SetInt(l2txs[i].Amount)
@@ -427,7 +430,6 @@ func (hdb *HistoryDB) addTxs(d meddler.DB, txs []txWrite) error {
 			to_idx,
 			amount,
 			amount_f,
-			token_id,
 			batch_num,
 			eth_block_num,
 			to_forge_l1_txs_num,
@@ -463,7 +465,7 @@ func (hdb *HistoryDB) GetAllL1UserTxs() ([]common.L1Tx, error) {
 	err := meddler.QueryAll(
 		hdb.dbRead, &txs,
 		`SELECT tx.id, tx.to_forge_l1_txs_num, tx.position, tx.user_origin,
-		tx.from_idx, tx.effective_from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx, tx.token_id,
+		tx.from_idx, tx.effective_from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx,
 		tx.amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.amount_success THEN tx.amount ELSE 0 END) AS effective_amount,
 		tx.deposit_amount, (CASE WHEN tx.batch_num IS NULL THEN NULL WHEN tx.deposit_amount_success THEN tx.deposit_amount ELSE 0 END) AS effective_deposit_amount,
 		tx.eth_block_num, tx.type, tx.batch_num
@@ -480,7 +482,7 @@ func (hdb *HistoryDB) GetAllL1CoordinatorTxs() ([]common.L1Tx, error) {
 	err := meddler.QueryAll(
 		hdb.dbRead, &txs,
 		`SELECT tx.id, tx.to_forge_l1_txs_num, tx.position, tx.user_origin,
-		tx.from_idx, tx.effective_from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx, tx.token_id,
+		tx.from_idx, tx.effective_from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx,
 		tx.amount, tx.amount AS effective_amount,
 		tx.deposit_amount, tx.deposit_amount AS effective_deposit_amount,
 		tx.eth_block_num, tx.type, tx.batch_num
@@ -495,8 +497,8 @@ func (hdb *HistoryDB) GetAllL2Txs() ([]common.L2Tx, error) {
 	err := meddler.QueryAll(
 		hdb.dbRead, &txs,
 		`SELECT tx.id, tx.batch_num, tx.position,
-		tx.from_idx, tx.to_idx, tx.amount, tx.token_id,
-		tx.fee, tx.nonce, tx.type, tx.eth_block_num
+		tx.from_idx, tx.to_idx, tx.amount,
+		tx.nonce, tx.type, tx.eth_block_num
 		FROM tx WHERE is_l1 = FALSE ORDER BY item_id;`,
 	)
 	return database.SlicePtrsToSlice(txs).([]common.L2Tx), common.Wrap(err)
@@ -508,7 +510,7 @@ func (hdb *HistoryDB) GetUnforgedL1UserTxs(toForgeL1TxsNum int64) ([]common.L1Tx
 	err := meddler.QueryAll(
 		hdb.dbRead, &txs, // only L1 user txs can have batch_num set to null
 		`SELECT tx.id, tx.to_forge_l1_txs_num, tx.position, tx.user_origin,
-		tx.from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx, tx.token_id,
+		tx.from_idx, tx.from_eth_addr, tx.from_bjj, tx.to_idx,
 		tx.amount, NULL AS effective_amount,
 		tx.deposit_amount, NULL AS effective_deposit_amount,
 		tx.eth_block_num, tx.type, tx.batch_num

sequencer/database/historydb/historydb_test.go

@@ -163,7 +163,7 @@ func TestBatches(t *testing.T) {
 			batch.Batch.GasPrice = big.NewInt(0)
 			batches = append(batches, batch.Batch)
 			forgeTxsNum := batch.Batch.ForgeL1TxsNum
-			if forgeTxsNum != nil && *lastL1TxsNum < *forgeTxsNum {
+			if forgeTxsNum != nil && (lastL1TxsNum == nil || *lastL1TxsNum < *forgeTxsNum) {
 				*lastL1TxsNum = *forgeTxsNum
 				lastL1BatchBlockNum = batch.Batch.EthBlockNum
 			}

sequencer/database/historydb/views.go

@@ -24,9 +24,9 @@ type txWrite struct {
 	ToIdx            common.AccountIdx  `meddler:"to_idx"`
 	Amount           *big.Int           `meddler:"amount,bigint"`
 	AmountFloat      float64            `meddler:"amount_f"`
-	TokenID          common.TokenID     `meddler:"token_id"`
-	BatchNum         *common.BatchNum   `meddler:"batch_num"`     // batchNum in which this tx was forged. If the tx is L2, this must be != 0
-	EthBlockNum      int64              `meddler:"eth_block_num"` // Ethereum Block Number in which this L1Tx was added to the queue
+	// TokenID          common.TokenID     `meddler:"token_id"`
+	BatchNum    *common.BatchNum `meddler:"batch_num"`     // batchNum in which this tx was forged. If the tx is L2, this must be != 0
+	EthBlockNum int64            `meddler:"eth_block_num"` // Ethereum Block Number in which this L1Tx was added to the queue
 	// L1
 	ToForgeL1TxsNum    *int64                 `meddler:"to_forge_l1_txs_num"` // toForgeL1TxsNum in which the tx was forged / will be forged
 	UserOrigin         *bool                  `meddler:"user_origin"`         // true if the tx was originated by a user, false if it was aoriginated by a coordinator. Note that this differ from the spec for implementation simplification purpposes

sequencer/database/migrations/0001.sql

@@ -495,7 +495,7 @@ BEGIN
             NEW.fee = (SELECT 0);
         END IF;
         -- Set token_id
-        NEW."token_id" = (SELECT token_id FROM account WHERE idx = NEW."from_idx");
+        -- NEW.token_id = (SELECT token_id FROM account WHERE idx = NEW.from_idx);
         -- Set from_{eth_addr,bjj}
         SELECT INTO NEW."from_eth_addr", NEW."from_bjj" eth_addr, bjj FROM account WHERE idx = NEW.from_idx;
     END IF;

sequencer/database/statedb/account_service.go

@@ -108,7 +108,7 @@ func (s *StateDB) GetAccount(idx common.AccountIdx) (*common.Account, error) {
 func accountsIter(db db.Storage, fn func(a *common.Account) (bool, error)) error {
 	idxDB := db.WithPrefix(PrefixKeyAccIdx)
 	if err := idxDB.Iterate(func(k []byte, v []byte) (bool, error) {
-		idx, err := common.IdxFromBytes(k)
+		idx, err := common.AccountIdxFromBytes(k)
 		if err != nil {
 			return false, common.Wrap(err)
 		}

sequencer/database/statedb/state_db.go

@@ -21,7 +21,7 @@ const (
 	TypeBatchBuilder = "batchbuilder"
 	// MaxNLevels is the maximum value of NLevels for the merkle tree,
 	// which comes from the fact that AccountIdx has 48 bits.
-	MaxNLevels = 48
+	MaxNLevels = 24
 )
 
 // Config of the StateDB

sequencer/eth/rollup.go

@@ -54,9 +54,9 @@ type RollupEventInitialize struct {
 
 // RollupEventL1UserTx is an event of the Rollup Smart Contract
 type RollupEventL1UserTx struct {
-	// ToForgeL1TxsNum int64 // QueueIndex       *big.Int
-	// Position        int   // TransactionIndex *big.Int
-	L1UserTx common.L1Tx
+	ToForgeL1TxsNum int64 // QueueIndex       *big.Int
+	Position        int   // TransactionIndex *big.Int
+	L1UserTx        common.L1Tx
 }
 
 // RollupEventL1UserTxAux is an event of the Rollup Smart Contract
@@ -654,7 +654,7 @@ func (c *RollupClient) RollupForgeBatchArgs(ethTxHash ethCommon.Hash,
 			copy(paddedFeeIdx[:],
 				aux.FeeIdxCoordinator[i*lenFeeIdxCoordinatorBytes:(i+1)*lenFeeIdxCoordinatorBytes])
 		}
-		feeIdxCoordinator, err := common.IdxFromBytes(paddedFeeIdx[:])
+		feeIdxCoordinator, err := common.AccountIdxFromBytes(paddedFeeIdx[:])
 		if err != nil {
 			return nil, nil, common.Wrap(err)
 		}

sequencer/go.mod

@@ -11,11 +11,10 @@ require (
 	github.com/gin-gonic/gin v1.7.2
 	github.com/go-playground/validator v9.31.0+incompatible
 	github.com/gobuffalo/packr/v2 v2.8.1
-	github.com/hermeznetwork/tracerr v0.3.2
 	github.com/iden3/go-iden3-crypto v0.0.6-0.20210308142348-8f85683b2cef
 	github.com/iden3/go-merkletree v0.0.0-20210308143313-8b63ca866189
 	github.com/jmoiron/sqlx v1.2.1-0.20200615141059-0794cb1f47ee
-	github.com/joho/godotenv v1.5.1
+	github.com/joho/godotenv v1.3.0
 	github.com/lib/pq v1.8.0
 	github.com/libp2p/go-libp2p-core v0.8.6
 	github.com/libp2p/go-libp2p-discovery v0.5.1
@@ -26,10 +25,10 @@ require (
 	github.com/prometheus/client_golang v1.10.0
 	github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351
 	github.com/russross/meddler v1.0.1
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli v1.22.1
 	go.uber.org/zap v1.16.0
-	golang.org/x/sync v0.8.0
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 )
 
 require (
@@ -61,7 +60,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.4.3 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/google/uuid v1.2.0 // indirect
@@ -83,10 +81,10 @@ require (
 	github.com/json-iterator/go v1.1.10 // indirect
 	github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 // indirect
 	github.com/karrick/godirwalk v1.15.8 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.4 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
-	github.com/kr/pretty v0.3.1 // indirect
-	github.com/kr/text v0.2.0 // indirect
+	github.com/kr/pretty v0.2.1 // indirect
+	github.com/kr/text v0.1.0 // indirect
 	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/libp2p/go-buffer-pool v0.0.2 // indirect
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
@@ -101,12 +99,10 @@ require (
 	github.com/libp2p/go-netroute v0.1.6 // indirect
 	github.com/libp2p/go-openssl v0.0.7 // indirect
 	github.com/libp2p/go-sockaddr v0.1.1 // indirect
-	github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e // indirect
 	github.com/markbates/errx v1.1.0 // indirect
 	github.com/markbates/oncer v1.0.0 // indirect
 	github.com/markbates/safe v1.0.1 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
@@ -134,7 +130,6 @@ require (
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/prometheus/tsdb v0.7.1 // indirect
 	github.com/rjeczalik/notify v0.9.2 // indirect
-	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/russross/blackfriday/v2 v2.0.1 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
@@ -151,17 +146,16 @@ require (
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
 	golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect
-	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/sys v0.25.0 // indirect
-	golang.org/x/term v0.24.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
-	golang.org/x/tools v0.21.0 // indirect
+	golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6 // indirect
+	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
+	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
+	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/gorp.v1 v1.7.2 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v2 v2.3.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )