encoding: eliminate unnecessary allocations (#5655)

crypto/merklesignature/persistentMerkleSignatureScheme.go

@@ -98,7 +98,8 @@ func (s *Secrets) Persist(store db.Accessor) error {
 		return fmt.Errorf("Secrets.Persist: %w", ErrKeyLifetimeIsZero)
 	}
 	round := indexToRound(s.FirstValid, s.KeyLifetime, 0)
-	encodedKey := protocol.GetEncodingBuf()
+	encodedBuf := protocol.GetEncodingBuf()
+	encodedKey := encodedBuf.Bytes()
 	err := store.Atomic(func(ctx context.Context, tx *sql.Tx) error {
 		err := InstallStateProofTable(tx) // assumes schema table already exists (created by partInstallDatabase)
 		if err != nil {
@@ -126,7 +127,7 @@ func (s *Secrets) Persist(store db.Accessor) error {
 
 		return nil
 	})
-	protocol.PutEncodingBuf(encodedKey)
+	protocol.PutEncodingBuf(encodedBuf.Update(encodedKey))
 	if err != nil {
 		return fmt.Errorf("Secrets.Persist: %w", err)
 	}

data/transactions/signedtxn.go

@@ -71,15 +71,17 @@ func (s SignedTxnInBlock) ID() {
 
 // GetEncodedLength returns the length in bytes of the encoded transaction
 func (s SignedTxn) GetEncodedLength() int {
-	enc := s.MarshalMsg(protocol.GetEncodingBuf())
-	defer protocol.PutEncodingBuf(enc)
+	buf := protocol.GetEncodingBuf()
+	enc := s.MarshalMsg(buf.Bytes())
+	defer protocol.PutEncodingBuf(buf.Update(enc))
 	return len(enc)
 }
 
 // GetEncodedLength returns the length in bytes of the encoded transaction
 func (s SignedTxnInBlock) GetEncodedLength() int {
-	enc := s.MarshalMsg(protocol.GetEncodingBuf())
-	defer protocol.PutEncodingBuf(enc)
+	buf := protocol.GetEncodingBuf()
+	enc := s.MarshalMsg(buf.Bytes())
+	defer protocol.PutEncodingBuf(buf.Update(enc))
 	return len(enc)
 }
 
@@ -116,16 +118,17 @@ func (s *SignedTxnInBlock) ToBeHashed() (protocol.HashID, []byte) {
 
 // Hash implements an optimized version of crypto.HashObj(s).
 func (s *SignedTxnInBlock) Hash() crypto.Digest {
-	enc := s.MarshalMsg(append(protocol.GetEncodingBuf(), []byte(protocol.SignedTxnInBlock)...))
-	defer protocol.PutEncodingBuf(enc)
-
+	buf := protocol.GetEncodingBuf()
+	enc := s.MarshalMsg(append(buf.Bytes(), []byte(protocol.SignedTxnInBlock)...))
+	defer protocol.PutEncodingBuf(buf.Update(enc))
 	return crypto.Hash(enc)
 }
 
 // HashSHA256 implements an optimized version of crypto.HashObj(s) using SHA256 instead of the default SHA512_256.
 func (s *SignedTxnInBlock) HashSHA256() crypto.Digest {
-	enc := s.MarshalMsg(append(protocol.GetEncodingBuf(), []byte(protocol.SignedTxnInBlock)...))
-	defer protocol.PutEncodingBuf(enc)
+	buf := protocol.GetEncodingBuf()
+	enc := s.MarshalMsg(append(buf.Bytes(), []byte(protocol.SignedTxnInBlock)...))
+	defer protocol.PutEncodingBuf(buf.Update(enc))
 
 	return sha256.Sum256(enc)
 }

data/transactions/teal.go

@@ -89,9 +89,13 @@ func (ed EvalDelta) Equal(o EvalDelta) bool {
 // tedious) field comparisons. == is not defined on almost any of the
 // subfields because of slices.
 func (stx SignedTxn) equal(o SignedTxn) bool {
-	stxenc := stx.MarshalMsg(protocol.GetEncodingBuf())
-	defer protocol.PutEncodingBuf(stxenc)
-	oenc := o.MarshalMsg(protocol.GetEncodingBuf())
-	defer protocol.PutEncodingBuf(oenc)
+	buf1 := protocol.GetEncodingBuf()
+	stxenc := stx.MarshalMsg(buf1.Bytes())
+	defer protocol.PutEncodingBuf(buf1.Update(stxenc))
+
+	buf2 := protocol.GetEncodingBuf()
+	oenc := o.MarshalMsg(buf2.Bytes())
+	defer protocol.PutEncodingBuf(buf2.Update(oenc))
+
 	return bytes.Equal(stxenc, oenc)
 }

data/transactions/transaction.go

@@ -21,6 +21,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"sync"
 
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/crypto"
@@ -177,31 +178,81 @@ func (tx Transaction) ToBeHashed() (protocol.HashID, []byte) {
 	return protocol.Transaction, protocol.Encode(&tx)
 }
 
+// txAllocSize returns the max possible size of a transaction without state proof fields.
+// It is used to preallocate a buffer for encoding a transaction.
+func txAllocSize() int {
+	return TransactionMaxSize() - StateProofTxnFieldsMaxSize()
+}
+
+// txEncodingPool holds temporary byte slice buffers used for encoding transaction messages.
+// Note, it prepends protocol.Transaction tag to the buffer economizing on subsequent append ops.
+var txEncodingPool = sync.Pool{
+	New: func() interface{} {
+		size := txAllocSize() + len(protocol.Transaction)
+		buf := make([]byte, len(protocol.Transaction), size)
+		copy(buf, []byte(protocol.Transaction))
+		return &txEncodingBuf{b: buf}
+	},
+}
+
+// getTxEncodingBuf returns a wrapped byte slice that can be used for encoding a
+// temporary message.  The byte slice length of encoded Transaction{} object.
+// The caller gets full ownership of the byte slice,
+// but is encouraged to return it using putEncodingBuf().
+func getTxEncodingBuf() *txEncodingBuf {
+	buf := txEncodingPool.Get().(*txEncodingBuf)
+	return buf
+}
+
+// putTxEncodingBuf places a byte slice into the pool of temporary buffers
+// for encoding.  The caller gives up ownership of the byte slice when
+// passing it to putTxEncodingBuf().
+func putTxEncodingBuf(buf *txEncodingBuf) {
+	buf.b = buf.b[:len(protocol.Transaction)]
+	txEncodingPool.Put(buf)
+}
+
+type txEncodingBuf struct {
+	b []byte
+}
+
 // ID returns the Txid (i.e., hash) of the transaction.
 func (tx Transaction) ID() Txid {
-	enc := tx.MarshalMsg(append(protocol.GetEncodingBuf(), []byte(protocol.Transaction)...))
-	defer protocol.PutEncodingBuf(enc)
+	buf := getTxEncodingBuf()
+	enc := tx.MarshalMsg(buf.b)
+	if cap(enc) > cap(buf.b) {
+		// use a bigger buffer as New's estimate was too small
+		buf.b = enc
+	}
+	defer putTxEncodingBuf(buf)
 	return Txid(crypto.Hash(enc))
 }
 
 // IDSha256 returns the digest (i.e., hash) of the transaction.
 // This is different from the canonical ID computed with Sum512_256 hashing function.
 func (tx Transaction) IDSha256() crypto.Digest {
-	enc := tx.MarshalMsg(append(protocol.GetEncodingBuf(), []byte(protocol.Transaction)...))
-	defer protocol.PutEncodingBuf(enc)
+	buf := getTxEncodingBuf()
+	enc := tx.MarshalMsg(buf.b)
+	if cap(enc) > cap(buf.b) {
+		buf.b = enc
+	}
+	defer putTxEncodingBuf(buf)
 	return sha256.Sum256(enc)
 }
 
 // InnerID returns something akin to Txid, but folds in the parent Txid and the
 // index of the inner call.
 func (tx Transaction) InnerID(parent Txid, index int) Txid {
-	input := append(protocol.GetEncodingBuf(), []byte(protocol.Transaction)...)
-	input = append(input, parent[:]...)
-	buf := make([]byte, 8)
-	binary.BigEndian.PutUint64(buf, uint64(index))
-	input = append(input, buf...)
+	buf := getTxEncodingBuf()
+	input := append(buf.b, parent[:]...)
+	var indexBuf [8]byte
+	binary.BigEndian.PutUint64(indexBuf[:], uint64(index))
+	input = append(input, indexBuf[:]...)
 	enc := tx.MarshalMsg(input)
-	defer protocol.PutEncodingBuf(enc)
+	if cap(enc) > cap(buf.b) {
+		buf.b = enc
+	}
+	defer putTxEncodingBuf(buf)
 	return Txid(crypto.Hash(enc))
 }
 

protocol/codec.go

@@ -288,21 +288,41 @@ func (d *MsgpDecoderBytes) Remaining() int {
 // encodingPool holds temporary byte slice buffers used for encoding messages.
 var encodingPool = sync.Pool{
 	New: func() interface{} {
-		return []byte{}
+		return &EncodingBuf{b: make([]byte, 0)}
 	},
 }
 
+// EncodingBuf is a wrapper for a byte slice that can be used for encoding
+type EncodingBuf struct {
+	b []byte
+}
+
+// Bytes returns the underlying byte slice
+func (eb *EncodingBuf) Bytes() []byte {
+	return eb.b
+}
+
+// Update updates the underlying byte slice to the given one if its capacity exceeds the current one.
+func (eb *EncodingBuf) Update(v []byte) *EncodingBuf {
+	if cap(eb.b) < cap(v) {
+		eb.b = v
+	}
+	return eb
+}
+
 // GetEncodingBuf returns a byte slice that can be used for encoding a
 // temporary message.  The byte slice has zero length but potentially
 // non-zero capacity.  The caller gets full ownership of the byte slice,
 // but is encouraged to return it using PutEncodingBuf().
-func GetEncodingBuf() []byte {
-	return encodingPool.Get().([]byte)[:0]
+func GetEncodingBuf() *EncodingBuf {
+	buf := encodingPool.Get().(*EncodingBuf)
+	buf.b = buf.b[:0]
+	return buf
 }
 
 // PutEncodingBuf places a byte slice into the pool of temporary buffers
 // for encoding.  The caller gives up ownership of the byte slice when
 // passing it to PutEncodingBuf().
-func PutEncodingBuf(s []byte) {
-	encodingPool.Put(s)
+func PutEncodingBuf(buf *EncodingBuf) {
+	encodingPool.Put(buf)
 }