Expired nonce pruning

signer/cosigner_nonce_cache.go

@@ -12,6 +12,7 @@ import (
 
 const defaultGetNoncesInterval = 3 * time.Second
 const defaultGetNoncesTimeout = 4 * time.Second
+const cachePreSize = 10000
 
 type CosignerNonceCache struct {
 	logger    cometlog.Logger
@@ -61,7 +62,7 @@ type NonceCache struct {
 
 func NewNonceCache() NonceCache {
 	return NonceCache{
-		cache: make(map[uuid.UUID]*CachedNonce, 10000),
+		cache: make(map[uuid.UUID]*CachedNonce, cachePreSize),
 	}
 }
 
@@ -98,6 +99,9 @@ type CachedNonce struct {
 	// UUID identifying this collection of nonces
 	UUID uuid.UUID
 
+	// Expiration time of this nonce
+	Expiration time.Time
+
 	// Cached nonces, cosigners which have this nonce in their metadata, ready to sign
 	Nonces []CosignerNoncesRel
 }
@@ -130,6 +134,9 @@ func (cnc *CosignerNonceCache) getUuids(n int) []uuid.UUID {
 }
 
 func (cnc *CosignerNonceCache) reconcile(ctx context.Context) {
+	// prune expired nonces
+	cnc.pruneNonces()
+
 	if !cnc.leader.IsLeader() {
 		return
 	}
@@ -191,6 +198,9 @@ func (cnc *CosignerNonceCache) LoadN(ctx context.Context, n int) {
 	nonces := make([]*CachedNonceSingle, len(cnc.cosigners))
 	var wg sync.WaitGroup
 	wg.Add(len(cnc.cosigners))
+
+	expiration := time.Now().Add(cnc.getNoncesInterval)
+
 	for i, p := range cnc.cosigners {
 		i := i
 		p := p
@@ -223,7 +233,8 @@ func (cnc *CosignerNonceCache) LoadN(ctx context.Context, n int) {
 	added := 0
 	for i, u := range uuids {
 		nonce := CachedNonce{
-			UUID: u,
+			UUID:       u,
+			Expiration: expiration,
 		}
 		num := uint8(0)
 		for _, n := range nonces {
@@ -302,6 +313,16 @@ CheckNoncesLoop:
 	return nil, fmt.Errorf("no nonces found involving cosigners %+v", cosignerInts)
 }
 
+func (cnc *CosignerNonceCache) pruneNonces() {
+	cnc.cache.mu.Lock()
+	defer cnc.cache.mu.Unlock()
+	for u, cn := range cnc.cache.cache {
+		if time.Now().After(cn.Expiration) {
+			delete(cnc.cache.cache, u)
+		}
+	}
+}
+
 func (cnc *CosignerNonceCache) clearNonce(uuid uuid.UUID) {
 	cnc.cache.mu.Lock()
 	defer cnc.cache.mu.Unlock()
@@ -334,5 +355,5 @@ func (cnc *CosignerNonceCache) ClearNonces(cosigner Cosigner) {
 func (cnc *CosignerNonceCache) ClearAllNonces() {
 	cnc.cache.mu.Lock()
 	defer cnc.cache.mu.Unlock()
-	cnc.cache.cache = make(map[uuid.UUID]*CachedNonce, 10000)
+	cnc.cache.cache = make(map[uuid.UUID]*CachedNonce, cachePreSize)
 }

signer/cosigner_nonce_cache_test.go

@@ -28,7 +28,7 @@ func TestNonceCacheDemand(t *testing.T) {
 
 	ctx, cancel := context.WithCancel(context.Background())
 
-	nonceCache.LoadN(ctx, 1000)
+	nonceCache.LoadN(ctx, 500)
 
 	go nonceCache.Start(ctx)
 

signer/local_cosigner.go

@@ -16,6 +16,8 @@ import (
 
 var _ Cosigner = &LocalCosigner{}
 
+const nonceExpiration = 10 * time.Second
+
 // LocalCosigner responds to sign requests.
 // It maintains a high watermark to avoid double-signing.
 // Signing is thread safe.
@@ -27,7 +29,7 @@ type LocalCosigner struct {
 	address       string
 	pendingDiskWG sync.WaitGroup
 
-	nonces map[uuid.UUID][]Nonces
+	nonces map[uuid.UUID]*NoncesWithExpiration
 	// protects the nonces map
 	noncesMu sync.RWMutex
 }
@@ -43,7 +45,7 @@ func NewLocalCosigner(
 		config:   config,
 		security: security,
 		address:  address,
-		nonces:   make(map[uuid.UUID][]Nonces),
+		nonces:   make(map[uuid.UUID]*NoncesWithExpiration),
 	}
 }
 
@@ -55,6 +57,31 @@ type ChainState struct {
 	signer ThresholdSigner
 }
 
+// StartNoncePruner periodically prunes nonces that have expired.
+func (cosigner *LocalCosigner) StartNoncePruner(ctx context.Context) {
+	ticker := time.NewTicker(nonceExpiration / 4)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			cosigner.pruneNonces()
+		}
+	}
+}
+
+// pruneNonces removes nonces that have expired.
+func (cosigner *LocalCosigner) pruneNonces() {
+	cosigner.noncesMu.Lock()
+	defer cosigner.noncesMu.Unlock()
+	now := time.Now()
+	for uuid, nonces := range cosigner.nonces {
+		if now.After(nonces.Expiration) {
+			delete(cosigner.nonces, uuid)
+		}
+	}
+}
+
 func (cosigner *LocalCosigner) combinedNonces(myID int, threshold uint8, uuid uuid.UUID) ([]Nonce, error) {
 	cosigner.noncesMu.RLock()
 	defer cosigner.noncesMu.RUnlock()
@@ -67,7 +94,7 @@ func (cosigner *LocalCosigner) combinedNonces(myID int, threshold uint8, uuid uu
 	combinedNonces := make([]Nonce, 0, threshold)
 
 	// calculate secret and public keys
-	for _, c := range nonces {
+	for _, c := range nonces.Nonces {
 		if len(c.Shares) == 0 || len(c.Shares[myID-1]) == 0 {
 			continue
 		}
@@ -289,6 +316,7 @@ func (cosigner *LocalCosigner) LoadSignStateIfNecessary(chainID string) error {
 	return nil
 }
 
+// GetNonces returns the nonces for the given UUIDs, generating if necessary.
 func (cosigner *LocalCosigner) GetNonces(
 	_ context.Context,
 	uuids []uuid.UUID,
@@ -355,7 +383,7 @@ func (cosigner *LocalCosigner) GetNonces(
 	return res, nil
 }
 
-func (cosigner *LocalCosigner) generateNoncesIfNecessary(uuid uuid.UUID) ([]Nonces, error) {
+func (cosigner *LocalCosigner) generateNoncesIfNecessary(uuid uuid.UUID) (*NoncesWithExpiration, error) {
 	// protects the meta map
 	cosigner.noncesMu.Lock()
 	defer cosigner.noncesMu.Unlock()
@@ -369,8 +397,13 @@ func (cosigner *LocalCosigner) generateNoncesIfNecessary(uuid uuid.UUID) ([]Nonc
 		return nil, err
 	}
 
-	cosigner.nonces[uuid] = newNonces
-	return newNonces, nil
+	res := NoncesWithExpiration{
+		Nonces:     newNonces,
+		Expiration: time.Now().Add(nonceExpiration),
+	}
+
+	cosigner.nonces[uuid] = &res
+	return &res, nil
 }
 
 // Get the ephemeral secret part for an ephemeral share
@@ -388,7 +421,7 @@ func (cosigner *LocalCosigner) getNonce(
 		return zero, err
 	}
 
-	ourCosignerMeta := meta[id-1]
+	ourCosignerMeta := meta.Nonces[id-1]
 	nonce, err := cosigner.security.EncryptAndSign(peerID, ourCosignerMeta.PubKey, ourCosignerMeta.Shares[peerID-1])
 	if err != nil {
 		return zero, err
@@ -414,7 +447,7 @@ func (cosigner *LocalCosigner) setNonce(uuid uuid.UUID, nonce CosignerNonce) err
 	cosigner.noncesMu.Lock()
 	defer cosigner.noncesMu.Unlock()
 
-	nonces, ok := cosigner.nonces[uuid]
+	n, ok := cosigner.nonces[uuid]
 	// generate metadata placeholder
 	if !ok {
 		return fmt.Errorf(
@@ -424,11 +457,11 @@ func (cosigner *LocalCosigner) setNonce(uuid uuid.UUID, nonce CosignerNonce) err
 	}
 
 	// set slot
-	if nonces[nonce.SourceID-1].Shares == nil {
-		nonces[nonce.SourceID-1].Shares = make([][]byte, len(cosigner.config.Config.ThresholdModeConfig.Cosigners))
+	if n.Nonces[nonce.SourceID-1].Shares == nil {
+		n.Nonces[nonce.SourceID-1].Shares = make([][]byte, len(cosigner.config.Config.ThresholdModeConfig.Cosigners))
 	}
-	nonces[nonce.SourceID-1].Shares[cosigner.GetID()-1] = nonceShare
-	nonces[nonce.SourceID-1].PubKey = noncePub
+	n.Nonces[nonce.SourceID-1].Shares[cosigner.GetID()-1] = nonceShare
+	n.Nonces[nonce.SourceID-1].PubKey = noncePub
 
 	return nil
 }

signer/threshold_signer.go

@@ -1,5 +1,7 @@
 package signer
 
+import "time"
+
 // Interface for the local signer whether it's a soft sign or HSM
 type ThresholdSigner interface {
 	// PubKey returns the public key bytes for the combination of all cosigners.
@@ -18,6 +20,11 @@ type Nonces struct {
 	Shares [][]byte
 }
 
+type NoncesWithExpiration struct {
+	Expiration time.Time
+	Nonces     []Nonces
+}
+
 // Nonce is the ephemeral information from another cosigner destined for this cosigner.
 type Nonce struct {
 	ID     int

signer/threshold_validator.go

@@ -106,6 +106,8 @@ func (pv *ThresholdValidator) Start(ctx context.Context) error {
 
 	go pv.nonceCache.Start(ctx)
 
+	go pv.myCosigner.StartNoncePruner(ctx)
+
 	return nil
 }