SoR History update (#416)

* first update to handle possible gaps in history

* extend tests to cover single gap case

* update getRandomSourceHistoryPage and add tests for backfilled pages

* add test for non-contiguous gap edge case

* enforce minimum source length

* add RandomBeaconHistory.Backfiller and integrate with SoR commitment

* bump Flow CLI version in CI workflow

* simplify the backfilling logic

* update gapStartIndex init value

* add event for missing and backfilled SoRs

* update assets

* set and get for the backfiller limit

* add a note about the hash output size

* reduce calls to the large array length

* go generate

* move all backfilling logic inside the backfiller resource

* move correct index discover to Backfiller resource

* add coverage.lcov to .gitignore

* re-add early return to optimize common case

* rewrap contract comments

* update .borrowBackfiller() access & add event test coverage

* tests: add non-continuous gaps events check and use backfilling constant

* PR review: use array length to compare with empty array and minor changes

* make generate

* optimize unnecessary function calls

* minor optimization

---------

Co-authored-by: Giovanni Sanchez <[email protected]>

.github/workflows/ci.yml

@@ -11,14 +11,14 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v1
         with:
-          go-version: '1.21.5'
+          go-version: "1.21.5"
       - uses: actions/setup-node@v3
         with:
           node-version: 16
-          cache: 'npm'
+          cache: "npm"
           cache-dependency-path: lib/js/test/package-lock.json
       - name: Install Flow CLI
-        run: sh -ci "$(curl -fsSL https://raw.githubusercontent.com/onflow/flow-cli/master/install.sh)" -- v1.5.0
+        run: sh -ci "$(curl -fsSL https://raw.githubusercontent.com/onflow/flow-cli/master/install.sh)" -- v1.15.0
       - name: Flow cli Version
         run: flow version
       - name: Update PATH

.gitignore

@@ -14,4 +14,5 @@ node_modules
 git
 
 coverage.json
+coverage.lcov
 flow.json

contracts/RandomBeaconHistory.cdc

@@ -3,16 +3,16 @@
 /// This contract stores the history of random sources generated by the Flow network. The defined Heartbeat resource is
 /// updated by the Flow Service Account at the end of every block with that block's source of randomness.
 ///
-/// While the source values are safely generated by the Random Beacon (non-predictable, unbiasable, verifiable) and transmitted into the execution 
-/// environment via the committing transaction, using the raw values from this contract does not guarantee non-revertible
-/// randomness. The Hearbeat is intended to be used in conjunction with a
-/// commit-reveal mechanism to provide an onchain source of non-revertible randomness.
-// It is also recommended to use the source values with a pseudo-random number 
-// generator (PRNG) to generate an arbitrary-long sequence of random values. 
-// 
-// For usage of randomness where result abortion is not an issue, it is recommended
-// to use the Cadence built-in function `revertibleRandom`, which is also based on 
-// the safe Random Beacon. 
+/// While the source values are safely generated by the Random Beacon (non-predictable, unbiasable, verifiable) and
+/// transmitted into the execution environment via the committing transaction, using the raw values from this contract
+/// does not guarantee non-revertible randomness. The Hearbeat is intended to be used in conjunction with a commit-reveal
+/// mechanism to provide an onchain source of non-revertible randomness.
+/// It is also recommended to use the source values with a pseudo-random number
+/// generator (PRNG) to generate an arbitrary-long sequence of random values.
+///
+/// For usage of randomness where result abortion is not an issue, it is recommended
+/// to use the Cadence built-in function `revertibleRandom`, which is also based on
+/// the safe Random Beacon.
 ///
 /// Read the full FLIP here: https://github.com/onflow/flips/pull/123
 ///
@@ -26,6 +26,19 @@ access(all) contract RandomBeaconHistory {
     /// The path of the Heartbeat resource in the deployment account
     access(all) let HeartbeatStoragePath: StoragePath
 
+    // Event emitted when missing SoRs from past heartbeats are detected and will be backfilled:
+    //  - `blockHeight` is the height where the gap is detected
+    //  - `gapStartHeight` is the height of the first missing entry detected
+    access(all) event RandomHistoryMissing(blockHeight: UInt64, gapStartHeight: UInt64)
+
+    // Event emitted when missing SoRs are backfilled on the current heartbeat:
+    //  - `blockHeight` is the height where the backfill happened, it also defines the SoR used to backfill
+    //  - `gapStartHeight` is the height of the first backfilled entry
+    //  - `count` is the number of backfilled entries
+    // Note that in very rare cases, the backfilled gap may not be contiguous. This event does not
+    // fully define the backfilled entries in this case.
+    access(all) event RandomHistoryBackfilled(blockHeight: UInt64, gapStartHeight: UInt64, count: UInt64)
+
     /* --- Hearbeat --- */
     //
     /// The Heartbeat resource containing each block's source of randomness in sequence
@@ -36,25 +49,174 @@ access(all) contract RandomBeaconHistory {
         ///
         /// @param randomSourceHistory The random source to record
         ///
+        /// The Flow protocol makes sure to call this function once per block as a system call. The transaction
+        /// comes at the end of each block so that the current block's entry becomes available only in the child
+        /// block.
+        ///
         access(all) fun heartbeat(randomSourceHistory: [UInt8]) {
 
+            assert(
+                // random source must be at least 128 bits
+                randomSourceHistory.length >= 128 / 8,
+                message: "Random source must be at least 128 bits"
+            )
+
             let currentBlockHeight = getCurrentBlock().height
+            // init lowestBlockHeight if it is not set yet
             if RandomBeaconHistory.lowestHeight == nil {
                 RandomBeaconHistory.lowestHeight = currentBlockHeight
             }
+            var maybeBackfiller = RandomBeaconHistory.borrowBackfiller()
+            // Create & save Backfiller if it is not saved yet
+            if maybeBackfiller == nil {
+                RandomBeaconHistory.account.save(<-create Backfiller(), to: /storage/randomBeaconHistoryBackfiller)
+                maybeBackfiller = RandomBeaconHistory.borrowBackfiller()
+            }
+            let backfiller = maybeBackfiller ?? panic("Problem borrowing backfiller")
+
+            // check for any existing gap and backfill using the input random source if needed.
+            backfiller.backfill(randomSource: randomSourceHistory)
 
+            // we are now at the correct index to record the source of randomness
+            // created by the protocol for the current block
             RandomBeaconHistory.randomSourceHistory.append(randomSourceHistory)
         }
     }
 
+    /* --- Backfiller --- */
+    //
+    /// A recovery mechanism designed to backfill missed sources of randomness in the event of a missed commitment due
+    /// to a system transaction failure.
+    ///
+    access(all) resource Backfiller {
+        /// Start index of the first gap in the `randomSourceHistory` array where random sources were not recorded,
+        /// because of a heartbeat failure.
+        /// There may be non contiguous gaps in the history, `gapStartIndex` is the start index of the lowest-height
+        /// gap.
+        /// If no gaps exist, `gapStartIndex` is equal to the `randomSourceHistory` array length.
+        access(contract) var gapStartIndex: UInt64
+        /// BackFilling is limited to a maximum number of entries per call to limit the computation cost.
+        /// This means a large gap may need a few calls to get fully backfilled.
+        access(contract) var maxEntriesPerCall: UInt64
+
+        init() {
+            self.gapStartIndex = UInt64(RandomBeaconHistory.randomSourceHistory.length)
+            self.maxEntriesPerCall = 100
+        }
+
+        access(all) view fun getMaxEntriesPerCall() : UInt64 {
+            return self.maxEntriesPerCall
+        }
+
+        access(all) fun setMaxEntriesPerCall(max: UInt64) {
+            assert(
+                max > 0,
+                message: "the maximum entry per call must be strictly positive"
+            )
+            self.maxEntriesPerCall = max
+        }
+
+        /// Finds the correct index to fill with the new random source. If a gap is detected, emits the
+        /// RandomHistoryMissing event.
+        ///
+        access(contract) view fun findGapAndReturnCorrectIndex(): UInt64 {
+
+            let currentBlockHeight = getCurrentBlock().height
+            // correct index to fill with the new random source
+            // so that eventually randomSourceHistory[correctIndex] = inputRandom
+            let lowestHeight = RandomBeaconHistory.lowestHeight!
+            let correctIndex = currentBlockHeight - lowestHeight
+
+            // if a new gap is detected, emit an event
+            var arrayLength = UInt64(RandomBeaconHistory.randomSourceHistory.length)
+            if correctIndex > arrayLength {
+                let gapStartHeight = lowestHeight + arrayLength
+                emit RandomHistoryMissing(blockHeight: currentBlockHeight, gapStartHeight: gapStartHeight)
+            }
+            return correctIndex
+        }
+
+        /// Backfills possible empty entries (gaps) in the history array starting from the stored `gapStartIndex`,
+        /// using `randomSource` as a seed for all entries.
+        /// If there are no gaps, `gapStartIndex` is just updated to `RandomBeaconHistory`'s length.
+        //
+        /// When backfilling, all entries use the same entropy. Each entry is extracted from `randomSource` using
+        /// successive hashing. This makes sure the entries are all distinct although they provide
+        /// the same entropy.
+        //
+        /// gaps only occur in the rare event of a system transaction failure. In this case, entries are still
+        /// filled using a source not known at the time of block execution, which guarantees unpredicatability.
+        access(contract) fun backfill(randomSource: [UInt8]) {
+
+            let correctIndex = self.findGapAndReturnCorrectIndex()
+            var arrayLength = UInt64(RandomBeaconHistory.randomSourceHistory.length)
+            // optional optimization for the happy common path: if no gaps are detected,
+            // backfilling isn't needed, return early
+            if correctIndex == self.gapStartIndex  {
+                self.gapStartIndex = arrayLength + 1
+                return
+            }
+
+            // If a new gap is detected in the current transaction, fill the gap with empty entries.
+            // This happens in the rare case where a new gap occurs because of a system transaction failure.
+            while correctIndex > UInt64(RandomBeaconHistory.randomSourceHistory.length) {
+                RandomBeaconHistory.randomSourceHistory.append([])
+            }
+
+            arrayLength = UInt64(RandomBeaconHistory.randomSourceHistory.length)
+            var newEntry = randomSource
+            var index = self.gapStartIndex
+            var count = 0 as UInt64
+            while count < self.maxEntriesPerCall {
+                // move to the next empty entry
+                while index < arrayLength && RandomBeaconHistory.randomSourceHistory[index].length > 0 {
+                    index = index + 1
+                }
+                // if we reach the end of the array then all existing gaps got filled
+                if index == arrayLength {
+                    break
+                }
+                // back fill the empty entry
+                // It is guaranteed that sha3 output (256 bits) is larger than the minimum
+                // required size of an SoR (128 bits)
+                newEntry = HashAlgorithm.SHA3_256.hash(newEntry)
+                RandomBeaconHistory.randomSourceHistory[index] = newEntry
+                index = index + 1
+                count = count + 1
+            }
+
+            // emit an event about backfilled entries
+            if count > 0 {
+                let gapStartHeight = RandomBeaconHistory.lowestHeight! + self.gapStartIndex
+                emit RandomHistoryBackfilled(
+                    blockHeight: getCurrentBlock().height,
+                    gapStartHeight: gapStartHeight,
+                    count: count
+                )
+            }
+
+            // no more backfilling is possible but we need to update `gapStartIndex`
+            // to:
+            //  - the next empty index if gaps still exist
+            //  - the length of the array at the end of the transaction if there are no gaps
+            while index < arrayLength && RandomBeaconHistory.randomSourceHistory[index].length > 0 {
+                index = index + 1
+            }
+            if index == arrayLength {
+                index = index + 1   // take into account the upcoming append of the SoR at the correct index
+            }
+            self.gapStartIndex = index
+        }
+    }
+
     /* --- RandomSourceHistory --- */
     //
     /// Represents a random source value for a given block height
     ///
     access(all) struct RandomSource {
         access(all) let blockHeight: UInt64
         access(all) let value: [UInt8]
-    
+
         init(blockHeight: UInt64, value: [UInt8]) {
             self.blockHeight = blockHeight
             self.value = value
@@ -70,7 +232,7 @@ access(all) contract RandomBeaconHistory {
         access(all) let perPage: UInt64
         access(all) let totalLength: UInt64
         access(all) let values: [RandomSource]
-    
+
         init(page: UInt64, perPage: UInt64, totalLength: UInt64, values: [RandomSource]) {
             self.page = page
             self.perPage = perPage
@@ -97,13 +259,17 @@ access(all) contract RandomBeaconHistory {
         }
         let index = blockHeight - self.lowestHeight!
         assert(
-            index >= 0 && index < UInt64(self.randomSourceHistory.length),
+            index >= 0,
             message: "Problem finding random source history index"
         )
+        assert(
+            index < UInt64(self.randomSourceHistory.length) && self.randomSourceHistory[index].length > 0,
+            message: "Source of randomness is currently not available but will be available soon"
+        )
         return RandomSource(blockHeight: blockHeight, value: self.randomSourceHistory[index])
     }
 
-    /// Retrieves a page from the history of random sources, ordered chronologically
+    /// Retrieves a page from the history of random sources recorded so far, ordered chronologically
     ///
     /// @param page: The page number to retrieve, 0-indexed
     /// @param perPage: The number of random sources to include per page
@@ -126,18 +292,23 @@ access(all) contract RandomBeaconHistory {
         if endIndex > totalLength {
             endIndex = totalLength
         }
+
         // Return empty page if request exceeds last page
         if startIndex == endIndex {
             return RandomSourceHistoryPage(page: page, perPage: perPage, totalLength: totalLength, values: values)
         }
 
         // Iterate over history and construct page RandomSource values
         let lowestHeight = self.lowestHeight!
-        for i, block in self.randomSourceHistory.slice(from: Int(startIndex), upTo: Int(endIndex)) {
+        for i, value in self.randomSourceHistory.slice(from: Int(startIndex), upTo: Int(endIndex)) {
+            assert(
+                value.length > 0,
+                message: "Source of randomness is currently not available but will be available soon"
+            )
             values.append(
                 RandomSource(
                     blockHeight: lowestHeight + startIndex + UInt64(i),
-                    value: self.randomSourceHistory[startIndex + UInt64(i)]
+                    value: value
                 )
             )
         }
@@ -158,6 +329,11 @@ access(all) contract RandomBeaconHistory {
         return self.lowestHeight ?? panic("History has not yet been initialized")
     }
 
+    /// Getter for the contract's Backfiller resource
+    access(contract) fun borrowBackfiller(): &Backfiller? {
+        return self.account.borrow<&Backfiller>(from: /storage/randomBeaconHistoryBackfiller)
+    }
+
     init() {
         self.lowestHeight = nil
         self.randomSourceHistory = []