-
Notifications
You must be signed in to change notification settings - Fork 300
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* start spec compliant share merging * refactor and finish unit testing * whoops * linter gods * fix initial changes and use constants * use constant * more polish * docs fix Co-authored-by: Ismail Khoffi <[email protected]> * review feedback: docs and out of range panic protection * review feedback: add panic protection from empty input * use constant instead of recalculating `ShareSize` Co-authored-by: John Adler <[email protected]> * don't redeclare existing var Co-authored-by: John Adler <[email protected]> * be more explicit with returned nil Co-authored-by: John Adler <[email protected]> * use constant instead of recalculating `ShareSize` Co-authored-by: John Adler <[email protected]> * review feedback: use consistent capitalization * stop accepting reserved namespaces as normal messages * use a descriptive var name for message length * linter and comparison fix * reorg tests, add test for parse delimiter, DataFromBlock and fix evidence marshal bug * catch error for linter * update test MakeShares to include length delimiters for the SHARE_RESERVED_BYTE * minor iteration change * refactor share splitting to fix bug * fix all bugs with third and final refactor * fix conflict * revert unnecessary changes * review feedback: better docs Co-authored-by: Ismail Khoffi <[email protected]> * reivew feedback: add comment for safeLen * review feedback: remove unnecessay comments * review feedback: split up share merging and splitting into their own files * review feedback: more descriptive var names * fix accidental change * add some constant docs * spelling error Co-authored-by: Ismail Khoffi <[email protected]> Co-authored-by: John Adler <[email protected]>
- Loading branch information
3 people
authored
Apr 6, 2021
1 parent
93c88a1
commit 13575a6
Showing
7 changed files
with
831 additions
and
161 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,332 @@ | ||
| package types | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/binary" | ||
| "errors" | ||
|
|
||
| "github.com/gogo/protobuf/proto" | ||
| tmbytes "github.com/lazyledger/lazyledger-core/libs/bytes" | ||
| tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" | ||
| "github.com/lazyledger/rsmt2d" | ||
| ) | ||
|
|
||
| // DataFromSquare extracts block data from an extended data square. | ||
| func DataFromSquare(eds *rsmt2d.ExtendedDataSquare) (Data, error) { | ||
| originalWidth := eds.Width() / 2 | ||
|
|
||
| // sort block data shares by namespace | ||
| var ( | ||
| sortedTxShares [][]byte | ||
| sortedISRShares [][]byte | ||
| sortedEvdShares [][]byte | ||
| sortedMsgShares [][]byte | ||
| ) | ||
|
|
||
| // iterate over each row index | ||
| for x := uint(0); x < originalWidth; x++ { | ||
| // iterate over each col index | ||
| for y := uint(0); y < originalWidth; y++ { | ||
| // sort the data of that share types via namespace | ||
| share := eds.Cell(x, y) | ||
| nid := share[:NamespaceSize] | ||
| switch { | ||
| case bytes.Equal(TxNamespaceID, nid): | ||
| sortedTxShares = append(sortedTxShares, share) | ||
|
|
||
| case bytes.Equal(IntermediateStateRootsNamespaceID, nid): | ||
| sortedISRShares = append(sortedISRShares, share) | ||
|
|
||
| case bytes.Equal(EvidenceNamespaceID, nid): | ||
| sortedEvdShares = append(sortedEvdShares, share) | ||
|
|
||
| case bytes.Equal(TailPaddingNamespaceID, nid): | ||
| continue | ||
|
|
||
| // ignore unused but reserved namespaces | ||
| case bytes.Compare(nid, MaxReservedNamespace) < 1: | ||
| continue | ||
|
|
||
| // every other namespaceID should be a message | ||
| default: | ||
| sortedMsgShares = append(sortedMsgShares, share) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // pass the raw share data to their respective parsers | ||
| txs, err := parseTxs(sortedTxShares) | ||
| if err != nil { | ||
| return Data{}, err | ||
| } | ||
|
|
||
| isrs, err := parseISRs(sortedISRShares) | ||
| if err != nil { | ||
| return Data{}, err | ||
| } | ||
|
|
||
| evd, err := parseEvd(sortedEvdShares) | ||
| if err != nil { | ||
| return Data{}, err | ||
| } | ||
|
|
||
| msgs, err := parseMsgs(sortedMsgShares) | ||
| if err != nil { | ||
| return Data{}, err | ||
| } | ||
|
|
||
| return Data{ | ||
| Txs: txs, | ||
| IntermediateStateRoots: isrs, | ||
| Evidence: evd, | ||
| Messages: msgs, | ||
| }, nil | ||
| } | ||
|
|
||
| // parseTxs collects all of the transactions from the shares provided | ||
| func parseTxs(shares [][]byte) (Txs, error) { | ||
| // parse the sharse | ||
| rawTxs, err := processContiguousShares(shares) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // convert to the Tx type | ||
| txs := make(Txs, len(rawTxs)) | ||
| for i := 0; i < len(txs); i++ { | ||
| txs[i] = Tx(rawTxs[i]) | ||
| } | ||
|
|
||
| return txs, nil | ||
| } | ||
|
|
||
| // parseISRs collects all the intermediate state roots from the shares provided | ||
| func parseISRs(shares [][]byte) (IntermediateStateRoots, error) { | ||
| rawISRs, err := processContiguousShares(shares) | ||
| if err != nil { | ||
| return IntermediateStateRoots{}, err | ||
| } | ||
|
|
||
| ISRs := make([]tmbytes.HexBytes, len(rawISRs)) | ||
| for i := 0; i < len(ISRs); i++ { | ||
| ISRs[i] = rawISRs[i] | ||
| } | ||
|
|
||
| return IntermediateStateRoots{RawRootsList: ISRs}, nil | ||
| } | ||
|
|
||
| // parseEvd collects all evidence from the shares provided. | ||
| func parseEvd(shares [][]byte) (EvidenceData, error) { | ||
| // the raw data returned does not have length delimiters or namespaces and | ||
| // is ready to be unmarshaled | ||
| rawEvd, err := processContiguousShares(shares) | ||
| if err != nil { | ||
| return EvidenceData{}, err | ||
| } | ||
|
|
||
| evdList := make(EvidenceList, len(rawEvd)) | ||
|
|
||
| // parse into protobuf bytes | ||
| for i := 0; i < len(rawEvd); i++ { | ||
| // unmarshal the evidence | ||
| var protoEvd tmproto.Evidence | ||
| err := proto.Unmarshal(rawEvd[i], &protoEvd) | ||
| if err != nil { | ||
| return EvidenceData{}, err | ||
| } | ||
| evd, err := EvidenceFromProto(&protoEvd) | ||
| if err != nil { | ||
| return EvidenceData{}, err | ||
| } | ||
|
|
||
| evdList[i] = evd | ||
| } | ||
|
|
||
| return EvidenceData{Evidence: evdList}, nil | ||
| } | ||
|
|
||
| // parseMsgs collects all messages from the shares provided | ||
| func parseMsgs(shares [][]byte) (Messages, error) { | ||
| msgList, err := parseMsgShares(shares) | ||
| if err != nil { | ||
| return MessagesEmpty, err | ||
| } | ||
|
|
||
| return Messages{ | ||
| MessagesList: msgList, | ||
| }, nil | ||
| } | ||
|
|
||
| // processContiguousShares takes raw shares and extracts out transactions, | ||
| // intermediate state roots, or evidence. The returned [][]byte do have | ||
| // namespaces or length delimiters and are ready to be unmarshalled | ||
| func processContiguousShares(shares [][]byte) (txs [][]byte, err error) { | ||
| if len(shares) == 0 { | ||
| return nil, nil | ||
| } | ||
|
|
||
| ss := newShareStack(shares) | ||
| return ss.resolve() | ||
| } | ||
|
|
||
| // shareStack hold variables for peel | ||
| type shareStack struct { | ||
| shares [][]byte | ||
| txLen uint64 | ||
| txs [][]byte | ||
| cursor int | ||
| } | ||
|
|
||
| func newShareStack(shares [][]byte) *shareStack { | ||
| return &shareStack{shares: shares} | ||
| } | ||
|
|
||
| func (ss *shareStack) resolve() ([][]byte, error) { | ||
| if len(ss.shares) == 0 { | ||
| return nil, nil | ||
| } | ||
| err := ss.peel(ss.shares[0][NamespaceSize+ShareReservedBytes:], true) | ||
| return ss.txs, err | ||
| } | ||
|
|
||
| // peel recursively parses each chunk of data (either a transaction, | ||
| // intermediate state root, or evidence) and adds it to the underlying slice of data. | ||
| func (ss *shareStack) peel(share []byte, delimited bool) (err error) { | ||
| if delimited { | ||
| var txLen uint64 | ||
| share, txLen, err = parseDelimiter(share) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if txLen == 0 { | ||
| return nil | ||
| } | ||
| ss.txLen = txLen | ||
| } | ||
| // safeLen describes the point in the share where it can be safely split. If | ||
| // split beyond this point, it is possible to break apart a length | ||
| // delimiter, which will result in incorrect share merging | ||
| safeLen := len(share) - binary.MaxVarintLen64 | ||
| if safeLen < 0 { | ||
| safeLen = 0 | ||
| } | ||
| if ss.txLen <= uint64(safeLen) { | ||
| ss.txs = append(ss.txs, share[:ss.txLen]) | ||
| share = share[ss.txLen:] | ||
| return ss.peel(share, true) | ||
| } | ||
| // add the next share to the current share to continue merging if possible | ||
| if len(ss.shares) > ss.cursor+1 { | ||
| ss.cursor++ | ||
| share := append(share, ss.shares[ss.cursor][NamespaceSize+ShareReservedBytes:]...) | ||
| return ss.peel(share, false) | ||
| } | ||
| // collect any remaining data | ||
| if ss.txLen <= uint64(len(share)) { | ||
| ss.txs = append(ss.txs, share[:ss.txLen]) | ||
| share = share[ss.txLen:] | ||
| return ss.peel(share, true) | ||
| } | ||
| return errors.New("failure to parse block data: transaction length exceeded data length") | ||
| } | ||
|
|
||
| // parseMsgShares iterates through raw shares and separates the contiguous chunks | ||
| // of data. It is only used for Messages, i.e. shares with a non-reserved namespace. | ||
| func parseMsgShares(shares [][]byte) ([]Message, error) { | ||
| if len(shares) == 0 { | ||
| return nil, nil | ||
| } | ||
|
|
||
| // set the first nid and current share | ||
| nid := shares[0][:NamespaceSize] | ||
| currentShare := shares[0][NamespaceSize:] | ||
|
|
||
| // find and remove the msg len delimiter | ||
| currentShare, msgLen, err := parseDelimiter(currentShare) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| var msgs []Message | ||
| for cursor := uint64(0); cursor < uint64(len(shares)); { | ||
| var msg Message | ||
| currentShare, nid, cursor, msgLen, msg, err = nextMsg( | ||
| shares, | ||
| currentShare, | ||
| nid, | ||
| cursor, | ||
| msgLen, | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if msg.Data != nil { | ||
| msgs = append(msgs, msg) | ||
| } | ||
| } | ||
|
|
||
| return msgs, nil | ||
| } | ||
|
|
||
| func nextMsg( | ||
| shares [][]byte, | ||
| current, | ||
| nid []byte, | ||
| cursor, | ||
| msgLen uint64, | ||
| ) ([]byte, []byte, uint64, uint64, Message, error) { | ||
| switch { | ||
| // the message uses all of the current share data and at least some of the | ||
| // next share | ||
| case msgLen > uint64(len(current)): | ||
| // add the next share to the current one and try again | ||
| cursor++ | ||
| current = append(current, shares[cursor][NamespaceSize:]...) | ||
| return nextMsg(shares, current, nid, cursor, msgLen) | ||
|
|
||
| // the msg we're looking for is contained in the current share | ||
| case msgLen <= uint64(len(current)): | ||
| msg := Message{nid, current[:msgLen]} | ||
| cursor++ | ||
|
|
||
| // call it a day if the work is done | ||
| if cursor >= uint64(len(shares)) { | ||
| return nil, nil, cursor, 0, msg, nil | ||
| } | ||
|
|
||
| nextNid := shares[cursor][:NamespaceSize] | ||
| next, msgLen, err := parseDelimiter(shares[cursor][NamespaceSize:]) | ||
| return next, nextNid, cursor, msgLen, msg, err | ||
| } | ||
| // this code is unreachable but the compiler doesn't know that | ||
| return nil, nil, 0, 0, MessageEmpty, nil | ||
| } | ||
|
|
||
| // parseDelimiter finds and returns the length delimiter of the message provided | ||
| // while also removing the delimiter bytes from the input | ||
| func parseDelimiter(input []byte) ([]byte, uint64, error) { | ||
| if len(input) == 0 { | ||
| return input, 0, nil | ||
| } | ||
|
|
||
| l := binary.MaxVarintLen64 | ||
| if len(input) < binary.MaxVarintLen64 { | ||
| l = len(input) | ||
| } | ||
|
|
||
| delimiter := zeroPadIfNecessary(input[:l], binary.MaxVarintLen64) | ||
|
|
||
| // read the length of the message | ||
| r := bytes.NewBuffer(delimiter) | ||
| msgLen, err := binary.ReadUvarint(r) | ||
| if err != nil { | ||
| return nil, 0, err | ||
| } | ||
|
|
||
| // calculate the number of bytes used by the delimiter | ||
| lenBuf := make([]byte, binary.MaxVarintLen64) | ||
| n := binary.PutUvarint(lenBuf, msgLen) | ||
|
|
||
| // return the input without the length delimiter | ||
| return input[n:], msgLen, nil | ||
| } |
Oops, something went wrong.