feat: Lazy share writers (#637)

* abci: PrepareProposal (#6544) * regenerate mocks, proto, mod/sum, and clean up remaining preprocesstxs * fix tests and revert to old go.mod * mockery * add processproposal proto/boilerplate/logic * gofmt * fix test * move UNKNOWN response behaviour to reject * fix test and add testing util code * pass full block data when proposing or processing proposals * linter * add the process proposal method to the e2e app * add missing kvstore abci method * pass block data and results for bass app * use correct kvstore process logic for kvstore app * add new lazy share writers * linter * remove unused arg * sort messages before exporting * formatting and bug fix * fix tests * allow for picking of square size when computing shares for the data square * remove accidental code duplication * fix test from using wrong formatting directive * linter * ci: backport lint configuration changes (#7225) * lint: cleanup pending lint errors (#7237) * linter * remove unused arg * sort messages before exporting * formatting and bug fix * fix tests * allow for picking of square size when computing shares for the data square * fix test from using wrong formatting directive * linter * ci: backport lint configuration changes (#7225) * lint: cleanup pending lint errors (#7237) * add new lazy share writers * fix rebase * linter * try ci with go 1.17 * Revert "try ci with go 1.17" This reverts commit 0f76b4d. * please work, linter gods * spelling Co-authored-by: Hlib Kanunnikov <[email protected]> * initialize pending share using the const share size for capacity * force the last reserve bytes to be zero * remove todo * add typecheck back to golang linter * Revert "ci: backport lint configuration changes (#7225)" This reverts commit 3517804. * Revert "lint: cleanup pending lint errors (#7237)" This reverts commit 5d80670. * add the link to issue back in * removed unfinished comment * switch fuzzer back to one minute * regenerate proto * better docs * fix encoding check to include the hash added to Data * add docs to CotiguousShareWriter * fix encoding check * explain why the share reserve byte is zero * use punctuation Co-authored-by: Ismail Khoffi <[email protected]> * use punctuation Co-authored-by: Ismail Khoffi <[email protected]> * use punctuation Co-authored-by: Ismail Khoffi <[email protected]> * use punctuation Co-authored-by: Ismail Khoffi <[email protected]> * use punctuation Co-authored-by: Ismail Khoffi <[email protected]> * use punctuation Co-authored-by: Ismail Khoffi <[email protected]> * use clearer wording for compute shares docs * more accurate docs Co-authored-by: Marko <[email protected]> Co-authored-by: mconcat <[email protected]> Co-authored-by: Sam Kleinman <[email protected]> Co-authored-by: Hlib Kanunnikov <[email protected]> Co-authored-by: Ismail Khoffi <[email protected]>
celestiaorg · Apr 25, 2022 · 7e29d65 · 7e29d65
1 parent 4c20ba2
commit 7e29d65
Show file tree

Hide file tree

Showing 13 changed files with 422 additions and 210 deletions.
diff --git a/.golangci.yml b/.golangci.yml
@@ -25,6 +25,7 @@ linters:
     - lll
     - misspell
     # - maligned
+    - misspell
     - nakedret
     - prealloc
     # - scopelint

diff --git a/blockchain/msgs_test.go b/blockchain/msgs_test.go
@@ -97,7 +97,7 @@ func TestBlockchainMessageVectors(t *testing.T) {
 			BlockRequest: &bcproto.BlockRequest{Height: math.MaxInt64}}},
 			"0a0a08ffffffffffffffff7f"},
 		{"BlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_BlockResponse{
-			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1a740a720a5b0a02080b1803220b088092b8c398feffffff012a0212003a20269ece38583f42aaf53fdd3abe1f570ab9b0d08284d080900966040a29df504c6a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512130a0b48656c6c6f20576f726c641a0022002801"},
+			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1a97010a94010a5b0a02080b1803220b088092b8c398feffffff012a0212003a20269ece38583f42aaf53fdd3abe1f570ab9b0d08284d080900966040a29df504c6a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85512350a0b48656c6c6f20576f726c641a00220028013220269ece38583f42aaf53fdd3abe1f570ab9b0d08284d080900966040a29df504c"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
 			NoBlockResponse: &bcproto.NoBlockResponse{Height: 1}}}, "12020801"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{

diff --git a/libs/protoio/io_test.go b/libs/protoio/io_test.go
@@ -97,10 +97,7 @@ func iotest(writer protoio.WriteCloser, reader protoio.ReadCloser) error {
 	if i != size {
 		panic("not enough messages read")
 	}
-	if err := reader.Close(); err != nil {
-		return err
-	}
-	return nil
+	return reader.Close()
 }
 
 type buffer struct {

diff --git a/node/node.go b/node/node.go
@@ -1046,11 +1046,8 @@ func (n *Node) ConfigureRPC() error {
 
 		Config: *n.config.RPC,
 	})
-	if err := rpccore.InitGenesisChunks(); err != nil {
-		return err
-	}
 
-	return nil
+	return rpccore.InitGenesisChunks()
 }
 
 func (n *Node) startRPC() ([]net.Listener, error) {
@@ -1395,11 +1392,8 @@ func saveGenesisDoc(db dbm.DB, genDoc *types.GenesisDoc) error {
 	if err != nil {
 		return fmt.Errorf("failed to save genesis doc due to marshaling error: %w", err)
 	}
-	if err := db.SetSync(genesisDocKey, b); err != nil {
-		return err
-	}
 
-	return nil
+	return db.SetSync(genesisDocKey, b)
 }
 
 func createAndStartPrivValidatorSocketClient(

diff --git a/pkg/prove/proof_test.go b/pkg/prove/proof_test.go
@@ -24,7 +24,8 @@ func TestTxInclusion(t *testing.T) {
 	}
 
 	// compute the data availability header
-	shares, _ := typicalBlockData.ComputeShares()
+	shares, _, err := typicalBlockData.ComputeShares(0)
+	require.NoError(t, err)
 
 	squareSize := uint(math.Sqrt(float64(len(shares))))
 
@@ -101,7 +102,9 @@ func Test_genRowShares(t *testing.T) {
 		Messages: generateRandomlySizedMessages(10, 1000),
 	}
 
-	allShares, _ := typicalBlockData.ComputeShares()
+	// compute the data availability header
+	allShares, _, err := typicalBlockData.ComputeShares(0)
+	require.NoError(t, err)
 	rawShares := allShares.RawShares()
 
 	originalSquareSize := uint(math.Sqrt(float64(len(rawShares))))
@@ -137,7 +140,9 @@ func Test_genOrigRowShares(t *testing.T) {
 		Messages: generateRandomlySizedMessages(10, 1500),
 	}
 
-	allShares, _ := typicalBlockData.ComputeShares()
+	// compute the data availability header
+	allShares, _, err := typicalBlockData.ComputeShares(0)
+	require.NoError(t, err)
 	rawShares := allShares.RawShares()
 
 	genShares := genOrigRowShares(typicalBlockData, 8, 0, 7)

diff --git a/proto/tendermint/abci/types.proto b/proto/tendermint/abci/types.proto
@@ -137,8 +137,8 @@ message RequestPrepareProposal {
 }
 
 message RequestProcessProposal {
-  tendermint.types.Header header = 1 [(gogoproto.nullable) = false];
-  tendermint.types.Data block_data = 2;
+  tendermint.types.Header header     = 1 [(gogoproto.nullable) = false];
+  tendermint.types.Data   block_data = 2;
 }
 
 //----------------------------------------
@@ -163,7 +163,7 @@ message Response {
     ResponseLoadSnapshotChunk  load_snapshot_chunk  = 15;
     ResponseApplySnapshotChunk apply_snapshot_chunk = 16;
     ResponsePrepareProposal    prepare_proposal     = 17;
-    ResponseProcessProposal     process_proposal     = 18;
+    ResponseProcessProposal    process_proposal     = 18;
   }
 }
 
@@ -299,13 +299,13 @@ message ResponsePrepareProposal {
 }
 
 message ResponseProcessProposal {
-  Result result = 1;
+  Result         result   = 1;
   repeated bytes evidence = 2;
 
   enum Result {
-    UNKNOWN       = 0;  // Unknown result, invalidate
-    ACCEPT        = 1;  // proposal verified, vote on the proposal
-    REJECT        = 2;  // proposal invalidated
+    UNKNOWN = 0;  // Unknown result, invalidate
+    ACCEPT  = 1;  // proposal verified, vote on the proposal
+    REJECT  = 2;  // proposal invalidated
   }
 }