speedup import block processing

README.md

@@ -1,6 +1,6 @@
 
 <h1 align="center">Ontology</h1>
-<h4 align="center">Version 1.6.0</h4>
+<h4 align="center">Version 1.8.0</h4>
 
 [![GoDoc](https://godoc.org/github.com/ontio/ontology?status.svg)](https://godoc.org/github.com/ontio/ontology)
 [![Go Report Card](https://goreportcard.com/badge/github.com/ontio/ontology)](https://goreportcard.com/report/github.com/ontio/ontology)
@@ -49,7 +49,7 @@ New features are still being rapidly developed, therefore the master branch may
 ## Build Development Environment
 The requirements to build Ontology are:
 
-- [Golang](https://golang.org/doc/install) version 1.9 or later
+- [Golang](https://golang.org/doc/install) version 1.11 or later
 - [Glide](https://glide.sh) (a third party package management tool for Golang)
 
 ## Download Ontology

README_CN.md

@@ -1,6 +1,6 @@
 
 <h1 align="center">Ontology </h1>
-<h4 align="center">Version 1.0 </h4>
+<h4 align="center">Version 1.8.0 </h4>
 
 [![GoDoc](https://godoc.org/github.com/ontio/ontology?status.svg)](https://godoc.org/github.com/ontio/ontology)
 [![Go Report Card](https://goreportcard.com/badge/github.com/ontio/ontology)](https://goreportcard.com/report/github.com/ontio/ontology)
@@ -52,7 +52,7 @@ Ontology MainNet 已经在2018年6月30日成功上线。<br>
 ## 构建开发环境
 成功编译ontology需要以下准备：
 
-* Golang版本在1.9及以上
+* Golang版本在1.11及以上
 * 安装第三方包管理工具glide
 * 正确的Go语言开发环境
 * Golang所支持的操作系统

cmd/import_cmd.go

@@ -136,39 +136,56 @@ func importBlocks(ctx *cli.Context) error {
 
 	PrintInfoMsg("Start import blocks.")
 
-	for i := uint32(startBlockHeight); i <= endBlockHeight; i++ {
-		size, err := serialization.ReadUint32(fReader)
-		if err != nil {
-			return fmt.Errorf("read block height:%d error:%s", i, err)
-		}
-		compressData := make([]byte, size)
-		_, err = io.ReadFull(fReader, compressData)
-		if err != nil {
-			return fmt.Errorf("read block data height:%d error:%s", i, err)
+	var readErr error
+	blocks := make(chan *types.Block, 10)
+	go func() {
+		defer close(blocks)
+		for i := uint32(startBlockHeight); i <= endBlockHeight; i++ {
+			size, err := serialization.ReadUint32(fReader)
+			if err != nil {
+				readErr = fmt.Errorf("read block height:%d error:%s", i, err)
+				break
+			}
+			compressData := make([]byte, size)
+			_, err = io.ReadFull(fReader, compressData)
+			if err != nil {
+				readErr = fmt.Errorf("read block data height:%d error:%s", i, err)
+				break
+			}
+
+			if i <= currBlockHeight {
+				continue
+			}
+
+			blockData, err := utils.DecompressBlockData(compressData, metadata.CompressType)
+			if err != nil {
+				readErr = fmt.Errorf("block height:%d decompress error:%s", i, err)
+				break
+			}
+			block, err := types.BlockFromRawBytes(blockData)
+			if err != nil {
+				readErr = fmt.Errorf("block height:%d deserialize error:%s", i, err)
+				break
+			}
+			blocks <- block
 		}
+	}()
 
-		if i <= currBlockHeight {
-			continue
-		}
-
-		blockData, err := utils.DecompressBlockData(compressData, metadata.CompressType)
-		if err != nil {
-			return fmt.Errorf("block height:%d decompress error:%s", i, err)
-		}
-		block, err := types.BlockFromRawBytes(blockData)
-		if err != nil {
-			return fmt.Errorf("block height:%d deserialize error:%s", i, err)
-		}
+	for block := range blocks {
 		execResult, err := ledger.DefLedger.ExecuteBlock(block)
 		if err != nil {
-			return fmt.Errorf("block height:%d ExecuteBlock error:%s", i, err)
+			return fmt.Errorf("block height:%d ExecuteBlock error:%s", block.Header.Height, err)
 		}
 		err = ledger.DefLedger.SubmitBlock(block, execResult)
 		if err != nil {
-			return fmt.Errorf("SubmitBlock block height:%d error:%s", i, err)
+			return fmt.Errorf("SubmitBlock block height:%d error:%s", block.Header.Height, err)
 		}
 		bar.Incr()
 	}
+	if readErr != nil {
+		return readErr
+	}
+
 	uiprogress.Stop()
 	PrintInfoMsg("Import block completed, current block height:%d.", ledger.DefLedger.GetCurrentBlockHeight())
 	return nil

cmd/sigsvr/handlers/sig_native_invoke_tx.go

@@ -19,7 +19,6 @@
 package handlers
 
 import (
-	"bytes"
 	"encoding/hex"
 	"encoding/json"
 	"github.com/ontio/ontology/cmd/abi"
@@ -102,14 +101,7 @@ func SigNativeInvokeTx(req *clisvrcom.CliRpcRequest, resp *clisvrcom.CliRpcRespo
 		return
 	}
 
-	buf := bytes.NewBuffer(nil)
-	err = immutable.Serialize(buf)
-	if err != nil {
-		log.Infof("Cli Qid:%s SigNativeInvokeTx tx Serialize error:%s", req.Qid, err)
-		resp.ErrorCode = clisvrcom.CLIERR_INTERNAL_ERR
-		return
-	}
 	resp.Result = &SigNativeInvokeTxRsp{
-		SignedTx: hex.EncodeToString(buf.Bytes()),
+		SignedTx: hex.EncodeToString(common.SerializeToBytes(immutable)),
 	}
 }

cmd/sigsvr/handlers/sig_neovm_invoke_tx_abi.go

@@ -18,7 +18,6 @@
 package handlers
 
 import (
-	"bytes"
 	"encoding/hex"
 	"encoding/json"
 	clisvrcom "github.com/ontio/ontology/cmd/sigsvr/common"
@@ -107,16 +106,7 @@ func SigNeoVMInvokeAbiTx(req *clisvrcom.CliRpcRequest, resp *clisvrcom.CliRpcRes
 		resp.ErrorCode = clisvrcom.CLIERR_INTERNAL_ERR
 		return
 	}
-	sink := common.ZeroCopySink{}
-	tx.Serialization(&sink)
-	buf := bytes.NewBuffer(nil)
-	err = tx.Serialize(buf)
-	if err != nil {
-		log.Infof("Cli Qid:%s SigNeoVMInvokeAbiTx tx Serialize error:%s", req.Qid, err)
-		resp.ErrorCode = clisvrcom.CLIERR_INTERNAL_ERR
-		return
-	}
 	resp.Result = &SigNeoVMInvokeTxAbiRsp{
-		SignedTx: hex.EncodeToString(buf.Bytes()),
+		SignedTx: hex.EncodeToString(common.SerializeToBytes(tx)),
 	}
 }

cmd/utils/ont.go

@@ -38,6 +38,7 @@ import (
 	"github.com/ontio/ontology/smartcontract/service/native/ont"
 	"github.com/ontio/ontology/smartcontract/service/native/utils"
 	cstates "github.com/ontio/ontology/smartcontract/states"
+	"io"
 	"math/rand"
 	"sort"
 	"strconv"
@@ -436,12 +437,7 @@ func Sign(data []byte, signer *account.Account) ([]byte, error) {
 
 //SendRawTransaction send a transaction to ontology network, and return hash of the transaction
 func SendRawTransaction(tx *types.Transaction) (string, error) {
-	var buffer bytes.Buffer
-	err := tx.Serialize(&buffer)
-	if err != nil {
-		return "", fmt.Errorf("serialize error:%s", err)
-	}
-	txData := hex.EncodeToString(buffer.Bytes())
+	txData := hex.EncodeToString(common.SerializeToBytes(tx))
 	return SendRawTransactionData(txData)
 }
 
@@ -641,13 +637,11 @@ func PrepareDeployContract(
 	if err != nil {
 		return nil, fmt.Errorf("NewDeployCodeTransaction error:%s", err)
 	}
-	tx, _ := mutable.IntoImmutable()
-	var buffer bytes.Buffer
-	err = tx.Serialize(&buffer)
+	tx, err := mutable.IntoImmutable()
 	if err != nil {
-		return nil, fmt.Errorf("tx serialize error:%s", err)
+		return nil, err
 	}
-	txData := hex.EncodeToString(buffer.Bytes())
+	txData := hex.EncodeToString(common.SerializeToBytes(tx))
 	return PrepareSendRawTransaction(txData)
 }
 
@@ -710,12 +704,7 @@ func PrepareInvokeNeoVMContract(
 		return nil, err
 	}
 
-	var buffer bytes.Buffer
-	err = tx.Serialize(&buffer)
-	if err != nil {
-		return nil, fmt.Errorf("tx serialize error:%s", err)
-	}
-	txData := hex.EncodeToString(buffer.Bytes())
+	txData := hex.EncodeToString(common.SerializeToBytes(tx))
 	return PrepareSendRawTransaction(txData)
 }
 
@@ -728,12 +717,7 @@ func PrepareInvokeCodeNeoVMContract(code []byte) (*cstates.PreExecResult, error)
 	if err != nil {
 		return nil, err
 	}
-	var buffer bytes.Buffer
-	err = tx.Serialize(&buffer)
-	if err != nil {
-		return nil, fmt.Errorf("tx serialize error:%s", err)
-	}
-	txData := hex.EncodeToString(buffer.Bytes())
+	txData := hex.EncodeToString(common.SerializeToBytes(tx))
 	return PrepareSendRawTransaction(txData)
 }
 
@@ -749,12 +733,7 @@ func PrepareInvokeWasmVMContract(contractAddress common.Address, params []interf
 		return nil, err
 	}
 
-	var buffer bytes.Buffer
-	err = tx.Serialize(&buffer)
-	if err != nil {
-		return nil, fmt.Errorf("tx serialize error:%s", err)
-	}
-	txData := hex.EncodeToString(buffer.Bytes())
+	txData := hex.EncodeToString(common.SerializeToBytes(tx))
 	return PrepareSendRawTransaction(txData)
 }
 
@@ -771,12 +750,7 @@ func PrepareInvokeNativeContract(
 	if err != nil {
 		return nil, err
 	}
-	var buffer bytes.Buffer
-	err = tx.Serialize(&buffer)
-	if err != nil {
-		return nil, fmt.Errorf("tx serialize error:%s", err)
-	}
-	txData := hex.EncodeToString(buffer.Bytes())
+	txData := hex.EncodeToString(common.SerializeToBytes(tx))
 	return PrepareSendRawTransaction(txData)
 }
 
@@ -833,10 +807,13 @@ func ParseWasmVMContractReturnTypeByteArray(hexStr string) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("common.HexToBytes:%s error:%s", hexStr, err)
 	}
-	bf := bytes.NewBuffer(hexbs)
-	bs, err := serialization.ReadVarBytes(bf)
-	if err != nil {
-		return "", fmt.Errorf("ParseWasmVMContractReturnTypeByteArray:%s error:%s", hexStr, err)
+	source := common.NewZeroCopySource(hexbs)
+	bs, _, irregular, eof := source.NextVarBytes()
+	if irregular {
+		return "", fmt.Errorf("ParseWasmVMContractReturnTypeByteArray:%s error:%s", hexStr, common.ErrIrregularData)
+	}
+	if eof {
+		return "", fmt.Errorf("ParseWasmVMContractReturnTypeByteArray:%s error:%s", hexStr, io.ErrUnexpectedEOF)
 	}
 	return common.ToHexString(bs), nil
 }
@@ -847,8 +824,15 @@ func ParseWasmVMContractReturnTypeString(hexStr string) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("common.HexToBytes:%s error:%s", hexStr, err)
 	}
-	bf := bytes.NewBuffer(hexbs)
-	return serialization.ReadString(bf)
+	source := common.NewZeroCopySource(hexbs)
+	data, _, irregular, eof := source.NextString()
+	if irregular {
+		return "", common.ErrIrregularData
+	}
+	if eof {
+		return "", io.ErrUnexpectedEOF
+	}
+	return data, nil
 }
 
 //ParseWasmVMContractReturnTypeInteger return integer value of smart contract execute code.

common/address.go

@@ -22,11 +22,11 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
-	"io"
 	"math/big"
 
 	"github.com/itchyny/base58-go"
 	"golang.org/x/crypto/ripemd160"
+	"io"
 )
 
 const ADDR_LEN = 20
@@ -41,16 +41,16 @@ func (self *Address) ToHexString() string {
 }
 
 // Serialize serialize Address into io.Writer
-func (self *Address) Serialize(w io.Writer) error {
-	_, err := w.Write(self[:])
-	return err
+func (self *Address) Serialization(sink *ZeroCopySink) {
+	sink.WriteAddress(*self)
 }
 
 // Deserialize deserialize Address from io.Reader
-func (self *Address) Deserialize(r io.Reader) error {
-	_, err := io.ReadFull(r, self[:])
-	if err != nil {
-		return errors.New("deserialize Address error")
+func (self *Address) Deserialization(source *ZeroCopySource) error {
+	var eof bool
+	*self, eof = source.NextAddress()
+	if eof {
+		return io.ErrUnexpectedEOF
 	}
 	return nil
 }

common/address_test.go

@@ -19,7 +19,6 @@
 package common
 
 import (
-	"bytes"
 	"crypto/rand"
 	"testing"
 
@@ -55,10 +54,11 @@ func TestAddress_Serialize(t *testing.T) {
 	var addr Address
 	rand.Read(addr[:])
 
-	buf := bytes.NewBuffer(nil)
-	addr.Serialize(buf)
+	sink := NewZeroCopySink(nil)
+	addr.Serialization(sink)
 
 	var addr2 Address
-	addr2.Deserialize(buf)
+	source := NewZeroCopySource(sink.Bytes())
+	addr2.Deserialization(source)
 	assert.Equal(t, addr, addr2)
 }

common/codec.go

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2018 The ontology Authors
+ * This file is part of The ontology library.
+ *
+ * The ontology is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The ontology is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with The ontology.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package common
+
+type Serializable interface {
+	Serialization(sink *ZeroCopySink)
+}
+
+func SerializeToBytes(values ...Serializable) []byte {
+	sink := NewZeroCopySink(nil)
+	for _, val := range values {
+		val.Serialization(sink)
+	}
+
+	return sink.Bytes()
+}