chain-libs Serialization/Deserialization update, remove usage of the …

…bincode crate (left only in the jcli) (#3712)

* fix build

* update with the latest lib

* update error

* fix Gossipe serde issue

* add comment

* remove usage of the  bincode crate

* bump version

* bump version

* update chain-libs

* bump ed25519-bip32

* update lib

explorer/src/main.rs

@@ -6,14 +6,11 @@ mod settings;
 
 use crate::indexer::Indexer;
 use anyhow::Context;
+use chain_core::{packer::Codec, property::Deserialize};
 use chain_impl_mockchain::block::Block;
 use chain_network::grpc::watch::client::{
     BlockSubscription, Client, SyncMultiverseStream, TipSubscription,
 };
-use chain_ser::{
-    deser::Deserialize,
-    mempack::{ReadBuf, Readable},
-};
 use db::ExplorerDb;
 use futures::stream::StreamExt;
 use futures_util::{future, pin_mut, FutureExt, TryFutureExt};
@@ -233,9 +230,9 @@ async fn bootstrap(mut sync_stream: SyncMultiverseStream) -> Result<ExplorerDb,
             .context("failed to decode Block received through bootstrap subscription")
             .map_err(Error::UnrecoverableError)?;
 
-        let reader = std::io::BufReader::new(bytes.as_ref());
+        let mut codec = Codec::new(bytes.as_ref());
 
-        let block = Block::deserialize(reader)
+        let block = Block::deserialize(&mut codec)
             .context("failed to decode Block received through bootstrap subscription")
             .map_err(Error::UnrecoverableError)?;
 
@@ -308,8 +305,8 @@ async fn rest_service(mut state: broadcast::Receiver<GlobalState>, settings: Set
 }
 
 async fn handle_tip(raw_tip: chain_network::data::Header, indexer: Indexer) -> Result<(), Error> {
-    let mut buf = ReadBuf::from(raw_tip.as_bytes());
-    let header = chain_impl_mockchain::block::Header::read(&mut buf)
+    let mut codec = Codec::new(raw_tip.as_bytes());
+    let header = chain_impl_mockchain::block::Header::deserialize(&mut codec)
         .context("failed to decode tip header")
         .map_err(Error::Other)?;
 
@@ -322,8 +319,8 @@ async fn handle_block(
     raw_block: chain_network::data::Block,
     indexer: Indexer,
 ) -> Result<(), Error> {
-    let reader = std::io::BufReader::new(raw_block.as_bytes());
-    let block = Block::deserialize(reader)
+    let mut codec = chain_core::packer::Codec::new(raw_block.as_bytes());
+    let block = Block::deserialize(&mut codec)
         .context("Failed to deserialize block from block subscription")
         .map_err(Error::Other)?;
 

jcli/src/jcli_lib/block/mod.rs

@@ -1,5 +1,8 @@
 use crate::jcli_lib::utils::io;
-use chain_core::property::{Block as _, Deserialize, Serialize};
+use chain_core::{
+    packer::Codec,
+    property::{Block as _, Deserialize, ReadError, Serialize, WriteError},
+};
 use chain_impl_mockchain::{
     block::Block,
     ledger::{self, Ledger},
@@ -30,13 +33,13 @@ pub enum Error {
         path: PathBuf,
     },
     #[error("block file corrupted")]
-    BlockFileCorrupted(#[source] std::io::Error),
+    BlockFileCorrupted(#[source] ReadError),
     #[error("genesis file corrupted")]
     GenesisFileCorrupted(#[source] serde_yaml::Error),
     #[error("generated block is not a valid genesis block")]
     GeneratedBlock0Invalid(#[from] ledger::Error),
     #[error("failed to serialize block")]
-    BlockSerializationFailed(#[source] std::io::Error),
+    BlockSerializationFailed(#[source] WriteError),
     #[error("failed to serialize genesis")]
     GenesisSerializationFailed(#[source] serde_yaml::Error),
     #[error("failed to build genesis from block 0")]
@@ -68,7 +71,7 @@ fn encode_block_0(common: Common) -> Result<(), Error> {
     let block = genesis.to_block();
     Ledger::new(block.id(), block.fragments())?;
     block
-        .serialize(common.open_output()?)
+        .serialize(&mut Codec::new(common.open_output()?))
         .map_err(Error::BlockSerializationFailed)
 }
 
@@ -133,7 +136,7 @@ pub fn open_block_file(input_file: &Option<PathBuf>) -> Result<impl BufRead, Err
 }
 
 pub fn load_block(block_reader: impl BufRead) -> Result<Block, Error> {
-    Block::deserialize(block_reader).map_err(Error::BlockFileCorrupted)
+    Block::deserialize(&mut Codec::new(block_reader)).map_err(Error::BlockFileCorrupted)
 }
 
 #[derive(StructOpt)]

jcli/src/jcli_lib/debug/block.rs

@@ -1,4 +1,5 @@
 use crate::jcli_lib::{debug::Error, utils::io};
+use chain_core::packer::Codec;
 use chain_core::property::Deserialize as _;
 use chain_impl_mockchain::block::Block as BlockMock;
 use std::io::{BufRead, BufReader};
@@ -21,7 +22,8 @@ impl Block {
         let mut hex_str = String::new();
         BufReader::new(reader).read_line(&mut hex_str)?;
         let bytes = hex::decode(hex_str.trim())?;
-        let message = BlockMock::deserialize(bytes.as_ref()).map_err(Error::MessageMalformed)?;
+        let message = BlockMock::deserialize(&mut Codec::new(bytes.as_slice()))
+            .map_err(Error::MessageMalformed)?;
         println!("{:#?}", message);
         Ok(())
     }

jcli/src/jcli_lib/debug/message.rs

@@ -1,5 +1,6 @@
 use crate::jcli_lib::{debug::Error, utils::io};
-use chain_core::property::Deserialize as _;
+use chain_core::packer::Codec;
+use chain_core::property::DeserializeFromSlice as _;
 use chain_impl_mockchain::fragment::Fragment as MockFragment;
 use std::io::{BufRead, BufReader};
 use std::path::PathBuf;
@@ -21,7 +22,8 @@ impl Message {
         let mut hex_str = String::new();
         BufReader::new(reader).read_line(&mut hex_str)?;
         let bytes = hex::decode(hex_str.trim())?;
-        let message = MockFragment::deserialize(bytes.as_ref()).map_err(Error::MessageMalformed)?;
+        let message = MockFragment::deserialize_from_slice(&mut Codec::new(bytes.as_slice()))
+            .map_err(Error::MessageMalformed)?;
         println!("{:#?}", message);
         Ok(())
     }

jcli/src/jcli_lib/debug/mod.rs

@@ -1,5 +1,6 @@
 mod block;
 mod message;
+use chain_core::property::ReadError;
 use hex::FromHexError;
 use std::path::PathBuf;
 use structopt::StructOpt;
@@ -27,7 +28,7 @@ pub enum Error {
     #[error("hex encoding malformed")]
     HexMalformed(#[from] FromHexError),
     #[error("message malformed")]
-    MessageMalformed(#[source] std::io::Error),
+    MessageMalformed(#[source] ReadError),
 }
 
 impl Debug {

jcli/src/jcli_lib/rest/mod.rs

@@ -3,6 +3,7 @@ pub mod v0;
 pub mod v1;
 
 use crate::jcli_lib::utils::{io::ReadYamlError, output_format};
+use chain_core::property::{ReadError, WriteError};
 pub use config::RestArgs;
 use hex::FromHexError;
 use structopt::StructOpt;
@@ -21,7 +22,9 @@ pub enum Rest {
 #[derive(Debug, Error)]
 pub enum Error {
     #[error("input is not a valid fragment")]
-    InputFragmentMalformed(#[source] std::io::Error),
+    InputFragmentMalformed(#[from] ReadError),
+    #[error("output is not a valid fragment")]
+    OutputFragmentMalformed(#[from] WriteError),
     #[error("formatting output failed")]
     OutputFormatFailed(#[from] output_format::Error),
     #[error("could not read input file")]

jcli/src/jcli_lib/rest/v0/message/mod.rs

@@ -2,7 +2,10 @@ use crate::jcli_lib::{
     rest::{Error, RestArgs},
     utils::{io, OutputFormat},
 };
-use chain_core::property::{Deserialize, Serialize};
+use chain_core::{
+    packer::Codec,
+    property::{DeserializeFromSlice as _, Serialize},
+};
 use chain_impl_mockchain::fragment::Fragment;
 use std::path::PathBuf;
 use structopt::StructOpt;
@@ -57,8 +60,7 @@ fn get_logs(args: RestArgs, output_format: OutputFormat) -> Result<(), Error> {
 fn post_message(args: RestArgs, file: Option<PathBuf>) -> Result<(), Error> {
     let msg_hex = io::read_line(&file)?;
     let msg_bin = hex::decode(&msg_hex)?;
-    let fragment =
-        Fragment::deserialize(msg_bin.as_slice()).map_err(Error::InputFragmentMalformed)?;
+    let fragment = Fragment::deserialize_from_slice(&mut Codec::new(msg_bin.as_slice()))?;
     let fragment_id = post_fragment(args, fragment)?;
     println!("{}", fragment_id);
     Ok(())

jcli/src/jcli_lib/transaction/add_witness.rs

@@ -3,7 +3,7 @@ use crate::jcli_lib::{
     utils::io,
 };
 use bech32::{self, FromBase32 as _};
-use chain_core::mempack::{ReadBuf, Readable as _};
+use chain_core::{packer::Codec, property::DeserializeFromSlice};
 use chain_impl_mockchain::transaction::Witness;
 use std::path::PathBuf;
 use structopt::StructOpt;
@@ -56,7 +56,7 @@ impl AddWitness {
                 source,
                 path: self.witness.clone(),
             })?;
-        Witness::read(&mut ReadBuf::from(&bytes)).map_err(|source| {
+        Witness::deserialize_from_slice(&mut Codec::new(bytes.as_slice())).map_err(|source| {
             Error::WitnessFileDeserializationFailed {
                 source,
                 path: self.witness.clone(),

jcli/src/jcli_lib/transaction/mod.rs

@@ -21,7 +21,7 @@ use crate::jcli_lib::{
     utils::{key_parser, output_format},
 };
 use crate::{block, rest, utils};
-use chain_core::property::Serialize as _;
+use chain_core::property::{ReadError, Serialize as _, WriteError};
 use chain_impl_mockchain as chain;
 use std::path::PathBuf;
 use structopt::StructOpt;
@@ -137,11 +137,11 @@ pub enum Error {
     #[error("could not parse data in witness file '{path}'")]
     WitnessFileDeserializationFailed {
         #[source]
-        source: chain_core::mempack::ReadError,
+        source: ReadError,
         path: PathBuf,
     },
     #[error("could not serialize witness data")]
-    WitnessFileSerializationFailed(#[source] std::io::Error),
+    WitnessFileSerializationFailed(#[source] WriteError),
     #[error("could not write info file '{path}'")]
     InfoFileWriteFailed {
         #[source]
@@ -183,7 +183,7 @@ pub enum Error {
     #[error("transaction finalization failed")]
     TxFinalizationFailed(#[from] chain::transaction::Error),
     #[error("serialization of message to bytes failed")]
-    MessageSerializationFailed(#[source] std::io::Error),
+    MessageSerializationFailed(#[source] WriteError),
     #[error("calculation of info failed")]
     InfoCalculationFailed(#[from] chain::value::ValueError),
     #[error("expected a single account, multisig is not supported yet")]

jormungandr-lib/Cargo.toml

@@ -7,7 +7,6 @@ edition = "2018"
 description = "Data structures and formats used by Jormungandr node API and configuration files"
 
 [dependencies]
-bincode = "1.3"
 serde = { version = "1.0", features = ["derive"] }
 serde_with = { version = "1.12", features = ["macros"] }
 chain-impl-mockchain = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master" }
@@ -31,6 +30,7 @@ base64 = "0.13.0"
 http = "0.2.2"
 
 [dev-dependencies]
+bincode = "1.3.3"
 quickcheck = "0.9"
 quickcheck_macros = "0.9"
 # FIXME required to work with quickcheck 0.9. Remove after migrating another crate or newer quickcheck