Supports to sync partial chunks (#4)

* refactor p2p signed message

* add new pubsub messages in network layer to find chunks

* handle find chunks pubsub message in router

* Supports to sync partial chunks

* add admin rpc to sync chunks

* limit number of chunks to sync at a time

* refactor code to sync file and chunks

* add more switches to trigger file sync

* fix ut failure

* refactor code

node/file_location_cache/src/test_util.rs

@@ -1,6 +1,6 @@
 use network::{
     libp2p::identity,
-    types::{AnnounceFile, SignedAnnounceFile},
+    types::{AnnounceFile, SignedAnnounceFile, SignedMessage},
     Multiaddr, PeerId,
 };
 use shared_types::{timestamp_now, TxID};
@@ -42,6 +42,6 @@ impl AnnounceFileBuilder {
         };
 
         let keypair = identity::Keypair::generate_secp256k1();
-        msg.into_signed(&keypair).unwrap()
+        SignedMessage::sign_message(msg, &keypair).unwrap()
     }
 }

node/network/src/behaviour/gossip_cache.rs

@@ -22,8 +22,12 @@ pub struct GossipCache {
     example: Option<Duration>,
     /// Timeout for FindFile messages.
     find_file: Option<Duration>,
+    /// Timeout for FindChunks messages.
+    find_chunks: Option<Duration>,
     /// Timeout for AnnounceFile.
     announce_file: Option<Duration>,
+    /// Timeout for AnnounceChunks.
+    announce_chunks: Option<Duration>,
 }
 
 #[derive(Default)]
@@ -33,8 +37,12 @@ pub struct GossipCacheBuilder {
     example: Option<Duration>,
     /// Timeout for blocks FindFile messages.
     find_file: Option<Duration>,
+    /// Timeout for blocks FindChunks messages.
+    find_chunks: Option<Duration>,
     /// Timeout for AnnounceFile messages.
     announce_file: Option<Duration>,
+    /// Timeout for AnnounceChunks messages.
+    announce_chunks: Option<Duration>,
 }
 
 #[allow(dead_code)]
@@ -58,26 +66,42 @@ impl GossipCacheBuilder {
         self
     }
 
+    /// Timeout for FindChunks messages.
+    pub fn find_chunks_timeout(mut self, timeout: Duration) -> Self {
+        self.find_chunks = Some(timeout);
+        self
+    }
+
     /// Timeout for AnnounceFile messages.
     pub fn announce_file_timeout(mut self, timeout: Duration) -> Self {
         self.announce_file = Some(timeout);
         self
     }
 
+    /// Timeout for AnnounceChunks messages.
+    pub fn announce_chunks_timeout(mut self, timeout: Duration) -> Self {
+        self.announce_chunks = Some(timeout);
+        self
+    }
+
     pub fn build(self) -> GossipCache {
         let GossipCacheBuilder {
             default_timeout,
             example,
             find_file,
+            find_chunks,
             announce_file,
+            announce_chunks,
         } = self;
 
         GossipCache {
             expirations: DelayQueue::default(),
             topic_msgs: HashMap::default(),
             example: example.or(default_timeout),
             find_file: find_file.or(default_timeout),
+            find_chunks: find_chunks.or(default_timeout),
             announce_file: announce_file.or(default_timeout),
+            announce_chunks: announce_chunks.or(default_timeout),
         }
     }
 }
@@ -94,7 +118,9 @@ impl GossipCache {
         let expire_timeout = match topic.kind() {
             GossipKind::Example => self.example,
             GossipKind::FindFile => self.find_file,
+            GossipKind::FindChunks => self.find_chunks,
             GossipKind::AnnounceFile => self.announce_file,
+            GossipKind::AnnounceChunks => self.announce_chunks,
         };
 
         let expire_timeout = match expire_timeout {

node/network/src/types/mod.rs

@@ -6,5 +6,8 @@ mod topics;
 pub type Enr = discv5::enr::Enr<discv5::enr::CombinedKey>;
 
 pub use globals::NetworkGlobals;
-pub use pubsub::{AnnounceFile, FindFile, PubsubMessage, SignedAnnounceFile, SnappyTransform};
+pub use pubsub::{
+    AnnounceChunks, AnnounceFile, FindChunks, FindFile, HasSignature, PubsubMessage,
+    SignedAnnounceChunks, SignedAnnounceFile, SignedMessage, SnappyTransform,
+};
 pub use topics::{GossipEncoding, GossipKind, GossipTopic, CORE_TOPICS};

node/network/src/types/pubsub.rs

@@ -120,6 +120,14 @@ pub struct FindFile {
     pub timestamp: u32,
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)]
+pub struct FindChunks {
+    pub tx_id: TxID,
+    pub index_start: u64, // inclusive
+    pub index_end: u64,   // exclusive
+    pub timestamp: u32,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Encode, Decode)]
 pub struct AnnounceFile {
     pub tx_id: TxID,
@@ -128,46 +136,65 @@ pub struct AnnounceFile {
     pub timestamp: u32,
 }
 
-impl AnnounceFile {
-    pub fn into_signed(self, keypair: &Keypair) -> Result<SignedAnnounceFile, SigningError> {
-        let raw = self.as_ssz_bytes();
-        let signature = keypair.sign(&raw)?;
-
-        Ok(SignedAnnounceFile {
-            inner: self,
-            signature,
-            resend_timestamp: 0,
-        })
-    }
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Encode, Decode)]
+pub struct AnnounceChunks {
+    pub tx_id: TxID,
+    pub index_start: u64, // inclusive
+    pub index_end: u64,   // exclusive
+    pub peer_id: WrappedPeerId,
+    pub at: WrappedMultiaddr,
+    pub timestamp: u32,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Encode, Decode)]
-pub struct SignedAnnounceFile {
-    pub inner: AnnounceFile,
+pub struct SignedMessage<T: Encode + Decode> {
+    pub inner: T,
     pub signature: Vec<u8>,
     pub resend_timestamp: u32,
 }
 
-impl SignedAnnounceFile {
-    pub fn verify_signature(&self, public_key: &PublicKey) -> bool {
-        let raw = self.inner.as_ssz_bytes();
-        public_key.verify(&raw, &self.signature)
+impl<T: Encode + Decode> SignedMessage<T> {
+    pub fn sign_message(msg: T, keypair: &Keypair) -> Result<SignedMessage<T>, SigningError> {
+        let raw = msg.as_ssz_bytes();
+        let signature = keypair.sign(&raw)?;
+
+        Ok(SignedMessage {
+            inner: msg,
+            signature,
+            resend_timestamp: 0,
+        })
     }
 }
 
-impl Deref for SignedAnnounceFile {
-    type Target = AnnounceFile;
+impl<T: Encode + Decode> Deref for SignedMessage<T> {
+    type Target = T;
 
     fn deref(&self) -> &Self::Target {
         &self.inner
     }
 }
 
+pub trait HasSignature {
+    fn verify_signature(&self, public_key: &PublicKey) -> bool;
+}
+
+impl<T: Encode + Decode> HasSignature for SignedMessage<T> {
+    fn verify_signature(&self, public_key: &PublicKey) -> bool {
+        let raw = self.inner.as_ssz_bytes();
+        public_key.verify(&raw, &self.signature)
+    }
+}
+
+pub type SignedAnnounceFile = SignedMessage<AnnounceFile>;
+pub type SignedAnnounceChunks = SignedMessage<AnnounceChunks>;
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum PubsubMessage {
     ExampleMessage(u64),
     FindFile(FindFile),
+    FindChunks(FindChunks),
     AnnounceFile(SignedAnnounceFile),
+    AnnounceChunks(SignedAnnounceChunks),
 }
 
 // Implements the `DataTransform` trait of gossipsub to employ snappy compression
@@ -242,7 +269,9 @@ impl PubsubMessage {
         match self {
             PubsubMessage::ExampleMessage(_) => GossipKind::Example,
             PubsubMessage::FindFile(_) => GossipKind::FindFile,
+            PubsubMessage::FindChunks(_) => GossipKind::FindChunks,
             PubsubMessage::AnnounceFile(_) => GossipKind::AnnounceFile,
+            PubsubMessage::AnnounceChunks(_) => GossipKind::AnnounceChunks,
         }
     }
 
@@ -267,9 +296,16 @@ impl PubsubMessage {
                     GossipKind::FindFile => Ok(PubsubMessage::FindFile(
                         FindFile::from_ssz_bytes(data).map_err(|e| format!("{:?}", e))?,
                     )),
+                    GossipKind::FindChunks => Ok(PubsubMessage::FindChunks(
+                        FindChunks::from_ssz_bytes(data).map_err(|e| format!("{:?}", e))?,
+                    )),
                     GossipKind::AnnounceFile => Ok(PubsubMessage::AnnounceFile(
                         SignedAnnounceFile::from_ssz_bytes(data).map_err(|e| format!("{:?}", e))?,
                     )),
+                    GossipKind::AnnounceChunks => Ok(PubsubMessage::AnnounceChunks(
+                        SignedAnnounceChunks::from_ssz_bytes(data)
+                            .map_err(|e| format!("{:?}", e))?,
+                    )),
                 }
             }
         }
@@ -285,7 +321,9 @@ impl PubsubMessage {
         match &self {
             PubsubMessage::ExampleMessage(data) => data.as_ssz_bytes(),
             PubsubMessage::FindFile(data) => data.as_ssz_bytes(),
+            PubsubMessage::FindChunks(data) => data.as_ssz_bytes(),
             PubsubMessage::AnnounceFile(data) => data.as_ssz_bytes(),
+            PubsubMessage::AnnounceChunks(data) => data.as_ssz_bytes(),
         }
     }
 }
@@ -299,9 +337,15 @@ impl std::fmt::Display for PubsubMessage {
             PubsubMessage::FindFile(msg) => {
                 write!(f, "FindFile message: {:?}", msg)
             }
+            PubsubMessage::FindChunks(msg) => {
+                write!(f, "FindChunks message: {:?}", msg)
+            }
             PubsubMessage::AnnounceFile(msg) => {
                 write!(f, "AnnounceFile message: {:?}", msg)
             }
+            PubsubMessage::AnnounceChunks(msg) => {
+                write!(f, "AnnounceChunks message: {:?}", msg)
+            }
         }
     }
 }

node/network/src/types/topics.rs

@@ -9,9 +9,16 @@ pub const TOPIC_PREFIX: &str = "eth2";
 pub const SSZ_SNAPPY_ENCODING_POSTFIX: &str = "ssz_snappy";
 pub const EXAMPLE_TOPIC: &str = "example";
 pub const FIND_FILE_TOPIC: &str = "find_file";
+pub const FIND_CHUNKS_TOPIC: &str = "find_chunks";
 pub const ANNOUNCE_FILE_TOPIC: &str = "announce_file";
+pub const ANNOUNCE_CHUNKS_TOPIC: &str = "announce_chunks";
 
-pub const CORE_TOPICS: [GossipKind; 2] = [GossipKind::FindFile, GossipKind::AnnounceFile];
+pub const CORE_TOPICS: [GossipKind; 4] = [
+    GossipKind::FindFile,
+    GossipKind::FindChunks,
+    GossipKind::AnnounceFile,
+    GossipKind::AnnounceChunks,
+];
 
 /// A gossipsub topic which encapsulates the type of messages that should be sent and received over
 /// the pubsub protocol and the way the messages should be encoded.
@@ -30,7 +37,9 @@ pub struct GossipTopic {
 pub enum GossipKind {
     Example,
     FindFile,
+    FindChunks,
     AnnounceFile,
+    AnnounceChunks,
 }
 
 /// The known encoding types for gossipsub messages.
@@ -67,7 +76,9 @@ impl GossipTopic {
             let kind = match topic_parts[2] {
                 EXAMPLE_TOPIC => GossipKind::Example,
                 FIND_FILE_TOPIC => GossipKind::FindFile,
+                FIND_CHUNKS_TOPIC => GossipKind::FindChunks,
                 ANNOUNCE_FILE_TOPIC => GossipKind::AnnounceFile,
+                ANNOUNCE_CHUNKS_TOPIC => GossipKind::AnnounceChunks,
                 _ => return Err(format!("Unknown topic: {}", topic)),
             };
 
@@ -93,7 +104,9 @@ impl From<GossipTopic> for String {
         let kind = match topic.kind {
             GossipKind::Example => EXAMPLE_TOPIC,
             GossipKind::FindFile => FIND_FILE_TOPIC,
+            GossipKind::FindChunks => FIND_CHUNKS_TOPIC,
             GossipKind::AnnounceFile => ANNOUNCE_FILE_TOPIC,
+            GossipKind::AnnounceChunks => ANNOUNCE_CHUNKS_TOPIC,
         };
 
         format!("/{}/{}/{}", TOPIC_PREFIX, kind, encoding)
@@ -109,7 +122,9 @@ impl std::fmt::Display for GossipTopic {
         let kind = match self.kind {
             GossipKind::Example => EXAMPLE_TOPIC,
             GossipKind::FindFile => FIND_FILE_TOPIC,
+            GossipKind::FindChunks => FIND_CHUNKS_TOPIC,
             GossipKind::AnnounceFile => ANNOUNCE_FILE_TOPIC,
+            GossipKind::AnnounceChunks => ANNOUNCE_CHUNKS_TOPIC,
         };
 
         write!(f, "/{}/{}/{}", TOPIC_PREFIX, kind, encoding)