diff --git a/Cargo.lock b/Cargo.lock index 92dd35b37..c204fb703 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -706,6 +706,7 @@ dependencies = [ "collab-derive", "collab-persistence", "collab-plugins", + "lib0", "nanoid", "parking_lot 0.12.1", "serde", diff --git a/collab-document/Cargo.toml b/collab-document/Cargo.toml index 091b13dd3..34c6f2caf 100644 --- a/collab-document/Cargo.toml +++ b/collab-document/Cargo.toml @@ -11,6 +11,7 @@ collab-derive = { path = "../collab-derive" } collab-persistence = { path = "../collab-persistence" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.94" +lib0 = { version = "0.16.3", features = ["lib0-serde"] } nanoid = "0.4.0" thiserror = "1.0.30" anyhow = "1.0" diff --git a/collab-document/src/blocks/block.rs b/collab-document/src/blocks/block.rs index e1d1e152b..4b18a5832 100644 --- a/collab-document/src/blocks/block.rs +++ b/collab-document/src/blocks/block.rs @@ -67,6 +67,8 @@ impl BlockOperation { map.insert_with_txn(txn, PARENT, block.parent); map.insert_with_txn(txn, CHILDREN, block.children); map.insert_with_txn(txn, DATA, json_str); + map.insert_with_txn(txn, EXTERNAL_ID, block.external_id); + map.insert_with_txn(txn, EXTERNAL_TYPE, block.external_type); // Create the children for each block. self @@ -106,7 +108,7 @@ impl BlockOperation { } /// Update the block with the given id. - /// Except \`data\` and \`parent\`, other fields can not be updated. + /// Except \`data\` and \`parent\` and \'external_id\' and \'external_type\' field, other fields can be updated. /// If you want to turn into other block, you should delete the block and create a new block. pub fn set_block_with_txn( &self, @@ -114,6 +116,8 @@ impl BlockOperation { id: &str, data: Option>, parent_id: Option<&str>, + external_id: Option, + external_type: Option, ) -> Result<(), DocumentError> { let map = self .root @@ -128,6 +132,15 @@ impl BlockOperation { if let Some(data) = data { map.insert_with_txn(txn, DATA, hashmap_to_json_str(data)?); } + + // Update external id and external type. + if let Some(external_id) = external_id { + map.insert_with_txn(txn, EXTERNAL_ID, external_id); + } + + if let Some(external_type) = external_type { + map.insert_with_txn(txn, EXTERNAL_TYPE, external_type); + } Ok(()) } } diff --git a/collab-document/src/blocks/entities.rs b/collab-document/src/blocks/entities.rs index cc44399ed..588ec6b85 100644 --- a/collab-document/src/blocks/entities.rs +++ b/collab-document/src/blocks/entities.rs @@ -1,6 +1,8 @@ use serde::Serialize; +use serde_json; use serde_json::Value; use std::collections::HashMap; +use std::hash::Hash; use std::ops::Deref; /// [Block] Struct. @@ -26,6 +28,10 @@ pub struct Block { pub struct DocumentMeta { /// Meta has a children map. pub children_map: HashMap>, + /// Meta has a text map. + /// - @key: [Block]'s `external_id` + /// - @value: text delta json string - "\[ { "insert": "Hello World!", "attributes": { "bold": true } } \]" + pub text_map: Option>, } #[derive(Debug, Clone, Serialize, PartialEq, Eq)] diff --git a/collab-document/src/blocks/mod.rs b/collab-document/src/blocks/mod.rs index 4d5517819..793fe01c8 100644 --- a/collab-document/src/blocks/mod.rs +++ b/collab-document/src/blocks/mod.rs @@ -2,10 +2,14 @@ mod block; mod children; mod entities; mod subscribe; +mod text; +mod text_entities; mod utils; pub use block::*; pub use children::*; pub use entities::*; pub use subscribe::*; +pub use text::*; +pub use text_entities::*; pub use utils::*; diff --git a/collab-document/src/blocks/text.rs b/collab-document/src/blocks/text.rs new file mode 100644 index 000000000..265fb288b --- /dev/null +++ b/collab-document/src/blocks/text.rs @@ -0,0 +1,66 @@ +use crate::blocks::text_entities::TextDelta; +use collab::preclude::*; +use std::collections::HashMap; + +pub struct TextOperation { + root: MapRefWrapper, +} + +impl TextOperation { + pub fn new(root: MapRefWrapper) -> Self { + Self { root } + } + + pub fn get_text_with_txn(&self, txn: &mut TransactionMut, text_id: &str) -> TextRefWrapper { + self + .root + .get_text_ref_with_txn(txn, text_id) + .unwrap_or_else(|| self.create_text_with_txn(txn, text_id)) + } + + pub fn create_text_with_txn(&self, txn: &mut TransactionMut, text_id: &str) -> TextRefWrapper { + self.root.insert_text_with_txn(txn, text_id) + } + + pub fn delete_text_with_txn(&self, txn: &mut TransactionMut, text_id: &str) { + self.root.delete_with_txn(txn, text_id); + } + + pub fn get_delta_with_txn(&self, txn: &T, text_id: &str) -> Option> { + let text_ref = self.root.get_text_ref_with_txn(txn, text_id)?; + Some( + text_ref + .get_delta_with_txn(txn) + .iter() + .map(|d| TextDelta::from(txn, d.to_owned())) + .collect(), + ) + } + + pub fn apply_delta_with_txn( + &self, + txn: &mut TransactionMut, + text_id: &str, + delta: Vec, + ) { + let text_ref = self.get_text_with_txn(txn, text_id); + let delta = delta.iter().map(|d| d.to_owned().to_delta()).collect(); + text_ref.apply_delta_with_txn(txn, delta); + } + + pub fn serialize_all_text_delta(&self) -> HashMap { + let txn = self.root.transact(); + self + .root + .iter(&txn) + .filter_map(|(k, _)| { + self.get_delta_with_txn(&txn, k).map(|delta| { + ( + k.to_string(), + serde_json::to_string(&delta).unwrap_or_default(), + ) + }) + }) + .collect() + } +} diff --git a/collab-document/src/blocks/text_entities.rs b/collab-document/src/blocks/text_entities.rs new file mode 100644 index 000000000..2da0aed94 --- /dev/null +++ b/collab-document/src/blocks/text_entities.rs @@ -0,0 +1,216 @@ +use collab::preclude::{Attrs, Delta, ReadTxn, Value as YrsValue}; +use lib0::any::Any; +use serde::de::{self, Deserialize, Deserializer, MapAccess, Visitor}; +use serde::ser::SerializeMap; +use serde::{Serialize, Serializer}; +use std::collections::HashMap; +use std::fmt; +use std::rc::Rc; + +const FIELD_INSERT: &str = "insert"; +const FIELD_DELETE: &str = "delete"; +const FIELD_RETAIN: &str = "retain"; +const FIELD_ATTRIBUTES: &str = "attributes"; +const FIELDS: &[&str] = &[FIELD_INSERT, FIELD_DELETE, FIELD_RETAIN, FIELD_ATTRIBUTES]; + +#[derive(Debug, Clone)] +pub enum TextDelta { + /// Determines a change that resulted in insertion of a piece of text, which optionally could have been + /// formatted with provided set of attributes. + Inserted(String, Option), + + /// Determines a change that resulted in removing a consecutive range of characters. + Deleted(u32), + + /// Determines a number of consecutive unchanged characters. Used to recognize non-edited spaces + /// between [Delta::Inserted] and/or [Delta::Deleted] chunks. Can contain an optional set of + /// attributes, which have been used to format an existing piece of text. + Retain(u32, Option), +} + +impl PartialEq for TextDelta { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Inserted(content1, attrs1), Self::Inserted(content2, attrs2)) => { + content1 == content2 && attrs1 == attrs2 + }, + (Self::Deleted(len1), Self::Deleted(len2)) => len1 == len2, + (Self::Retain(len1, attrs1), Self::Retain(len2, attrs2)) => len1 == len2 && attrs1 == attrs2, + _ => false, + } + } +} + +impl Eq for TextDelta {} + +impl TextDelta { + pub fn from(txn: &T, value: Delta) -> Self { + match value { + Delta::Inserted(content, attrs) => { + let content = content.to_string(txn); + Self::Inserted(content, attrs.map(|attrs| *attrs)) + }, + Delta::Deleted(len) => Self::Deleted(len), + Delta::Retain(len, attrs) => Self::Retain(len, attrs.map(|attrs| *attrs)), + } + } + + pub fn to_delta(self) -> Delta { + match self { + Self::Inserted(content, attrs) => { + let content = YrsValue::from(content); + Delta::Inserted(content, attrs.map(Box::new)) + }, + Self::Deleted(len) => Delta::Deleted(len), + Self::Retain(len, attrs) => Delta::Retain(len, attrs.map(Box::new)), + } + } +} + +impl Serialize for TextDelta { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + Self::Inserted(content, attrs) => { + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry(FIELD_INSERT, content)?; + if let Some(attrs) = attrs { + let attrs_hash = attrs + .iter() + .map(|(k, v)| (k.to_string(), v.to_owned())) + .collect::>(); + map.serialize_entry(FIELD_ATTRIBUTES, &attrs_hash)?; + } + map.end() + }, + Self::Deleted(len) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry(FIELD_DELETE, len)?; + map.end() + }, + Self::Retain(len, attrs) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry(FIELD_RETAIN, len)?; + if let Some(attrs) = attrs { + let attrs_hash = attrs + .iter() + .map(|(k, v)| (k.to_string(), v.to_owned())) + .collect::>(); + map.serialize_entry(FIELD_ATTRIBUTES, &attrs_hash)?; + } + map.end() + }, + } + } +} + +impl<'de> Deserialize<'de> for TextDelta { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct TextDeltaVisitor; + + impl<'de> Visitor<'de> for TextDeltaVisitor { + type Value = TextDelta; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a valid TextDelta") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut delta_type: Option = None; + let mut content: Option = None; + let mut len: Option = None; + let mut attrs: Option, Any>> = None; + + while let Some(key) = map.next_key::()? { + match key.as_str() { + FIELD_INSERT => { + if delta_type.is_some() { + return Err(de::Error::duplicate_field(FIELD_INSERT)); + } + content = Some(map.next_value()?); + delta_type = Some(key); + }, + FIELD_DELETE => { + if delta_type.is_some() { + return Err(de::Error::duplicate_field(FIELD_DELETE)); + } + len = Some(map.next_value()?); + delta_type = Some(key); + }, + FIELD_RETAIN => { + if delta_type.is_some() { + return Err(de::Error::duplicate_field(FIELD_RETAIN)); + } + len = Some(map.next_value()?); + delta_type = Some(key); + }, + FIELD_ATTRIBUTES => { + if attrs.is_none() { + attrs = Some(HashMap::new()); + } + let attrs_val = map.next_value::>()?; + attrs_val.iter().for_each(|(key, val)| { + attrs + .as_mut() + .unwrap() + .insert(Rc::from(key.to_string()), val.clone()); + }); + }, + _ => { + return Err(de::Error::unknown_field(key.as_str(), FIELDS)); + }, + } + } + + match delta_type { + Some(delta_type) => match delta_type.as_str() { + FIELD_INSERT => { + if let Some(attrs) = attrs { + Ok(TextDelta::Inserted( + content.ok_or_else(|| de::Error::missing_field(FIELD_INSERT))?, + Some(attrs), + )) + } else { + Ok(TextDelta::Inserted( + content.ok_or_else(|| de::Error::missing_field(FIELD_INSERT))?, + None, + )) + } + }, + FIELD_DELETE => Ok(TextDelta::Deleted( + len.ok_or_else(|| de::Error::missing_field(FIELD_DELETE))? as u32, + )), + FIELD_RETAIN => { + if let Some(attrs) = attrs { + Ok(TextDelta::Retain( + len.ok_or_else(|| de::Error::missing_field(FIELD_RETAIN))? as u32, + Some(attrs), + )) + } else { + Ok(TextDelta::Retain( + len.ok_or_else(|| de::Error::missing_field(FIELD_RETAIN))? as u32, + None, + )) + } + }, + _ => Err(de::Error::unknown_variant( + &delta_type, + &[FIELD_INSERT, FIELD_DELETE, FIELD_RETAIN], + )), + }, + None => Err(de::Error::missing_field("delta type")), + } + } + } + + deserializer.deserialize_map(TextDeltaVisitor) + } +} diff --git a/collab-document/src/blocks/utils.rs b/collab-document/src/blocks/utils.rs index fb4271e24..61d6e982f 100644 --- a/collab-document/src/blocks/utils.rs +++ b/collab-document/src/blocks/utils.rs @@ -1,3 +1,4 @@ +use crate::blocks::text_entities::TextDelta; use crate::blocks::{BlockEvent, BlockEventPayload, DeltaType}; use crate::error::DocumentError; use collab::preclude::{Array, EntryChange, Event, Map, PathSegment, TransactionMut, YrsValue}; @@ -25,6 +26,21 @@ pub fn parse_event(txn: &TransactionMut, event: &Event) -> BlockEvent { }) .collect::>(); let delta = match event { + Event::Text(val) => { + let id = path.last().unwrap().to_string(); + let delta = val + .delta(txn) + .iter() + .map(|v| TextDelta::from(txn, v.to_owned())) + .collect::>(); + let value = serde_json::to_string(&delta).unwrap_or_default(); + vec![BlockEventPayload { + value, + id, + path, + command: DeltaType::Updated, + }] + }, Event::Array(_val) => { // Here use unwrap is safe, because we have checked the type of event. let id = path.last().unwrap().to_string(); @@ -92,3 +108,12 @@ fn parse_yrs_value(txn: &TransactionMut, value: &YrsValue) -> String { _ => "".to_string(), } } + +pub fn gen_empty_text_delta() -> String { + let delta = vec![TextDelta::Inserted("".to_string(), None)]; + serde_json::to_string(&delta).unwrap_or_default() +} + +pub fn deserialize_text_delta(delta: &str) -> serde_json::Result> { + serde_json::from_str::>(delta) +} diff --git a/collab-document/src/document.rs b/collab-document/src/document.rs index c9df44a4d..b73cd6797 100644 --- a/collab-document/src/document.rs +++ b/collab-document/src/document.rs @@ -10,8 +10,8 @@ use serde_json::Value; use tokio_stream::wrappers::WatchStream; use crate::blocks::{ - Block, BlockAction, BlockActionType, BlockEvent, BlockOperation, ChildrenOperation, DocumentData, - DocumentMeta, RootDeepSubscription, + deserialize_text_delta, Block, BlockAction, BlockActionType, BlockEvent, BlockOperation, + ChildrenOperation, DocumentData, DocumentMeta, RootDeepSubscription, TextDelta, TextOperation, }; use crate::error::DocumentError; @@ -25,8 +25,12 @@ const PAGE_ID: &str = "page_id"; const BLOCKS: &str = "blocks"; /// Document's meta data. const META: &str = "meta"; -/// [Block]'s children map. And it's also in [META]. +/// [Block]'s relation map. And it's also in [META]. +/// The key is the parent block's children_id, and the value is the children block's id. const CHILDREN_MAP: &str = "children_map"; +/// [Block]'s yText map. And it's also in [META]. +/// The key is the text block's external_id, and the value is the text block's yText. +const TEXT_MAP: &str = "text_map"; pub struct Document { inner: Arc, @@ -34,6 +38,7 @@ pub struct Document { subscription: RootDeepSubscription, children_operation: ChildrenOperation, block_operation: BlockOperation, + text_operation: TextOperation, } impl Document { @@ -103,14 +108,52 @@ impl Document { let blocks = self.block_operation.get_all_blocks(); let children_map = self.children_operation.get_all_children(); + let text_map = self.text_operation.serialize_all_text_delta(); let document_data = DocumentData { page_id, blocks, - meta: DocumentMeta { children_map }, + meta: DocumentMeta { + children_map, + text_map: Some(text_map), + }, }; Ok(document_data) } + /// Create a yText for incremental synchronization. + /// - @param text_id: The text block's external_id. + /// - @param delta: The text block's delta. "\[{"insert": "Hello", "attributes": { "bold": true, "italic": true } }, {"insert": " World!"}]". + pub fn create_text(&self, text_id: &str, delta: String) { + self.inner.lock().with_origin_transact_mut(|txn| { + let delta = deserialize_text_delta(&delta).ok(); + if let Some(delta) = delta { + self + .text_operation + .apply_delta_with_txn(txn, text_id, delta); + } else { + self.text_operation.create_text_with_txn(txn, text_id); + } + }) + } + + /// Apply a delta to the yText. + /// - @param text_id: The text block's external_id. + /// - @param delta: The text block's delta. "\[{"insert": "Hello", "attributes": { "bold": true, "italic": true } }, {"insert": " World!"}]". + pub fn apply_text_delta(&self, text_id: &str, delta: String) { + self.inner.lock().with_origin_transact_mut(|txn| { + let delta = deserialize_text_delta(&delta).ok(); + if let Some(delta) = delta { + self + .text_operation + .apply_delta_with_txn(txn, text_id, delta); + } else { + self + .text_operation + .apply_delta_with_txn(txn, text_id, vec![]); + } + }) + } + /// Apply actions to the document. pub fn apply_action(&self, actions: Vec) { self.inner.lock().with_origin_transact_mut(|txn| { @@ -215,6 +258,8 @@ impl Document { None => return Err(DocumentError::BlockIsNotFound), }; + let external_id = &block.external_id; + // Delete all the children of this block. let children = self .children_operation @@ -230,6 +275,10 @@ impl Document { let parent_id = &block.parent; self.delete_block_from_parent(txn, block_id, parent_id); + // Delete the text + if let Some(external_id) = external_id { + self.text_operation.delete_text_with_txn(txn, external_id); + } // Delete the block self .block_operation @@ -264,9 +313,14 @@ impl Document { Some(block) => block, None => return Err(DocumentError::BlockIsNotFound), }; - self - .block_operation - .set_block_with_txn(txn, &block.id, Some(data), None) + self.block_operation.set_block_with_txn( + txn, + &block.id, + Some(data), + None, + block.external_id, + block.external_type, + ) } /// move the block to the new parent. @@ -324,9 +378,14 @@ impl Document { .insert_child_with_txn(txn, &new_parent_children_id, block_id, index); // Update the parent of the block. - self - .block_operation - .set_block_with_txn(txn, block_id, Some(block.data), Some(&new_parent.id)) + self.block_operation.set_block_with_txn( + txn, + block_id, + Some(block.data), + Some(&new_parent.id), + None, + None, + ) } pub fn redo(&self) -> bool { @@ -372,8 +431,8 @@ impl Document { data: Option, ) -> Result { let mut collab_guard = collab.lock(); - let (root, block_operation, children_operation) = - collab_guard.with_origin_transact_mut(|txn| { + let (root, block_operation, children_operation, text_operation) = collab_guard + .with_origin_transact_mut(|txn| { // { document: {:} } let root = collab_guard.insert_map_with_txn(txn, ROOT); // { document: { blocks: {:} } } @@ -382,6 +441,9 @@ impl Document { let meta = root.insert_map_with_txn(txn, META); // {document: { blocks: {:}, meta: { children_map: {:} } } let children_map = meta.insert_map_with_txn(txn, CHILDREN_MAP); + // { document: { blocks: {:}, meta: { text_map: {:} } } + let text_map = meta.insert_map_with_txn(txn, TEXT_MAP); + let text_operation = TextOperation::new(text_map); let children_operation = ChildrenOperation::new(children_map); let block_operation = BlockOperation::new(blocks, children_operation.clone()); @@ -400,9 +462,16 @@ impl Document { map.push_back(txn, child_id.to_string()); }); } + if let Some(text_map) = data.meta.text_map { + for (id, delta) in text_map { + let delta = serde_json::from_str(&delta) + .unwrap_or_else(|_| vec![TextDelta::Inserted("insert".to_string(), None)]); + text_operation.apply_delta_with_txn(txn, &id, delta) + } + } } - Ok::<_, DocumentError>((root, block_operation, children_operation)) + Ok::<_, DocumentError>((root, block_operation, children_operation, text_operation)) })?; collab_guard.enable_undo_redo(); @@ -415,6 +484,7 @@ impl Document { root, block_operation, children_operation, + text_operation, subscription, }; Ok(document) @@ -422,23 +492,26 @@ impl Document { fn open_document_with_collab(collab: Arc) -> Result { let mut collab_guard = collab.lock(); - let (root, block_operation, children_operation, subscription) = { - let txn = collab_guard.transact(); - let root = collab_guard - .get_map_with_txn(&txn, vec![ROOT]) - .ok_or_else(|| { - DocumentError::Internal(anyhow::anyhow!("Unexpected empty document value")) - })?; - let blocks = collab_guard - .get_map_with_txn(&txn, vec![ROOT, BLOCKS]) - .ok_or(DocumentError::BlockIsNotFound)?; - let children_map = collab_guard - .get_map_with_txn(&txn, vec![ROOT, META, CHILDREN_MAP]) - .ok_or_else(|| DocumentError::Internal(anyhow::anyhow!("Unexpected empty child map")))?; - let children_operation = ChildrenOperation::new(children_map); - let block_operation = BlockOperation::new(blocks, children_operation.clone()); - let subscription = RootDeepSubscription::default(); - (root, block_operation, children_operation, subscription) + let (root, block_operation, children_operation, text_operation, subscription) = { + collab_guard.with_origin_transact_mut(|txn| { + let root = collab_guard.insert_map_with_txn_if_not_exist(txn, ROOT); + let blocks = root.insert_map_with_txn_if_not_exist(txn, BLOCKS); + let meta = root.insert_map_with_txn_if_not_exist(txn, META); + + let children_map = meta.insert_map_with_txn_if_not_exist(txn, CHILDREN_MAP); + let text_map = meta.insert_map_with_txn_if_not_exist(txn, TEXT_MAP); + let children_operation = ChildrenOperation::new(children_map); + let text_operation = TextOperation::new(text_map); + let block_operation = BlockOperation::new(blocks, children_operation.clone()); + let subscription = RootDeepSubscription::default(); + ( + root, + block_operation, + children_operation, + text_operation, + subscription, + ) + }) }; collab_guard.enable_undo_redo(); @@ -449,6 +522,7 @@ impl Document { root, block_operation, children_operation, + text_operation, subscription, }) } diff --git a/collab-document/src/document_data.rs b/collab-document/src/document_data.rs index 85efcd646..2f43d902f 100644 --- a/collab-document/src/document_data.rs +++ b/collab-document/src/document_data.rs @@ -1,4 +1,4 @@ -use crate::blocks::{Block, DocumentData, DocumentMeta}; +use crate::blocks::{gen_empty_text_delta, Block, DocumentData, DocumentMeta}; use nanoid::nanoid; use std::collections::HashMap; @@ -27,7 +27,8 @@ pub fn default_document_data() -> DocumentData { let text_type = PARAGRAPH_BLOCK_TYPE.to_string(); let mut blocks: HashMap = HashMap::new(); - let mut meta: HashMap> = HashMap::new(); + let mut children_map: HashMap> = HashMap::new(); + let mut text_map: HashMap = HashMap::new(); // page block let page_id = nanoid!(10); @@ -46,25 +47,33 @@ pub fn default_document_data() -> DocumentData { // text block let text_block_id = nanoid!(10); let text_block_children_id = nanoid!(10); + let text_external_id = nanoid!(10); let text_block = Block { id: text_block_id.clone(), ty: text_type, parent: page_id.clone(), children: text_block_children_id.clone(), - external_id: None, - external_type: None, + external_id: Some(text_external_id.clone()), + external_type: Some("text".to_string()), data: HashMap::new(), }; blocks.insert(text_block_id.clone(), text_block); - // meta - meta.insert(children_id, vec![text_block_id]); - meta.insert(text_block_children_id, vec![]); + // children_map + children_map.insert(children_id, vec![text_block_id]); + children_map.insert(text_block_children_id, vec![]); + + // text_map + let empty_text_delta = gen_empty_text_delta(); + text_map.insert(text_external_id, empty_text_delta); DocumentData { page_id, blocks, - meta: DocumentMeta { children_map: meta }, + meta: DocumentMeta { + children_map, + text_map: Some(text_map), + }, } } @@ -74,7 +83,8 @@ pub fn default_document_data2() -> DocumentData { let text_type = PARAGRAPH_BLOCK_TYPE.to_string(); let mut blocks: HashMap = HashMap::new(); - let mut meta: HashMap> = HashMap::new(); + let mut children_map: HashMap> = HashMap::new(); + let mut text_map: HashMap = HashMap::new(); // page block let page_id = nanoid!(10); @@ -93,23 +103,31 @@ pub fn default_document_data2() -> DocumentData { // text block let text_block_id = nanoid!(10); let text_block_children_id = nanoid!(10); + let text_external_id = nanoid!(10); let text_block = Block { id: text_block_id.clone(), ty: text_type, parent: page_id.clone(), children: text_block_children_id.clone(), - external_id: None, - external_type: None, + external_id: Some(text_external_id.clone()), + external_type: Some("text".to_string()), data: HashMap::new(), }; blocks.insert(text_block_id.clone(), text_block); - // meta - meta.insert(children_id, vec![text_block_id]); - meta.insert(text_block_children_id, vec![]); + // children_map + children_map.insert(children_id, vec![text_block_id]); + children_map.insert(text_block_children_id, vec![]); + + // text_map + let empty_text_delta = gen_empty_text_delta(); + text_map.insert(text_external_id, empty_text_delta); DocumentData { page_id, blocks, - meta: DocumentMeta { children_map: meta }, + meta: DocumentMeta { + children_map, + text_map: Some(text_map), + }, } } diff --git a/collab-document/tests/blocks/block_test.rs b/collab-document/tests/blocks/block_test.rs index 7fee56224..8ef22a971 100644 --- a/collab-document/tests/blocks/block_test.rs +++ b/collab-document/tests/blocks/block_test.rs @@ -35,7 +35,7 @@ async fn open_document_test() { async fn subscribe_insert_change_test() { let mut test = BlockTestCore::new(); test.subscribe(|e, _| { - assert_eq!(e.len(), 3); + println!("event: {:?}", e); }); let page = test.get_page(); let page_id = page.id.as_str(); @@ -47,19 +47,20 @@ async fn subscribe_insert_change_test() { async fn subscribe_update_change_test() { let mut test = BlockTestCore::new(); test.subscribe(|e, _| { - assert_eq!(e.len(), 1); + println!("event: {:?}", e); }); let page = test.get_page(); let page_id = page.id.as_str(); - let text = "Hello World Updated".to_string(); - test.update_text_block(text, page_id); + let mut data = HashMap::new(); + data.insert("text".to_string(), json!("Hello World Updated")); + test.update_block_data(page_id, data); } #[tokio::test] async fn subscribe_delete_change_test() { let mut test = BlockTestCore::new(); test.subscribe(|e, _| { - assert_eq!(e.len(), 3); + println!("event: {:?}", e); }); let page = test.get_page(); let page_id = page.id.as_str(); @@ -164,10 +165,12 @@ async fn update_block_data_test() { let page_children = test.get_block_children(page_id); let block_id = page_children[0].id.as_str(); let update_text = "Hello World Updated".to_string(); - test.update_text_block(update_text.clone(), block_id); + let mut update_data = HashMap::new(); + update_data.insert("text".to_string(), json!(update_text)); + test.update_block_data(block_id, update_data); let block = test.get_block(block_id); let mut expected_data = HashMap::new(); - expected_data.insert("delta".to_string(), json!([{ "insert": update_text }])); + expected_data.insert("text".to_string(), json!(update_text)); assert_eq!(block.data, expected_data); } diff --git a/collab-document/tests/blocks/block_test_core.rs b/collab-document/tests/blocks/block_test_core.rs index 6ac885b54..59a6d25c2 100644 --- a/collab-document/tests/blocks/block_test_core.rs +++ b/collab-document/tests/blocks/block_test_core.rs @@ -4,12 +4,13 @@ use std::sync::Arc; use collab::core::collab::MutexCollab; use collab::preclude::CollabBuilder; use collab_document::blocks::{ - Block, BlockAction, BlockActionPayload, BlockActionType, BlockEvent, DocumentData, DocumentMeta, + gen_empty_text_delta, Block, BlockAction, BlockActionPayload, BlockActionType, BlockEvent, + DocumentData, DocumentMeta, }; use collab_document::document::Document; use collab_persistence::kv::rocks_kv::RocksCollabDB; use nanoid::nanoid; -use serde_json::json; +use serde_json::{json, Value}; use collab_plugins::local_storage::rocksdb::RocksdbDiskPlugin; @@ -19,7 +20,7 @@ pub const TEXT_BLOCK_TYPE: &str = "paragraph"; pub struct BlockTestCore { pub db: Arc, - document: Document, + pub document: Document, pub collab: Arc, } @@ -69,14 +70,8 @@ impl BlockTestCore { pub fn get_default_data() -> DocumentData { let mut blocks = HashMap::new(); let mut children_map = HashMap::new(); - - let mut data = HashMap::new(); - data.insert( - "delta".to_string(), - json!([{ - "insert": "Hello World" - }]), - ); + let mut text_map = HashMap::new(); + let data = HashMap::new(); let page_id = generate_id(); let page_children_id = generate_id(); blocks.insert( @@ -96,6 +91,9 @@ impl BlockTestCore { children_map.insert(page_children_id, vec![first_text_id.clone()]); let first_text_children_id = generate_id(); children_map.insert(first_text_children_id.clone(), vec![]); + let first_text_external_id = generate_id(); + let empty_text_delta = gen_empty_text_delta(); + text_map.insert(first_text_external_id.clone(), empty_text_delta); blocks.insert( first_text_id.clone(), Block { @@ -103,12 +101,15 @@ impl BlockTestCore { ty: TEXT_BLOCK_TYPE.to_string(), parent: page_id.clone(), children: first_text_children_id, - data: data.clone(), - external_id: None, - external_type: None, + data, + external_id: Some(first_text_external_id), + external_type: Some("text".to_string()), }, ); - let meta = DocumentMeta { children_map }; + let meta = DocumentMeta { + children_map, + text_map: Some(text_map), + }; DocumentData { page_id, blocks, @@ -136,6 +137,15 @@ impl BlockTestCore { .unwrap_or_else(|| panic!("get block error: {}", block_id)) } + pub fn get_text_delta_with_text_id(&self, text_id: &str) -> String { + let document_data = self.get_document_data(); + let text_map = document_data.meta.text_map.unwrap(); + text_map + .get(text_id) + .unwrap_or_else(|| panic!("get text delta error: {}", text_id)) + .clone() + } + pub fn get_block_children(&self, block_id: &str) -> Vec { let block = self.get_block(block_id); let block_children_id = block.children; @@ -152,16 +162,27 @@ impl BlockTestCore { children } + pub fn create_text(&self, delta: String) -> String { + let external_id = generate_id(); + self.document.create_text(&external_id, delta); + external_id + } + + pub fn apply_text_delta(&self, text_id: &str, delta: String) { + self.document.apply_text_delta(text_id, delta); + } + pub fn get_text_block(&self, text: String, parent_id: &str) -> Block { - let mut data = HashMap::new(); - data.insert("delta".to_string(), json!([{ "insert": text }])); + let data = HashMap::new(); + let delta = json!([{ "insert": text }]).to_string(); + let external_id = self.create_text(delta); Block { id: generate_id(), ty: TEXT_BLOCK_TYPE.to_string(), parent: parent_id.to_string(), children: generate_id(), - external_id: None, - external_type: None, + external_id: Some(external_id), + external_type: Some("text".to_string()), data, } } @@ -176,10 +197,8 @@ impl BlockTestCore { }) } - pub fn update_text_block(&self, text: String, block_id: &str) { + pub fn update_block_data(&self, block_id: &str, data: HashMap) { let block = self.get_block(block_id); - let mut data = block.data; - data.insert("delta".to_string(), json!([{ "insert": text }])); self.document.with_transact_mut(|txn| { self diff --git a/collab-document/tests/blocks/mod.rs b/collab-document/tests/blocks/mod.rs index 12c860fca..c3a41d4ee 100644 --- a/collab-document/tests/blocks/mod.rs +++ b/collab-document/tests/blocks/mod.rs @@ -1,2 +1,3 @@ mod block_test; mod block_test_core; +mod text_test; diff --git a/collab-document/tests/blocks/text_test.rs b/collab-document/tests/blocks/text_test.rs new file mode 100644 index 000000000..03d803774 --- /dev/null +++ b/collab-document/tests/blocks/text_test.rs @@ -0,0 +1,251 @@ +use crate::blocks::block_test_core::BlockTestCore; +use collab::preclude::{Attrs, Delta, YrsValue}; +use collab_document::blocks::{deserialize_text_delta, TextDelta}; + +use serde_json::json; +use std::rc::Rc; + +#[tokio::test] +async fn insert_text_test() { + let test = BlockTestCore::new(); + let origin_delta = json!([{"insert": "Hello World"}]).to_string(); + let text_id = test.create_text(origin_delta.clone()); + let delta = test.get_text_delta_with_text_id(&text_id); + assert_eq!( + deserialize_text_delta(&delta).unwrap(), + deserialize_text_delta(&origin_delta).unwrap() + ); +} + +#[tokio::test] +async fn insert_empty_text_test() { + let test = BlockTestCore::new(); + let origin_delta = "".to_string(); + let text_id = test.create_text(origin_delta); + let delta = test.get_text_delta_with_text_id(&text_id); + let result = deserialize_text_delta(&delta); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), vec![]); +} + +#[tokio::test] +async fn apply_empty_delta_test() { + let test = BlockTestCore::new(); + let origin_delta = json!([{"insert": "Hello World"}]).to_string(); + let text_id = test.create_text(origin_delta); + let origin_delta = test.get_text_delta_with_text_id(&text_id); + let delta = "".to_string(); + test.apply_text_delta(&text_id, delta); + let delta = test.get_text_delta_with_text_id(&text_id); + assert_eq!( + deserialize_text_delta(&delta).unwrap(), + deserialize_text_delta(&origin_delta).unwrap() + ); +} +#[tokio::test] +async fn format_text_test() { + let test = BlockTestCore::new(); + let origin_delta = json!([{"insert": "Hello World", "attributes": { "bold": true }}]).to_string(); + let text_id = test.create_text(origin_delta.clone()); + let delta = test.get_text_delta_with_text_id(&text_id); + assert_eq!( + deserialize_text_delta(&delta).unwrap(), + deserialize_text_delta(&origin_delta).unwrap() + ); +} + +#[tokio::test] +async fn apply_retain_delta_test() { + let test = BlockTestCore::new(); + let text = "Hello World".to_string(); + let length = text.len() as u32; + let origin_delta = json!([{"insert": "Hello World"}]).to_string(); + let text_id = test.create_text(origin_delta); + let origin_delta = test.get_text_delta_with_text_id(&text_id); + + // retain text + let retain_delta = json!([{ "retain": length }]).to_string(); + test.apply_text_delta(&text_id, retain_delta); + let delta = test.get_text_delta_with_text_id(&text_id); + assert_eq!( + deserialize_text_delta(&delta).unwrap(), + deserialize_text_delta(&origin_delta).unwrap() + ); + + // retain text and format + let format_delta = json!([ + {"retain": length, "attributes": { "bold": true, "italic": true }} + ]) + .to_string(); + test.apply_text_delta(&text_id, format_delta); + let delta = test.get_text_delta_with_text_id(&text_id); + let expect = json!( + [{"insert": "Hello World", "attributes": { "bold": true, "italic": true }}] + ) + .to_string(); + assert_eq!( + deserialize_text_delta(&delta).unwrap(), + deserialize_text_delta(&expect).unwrap() + ); + + // retain text and clear format + let clear_format_delta = json!([ + {"retain": length, "attributes": { "bold": null, "italic": null }} + ]) + .to_string(); + test.apply_text_delta(&text_id, clear_format_delta); + let delta = test.get_text_delta_with_text_id(&text_id); + let expect = json!( + [{"insert": "Hello World"}] + ) + .to_string(); + assert_eq!( + deserialize_text_delta(&delta).unwrap(), + deserialize_text_delta(&expect).unwrap() + ); +} + +#[tokio::test] +async fn apply_delete_delta_test() { + let test = BlockTestCore::new(); + let origin_delta = json!([{"insert": "Hello World", "attributes": { "bold": true }}]).to_string(); + let text_id = test.create_text(origin_delta); + let delete_delta = json!([ + {"retain": 6}, + {"delete": 5}, + ]) + .to_string(); + test.apply_text_delta(&text_id, delete_delta); + let delta = test.get_text_delta_with_text_id(&text_id); + let expect = json!([{"insert": "Hello ", "attributes": { "bold": true }}]).to_string(); + + assert_eq!( + deserialize_text_delta(&delta).unwrap(), + deserialize_text_delta(&expect).unwrap() + ); +} + +#[tokio::test] +async fn apply_insert_delta_test() { + let test = BlockTestCore::new(); + let text = "Hello World".to_string(); + let delta = json!([{ "insert": text }]).to_string(); + let text_id = test.create_text(delta); + let insert_delta = json!([{ + "retain": 6, + }, { + "insert": "World ", + }]) + .to_string(); + test.apply_text_delta(&text_id, insert_delta); + let delta = test.get_text_delta_with_text_id(&text_id); + let expect = json!([{"insert": "Hello World World"}]).to_string(); + assert_eq!( + deserialize_text_delta(&delta).unwrap(), + deserialize_text_delta(&expect).unwrap() + ); +} + +#[tokio::test] +async fn subscribe_apply_delta_test() { + let mut test = BlockTestCore::new(); + test.subscribe(|e, _| { + println!("event: {:?}", e); + }); + let text = "Hello World".to_string(); + let delta = json!([{ "insert": text }]).to_string(); + let text_id = test.create_text(delta); + let delta = json!([{ + "retain": 6, + }, { + "insert": "World ", + }]) + .to_string(); + test.apply_text_delta(&text_id, delta); +} + +#[tokio::test] +async fn delta_equal_test() { + let delta = json!([{"insert": "Hello World"}, { "retain": 6, "attributes": { "bold": true } }, { "delete": 4 } ]).to_string(); + let delta2 = json!([{"insert": "Hello World"}, { "attributes": { "bold": true }, "retain": 6 }, { "delete": 4 } ]).to_string(); + + assert_eq!( + deserialize_text_delta(&delta).unwrap(), + deserialize_text_delta(&delta2).unwrap() + ); +} + +#[tokio::test] +async fn text_delta_from_delta_test() { + let test = BlockTestCore::new(); + let _delta = vec![ + Delta::Inserted(YrsValue::from("Hello World"), None), + Delta::Retain( + 6, + Some(Box::from(Attrs::from([(Rc::from("bold"), true.into())]))), + ), + Delta::Deleted(4), + ]; + test.document.with_transact_mut(|txn| { + let insert_delta = Delta::Inserted(YrsValue::from("Hello World"), None); + let text_delta = TextDelta::from(txn, insert_delta); + assert_eq!( + text_delta, + TextDelta::Inserted("Hello World".to_string(), None) + ); + + let attrs = Attrs::from([(Rc::from("bold"), true.into())]); + let retain_delta = Delta::Retain(6, Some(Box::from(attrs.clone()))); + let text_delta = TextDelta::from(txn, retain_delta); + assert_eq!(text_delta, TextDelta::Retain(6, Some(attrs))); + + let delete_delta = Delta::Deleted(4); + let text_delta = TextDelta::from(txn, delete_delta); + assert_eq!(text_delta, TextDelta::Deleted(4)); + }) +} + +#[tokio::test] +async fn serialize_delta_test() { + let delta = TextDelta::Inserted("Hello World".to_string(), None); + let json = serde_json::to_string(&delta).unwrap(); + assert_eq!(json, r#"{"insert":"Hello World"}"#); + + let delta = TextDelta::Retain(6, Some(Attrs::from([(Rc::from("bold"), true.into())]))); + let json = serde_json::to_string(&delta).unwrap(); + assert_eq!(json, r#"{"retain":6,"attributes":{"bold":true}}"#); + + let delta = TextDelta::Deleted(4); + let json = serde_json::to_string(&delta).unwrap(); + assert_eq!(json, r#"{"delete":4}"#); +} + +#[tokio::test] +async fn deserialize_delta_test() { + let json = r#"{"insert":"Hello World"}"#; + let delta: TextDelta = serde_json::from_str(json).unwrap(); + assert_eq!(delta, TextDelta::Inserted("Hello World".to_string(), None)); + + let json = r#"{"retain":6,"attributes":{"bold":true}}"#; + let delta: TextDelta = serde_json::from_str(json).unwrap(); + assert_eq!( + delta, + TextDelta::Retain(6, Some(Attrs::from([(Rc::from("bold"), true.into())]))) + ); + + let json = r#"{"delete":4}"#; + let delta: TextDelta = serde_json::from_str(json).unwrap(); + assert_eq!(delta, TextDelta::Deleted(4)); + + let json = r#"{ "unexpected_field": "value" }"#; + let result: Result = serde_json::from_str(json); + assert!(result.is_err()); + + let json = r#"{}"#; + let result: Result = serde_json::from_str(json); + assert!(result.is_err()); + + let json = r#""#; + let result: Result = serde_json::from_str(json); + assert!(result.is_err()); +} diff --git a/collab-document/tests/document/document_data_test.rs b/collab-document/tests/document/document_data_test.rs new file mode 100644 index 000000000..e6480c04c --- /dev/null +++ b/collab-document/tests/document/document_data_test.rs @@ -0,0 +1,19 @@ +use collab_document::document_data::{default_document_data, default_document_data2}; + +#[tokio::test] +async fn get_default_data_test() { + let data = default_document_data(); + assert!(!data.page_id.is_empty()); + assert!(!data.blocks.is_empty()); + assert!(!data.meta.children_map.is_empty()); + assert!(data.meta.text_map.is_some()); + assert!(!data.meta.text_map.unwrap().is_empty()); + + let data = default_document_data2(); + println!("{:?}", data); + assert!(!data.page_id.is_empty()); + assert_eq!(data.blocks.len(), 2); + assert_eq!(data.meta.children_map.len(), 2); + assert!(data.meta.text_map.is_some()); + assert_eq!(data.meta.text_map.unwrap().len(), 1); +} diff --git a/collab-document/tests/document/mod.rs b/collab-document/tests/document/mod.rs index cecf008b5..a6834721c 100644 --- a/collab-document/tests/document/mod.rs +++ b/collab-document/tests/document/mod.rs @@ -1,3 +1,4 @@ +mod document_data_test; mod document_test; mod redo_undo_test; mod restore_test; diff --git a/collab-document/tests/util.rs b/collab-document/tests/util.rs index 3bc8a2a57..911468ce2 100644 --- a/collab-document/tests/util.rs +++ b/collab-document/tests/util.rs @@ -8,7 +8,9 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Once}; use collab::preclude::CollabBuilder; -use collab_document::blocks::{Block, BlockAction, DocumentData, DocumentMeta}; +use collab_document::blocks::{ + gen_empty_text_delta, Block, BlockAction, DocumentData, DocumentMeta, +}; use collab_document::document::Document; use collab_document::error::DocumentError; use collab_persistence::kv::rocks_kv::RocksCollabDB; @@ -42,6 +44,7 @@ impl DocumentTest { let mut blocks = HashMap::new(); let mut children_map = HashMap::new(); + let mut text_map = HashMap::new(); let mut data = HashMap::new(); data.insert("delta".to_string(), json!([])); @@ -64,6 +67,9 @@ impl DocumentTest { children_map.insert(page_children_id, vec![first_text_id.clone()]); let first_text_children_id = nanoid!(10); children_map.insert(first_text_children_id.clone(), vec![]); + let first_text_external_id = nanoid!(10); + let empty_text_delta = gen_empty_text_delta(); + text_map.insert(first_text_external_id.clone(), empty_text_delta); blocks.insert( first_text_id.clone(), Block { @@ -72,11 +78,14 @@ impl DocumentTest { parent: page_id.clone(), children: first_text_children_id, data: data.clone(), - external_id: None, - external_type: None, + external_id: Some(first_text_external_id), + external_type: Some("text".to_string()), }, ); - let meta = DocumentMeta { children_map }; + let meta = DocumentMeta { + children_map, + text_map: Some(text_map), + }; let document_data = DocumentData { page_id, blocks, diff --git a/collab/src/core/text_wrapper.rs b/collab/src/core/text_wrapper.rs index 3fdadd014..9da910916 100644 --- a/collab/src/core/text_wrapper.rs +++ b/collab/src/core/text_wrapper.rs @@ -2,23 +2,11 @@ use crate::preclude::{CollabContext, YrsDelta}; use std::ops::{Deref, DerefMut}; use std::sync::Arc; use yrs::types::text::{TextEvent, YChange}; -use yrs::types::{Attrs, Delta}; +use yrs::types::Delta; use yrs::{ReadTxn, Subscription, Text, TextRef, Transaction, TransactionMut}; pub type TextSubscriptionCallback = Arc; pub type TextSubscription = Subscription; -pub enum TextDelta { - Inserted(String, Attrs), - - /// Determines a change that resulted in removing a consecutive range of characters. - Deleted(u32), - - /// Determines a number of consecutive unchanged characters. Used to recognize non-edited spaces - /// between [Delta::Inserted] and/or [Delta::Deleted] chunks. Can contain an optional set of - /// attributes, which have been used to format an existing piece of text. - Retain(u32, Attrs), -} - pub struct TextRefWrapper { text_ref: TextRef, collab_ctx: CollabContext, @@ -53,23 +41,29 @@ impl TextRefWrapper { deltas } - pub fn apply_delta_with_txn(&self, txn: &mut TransactionMut, delta: Vec) { + pub fn apply_delta_with_txn(&self, txn: &mut TransactionMut, delta: Vec) { let mut index = 0; for d in delta { match d { - TextDelta::Inserted(content, attrs) => { - let value = content.to_string(); + Delta::Inserted(content, attrs) => { + let value = content.to_string(txn); let len = value.len() as u32; self.text_ref.insert(txn, index, &value); - self.text_ref.format(txn, index, len, attrs); + attrs.map(|attrs| { + self.text_ref.format(txn, index, len, *attrs); + Some(()) + }); index += len; }, - TextDelta::Deleted(len) => { + Delta::Deleted(len) => { self.text_ref.remove_range(txn, index, len); index += len; }, - TextDelta::Retain(len, attrs) => { - self.text_ref.format(txn, index, len, attrs); + Delta::Retain(len, attrs) => { + attrs.map(|attrs| { + self.text_ref.format(txn, index, len, *attrs); + Some(()) + }); index += len; }, }