feat: sync the documents and databases after batch importing document…

…s and databases (AppFlowy-IO#5644) * feat: support batch import * feat: support batch import database * chore: revert launch.json * chore: fix rust ci * fix: rust ci
tantanorange · Jul 1, 2024 · 2b8dca2 · 2b8dca2
1 parent c78f23e
commit 2b8dca2
Show file tree

Hide file tree

Showing 14 changed files with 285 additions and 158 deletions.
diff --git a/frontend/appflowy_flutter/lib/workspace/application/settings/share/import_service.dart b/frontend/appflowy_flutter/lib/workspace/application/settings/share/import_service.dart
@@ -4,34 +4,29 @@ import 'package:appflowy_backend/protobuf/flowy-folder/import.pb.dart';
 import 'package:appflowy_backend/protobuf/flowy-folder/view.pbenum.dart';
 import 'package:appflowy_result/appflowy_result.dart';
 
+class ImportPayload {
+  ImportPayload({
+    required this.name,
+    required this.data,
+    required this.layout,
+  });
+
+  final String name;
+  final List<int> data;
+  final ViewLayoutPB layout;
+}
+
 class ImportBackendService {
-  static Future<FlowyResult<void, FlowyError>> importData(
-    List<int> data,
-    String name,
+  static Future<FlowyResult<void, FlowyError>> importPages(
     String parentViewId,
-    ImportTypePB importType,
+    List<ImportValuePayloadPB> values,
   ) async {
-    final payload = ImportPB.create()
-      ..data = data
-      ..parentViewId = parentViewId
-      ..viewLayout = importType.toLayout()
-      ..name = name
-      ..importType = importType;
-    return FolderEventImportData(payload).send();
-  }
-}
+    final request = ImportPayloadPB(
+      parentViewId: parentViewId,
+      values: values,
+      syncAfterCreate: true,
+    );
 
-extension on ImportTypePB {
-  ViewLayoutPB toLayout() {
-    switch (this) {
-      case ImportTypePB.HistoryDocument:
-        return ViewLayoutPB.Document;
-      case ImportTypePB.HistoryDatabase ||
-            ImportTypePB.CSV ||
-            ImportTypePB.RawDatabase:
-        return ViewLayoutPB.Grid;
-      default:
-        throw UnimplementedError('Unsupported import type $this');
-    }
+    return FolderEventImportData(request).send();
   }
 }
diff --git a/...nd/appflowy_flutter/lib/workspace/presentation/home/menu/sidebar/import/import_panel.dart b/...nd/appflowy_flutter/lib/workspace/presentation/home/menu/sidebar/import/import_panel.dart
@@ -7,8 +7,6 @@ import 'package:appflowy/plugins/document/presentation/editor_plugins/migration/
 import 'package:appflowy/startup/startup.dart';
 import 'package:appflowy/workspace/application/settings/share/import_service.dart';
 import 'package:appflowy/workspace/presentation/home/menu/sidebar/import/import_type.dart';
-import 'package:appflowy/workspace/presentation/home/toast.dart';
-import 'package:appflowy_backend/log.dart';
 import 'package:appflowy_backend/protobuf/flowy-folder/protobuf.dart';
 import 'package:appflowy_editor/appflowy_editor.dart' hide Log;
 import 'package:easy_localization/easy_localization.dart';
@@ -153,6 +151,8 @@ class _ImportPanelState extends State<ImportPanel> {
 
     showLoading.value = true;
 
+    final importValues = <ImportValuePayloadPB>[];
+
     for (final file in result.files) {
       final path = file.path;
       if (path == null) {
@@ -166,59 +166,52 @@ class _ImportPanelState extends State<ImportPanel> {
         case ImportType.historyDocument:
           final bytes = _documentDataFrom(importType, data);
           if (bytes != null) {
-            final result = await ImportBackendService.importData(
-              bytes,
-              name,
-              parentViewId,
-              ImportTypePB.HistoryDocument,
+            importValues.add(
+              ImportValuePayloadPB.create()
+                ..name = name
+                ..data = bytes
+                ..viewLayout = ViewLayoutPB.Document
+                ..importType = ImportTypePB.HistoryDocument,
             );
-            result.onFailure((error) {
-              showSnackBarMessage(context, error.msg);
-              Log.error('Failed to import markdown $error');
-            });
           }
           break;
         case ImportType.historyDatabase:
-          final result = await ImportBackendService.importData(
-            utf8.encode(data),
-            name,
-            parentViewId,
-            ImportTypePB.HistoryDatabase,
+          importValues.add(
+            ImportValuePayloadPB.create()
+              ..name = name
+              ..data = utf8.encode(data)
+              ..viewLayout = ViewLayoutPB.Grid
+              ..importType = ImportTypePB.HistoryDatabase,
           );
-          result.onFailure((error) {
-            showSnackBarMessage(context, error.msg);
-            Log.error('Failed to import history database $error');
-          });
           break;
         case ImportType.databaseRawData:
-          final result = await ImportBackendService.importData(
-            utf8.encode(data),
-            name,
-            parentViewId,
-            ImportTypePB.RawDatabase,
+          importValues.add(
+            ImportValuePayloadPB.create()
+              ..name = name
+              ..data = utf8.encode(data)
+              ..viewLayout = ViewLayoutPB.Grid
+              ..importType = ImportTypePB.RawDatabase,
           );
-          result.onFailure((error) {
-            showSnackBarMessage(context, error.msg);
-            Log.error('Failed to import database raw data $error');
-          });
           break;
         case ImportType.databaseCSV:
-          final result = await ImportBackendService.importData(
-            utf8.encode(data),
-            name,
-            parentViewId,
-            ImportTypePB.CSV,
+          importValues.add(
+            ImportValuePayloadPB.create()
+              ..name = name
+              ..data = utf8.encode(data)
+              ..viewLayout = ViewLayoutPB.Grid
+              ..importType = ImportTypePB.CSV,
           );
-          result.onFailure((error) {
-            showSnackBarMessage(context, error.msg);
-            Log.error('Failed to import CSV $error');
-          });
           break;
         default:
           assert(false, 'Unsupported Type $importType');
       }
     }
 
+    await ImportBackendService.importPages(
+      parentViewId,
+      importValues,
+    );
+
     showLoading.value = false;
     widget.importCallback(importType, '', null);
   }

diff --git a/frontend/rust-lib/event-integration-test/src/folder_event.rs b/frontend/rust-lib/event-integration-test/src/folder_event.rs
@@ -259,13 +259,14 @@ impl EventIntegrationTest {
       .parse::<ViewPB>()
   }
 
-  pub async fn import_data(&self, data: ImportPB) -> ViewPB {
+  pub async fn import_data(&self, data: ImportPayloadPB) -> Vec<ViewPB> {
     EventBuilder::new(self.clone())
       .event(FolderEvent::ImportData)
       .payload(data)
       .async_send()
       .await
-      .parse::<ViewPB>()
+      .parse::<RepeatedViewPB>()
+      .items
   }
 
   pub async fn get_view_ancestors(&self, view_id: &str) -> Vec<ViewPB> {

diff --git a/frontend/rust-lib/event-integration-test/tests/folder/local_test/import_test.rs b/frontend/rust-lib/event-integration-test/tests/folder/local_test/import_test.rs
@@ -1,7 +1,7 @@
 use crate::util::unzip;
 use event_integration_test::EventIntegrationTest;
 use flowy_core::DEFAULT_NAME;
-use flowy_folder::entities::{ImportPB, ImportTypePB, ViewLayoutPB};
+use flowy_folder::entities::{ImportPayloadPB, ImportTypePB, ImportValuePayloadPB, ViewLayoutPB};
 
 #[tokio::test]
 async fn import_492_row_csv_file_test() {
@@ -16,8 +16,9 @@ async fn import_492_row_csv_file_test() {
   let workspace_id = test.get_current_workspace().await.id;
   let import_data = gen_import_data(file_name, csv_string, workspace_id);
 
-  let view = test.import_data(import_data).await;
-  let database = test.get_database(&view.id).await;
+  let views = test.import_data(import_data).await;
+  let view_id = views[0].clone().id;
+  let database = test.get_database(&view_id).await;
   assert_eq!(database.rows.len(), 492);
   drop(cleaner);
 }
@@ -35,21 +36,24 @@ async fn import_10240_row_csv_file_test() {
   let workspace_id = test.get_current_workspace().await.id;
   let import_data = gen_import_data(file_name, csv_string, workspace_id);
 
-  let view = test.import_data(import_data).await;
-  let database = test.get_database(&view.id).await;
+  let views = test.import_data(import_data).await;
+  let view_id = views[0].clone().id;
+  let database = test.get_database(&view_id).await;
   assert_eq!(database.rows.len(), 10240);
 
   drop(cleaner);
 }
 
-fn gen_import_data(file_name: String, csv_string: String, workspace_id: String) -> ImportPB {
-  let import_data = ImportPB {
+fn gen_import_data(file_name: String, csv_string: String, workspace_id: String) -> ImportPayloadPB {
+  ImportPayloadPB {
     parent_view_id: workspace_id.clone(),
-    name: file_name,
-    data: Some(csv_string.as_bytes().to_vec()),
-    file_path: None,
-    view_layout: ViewLayoutPB::Grid,
-    import_type: ImportTypePB::CSV,
-  };
-  import_data
+    sync_after_create: false,
+    values: vec![ImportValuePayloadPB {
+      name: file_name,
+      data: Some(csv_string.as_bytes().to_vec()),
+      file_path: None,
+      view_layout: ViewLayoutPB::Grid,
+      import_type: ImportTypePB::CSV,
+    }],
+  }
 }
diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs
@@ -1,4 +1,5 @@
 use bytes::Bytes;
+use collab_entity::EncodedCollab;
 use collab_integrate::collab_builder::AppFlowyCollabBuilder;
 use collab_integrate::CollabKVDB;
 use flowy_chat::chat_manager::ChatManager;
@@ -229,15 +230,15 @@ impl FolderOperationHandler for DocumentFolderOperation {
     _name: &str,
     _import_type: ImportType,
     bytes: Vec<u8>,
-  ) -> FutureResult<(), FlowyError> {
+  ) -> FutureResult<EncodedCollab, FlowyError> {
     let view_id = view_id.to_string();
     let manager = self.0.clone();
     FutureResult::new(async move {
       let data = DocumentDataPB::try_from(Bytes::from(bytes))?;
-      manager
+      let encoded_collab = manager
         .create_document(uid, &view_id, Some(data.into()))
         .await?;
-      Ok(())
+      Ok(encoded_collab)
     })
   }
 
@@ -392,7 +393,7 @@ impl FolderOperationHandler for DatabaseFolderOperation {
     _name: &str,
     import_type: ImportType,
     bytes: Vec<u8>,
-  ) -> FutureResult<(), FlowyError> {
+  ) -> FutureResult<EncodedCollab, FlowyError> {
     let database_manager = self.0.clone();
     let view_id = view_id.to_string();
     let format = match import_type {
@@ -406,11 +407,10 @@ impl FolderOperationHandler for DatabaseFolderOperation {
         String::from_utf8(bytes).map_err(|err| FlowyError::internal().with_context(err))
       })
       .await??;
-
-      database_manager
+      let result = database_manager
         .import_csv(view_id, content, format)
         .await?;
-      Ok(())
+      Ok(result.encoded_collab)
     })
   }
 
@@ -531,7 +531,7 @@ impl FolderOperationHandler for ChatFolderOperation {
     _name: &str,
     _import_type: ImportType,
     _bytes: Vec<u8>,
-  ) -> FutureResult<(), FlowyError> {
+  ) -> FutureResult<EncodedCollab, FlowyError> {
     FutureResult::new(async move { Err(FlowyError::not_support()) })
   }
 

diff --git a/frontend/rust-lib/flowy-database2/src/manager.rs b/frontend/rust-lib/flowy-database2/src/manager.rs
@@ -3,7 +3,7 @@ use std::collections::HashMap;
 use std::sync::{Arc, Weak};
 
 use collab::core::collab::{DataSource, MutexCollab};
-use collab_database::database::DatabaseData;
+use collab_database::database::{DatabaseData, MutexDatabase};
 use collab_database::error::DatabaseError;
 use collab_database::rows::RowId;
 use collab_database::views::{CreateDatabaseParams, CreateViewParams, DatabaseLayout};
@@ -309,10 +309,13 @@ impl DatabaseManager {
     Ok(())
   }
 
-  pub async fn create_database_with_params(&self, params: CreateDatabaseParams) -> FlowyResult<()> {
+  pub async fn create_database_with_params(
+    &self,
+    params: CreateDatabaseParams,
+  ) -> FlowyResult<Arc<MutexDatabase>> {
     let wdb = self.get_database_indexer().await?;
-    let _ = wdb.create_database(params)?;
-    Ok(())
+    let database = wdb.create_database(params)?;
+    Ok(database)
   }
 
   /// A linked view is a view that is linked to existing database.
@@ -362,11 +365,19 @@ impl DatabaseManager {
       return Err(FlowyError::internal().with_context("The number of rows exceeds the limit"));
     }
 
+    let view_id = params.inline_view_id.clone();
+    let database_id = params.database_id.clone();
+    let database = self.create_database_with_params(params).await?;
+    let encoded_collab = database
+      .lock()
+      .get_collab()
+      .lock()
+      .encode_collab_v1(|collab| CollabType::Database.validate_require_data(collab))?;
     let result = ImportResult {
-      database_id: params.database_id.clone(),
-      view_id: params.inline_view_id.clone(),
+      database_id,
+      view_id,
+      encoded_collab,
     };
-    self.create_database_with_params(params).await?;
     Ok(result)
   }
 

diff --git a/frontend/rust-lib/flowy-database2/src/services/share/csv/import.rs b/frontend/rust-lib/flowy-database2/src/services/share/csv/import.rs
@@ -5,6 +5,7 @@ use collab_database::fields::Field;
 use collab_database::rows::{new_cell_builder, Cell, CreateRowParams};
 use collab_database::views::{CreateDatabaseParams, CreateViewParams, DatabaseLayout};
 
+use collab_entity::EncodedCollab;
 use flowy_error::{FlowyError, FlowyResult};
 
 use crate::entities::FieldType;
@@ -166,6 +167,7 @@ impl FieldsRows {
 pub struct ImportResult {
   pub database_id: String,
   pub view_id: String,
+  pub encoded_collab: EncodedCollab,
 }
 
 #[cfg(test)]

diff --git a/frontend/rust-lib/flowy-document/src/manager.rs b/frontend/rust-lib/flowy-document/src/manager.rs
@@ -110,25 +110,30 @@ impl DocumentManager {
     uid: i64,
     doc_id: &str,
     data: Option<DocumentData>,
-  ) -> FlowyResult<()> {
+  ) -> FlowyResult<EncodedCollab> {
     if self.is_doc_exist(doc_id).await.unwrap_or(false) {
       Err(FlowyError::new(
         ErrorCode::RecordAlreadyExists,
         format!("document {} already exists", doc_id),
       ))
     } else {
-      let doc_state = doc_state_from_document_data(
+      let encoded_collab = doc_state_from_document_data(
         doc_id,
         data.unwrap_or_else(|| default_document_data(doc_id)),
       )
-      .await?
-      .doc_state
-      .to_vec();
+      .await?;
+      let doc_state = encoded_collab.doc_state.to_vec();
       let collab = self
-        .collab_for_document(uid, doc_id, DataSource::DocStateV1(doc_state), false)
+        .collab_for_document(
+          uid,
+          doc_id,
+          DataSource::DocStateV1(doc_state.clone()),
+          false,
+        )
         .await?;
       collab.lock().flush();
-      Ok(())
+
+      Ok(encoded_collab)
     }
   }