🐛 Fix document import for documents with >1000 changes

backend/openapi-schema.yml

@@ -133,12 +133,6 @@ components:
       type: object
     Body_import_document_api_v1_documents_import__post:
       properties:
-        document_updates:
-          items:
-            format: binary
-            type: string
-          title: Document Updates
-          type: array
         media_file:
           format: binary
           title: Media File
@@ -147,7 +141,6 @@ components:
           title: Name
           type: string
       required:
-      - document_updates
       - media_file
       - name
       title: Body_import_document_api_v1_documents_import__post

backend/transcribee_backend/routers/document.py

@@ -50,7 +50,6 @@
     DocumentMediaFile,
     DocumentMediaTag,
     DocumentShareToken,
-    DocumentUpdate,
     Task,
     TaskType,
     UserToken,
@@ -360,13 +359,11 @@ async def create_document(
 
 @document_router.post("/import/")
 def import_document(
-    document_updates: List[UploadFile] = File(),
     media_file: UploadFile = File(),
     token: UserToken = Depends(get_user_token),
     session: Session = Depends(get_session),
     name: str = Form(),
 ):
-
     document = Document(
         name=name,
         user_id=token.user_id,
@@ -399,13 +396,6 @@ def import_document(
     )
     session.add(reencode_task)
 
-    for document_update in document_updates:
-        session.add(
-            DocumentUpdate(
-                document_id=document.id, change_bytes=document_update.file.read()
-            )
-        )
-
     session.commit()
     return document.as_api_document()
 

frontend/src/editor/automerge_websocket_editor.ts

@@ -17,7 +17,7 @@ enum MessageSyncType {
 }
 
 export function useAutomergeWebsocketEditor(
-  url: string | URL,
+  url: string,
   { onInitialSyncComplete }: { onInitialSyncComplete: (editor?: Editor) => void },
 ): [Editor?, Paragraph[]?] {
   const debug = useDebugMode();
@@ -45,7 +45,7 @@ export function useAutomergeWebsocketEditor(
   const [_, navigate] = useLocation();
 
   useEffect(() => {
-    const ws = new ReconnectingWebSocket(url.toString(), [], { debug });
+    const ws = new ReconnectingWebSocket(url, [], { debug });
 
     let bytesReceived = 0;
     console.time('initialSync');
@@ -114,7 +114,7 @@ export function useAutomergeWebsocketEditor(
       wsRef.current = null;
       ws.close();
     };
-  }, [url.toString(), setEditorAndInitialValue]);
+  }, [url, setEditorAndInitialValue]);
 
   return [editorAndInitialValue?.editor, editorAndInitialValue?.initialValue];
 }

frontend/src/openapi-schema.ts

@@ -193,8 +193,6 @@ export interface components {
     };
     /** Body_import_document_api_v1_documents_import__post */
     Body_import_document_api_v1_documents_import__post: {
-      /** Document Updates */
-      document_updates: (string)[];
       /**
        * Media File
        * Format: binary

frontend/src/pages/document.tsx

@@ -16,8 +16,7 @@ import { BiPencil } from 'react-icons/bi';
 import { SubmitHandler, useForm } from 'react-hook-form';
 import { Helmet } from 'react-helmet';
 import { ShareModal } from '../editor/share';
-import { useAuthData } from '../utils/auth';
-import { getAuthToken, getShareToken } from '../api';
+import { getDocumentWsUrl, useAuthData } from '../utils/auth';
 import { ExportModal } from '../editor/export';
 
 const LazyDebugPanel = lazy(() =>
@@ -114,18 +113,7 @@ export function DocumentPage({
   const debugMode = useDebugMode();
   const { isLoggedIn } = useAuthData();
 
-  const url = new URL(`/api/v1/documents/sync/${documentId}/`, window.location.href);
-  url.protocol = url.protocol.replace('http', 'ws');
-
-  const authToken = getAuthToken();
-  if (authToken) {
-    url.searchParams.append('authorization', `Token ${authToken}`);
-  }
-
-  const shareToken = getShareToken();
-  if (shareToken) {
-    url.searchParams.append('share_token', shareToken);
-  }
+  const url = getDocumentWsUrl(documentId);
 
   const [editor, initialValue] = useAutomergeWebsocketEditor(url, {
     onInitialSyncComplete: () => {

frontend/src/pages/new_document.tsx

@@ -3,6 +3,7 @@ import { SubmitHandler, useForm } from 'react-hook-form';
 import clsx from 'clsx';
 import { useLocation } from 'wouter';
 
+import ReconnectingWebSocket from 'reconnecting-websocket';
 import { createDocument, importDocument } from '../api/document';
 import { Dialog, DialogTitle } from '../components/dialog';
 import { FormControl, Input, Select } from '../components/form';
@@ -11,6 +12,7 @@ import { AppCenter } from '../components/app';
 import { useGetConfig } from '../api/config';
 import { BlobReader, BlobWriter, ZipReader, Entry } from '@zip.js/zip.js';
 import * as Automerge from '@automerge/automerge';
+import { getDocumentWsUrl } from '../utils/auth';
 
 type FieldValues = {
   name: string;
@@ -97,7 +99,7 @@ export function NewDocumentPage() {
       setLoading(true);
       let response;
       if (isImport) {
-        type DocumentImportParamPeters = Parameters<typeof importDocument>[0];
+        type DocumentImportParameters = Parameters<typeof importDocument>[0];
         const zipReader = new ZipReader(new BlobReader(data.audioFile[0]));
         const entries = await zipReader.getEntries();
         const [automergeFile, mediaFile] = await Promise.all([
@@ -114,13 +116,16 @@ export function NewDocumentPage() {
         }
         const doc = Automerge.load(new Uint8Array(await automergeFile.arrayBuffer()));
         const changes = Automerge.getChanges(Automerge.init(), doc).map((x) => new Blob([x]));
-        const documentParameters: DocumentImportParamPeters = {
+        const documentParameters: DocumentImportParameters = {
           name: data.name,
           media_file: mediaFile,
-          document_updates: changes,
         };
 
         response = await importDocument(documentParameters);
+        const ws = new ReconnectingWebSocket(getDocumentWsUrl(response.data.id), []);
+        for (const change of changes) {
+          ws.send(change);
+        }
       } else {
         type DocumentCreateParameters = Parameters<typeof createDocument>[0];
         const documentParameters: DocumentCreateParameters = {

frontend/src/utils/auth.ts

@@ -1,6 +1,22 @@
-import { getShareToken } from '../api';
+import { getAuthToken, getShareToken } from '../api';
 import { useGetMe } from '../api/user';
 
+export function getDocumentWsUrl(documentId: string) {
+  const url = new URL(`/api/v1/documents/sync/${documentId}/`, window.location.href);
+  url.protocol = url.protocol.replace('http', 'ws');
+
+  const authToken = getAuthToken();
+  if (authToken) {
+    url.searchParams.append('authorization', `Token ${authToken}`);
+  }
+
+  const shareToken = getShareToken();
+  if (shareToken) {
+    url.searchParams.append('share_token', shareToken);
+  }
+  return url.toString();
+}
+
 export function useAuthData(): {
   isLoading: boolean;
   isLoggedIn: boolean;