feat: remove yrs

Cargo.toml

@@ -1,12 +1,8 @@
 [workspace]
 
 members = [
-    "apps/cloud",
     "apps/doc_merger",
     "apps/keck",
-    "libs/cloud-database",
-    "libs/cloud-database/migration",
-    "libs/cloud-infra",
     "libs/jwst",
     # "libs/jwst-binding/jwst-ffi",
     "libs/jwst-binding/jwst-jni",
@@ -16,7 +12,6 @@ members = [
     # "libs/jwst-binding/jwst-wasm",
     "libs/jwst-codec",
     "libs/jwst-codec-util",
-    #"libs/jwst-codec/fuzz",
     "libs/jwst-core",
     "libs/jwst-logger",
     "libs/jwst-rpc",
@@ -47,9 +42,6 @@ lto = false
 inherits = "fast-release"
 debug = true
 
-# [profile.release.package.jwst-codec-fuzz]
-# debug = 1
-
 [profile.dev.package.image]
 opt-level = 3
 

apps/android/app/src/main/java/com/example/jwst_demo/MainActivity.kt

@@ -5,10 +5,12 @@ import android.os.Bundle
 import android.util.Log
 import androidx.annotation.RequiresApi
 import androidx.appcompat.app.AppCompatActivity
+import com.toeverything.jwst.Block
 import com.toeverything.jwst.Storage
 import java.io.File
 import java.util.*
-import com.toeverything.jwst.JwstVecOfStrings
+import com.toeverything.jwst.Workspace
+import kotlin.jvm.optionals.getOrNull
 
 fun <T> Optional<T>.unwrap(): T? = orElse(null)
 
@@ -21,112 +23,119 @@ class MainActivity : AppCompatActivity() {
 
         val database = File(filesDir, "jwst.db")
         val storage = Storage(database.absolutePath, "ws://10.0.2.2:3000/collaboration", "debug")
-        storage.getWorkspace("test").unwrap()?.let { workspace ->
-            workspace.setCallback { block_ids -> Log.i("jwst", "change: $block_ids") }
-            workspace.withTrx { trx -> workspace.get(trx, "a").unwrap() }?.let { block ->
-                // load the existing block on the second startup program.
-                val content = workspace.withTrx { trx -> block.get(trx, "a key") }?.get()
-                this.title = (content as String) + " exists"
-                workspace.withTrx { trx -> workspace.get(trx, "root").get().children(trx) }
-                    ?.joinToString { it }
-                    ?.let { Log.i("jwst", it) }
-                workspace.withTrx { trx ->
-                    Thread.sleep(1000)
-                    trx.create("child11", "child");
-                }
-            } ?: run {
-                // create a new block on the first startup program.
-                workspace.withTrx { trx ->
-                    val block = trx.create("a", "b")
-                    block.set(trx, "a key", "a value")
-                }
-
-                val content = workspace.withTrx { trx ->
-                    val block = workspace.get(trx, "a").get()
-                    block.get(trx, "a key").get()
-                }
-                this.title = content as String
-
-                // new lot of block and insert into children
-                workspace.withTrx { trx ->
-                    val block1 = trx.create("root", "root")
-                    val block2 = trx.create("child1", "child")
-                    val block3 = trx.create("child2", "child")
-                    val block4 = trx.create("child3", "child")
-                    val block5 = trx.create("child4", "child")
-                    val block6 = trx.create("child5", "child")
-                    val block7 = trx.create("child6", "child")
-                    val block8 = trx.create("child7", "child")
-                    val block9 = trx.create("child8", "child")
-                    val block10 = trx.create("child9", "child")
-                    val block11 = trx.create("child10", "child")
-
-                    block1.insertChildrenAt(trx, block2, 0)
-                    block1.insertChildrenAt(trx, block3, 1)
-                    block1.insertChildrenAt(trx, block4, 2)
-                    block1.insertChildrenAt(trx, block5, 3)
-                    block1.insertChildrenAt(trx, block6, 4)
-                    block1.insertChildrenAt(trx, block7, 5)
-                    block1.insertChildrenAt(trx, block8, 6)
-                    block1.insertChildrenAt(trx, block9, 7)
-                    block1.insertChildrenAt(trx, block10, 8)
-                    block1.insertChildrenAt(trx, block11, 9)
-                }
-            }
-
-            Log.i("jwst", "getSyncState " + storage.getSyncState())
-            Log.i("jwst", "isOffline " + storage.isOffline())
-            Log.i("jwst", "isConnected " + storage.isConnected())
-            Log.i("jwst", "isFinished " + storage.isFinished())
-            Log.i("jwst", "isError " + storage.isError())
 
+        storage.getWorkspace("test").unwrap()?.let { workspace ->
+            setupWorkspace(workspace)
 
-            workspace.withTrx { trx ->
-                trx.create("test", "list")
-                trx.create("test2", "list")
-            }
+            workspace.create("test", "list")
+            workspace.create("test2", "list")
 
             val blocks = workspace.getBlocksByFlavour("list")
             Log.i("jwst", "getBlocksByFlavour: $blocks")
 
             // search demo
             Log.i("jwst", "search demo")
-            workspace.withTrx { trx ->
-                val block = trx.create("search_test", "search_test_flavour")
-                block.set(trx, "title", "introduction")
-                block.set(trx, "text", "hello every one")
-                block.set(trx, "index", "this is index")
-            }
+
+            val block = workspace.create("search_test", "search_test_flavour")
+            block.set("title", "introduction")
+            block.set("text", "hello every one")
+            block.set("index", "this is index")
 
             var indexFields = arrayOf("title", "text")
             workspace.setSearchIndex(indexFields)
-            Log.i("jwst",  "search index: " + workspace.getSearchIndex().joinToString(" "))
+            Log.i("jwst", "search index: " + workspace.getSearchIndex().joinToString(" "))
 
             val searchResult1 = "search result1: " + workspace.search("duc")
-            Log.i("jwst",  searchResult1)
+            Log.i("jwst", searchResult1)
 
             val searchResult2 = "search result2: " + workspace.search("this")
             Log.i("jwst", searchResult2)
 
             var indexFields2 = arrayOf("index")
             workspace.setSearchIndex(indexFields2)
 
-            Log.i("jwst",  "search index: " + workspace.getSearchIndex().joinToString(" "))
+            Log.i("jwst", "search index: " + workspace.getSearchIndex().joinToString(" "))
 
             val searchResult3 = "search result3: " + workspace.search("this")
             Log.i("jwst", searchResult3)
 
             while (true) {
-                workspace.withTrx { trx ->
-                    Log.i("jwst", " getting root")
-                    workspace.get(trx, "root").unwrap()?.let { block ->
-                        block.get(trx, "test").ifPresent { value ->
-                            Log.i("jwst", "test: $value")
-                        }
+
+                Log.i("jwst", " getting root")
+                workspace.get("root").unwrap()?.let { block ->
+                    block.get("test").ifPresent { value ->
+                        Log.i("jwst", "test: $value")
                     }
                 }
+
                 Thread.sleep(1000)
             }
         }
     }
-}
+
+    private fun setupWorkspace(workspace: Workspace) {
+        workspace.setCallback { blockIds ->
+            Log.i("jwst", "change: $blockIds")
+        }
+
+        val existingBlock = workspace.get("a").getOrNull()
+
+        if (existingBlock != null) {
+            handleExistingBlock(existingBlock, workspace)
+        } else {
+            handleNewBlock(workspace)
+        }
+
+        while (true) {
+            Log.i("jwst", " getting root")
+            workspace.get("root").unwrap()?.let { block ->
+                block.get("test").ifPresent { value ->
+                    Log.i("jwst", "test: $value")
+                }
+            }
+            Thread.sleep(1000)
+        }
+    }
+
+
+    private fun handleExistingBlock(block: Block, workspace: Workspace) {
+        val content = block.get("a key").get()
+        this.title = "$content exists"
+
+        val children = workspace.get("root").get().children().joinToString { it.toString() }
+        Log.i("jwst", children)
+
+        Thread.sleep(1000)
+        workspace.create("child11", "child")
+    }
+
+    private fun handleNewBlock(workspace: Workspace) {
+        val block = workspace.create("a", "b")
+        block.set("a key", "a value")
+
+        val content = workspace.get("a").get().get("a key").get()
+        this.title = content as String
+
+        // Create and insert blocks into children
+        createAndInsertChildren(workspace)
+    }
+
+    private fun createAndInsertChildren(workspace: Workspace) {
+        val root = workspace.create("root", "root")
+        val children = (1..10).map { i ->
+            workspace.create("child$i", "child")
+        }
+
+        children.forEachIndexed { index, child ->
+            root.insertChildrenAt(child, index.toLong())
+        }
+    }
+
+    private fun logWorkspaceStatus(storage: Storage) {
+        Log.i("jwst", "getSyncState ${storage.getSyncState()}")
+        Log.i("jwst", "isOffline ${storage.isOffline()}")
+        Log.i("jwst", "isConnected ${storage.isConnected()}")
+        Log.i("jwst", "isFinished ${storage.isFinished()}")
+        Log.i("jwst", "isError ${storage.isError()}")
+    }
+}

apps/doc_merger/Cargo.toml

@@ -10,4 +10,5 @@ license = "AGPL-3.0-only"
 [dependencies]
 clap = { version = "4.3.19", features = ["derive"] }
 jwst-codec = { workspace = true }
+serde_json = "1.0.105"
 yrs = "0.16.5"

apps/doc_merger/src/main.rs

@@ -52,29 +52,56 @@
 
 fn main() {
     let args = Args::parse();
-    jwst_merge(&args);
-    yrs_merge(&args);
+    jwst_merge(
+        &args.path,
+        &args.output.clone().unwrap_or_else(|| format!("{}.jwst", args.path)),
+    );
+    std::io::stdin().read_line(&mut String::new()).unwrap();
+    yrs_merge(
+        &args.path,
+        &args.output.clone().unwrap_or_else(|| format!("{}.yrs", args.path)),
+    );
 }
 
-fn jwst_merge(args: &Args) {
-    let updates = load_path(&args.path).unwrap();
+fn jwst_merge(path: &str, output: &str) {
+    let updates = load_path(path).unwrap();
 
     let mut doc = Doc::default();
     for (i, update) in updates.iter().enumerate() {
         println!("apply update{i} {} bytes", update.len());
         doc.apply_update_from_binary(update.clone()).unwrap();
     }
-    let binary = doc.encode_update_v1().unwrap();
-    println!("merged {} bytes", binary.len());
-    write(
-        args.output.clone().unwrap_or_else(|| format!("{}.jwst", args.path)),
-        binary,
-    )
-    .unwrap();
+    doc.gc().unwrap();
+
+    let binary = {
+        // let json =
+        // serde_json::to_string_pretty(&doc.get_map("space:blocks").unwrap()).unwrap();
+        // println!("json {} bytes", json.len());
+        let binary = doc.encode_update_v1().unwrap();
+        drop(doc);
+        println!("merged {} bytes", binary.len());
+        std::io::stdin().read_line(&mut String::new()).unwrap();
+        binary
+    };
+
+    {
+        let mut doc = Doc::default();
+        doc.apply_update_from_binary(binary.clone()).unwrap();
+        let new_binary = doc.encode_update_v1().unwrap();
+        let new_json = serde_json::to_string_pretty(&doc.get_map("space:blocks").unwrap()).unwrap();
+
+        println!(
+            "re-encoded {} bytes,  new json {} bytes",
+            new_binary.len(),
+            new_json.len()
+        );
+        std::io::stdin().read_line(&mut String::new()).unwrap();
+    }
+    write(output, binary).unwrap();
 }
 
-fn yrs_merge(args: &Args) {
-    let updates = load_path(&args.path).unwrap();
+fn yrs_merge(path: &str, output: &str) {
+    let updates = load_path(&path).unwrap();
 
     let doc = yrs::Doc::new();
     for (i, update) in updates.iter().enumerate() {
@@ -86,9 +113,15 @@
         .encode_state_as_update_v1(&StateVector::default())
         .unwrap();
     println!("merged {} bytes", binary.len());
-    write(
-        args.output.clone().unwrap_or_else(|| format!("{}.yrs", args.path)),
-        binary,
-    )
-    .unwrap();
+    write(output, binary).unwrap();
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_gc() {
+        jwst_merge("/Users/ds/Downloads/out", "/Users/ds/Downloads/out.jwst");
+    }
 }

apps/keck/Cargo.toml

@@ -20,10 +20,9 @@ schema = ["utoipa-swagger-ui"]
 
 [dependencies]
 anyhow = "1.0.70"
-axum = { version = "0.6.16", features = ["headers", "ws"] }
+axum = { version = "0.6.20", features = ["headers", "ws"] }
 cfg-if = "1.0.0"
 futures = "0.3.28"
-lib0 = { version = "0.16.5", features = ["lib0-serde"] }
 log = { version = "0.4.17", features = [
   "max_level_trace",
   "release_max_level_info",
@@ -43,27 +42,25 @@ tower = "0.4.13"
 tower-http = { version = "0.4.0", features = ["cors"] }
 thiserror = "1.0.40"
 time = "0.3.20"
-tokio = { version = "1.27.0", features = [
+tokio = { version = "=1.28.0", features = [
   "macros",
   "rt-multi-thread",
   "signal",
 ] }
-utoipa = { version = "3.3.0", features = ["axum_extras"], optional = true }
-utoipa-swagger-ui = { version = "3.1.3", features = ["axum"], optional = true }
-yrs = "0.16.5"
-libc = "0.2.141"
+utoipa = { version = "3.5.0", features = ["axum_extras"], optional = true }
+utoipa-swagger-ui = { version = "3.1.5", optional = true }
+libc = "0.2.147"
 rand = "0.8.5"
-reqwest = { version = "0.11.14", default-features = false, features = [
+reqwest = { version = "0.11.19", default-features = false, features = [
   "json",
   "rustls-tls",
 ] }
 
 # ======= workspace dependencies =======
-cloud-infra = { path = "../../libs/cloud-infra" }
-jwst = { workspace = true, features = ["workspace-auto-subscribe"] }
+jwst-core = { workspace = true }
 jwst-logger = { workspace = true }
 jwst-rpc = { workspace = true }
 jwst-storage = { workspace = true }
 
 [dev-dependencies]
-axum-test-helper = "0.2.0"
+axum-test-helper = "0.3.0"