From 1d188c647cda108f8e42ff8ae0c475cc5808542e Mon Sep 17 00:00:00 2001 From: DarkSky Date: Thu, 17 Aug 2023 23:17:54 +0800 Subject: [PATCH 01/49] feat: remove yrs --- Cargo.lock | 95 +++ Cargo.toml | 7 + .../RustXcframework.xcframework/Info.plist | 14 +- .../Sources/OctoBase/jwst-core-swift.swift | 647 +++++++++++++++++ libs/jwst-binding/jwst-core-swift/.gitignore | 2 + libs/jwst-binding/jwst-core-swift/Cargo.toml | 35 + libs/jwst-binding/jwst-core-swift/binding.h | 105 +++ libs/jwst-binding/jwst-core-swift/build.rs | 14 + .../jwst-swift-integrate/Cargo.toml | 9 + .../jwst-swift-integrate/src/main.rs | 62 ++ .../jwst-binding/jwst-core-swift/src/block.rs | 186 +++++ .../jwst-core-swift/src/dynamic_value.rs | 71 ++ libs/jwst-binding/jwst-core-swift/src/lib.rs | 143 ++++ .../jwst-core-swift/src/storage.rs | 213 ++++++ .../jwst-core-swift/src/workspace.rs | 74 ++ libs/jwst-core-rpc/Cargo.toml | 56 ++ libs/jwst-core-rpc/src/broadcast.rs | 109 +++ libs/jwst-core-rpc/src/client/mod.rs | 98 +++ libs/jwst-core-rpc/src/client/webrtc.rs | 96 +++ libs/jwst-core-rpc/src/client/websocket.rs | 103 +++ .../src/connector/axum_socket.rs | 84 +++ libs/jwst-core-rpc/src/connector/memory.rs | 125 ++++ libs/jwst-core-rpc/src/connector/mod.rs | 21 + .../src/connector/tungstenite_socket.rs | 77 ++ libs/jwst-core-rpc/src/connector/webrtc.rs | 186 +++++ libs/jwst-core-rpc/src/context.rs | 218 ++++++ libs/jwst-core-rpc/src/handler.rs | 481 +++++++++++++ libs/jwst-core-rpc/src/lib.rs | 53 ++ libs/jwst-core-rpc/src/types.rs | 22 + .../src/utils/memory_workspace.rs | 56 ++ libs/jwst-core-rpc/src/utils/mod.rs | 7 + .../jwst-core-rpc/src/utils/server_context.rs | 75 ++ libs/jwst-core-storage/Cargo.toml | 43 ++ libs/jwst-core-storage/src/entities/blobs.rs | 21 + .../src/entities/bucket_blobs.rs | 19 + .../src/entities/diff_log.rs | 18 + libs/jwst-core-storage/src/entities/docs.rs | 21 + libs/jwst-core-storage/src/entities/mod.rs | 9 + .../src/entities/optimized_blobs.rs | 23 + .../jwst-core-storage/src/entities/prelude.rs | 7 + libs/jwst-core-storage/src/lib.rs | 39 + .../src/migration/Cargo.toml | 16 + .../jwst-core-storage/src/migration/README.md | 41 ++ .../src/migration/src/lib.rs | 25 + .../m20220101_000001_initial_blob_table.rs | 57 ++ .../src/m20220101_000002_initial_doc_table.rs | 60 ++ .../m20230321_000001_blob_optimized_table.rs | 71 ++ ...230614_000001_initial_bucket_blob_table.rs | 60 ++ .../src/m20230626_023319_doc_guid.rs | 58 ++ ...m20230814_061223_initial_diff_log_table.rs | 47 ++ .../src/migration/src/main.rs | 6 + .../src/migration/src/schema.rs | 50 ++ libs/jwst-core-storage/src/rate_limiter.rs | 81 +++ .../src/storage/blobs/bucket_local_db.rs | 397 +++++++++++ .../src/storage/blobs/local_db.rs | 306 ++++++++ .../src/storage/blobs/mod.rs | 669 ++++++++++++++++++ .../src/storage/blobs/utils.rs | 252 +++++++ libs/jwst-core-storage/src/storage/difflog.rs | 51 ++ .../src/storage/docs/database.rs | 634 +++++++++++++++++ .../jwst-core-storage/src/storage/docs/mod.rs | 148 ++++ .../src/storage/docs/utils.rs | 26 + libs/jwst-core-storage/src/storage/mod.rs | 262 +++++++ libs/jwst-core-storage/src/storage/test.rs | 44 ++ libs/jwst-core-storage/src/types.rs | 28 + 64 files changed, 7126 insertions(+), 7 deletions(-) create mode 100644 apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-core-swift.swift create mode 100644 libs/jwst-binding/jwst-core-swift/.gitignore create mode 100644 libs/jwst-binding/jwst-core-swift/Cargo.toml create mode 100644 libs/jwst-binding/jwst-core-swift/binding.h create mode 100644 libs/jwst-binding/jwst-core-swift/build.rs create mode 100644 libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/Cargo.toml create mode 100644 libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs create mode 100644 libs/jwst-binding/jwst-core-swift/src/block.rs create mode 100644 libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs create mode 100644 libs/jwst-binding/jwst-core-swift/src/lib.rs create mode 100644 libs/jwst-binding/jwst-core-swift/src/storage.rs create mode 100644 libs/jwst-binding/jwst-core-swift/src/workspace.rs create mode 100644 libs/jwst-core-rpc/Cargo.toml create mode 100644 libs/jwst-core-rpc/src/broadcast.rs create mode 100644 libs/jwst-core-rpc/src/client/mod.rs create mode 100644 libs/jwst-core-rpc/src/client/webrtc.rs create mode 100644 libs/jwst-core-rpc/src/client/websocket.rs create mode 100644 libs/jwst-core-rpc/src/connector/axum_socket.rs create mode 100644 libs/jwst-core-rpc/src/connector/memory.rs create mode 100644 libs/jwst-core-rpc/src/connector/mod.rs create mode 100644 libs/jwst-core-rpc/src/connector/tungstenite_socket.rs create mode 100644 libs/jwst-core-rpc/src/connector/webrtc.rs create mode 100644 libs/jwst-core-rpc/src/context.rs create mode 100644 libs/jwst-core-rpc/src/handler.rs create mode 100644 libs/jwst-core-rpc/src/lib.rs create mode 100644 libs/jwst-core-rpc/src/types.rs create mode 100644 libs/jwst-core-rpc/src/utils/memory_workspace.rs create mode 100644 libs/jwst-core-rpc/src/utils/mod.rs create mode 100644 libs/jwst-core-rpc/src/utils/server_context.rs create mode 100644 libs/jwst-core-storage/Cargo.toml create mode 100644 libs/jwst-core-storage/src/entities/blobs.rs create mode 100644 libs/jwst-core-storage/src/entities/bucket_blobs.rs create mode 100644 libs/jwst-core-storage/src/entities/diff_log.rs create mode 100644 libs/jwst-core-storage/src/entities/docs.rs create mode 100644 libs/jwst-core-storage/src/entities/mod.rs create mode 100644 libs/jwst-core-storage/src/entities/optimized_blobs.rs create mode 100644 libs/jwst-core-storage/src/entities/prelude.rs create mode 100644 libs/jwst-core-storage/src/lib.rs create mode 100644 libs/jwst-core-storage/src/migration/Cargo.toml create mode 100644 libs/jwst-core-storage/src/migration/README.md create mode 100644 libs/jwst-core-storage/src/migration/src/lib.rs create mode 100644 libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs create mode 100644 libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs create mode 100644 libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs create mode 100644 libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs create mode 100644 libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs create mode 100644 libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs create mode 100644 libs/jwst-core-storage/src/migration/src/main.rs create mode 100644 libs/jwst-core-storage/src/migration/src/schema.rs create mode 100644 libs/jwst-core-storage/src/rate_limiter.rs create mode 100644 libs/jwst-core-storage/src/storage/blobs/bucket_local_db.rs create mode 100644 libs/jwst-core-storage/src/storage/blobs/local_db.rs create mode 100644 libs/jwst-core-storage/src/storage/blobs/mod.rs create mode 100644 libs/jwst-core-storage/src/storage/blobs/utils.rs create mode 100644 libs/jwst-core-storage/src/storage/difflog.rs create mode 100644 libs/jwst-core-storage/src/storage/docs/database.rs create mode 100644 libs/jwst-core-storage/src/storage/docs/mod.rs create mode 100644 libs/jwst-core-storage/src/storage/docs/utils.rs create mode 100644 libs/jwst-core-storage/src/storage/mod.rs create mode 100644 libs/jwst-core-storage/src/storage/test.rs create mode 100644 libs/jwst-core-storage/src/types.rs diff --git a/Cargo.lock b/Cargo.lock index 84a659f4d..68e7e1c0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2762,6 +2762,101 @@ dependencies = [ "tracing", ] +[[package]] +name = "jwst-core-rpc" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert-json-diff", + "async-trait", + "axum", + "byteorder", + "bytes", + "chrono", + "futures", + "indicatif", + "jwst-codec", + "jwst-core", + "jwst-core-storage", + "jwst-logger", + "lru_time_cache", + "nanoid", + "nom", + "rand 0.8.5", + "reqwest", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tokio", + "tokio-tungstenite 0.20.0", + "url", + "webrtc", +] + +[[package]] +name = "jwst-core-storage" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "dotenvy", + "futures", + "governor", + "image", + "jwst-codec", + "jwst-core", + "jwst-core-storage-migration", + "jwst-logger", + "opendal", + "path-ext", + "rand 0.8.5", + "sea-orm", + "sea-orm-migration", + "sha2", + "thiserror", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "jwst-core-storage-migration" +version = "0.1.0" +dependencies = [ + "sea-orm-migration", + "tokio", +] + +[[package]] +name = "jwst-core-swift" +version = "0.1.0" +dependencies = [ + "chrono", + "futures", + "jwst-core", + "jwst-core-rpc", + "jwst-core-storage", + "jwst-logger", + "nanoid", + "regex", + "reqwest", + "serde", + "serde_json", + "swift-bridge", + "swift-bridge-build", + "tokio", +] + +[[package]] +name = "jwst-core-swift-integrate" +version = "0.1.0" +dependencies = [ + "swift-bridge-build", +] + [[package]] name = "jwst-jni" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 35263cb54..9d8b86b9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,16 @@ members = [ # "libs/jwst-binding/jwst-py", "libs/jwst-binding/jwst-swift", "libs/jwst-binding/jwst-swift/jwst-swift-integrate", + "libs/jwst-binding/jwst-core-swift", + "libs/jwst-binding/jwst-core-swift/jwst-swift-integrate", # "libs/jwst-binding/jwst-wasm", "libs/jwst-codec", "libs/jwst-codec-util", #"libs/jwst-codec/fuzz", "libs/jwst-core", + "libs/jwst-core-rpc", + "libs/jwst-core-storage", + "libs/jwst-core-storage/src/migration", "libs/jwst-logger", "libs/jwst-rpc", "libs/jwst-storage", @@ -29,6 +34,8 @@ resolver = "2" jwst = { workspace = true, path = "libs/jwst" } jwst-codec = { workspace = true, path = "libs/jwst-codec" } jwst-core = { workspace = true, path = "libs/jwst-core" } +jwst-core-rpc = { workspace = true, path = "libs/jwst-core-rpc" } +jwst-core-storage = { workspace = true, path = "libs/jwst-core-storage" } jwst-logger = { workspace = true, path = "libs/jwst-logger" } jwst-rpc = { workspace = true, path = "libs/jwst-rpc" } jwst-storage = { workspace = true, path = "libs/jwst-storage" } diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist index 067a2b752..b3d1aee06 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist @@ -8,7 +8,7 @@ HeadersPath Headers LibraryIdentifier - macos-arm64 + ios-arm64-simulator LibraryPath liboctobase.a SupportedArchitectures @@ -16,13 +16,15 @@ arm64 SupportedPlatform - macos + ios + SupportedPlatformVariant + simulator HeadersPath Headers LibraryIdentifier - ios-arm64 + macos-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -30,13 +32,13 @@ arm64 SupportedPlatform - ios + macos HeadersPath Headers LibraryIdentifier - ios-arm64-simulator + ios-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -45,8 +47,6 @@ SupportedPlatform ios - SupportedPlatformVariant - simulator CFBundlePackageType diff --git a/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-core-swift.swift b/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-core-swift.swift new file mode 100644 index 000000000..0556654ee --- /dev/null +++ b/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-core-swift.swift @@ -0,0 +1,647 @@ +import RustXcframework + +public class Block: BlockRefMut { + var isOwned: Bool = true + + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } + + deinit { + if isOwned { + __swift_bridge__$Block$_free(ptr) + } + } +} +public class BlockRefMut: BlockRef { + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } +} +public class BlockRef { + var ptr: UnsafeMutableRawPointer + + public init(ptr: UnsafeMutableRawPointer) { + self.ptr = ptr + } +} +extension BlockRef { + public func get(_ block_id: GenericIntoRustString) -> Optional { + { let val = __swift_bridge__$Block$get(ptr, { let rustString = block_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return DynamicValue(ptr: val!) } else { return nil } }() + } + + public func children() -> RustVec { + RustVec(ptr: __swift_bridge__$Block$children(ptr)) + } + + public func push_children(_ block: BlockRef) { + __swift_bridge__$Block$push_children(ptr, block.ptr) + } + + public func insert_children_at(_ block: BlockRef, _ pos: UInt32) { + __swift_bridge__$Block$insert_children_at(ptr, block.ptr, pos) + } + + public func insert_children_before(_ block: BlockRef, _ reference: GenericToRustStr) { + reference.toRustStr({ referenceAsRustStr in + __swift_bridge__$Block$insert_children_before(ptr, block.ptr, referenceAsRustStr) + }) + } + + public func insert_children_after(_ block: BlockRef, _ reference: GenericToRustStr) { + reference.toRustStr({ referenceAsRustStr in + __swift_bridge__$Block$insert_children_after(ptr, block.ptr, referenceAsRustStr) + }) + } + + public func remove_children(_ block: BlockRef) { + __swift_bridge__$Block$remove_children(ptr, block.ptr) + } + + public func exists_children(_ block_id: GenericToRustStr) -> Int32 { + return block_id.toRustStr({ block_idAsRustStr in + __swift_bridge__$Block$exists_children(ptr, block_idAsRustStr) + }) + } + + public func parent() -> RustString { + RustString(ptr: __swift_bridge__$Block$parent(ptr)) + } + + public func updated() -> UInt64 { + __swift_bridge__$Block$updated(ptr) + } + + public func id() -> RustString { + RustString(ptr: __swift_bridge__$Block$id(ptr)) + } + + public func flavour() -> RustString { + RustString(ptr: __swift_bridge__$Block$flavour(ptr)) + } + + public func created() -> UInt64 { + __swift_bridge__$Block$created(ptr) + } + + public func set_bool(_ key: GenericIntoRustString, _ value: Bool) { + __swift_bridge__$Block$set_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), value) + } + + public func set_string(_ key: GenericIntoRustString, _ value: GenericIntoRustString) { + __swift_bridge__$Block$set_string(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = value.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) + } + + public func set_float(_ key: GenericIntoRustString, _ value: Double) { + __swift_bridge__$Block$set_float(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), value) + } + + public func set_integer(_ key: GenericIntoRustString, _ value: Int64) { + __swift_bridge__$Block$set_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), value) + } + + public func set_null(_ key: GenericIntoRustString) { + __swift_bridge__$Block$set_null(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) + } + + public func is_bool(_ key: GenericIntoRustString) -> Bool { + __swift_bridge__$Block$is_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) + } + + public func is_string(_ key: GenericIntoRustString) -> Bool { + __swift_bridge__$Block$is_string(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) + } + + public func is_float(_ key: GenericIntoRustString) -> Bool { + __swift_bridge__$Block$is_float(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) + } + + public func is_integer(_ key: GenericIntoRustString) -> Bool { + __swift_bridge__$Block$is_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) + } + + public func get_bool(_ key: GenericIntoRustString) -> Optional { + { let val = __swift_bridge__$Block$get_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() + } + + public func get_string(_ key: GenericIntoRustString) -> Optional { + { let val = __swift_bridge__$Block$get_string(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return RustString(ptr: val!) } else { return nil } }() + } + + public func get_float(_ key: GenericIntoRustString) -> Optional { + { let val = __swift_bridge__$Block$get_float(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() + } + + public func get_integer(_ key: GenericIntoRustString) -> Optional { + { let val = __swift_bridge__$Block$get_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() + } +} +extension Block: Vectorizable { + public static func vecOfSelfNew() -> UnsafeMutableRawPointer { + __swift_bridge__$Vec_Block$new() + } + + public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { + __swift_bridge__$Vec_Block$drop(vecPtr) + } + + public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Block) { + __swift_bridge__$Vec_Block$push(vecPtr, {value.isOwned = false; return value.ptr;}()) + } + + public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { + let pointer = __swift_bridge__$Vec_Block$pop(vecPtr) + if pointer == nil { + return nil + } else { + return (Block(ptr: pointer!) as! Self) + } + } + + public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_Block$get(vecPtr, index) + if pointer == nil { + return nil + } else { + return BlockRef(ptr: pointer!) + } + } + + public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_Block$get_mut(vecPtr, index) + if pointer == nil { + return nil + } else { + return BlockRefMut(ptr: pointer!) + } + } + + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { + __swift_bridge__$Vec_Block$len(vecPtr) + } +} + + +public class DynamicValueMap: DynamicValueMapRefMut { + var isOwned: Bool = true + + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } + + deinit { + if isOwned { + __swift_bridge__$DynamicValueMap$_free(ptr) + } + } +} +public class DynamicValueMapRefMut: DynamicValueMapRef { + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } +} +public class DynamicValueMapRef { + var ptr: UnsafeMutableRawPointer + + public init(ptr: UnsafeMutableRawPointer) { + self.ptr = ptr + } +} +extension DynamicValueMap: Vectorizable { + public static func vecOfSelfNew() -> UnsafeMutableRawPointer { + __swift_bridge__$Vec_DynamicValueMap$new() + } + + public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { + __swift_bridge__$Vec_DynamicValueMap$drop(vecPtr) + } + + public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: DynamicValueMap) { + __swift_bridge__$Vec_DynamicValueMap$push(vecPtr, {value.isOwned = false; return value.ptr;}()) + } + + public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { + let pointer = __swift_bridge__$Vec_DynamicValueMap$pop(vecPtr) + if pointer == nil { + return nil + } else { + return (DynamicValueMap(ptr: pointer!) as! Self) + } + } + + public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_DynamicValueMap$get(vecPtr, index) + if pointer == nil { + return nil + } else { + return DynamicValueMapRef(ptr: pointer!) + } + } + + public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_DynamicValueMap$get_mut(vecPtr, index) + if pointer == nil { + return nil + } else { + return DynamicValueMapRefMut(ptr: pointer!) + } + } + + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { + __swift_bridge__$Vec_DynamicValueMap$len(vecPtr) + } +} + + +public class DynamicValue: DynamicValueRefMut { + var isOwned: Bool = true + + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } + + deinit { + if isOwned { + __swift_bridge__$DynamicValue$_free(ptr) + } + } +} +public class DynamicValueRefMut: DynamicValueRef { + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } +} +public class DynamicValueRef { + var ptr: UnsafeMutableRawPointer + + public init(ptr: UnsafeMutableRawPointer) { + self.ptr = ptr + } +} +extension DynamicValueRef { + public func as_bool() -> Optional { + { let val = __swift_bridge__$DynamicValue$as_bool(ptr); if val.is_some { return val.val } else { return nil } }() + } + + public func as_number() -> Optional { + { let val = __swift_bridge__$DynamicValue$as_number(ptr); if val.is_some { return val.val } else { return nil } }() + } + + public func as_int() -> Optional { + { let val = __swift_bridge__$DynamicValue$as_int(ptr); if val.is_some { return val.val } else { return nil } }() + } + + public func as_string() -> Optional { + { let val = __swift_bridge__$DynamicValue$as_string(ptr); if val != nil { return RustString(ptr: val!) } else { return nil } }() + } + + public func as_map() -> Optional { + { let val = __swift_bridge__$DynamicValue$as_map(ptr); if val != nil { return DynamicValueMap(ptr: val!) } else { return nil } }() + } + + public func as_array() -> Optional> { + { let val = __swift_bridge__$DynamicValue$as_array(ptr); if val != nil { return RustVec(ptr: val!) } else { return nil } }() + } + + public func as_buffer() -> Optional> { + { let val = __swift_bridge__$DynamicValue$as_buffer(ptr); if val != nil { return RustVec(ptr: val!) } else { return nil } }() + } +} +extension DynamicValue: Vectorizable { + public static func vecOfSelfNew() -> UnsafeMutableRawPointer { + __swift_bridge__$Vec_DynamicValue$new() + } + + public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { + __swift_bridge__$Vec_DynamicValue$drop(vecPtr) + } + + public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: DynamicValue) { + __swift_bridge__$Vec_DynamicValue$push(vecPtr, {value.isOwned = false; return value.ptr;}()) + } + + public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { + let pointer = __swift_bridge__$Vec_DynamicValue$pop(vecPtr) + if pointer == nil { + return nil + } else { + return (DynamicValue(ptr: pointer!) as! Self) + } + } + + public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_DynamicValue$get(vecPtr, index) + if pointer == nil { + return nil + } else { + return DynamicValueRef(ptr: pointer!) + } + } + + public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_DynamicValue$get_mut(vecPtr, index) + if pointer == nil { + return nil + } else { + return DynamicValueRefMut(ptr: pointer!) + } + } + + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { + __swift_bridge__$Vec_DynamicValue$len(vecPtr) + } +} + + +public class Workspace: WorkspaceRefMut { + var isOwned: Bool = true + + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } + + deinit { + if isOwned { + __swift_bridge__$Workspace$_free(ptr) + } + } +} +public class WorkspaceRefMut: WorkspaceRef { + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } +} +public class WorkspaceRef { + var ptr: UnsafeMutableRawPointer + + public init(ptr: UnsafeMutableRawPointer) { + self.ptr = ptr + } +} +extension WorkspaceRef { + public func id() -> RustString { + RustString(ptr: __swift_bridge__$Workspace$id(ptr)) + } + + public func client_id() -> UInt64 { + __swift_bridge__$Workspace$client_id(ptr) + } + + public func get(_ block_id: GenericIntoRustString) -> Optional { + { let val = __swift_bridge__$Workspace$get(ptr, { let rustString = block_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return Block(ptr: val!) } else { return nil } }() + } + + public func create(_ block_id: GenericIntoRustString, _ flavour: GenericIntoRustString) -> Block { + Block(ptr: __swift_bridge__$Workspace$create(ptr, { let rustString = block_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = flavour.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) + } + + public func search(_ query: GenericIntoRustString) -> RustString { + RustString(ptr: __swift_bridge__$Workspace$search(ptr, { let rustString = query.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) + } + + public func get_blocks_by_flavour(_ flavour: GenericToRustStr) -> RustVec { + return flavour.toRustStr({ flavourAsRustStr in + RustVec(ptr: __swift_bridge__$Workspace$get_blocks_by_flavour(ptr, flavourAsRustStr)) + }) + } + + public func get_search_index() -> RustVec { + RustVec(ptr: __swift_bridge__$Workspace$get_search_index(ptr)) + } + + public func set_search_index(_ fields: RustVec) -> Bool { + __swift_bridge__$Workspace$set_search_index(ptr, { let val = fields; val.isOwned = false; return val.ptr }()) + } +} +extension Workspace: Vectorizable { + public static func vecOfSelfNew() -> UnsafeMutableRawPointer { + __swift_bridge__$Vec_Workspace$new() + } + + public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { + __swift_bridge__$Vec_Workspace$drop(vecPtr) + } + + public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Workspace) { + __swift_bridge__$Vec_Workspace$push(vecPtr, {value.isOwned = false; return value.ptr;}()) + } + + public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { + let pointer = __swift_bridge__$Vec_Workspace$pop(vecPtr) + if pointer == nil { + return nil + } else { + return (Workspace(ptr: pointer!) as! Self) + } + } + + public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_Workspace$get(vecPtr, index) + if pointer == nil { + return nil + } else { + return WorkspaceRef(ptr: pointer!) + } + } + + public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_Workspace$get_mut(vecPtr, index) + if pointer == nil { + return nil + } else { + return WorkspaceRefMut(ptr: pointer!) + } + } + + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { + __swift_bridge__$Vec_Workspace$len(vecPtr) + } +} + + +public class JwstWorkSpaceResult: JwstWorkSpaceResultRefMut { + var isOwned: Bool = true + + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } + + deinit { + if isOwned { + __swift_bridge__$JwstWorkSpaceResult$_free(ptr) + } + } +} +public class JwstWorkSpaceResultRefMut: JwstWorkSpaceResultRef { + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } +} +public class JwstWorkSpaceResultRef { + var ptr: UnsafeMutableRawPointer + + public init(ptr: UnsafeMutableRawPointer) { + self.ptr = ptr + } +} +extension JwstWorkSpaceResult: Vectorizable { + public static func vecOfSelfNew() -> UnsafeMutableRawPointer { + __swift_bridge__$Vec_JwstWorkSpaceResult$new() + } + + public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { + __swift_bridge__$Vec_JwstWorkSpaceResult$drop(vecPtr) + } + + public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: JwstWorkSpaceResult) { + __swift_bridge__$Vec_JwstWorkSpaceResult$push(vecPtr, {value.isOwned = false; return value.ptr;}()) + } + + public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { + let pointer = __swift_bridge__$Vec_JwstWorkSpaceResult$pop(vecPtr) + if pointer == nil { + return nil + } else { + return (JwstWorkSpaceResult(ptr: pointer!) as! Self) + } + } + + public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_JwstWorkSpaceResult$get(vecPtr, index) + if pointer == nil { + return nil + } else { + return JwstWorkSpaceResultRef(ptr: pointer!) + } + } + + public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_JwstWorkSpaceResult$get_mut(vecPtr, index) + if pointer == nil { + return nil + } else { + return JwstWorkSpaceResultRefMut(ptr: pointer!) + } + } + + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { + __swift_bridge__$Vec_JwstWorkSpaceResult$len(vecPtr) + } +} + + +public class Storage: StorageRefMut { + var isOwned: Bool = true + + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } + + deinit { + if isOwned { + __swift_bridge__$Storage$_free(ptr) + } + } +} +extension Storage { + public convenience init(_ path: GenericIntoRustString) { + self.init(ptr: __swift_bridge__$Storage$new({ let rustString = path.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) + } + + public convenience init(_ path: GenericIntoRustString, _ level: GenericIntoRustString) { + self.init(ptr: __swift_bridge__$Storage$new_with_log_level({ let rustString = path.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = level.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) + } +} +public class StorageRefMut: StorageRef { + public override init(ptr: UnsafeMutableRawPointer) { + super.init(ptr: ptr) + } +} +extension StorageRefMut { + public func connect(_ workspace_id: GenericIntoRustString, _ remote: GenericIntoRustString) -> Optional { + { let val = __swift_bridge__$Storage$connect(ptr, { let rustString = workspace_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = remote.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return Workspace(ptr: val!) } else { return nil } }() + } +} +public class StorageRef { + var ptr: UnsafeMutableRawPointer + + public init(ptr: UnsafeMutableRawPointer) { + self.ptr = ptr + } +} +extension StorageRef { + public func error() -> Optional { + { let val = __swift_bridge__$Storage$error(ptr); if val != nil { return RustString(ptr: val!) } else { return nil } }() + } + + public func is_offline() -> Bool { + __swift_bridge__$Storage$is_offline(ptr) + } + + public func is_connected() -> Bool { + __swift_bridge__$Storage$is_connected(ptr) + } + + public func is_finished() -> Bool { + __swift_bridge__$Storage$is_finished(ptr) + } + + public func is_error() -> Bool { + __swift_bridge__$Storage$is_error(ptr) + } + + public func get_sync_state() -> RustString { + RustString(ptr: __swift_bridge__$Storage$get_sync_state(ptr)) + } + + public func get_last_synced() -> RustVec { + RustVec(ptr: __swift_bridge__$Storage$get_last_synced(ptr)) + } +} +extension Storage: Vectorizable { + public static func vecOfSelfNew() -> UnsafeMutableRawPointer { + __swift_bridge__$Vec_Storage$new() + } + + public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { + __swift_bridge__$Vec_Storage$drop(vecPtr) + } + + public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Storage) { + __swift_bridge__$Vec_Storage$push(vecPtr, {value.isOwned = false; return value.ptr;}()) + } + + public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { + let pointer = __swift_bridge__$Vec_Storage$pop(vecPtr) + if pointer == nil { + return nil + } else { + return (Storage(ptr: pointer!) as! Self) + } + } + + public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_Storage$get(vecPtr, index) + if pointer == nil { + return nil + } else { + return StorageRef(ptr: pointer!) + } + } + + public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let pointer = __swift_bridge__$Vec_Storage$get_mut(vecPtr, index) + if pointer == nil { + return nil + } else { + return StorageRefMut(ptr: pointer!) + } + } + + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { + __swift_bridge__$Vec_Storage$len(vecPtr) + } +} + + + diff --git a/libs/jwst-binding/jwst-core-swift/.gitignore b/libs/jwst-binding/jwst-core-swift/.gitignore new file mode 100644 index 000000000..7a2cab5f8 --- /dev/null +++ b/libs/jwst-binding/jwst-core-swift/.gitignore @@ -0,0 +1,2 @@ +generated +octobase \ No newline at end of file diff --git a/libs/jwst-binding/jwst-core-swift/Cargo.toml b/libs/jwst-binding/jwst-core-swift/Cargo.toml new file mode 100644 index 000000000..3b83939d4 --- /dev/null +++ b/libs/jwst-binding/jwst-core-swift/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "jwst-core-swift" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +chrono = "0.4.26" +futures = "0.3.28" +swift-bridge = "0.1.51" +tokio = "1.27.0" +nanoid = "0.4.0" +serde = { version = "1.0.183", features = ["derive"] } +serde_json = "1.0.104" + +# ======= workspace dependencies ======= +jwst-core = { workspace = true } +jwst-core-rpc = { workspace = true } +jwst-core-storage = { workspace = true, features = ["sqlite"] } +jwst-logger = { workspace = true } + +[lib] +name = "octobase" +crate-type = ["staticlib"] + +[build-dependencies] +swift-bridge-build = "0.1.51" + +[dev-dependencies] +reqwest = { version = "0.11.14", default-features = false, features = [ + "json", + "rustls-tls", +] } +regex = "1.7.1" diff --git a/libs/jwst-binding/jwst-core-swift/binding.h b/libs/jwst-binding/jwst-core-swift/binding.h new file mode 100644 index 000000000..707d2f418 --- /dev/null +++ b/libs/jwst-binding/jwst-core-swift/binding.h @@ -0,0 +1,105 @@ + +#ifndef JWST_FFI_H +#define JWST_FFI_H +typedef struct JWSTWorkspace {} JWSTWorkspace; +typedef struct JWSTBlock {} JWSTBlock; +typedef struct YTransaction {} YTransaction; + + +#include +#include +#include +#include + +#define BLOCK_TAG_NUM 1 + +#define BLOCK_TAG_INT 2 + +#define BLOCK_TAG_BOOL 3 + +#define BLOCK_TAG_STR 4 + +typedef struct BlockChildren { + uintptr_t len; + char **data; +} BlockChildren; + +typedef union BlockValue { + double num; + int64_t int; + bool bool; + char *str; +} BlockValue; + +typedef struct BlockContent { + int8_t tag; + union BlockValue value; +} BlockContent; + +JWSTBlock *block_new(const JWSTWorkspace *workspace, + const char *block_id, + const char *flavor, + uint64_t operator_); + +void block_destroy(JWSTBlock *block); + +uint64_t block_get_created(const JWSTBlock *block); + +uint64_t block_get_updated(const JWSTBlock *block); + +char *block_get_flavor(const JWSTBlock *block); + +struct BlockChildren *block_get_children(const JWSTBlock *block); + +void block_push_children(const JWSTBlock *block, YTransaction *trx, const JWSTBlock *child); + +void block_insert_children_at(const JWSTBlock *block, + YTransaction *trx, + const JWSTBlock *child, + uint32_t pos); + +void block_insert_children_before(const JWSTBlock *block, + YTransaction *trx, + const JWSTBlock *child, + const char *reference); + +void block_insert_children_after(const JWSTBlock *block, + YTransaction *trx, + const JWSTBlock *child, + const char *reference); + +void block_children_destroy(struct BlockChildren *children); + +struct BlockContent *block_get_content(const JWSTBlock *block, const char *key); + +void block_set_content(JWSTBlock *block, + const char *key, + YTransaction *trx, + struct BlockContent content); + +void block_content_destroy(struct BlockContent *content); + +JWSTWorkspace *workspace_new(const char *id); + +void workspace_destroy(JWSTWorkspace *workspace); + +JWSTBlock *workspace_get_block(const JWSTWorkspace *workspace, const char *block_id); + +JWSTBlock *workspace_create_block(const JWSTWorkspace *workspace, + const char *block_id, + const char *flavor); + +bool workspace_remove_block(const JWSTWorkspace *workspace, const char *block_id); + +bool workspace_exists_block(const JWSTWorkspace *workspace, const char *block_id); + +void trx_commit(YTransaction *trx); + +Subscription *workspace_observe(JWSTWorkspace *workspace, + void *env, + void (*func)(void*, const YTransaction*, const UpdateEvent*)); + +void workspace_unobserve(Subscription *subscription); + + +#endif diff --git a/libs/jwst-binding/jwst-core-swift/build.rs b/libs/jwst-binding/jwst-core-swift/build.rs new file mode 100644 index 000000000..1ea6edd07 --- /dev/null +++ b/libs/jwst-binding/jwst-core-swift/build.rs @@ -0,0 +1,14 @@ +const XCODE_CONFIGURATION_ENV: &str = "CONFIGURATION"; + +fn main() { + let out_dir = "./generated"; + + let bridges = vec!["src/lib.rs"]; + for path in &bridges { + println!("cargo:rerun-if-changed={path}"); + } + println!("cargo:rerun-if-env-changed={XCODE_CONFIGURATION_ENV}"); + + swift_bridge_build::parse_bridges(bridges) + .write_all_concatenated(out_dir, env!("CARGO_PKG_NAME")); +} diff --git a/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/Cargo.toml b/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/Cargo.toml new file mode 100644 index 000000000..a17051532 --- /dev/null +++ b/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "jwst-core-swift-integrate" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +swift-bridge-build = "0.1.48" diff --git a/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs b/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs new file mode 100644 index 000000000..3733bc14c --- /dev/null +++ b/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs @@ -0,0 +1,62 @@ +use std::collections::HashMap; +use std::path::PathBuf; +use std::process::Command; +use swift_bridge_build::ApplePlatform as Platform; +use swift_bridge_build::{create_package, CreatePackageConfig}; + +fn main() { + let common_commands = [ + "-p", + "jwst-core-swift", + "--target", + "aarch64-apple-ios", + "--target", + "aarch64-apple-ios-sim", + "--target", + "aarch64-apple-darwin", + ]; + Command::new("rustup") + .args(["target", "add", "aarch64-apple-ios"]) + .status() + .expect("Failed to add target aarch64-apple-ios"); + Command::new("rustup") + .args(["target", "add", "aarch64-apple-ios-sim"]) + .status() + .expect("Failed to add target aarch64-apple-ios-sim"); + Command::new("rustup") + .args(["target", "add", "aarch64-apple-darwin"]) + .status() + .expect("Failed to add target aarch64-apple-darwin"); + Command::new("cargo") + .args(if cfg!(debug_assertions) { + ["build"].iter().chain(common_commands.iter()) + } else { + ["build", "--release"].iter().chain(common_commands.iter()) + }) + .status() + .expect("Failed to build jwst-core-swift"); + let dir = if cfg!(debug_assertions) { + "debug" + } else { + "release" + }; + create_package(CreatePackageConfig { + bridge_dir: PathBuf::from("libs/jwst-binding/jwst-core-swift/generated"), + paths: HashMap::from([ + ( + Platform::IOS, + PathBuf::from(format!("target/aarch64-apple-ios/{dir}/liboctobase.a")), + ), + ( + Platform::Simulator, + PathBuf::from(format!("target/aarch64-apple-ios-sim/{dir}/liboctobase.a",)), + ), + ( + Platform::MacOS, + PathBuf::from(format!("target/aarch64-apple-darwin/{dir}/liboctobase.a")), + ), + ]), + out_dir: PathBuf::from("apps/swift/OctoBaseSwift"), + package_name: "OctoBase".to_string(), + }); +} diff --git a/libs/jwst-binding/jwst-core-swift/src/block.rs b/libs/jwst-binding/jwst-core-swift/src/block.rs new file mode 100644 index 000000000..9e81cc7ba --- /dev/null +++ b/libs/jwst-binding/jwst-core-swift/src/block.rs @@ -0,0 +1,186 @@ +use super::*; +use jwst_core::{Any, Block as JwstBlock}; + +pub struct Block { + pub(crate) block: JwstBlock, +} + +impl Block { + pub fn new(block: JwstBlock) -> Self { + Self { block } + } + + pub fn get(&self, key: String) -> Option { + self.block.get(&key).map(DynamicValue::new) + } + + pub fn children(&self) -> Vec { + self.block.children() + } + + pub fn push_children(&self, target_block: &Block) { + let mut block = self.block.clone(); + let mut target_block = target_block.block.clone(); + + block + .push_children(&mut target_block) + .expect("failed to push children"); + } + + pub fn insert_children_at(&self, target_block: &Block, pos: u32) { + let mut block = self.block.clone(); + let mut target_block = target_block.block.clone(); + + block + .insert_children_at(&mut target_block, pos as u64) + .expect("failed to insert children at position"); + } + + pub fn insert_children_before(&self, target_block: &Block, reference: &str) { + let mut block = self.block.clone(); + let mut target_block = target_block.block.clone(); + + block + .insert_children_before(&mut target_block, &reference) + .expect("failed to insert children before"); + } + + pub fn insert_children_after(&self, target_block: &Block, reference: &str) { + let mut block = self.block.clone(); + let mut target_block = target_block.block.clone(); + + block + .insert_children_after(&mut target_block, &reference) + .expect("failed to insert children after"); + } + + pub fn remove_children(&self, target_block: &Block) { + let mut block = self.block.clone(); + let mut target_block = target_block.block.clone(); + + block + .remove_children(&mut target_block) + .expect("failed to remove jwst block"); + } + + pub fn exists_children(&self, block_id: &str) -> i32 { + self.block + .exists_children(block_id) + .map(|i| i as i32) + .unwrap_or(-1) + } + + pub fn parent(&self) -> String { + self.block.parent().unwrap() + } + + pub fn updated(&self) -> u64 { + self.block.updated() + } + + pub fn id(&self) -> String { + self.block.block_id() + } + + pub fn flavour(&self) -> String { + self.block.flavour() + } + + pub fn created(&self) -> u64 { + self.block.created() + } + + pub fn set_bool(&self, key: String, value: bool) { + let mut block = self.block.clone(); + block + .set(&key, value) + .expect(&format!("failed to set bool: {} {}", key, value)) + } + + pub fn set_string(&self, key: String, value: String) { + let mut block = self.block.clone(); + block + .set(&key, value.clone()) + .expect(&format!("failed to set string: {} {}", key, value)) + } + + pub fn set_float(&self, key: String, value: f64) { + let mut block = self.block.clone(); + block + .set(&key, value) + .expect(&format!("failed to set float: {} {}", key, value)); + } + + pub fn set_integer(&self, key: String, value: i64) { + let mut block = self.block.clone(); + block + .set(&key, value) + .expect(&format!("failed to set integer: {} {}", key, value)); + } + + pub fn set_null(&self, key: String) { + let mut block = self.block.clone(); + block + .set(&key, Any::Null) + .expect(&format!("failed to set null: {}", key)); + } + + pub fn is_bool(&self, key: String) -> bool { + self.block + .get(&key) + .map(|a| matches!(a, Any::True | Any::False)) + .unwrap_or(false) + } + + pub fn is_string(&self, key: String) -> bool { + self.block + .get(&key) + .map(|a| matches!(a, Any::String(_))) + .unwrap_or(false) + } + + pub fn is_float(&self, key: String) -> bool { + self.block + .get(&key) + .map(|a| matches!(a, Any::Float32(_) | Any::Float64(_))) + .unwrap_or(false) + } + + pub fn is_integer(&self, key: String) -> bool { + self.block + .get(&key) + .map(|a| matches!(a, Any::Integer(_) | Any::BigInt64(_))) + .unwrap_or(false) + } + + pub fn get_bool(&self, key: String) -> Option { + self.block.get(&key).and_then(|a| match a { + Any::True => Some(true), + Any::False => Some(false), + _ => None, + }) + } + + pub fn get_string(&self, key: String) -> Option { + self.block.get(&key).and_then(|a| match a { + Any::String(s) => Some(s), + _ => None, + }) + } + + pub fn get_float(&self, key: String) -> Option { + self.block.get(&key).and_then(|a| match a { + Any::Float32(f) => Some(f.0 as f64), + Any::Float64(f) => Some(f.0), + _ => None, + }) + } + + pub fn get_integer(&self, key: String) -> Option { + self.block.get(&key).and_then(|a| match a { + Any::Integer(i) => Some(i as i64), + Any::BigInt64(i) => Some(i), + _ => None, + }) + } +} diff --git a/libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs b/libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs new file mode 100644 index 000000000..453fc0983 --- /dev/null +++ b/libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs @@ -0,0 +1,71 @@ +use jwst_core::Any; +use std::collections::HashMap; + +pub type DynamicValueMap = HashMap; + +pub struct DynamicValue { + any: Any, +} + +impl DynamicValue { + pub fn new(any: Any) -> Self { + Self { any } + } + + pub fn as_bool(&self) -> Option { + match self.any { + Any::True => Some(true), + Any::False => Some(false), + _ => None, + } + } + + pub fn as_number(&self) -> Option { + match self.any { + Any::Float32(value) => Some(value.0 as f64), + Any::Float64(value) => Some(value.0), + _ => None, + } + } + + pub fn as_int(&self) -> Option { + match self.any { + Any::Integer(value) => Some(value as i64), + Any::BigInt64(value) => Some(value), + _ => None, + } + } + + pub fn as_string(&self) -> Option { + match &self.any { + Any::String(value) => Some(value.to_string()), + _ => None, + } + } + + pub fn as_buffer(&self) -> Option> { + match &self.any { + Any::Binary(value) => Some(value.clone()), + _ => None, + } + } + + pub fn as_array(&self) -> Option> { + match &self.any { + Any::Array(value) => Some(value.iter().map(|a| DynamicValue::new(a.clone())).collect()), + _ => None, + } + } + + pub fn as_map(&self) -> Option> { + match &self.any { + Any::Object(value) => Some( + value + .iter() + .map(|(key, value)| (key.clone(), DynamicValue::new(value.clone()))) + .collect(), + ), + _ => None, + } + } +} diff --git a/libs/jwst-binding/jwst-core-swift/src/lib.rs b/libs/jwst-binding/jwst-core-swift/src/lib.rs new file mode 100644 index 000000000..9d96d84c3 --- /dev/null +++ b/libs/jwst-binding/jwst-core-swift/src/lib.rs @@ -0,0 +1,143 @@ +mod block; +mod dynamic_value; +mod storage; +mod workspace; + +pub use block::Block; +pub use dynamic_value::{DynamicValue, DynamicValueMap}; +pub use storage::Storage; +pub use workspace::Workspace; + +use jwst_core::{error, warn, JwstError}; +use jwst_logger::init_logger_with; + +type JwstWorkSpaceResult = Result; + +#[swift_bridge::bridge] +mod ffi { + extern "Rust" { + type Block; + + fn get(self: &Block, block_id: String) -> Option; + + pub fn children(self: &Block) -> Vec; + + pub fn push_children(self: &Block, block: &Block); + + pub fn insert_children_at(&self, block: &Block, pos: u32); + + pub fn insert_children_before(self: &Block, block: &Block, reference: &str); + + pub fn insert_children_after(self: &Block, block: &Block, reference: &str); + + pub fn remove_children(self: &Block, block: &Block); + + pub fn exists_children(self: &Block, block_id: &str) -> i32; + + pub fn parent(self: &Block) -> String; + + pub fn updated(self: &Block) -> u64; + + pub fn id(self: &Block) -> String; + + pub fn flavour(self: &Block) -> String; + + pub fn created(self: &Block) -> u64; + + pub fn set_bool(self: &Block, key: String, value: bool); + + pub fn set_string(self: &Block, key: String, value: String); + + pub fn set_float(self: &Block, key: String, value: f64); + + pub fn set_integer(self: &Block, key: String, value: i64); + + pub fn set_null(self: &Block, key: String); + + pub fn is_bool(self: &Block, key: String) -> bool; + + pub fn is_string(self: &Block, key: String) -> bool; + + pub fn is_float(&self, key: String) -> bool; + + pub fn is_integer(&self, key: String) -> bool; + + pub fn get_bool(&self, key: String) -> Option; + + pub fn get_string(&self, key: String) -> Option; + + pub fn get_float(&self, key: String) -> Option; + + pub fn get_integer(&self, key: String) -> Option; + } + + extern "Rust" { + type DynamicValue; + type DynamicValueMap; + + fn as_bool(self: &DynamicValue) -> Option; + + fn as_number(self: &DynamicValue) -> Option; + + fn as_int(self: &DynamicValue) -> Option; + + fn as_string(self: &DynamicValue) -> Option; + + fn as_map(self: &DynamicValue) -> Option; + + fn as_array(self: &DynamicValue) -> Option>; + + fn as_buffer(self: &DynamicValue) -> Option>; + } + + extern "Rust" { + type Workspace; + + fn id(self: &Workspace) -> String; + + fn client_id(self: &Workspace) -> u64; + + fn get(self: &Workspace, block_id: String) -> Option; + + fn create(self: &Workspace, block_id: String, flavour: String) -> Block; + + fn search(self: &Workspace, query: String) -> String; + + fn get_blocks_by_flavour(self: &Workspace, flavour: &str) -> Vec; + + fn get_search_index(self: &Workspace) -> Vec; + + fn set_search_index(self: &Workspace, fields: Vec) -> bool; + } + + extern "Rust" { + type JwstWorkSpaceResult; + } + + extern "Rust" { + type Storage; + + #[swift_bridge(init)] + fn new(path: String) -> Storage; + + #[swift_bridge(init)] + fn new_with_log_level(path: String, level: String) -> Storage; + + fn error(self: &Storage) -> Option; + + fn is_offline(self: &Storage) -> bool; + + fn is_connected(self: &Storage) -> bool; + + fn is_finished(self: &Storage) -> bool; + + fn is_error(self: &Storage) -> bool; + + fn get_sync_state(self: &Storage) -> String; + + fn connect(self: &mut Storage, workspace_id: String, remote: String) -> Option; + + fn get_last_synced(self: &Storage) -> Vec; + + } +} diff --git a/libs/jwst-binding/jwst-core-swift/src/storage.rs b/libs/jwst-binding/jwst-core-swift/src/storage.rs new file mode 100644 index 000000000..3cfbac253 --- /dev/null +++ b/libs/jwst-binding/jwst-core-swift/src/storage.rs @@ -0,0 +1,213 @@ +use super::*; +use futures::TryFutureExt; +use jwst_core_rpc::{ + start_websocket_client_sync, BroadcastChannels, CachedLastSynced, RpcContextImpl, SyncState, +}; +use jwst_core_storage::{BlobStorageType, JwstStorage as AutoStorage, JwstStorageResult}; +use nanoid::nanoid; +use std::sync::{Arc, RwLock}; +use tokio::{ + runtime::{Builder, Runtime}, + sync::mpsc::channel, +}; + +#[derive(Clone)] +pub struct Storage { + storage: Arc, + channel: Arc, + error: Option, + sync_state: Arc>, + last_sync: CachedLastSynced, +} + +impl Storage { + pub fn new(path: String) -> Self { + Self::new_with_log_level(path, "info".to_string()) + } + + pub fn new_with_log_level(path: String, level: String) -> Self { + init_logger_with( + &format!("{level},mio=off,hyper=off,rustls=off,tantivy=off,sqlx::query=off"), + false, + ); + + let rt = Runtime::new().unwrap(); + + let storage = rt + .block_on( + AutoStorage::new_with_migration( + &format!("sqlite:{path}?mode=rwc"), + BlobStorageType::DB, + ) + .or_else(|e| { + warn!( + "Failed to open storage, falling back to memory storage: {}", + e + ); + AutoStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) + }), + ) + .unwrap(); + + Self { + storage: Arc::new(storage), + channel: Arc::default(), + error: None, + sync_state: Arc::new(RwLock::new(SyncState::Offline)), + last_sync: CachedLastSynced::default(), + } + } + + pub fn error(&self) -> Option { + self.error.clone() + } + + pub fn is_offline(&self) -> bool { + let sync_state = self.sync_state.read().unwrap(); + matches!(*sync_state, SyncState::Offline) + } + + pub fn is_connected(&self) -> bool { + let sync_state = self.sync_state.read().unwrap(); + matches!(*sync_state, SyncState::Connected) + } + + pub fn is_finished(&self) -> bool { + let sync_state = self.sync_state.read().unwrap(); + matches!(*sync_state, SyncState::Finished) + } + + pub fn is_error(&self) -> bool { + let sync_state = self.sync_state.read().unwrap(); + matches!(*sync_state, SyncState::Error(_)) + } + + pub fn get_sync_state(&self) -> String { + let sync_state = self.sync_state.read().unwrap(); + match sync_state.clone() { + SyncState::Offline => "offline".to_string(), + SyncState::Connected => "connected".to_string(), + SyncState::Finished => "finished".to_string(), + SyncState::Error(e) => format!("Error: {e}"), + } + } + + pub fn connect(&mut self, workspace_id: String, remote: String) -> Option { + match self.sync(workspace_id, remote) { + Ok(workspace) => Some(workspace), + Err(e) => { + error!("Failed to connect to workspace: {:?}", e); + self.error = Some(e.to_string()); + None + } + } + } + + fn sync(&mut self, workspace_id: String, remote: String) -> JwstStorageResult { + let rt = Arc::new( + Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .thread_name("jwst-core-swift") + .build() + .map_err(JwstError::Io)?, + ); + let is_offline = remote.is_empty(); + + let workspace = rt.block_on(async { self.get_workspace(&workspace_id).await }); + + match workspace { + Ok(mut workspace) => { + if is_offline { + let identifier = nanoid!(); + let (last_synced_tx, last_synced_rx) = channel::(128); + self.last_sync.add_receiver(rt.clone(), last_synced_rx); + + rt.block_on(async { + self.join_broadcast(&mut workspace, identifier.clone(), last_synced_tx) + .await; + }); + } else { + self.last_sync = start_websocket_client_sync( + rt.clone(), + Arc::new(self.clone()), + self.sync_state.clone(), + remote, + workspace_id.clone(), + ); + } + + Ok(Workspace { workspace }) + } + Err(e) => Err(e), + } + } + + pub fn get_last_synced(&self) -> Vec { + self.last_sync.pop() + } +} + +impl RpcContextImpl<'_> for Storage { + fn get_storage(&self) -> &AutoStorage { + &self.storage + } + + fn get_channel(&self) -> &BroadcastChannels { + &self.channel + } +} + +#[cfg(test)] +mod tests { + use crate::{Storage, Workspace}; + use tokio::runtime::Runtime; + + #[test] + #[ignore = "need manually start collaboration server"] + fn collaboration_test() { + let (workspace_id, block_id) = ("1", "1"); + let workspace = get_workspace(workspace_id, None); + let block = workspace.create(block_id.to_string(), "list".to_string()); + block.set_bool("bool_prop".to_string(), true); + block.set_float("float_prop".to_string(), 1.0); + block.push_children(&workspace.create("2".to_string(), "list".to_string())); + + let resp = get_block_from_server(workspace_id.to_string(), block.id().to_string()); + assert!(!resp.is_empty()); + let prop_extractor = + r#"("prop:bool_prop":true)|("prop:float_prop":1\.0)|("sys:children":\["2"\])"#; + let re = regex::Regex::new(prop_extractor).unwrap(); + assert_eq!(re.find_iter(resp.as_str()).count(), 3); + } + + fn get_workspace(workspace_id: &str, offline: Option<()>) -> Workspace { + let mut storage = Storage::new("memory".to_string()); + storage + .connect( + workspace_id.to_string(), + if offline.is_some() { + "".to_string() + } else { + format!("ws://localhost:3000/collaboration/{workspace_id}").to_string() + }, + ) + .unwrap() + } + + fn get_block_from_server(workspace_id: String, block_id: String) -> String { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + let client = reqwest::Client::builder().no_proxy().build().unwrap(); + let resp = client + .get(format!( + "http://localhost:3000/api/block/{}/{}", + workspace_id, block_id + )) + .send() + .await + .unwrap(); + resp.text().await.unwrap() + }) + } +} diff --git a/libs/jwst-binding/jwst-core-swift/src/workspace.rs b/libs/jwst-binding/jwst-core-swift/src/workspace.rs new file mode 100644 index 000000000..b2e87c4e9 --- /dev/null +++ b/libs/jwst-binding/jwst-core-swift/src/workspace.rs @@ -0,0 +1,74 @@ +use super::*; +use jwst_core::Workspace as JwstWorkspace; + +pub struct Workspace { + pub(crate) workspace: JwstWorkspace, +} + +impl Workspace { + pub fn new(id: String) -> Self { + Self { + workspace: JwstWorkspace::new(&id).unwrap(), + } + } + + pub fn id(&self) -> String { + self.workspace.id() + } + + pub fn client_id(&self) -> u64 { + self.workspace.client_id() + } + + pub fn get(&self, block_id: String) -> Option { + let mut workspace = self.workspace.clone(); + + workspace + .get_blocks() + .ok() + .and_then(|s| s.get(&block_id)) + .map(Block::new) + } + + pub fn create(&self, block_id: String, flavour: String) -> Block { + let mut workspace = self.workspace.clone(); + + Block::new( + workspace + .get_blocks() + .and_then(|mut b| b.create(block_id, flavour)) + .expect("failed to create jwst block"), + ) + } + + pub fn search(self: &Workspace, query: String) -> String { + // self.workspace.search_result(query) + "".to_owned() + } + + pub fn get_blocks_by_flavour(&self, flavour: &str) -> Vec { + let mut workspace = self.workspace.clone(); + + workspace + .get_blocks() + .map(|s| { + s.get_blocks_by_flavour(flavour) + .into_iter() + .map(Block::new) + .collect() + }) + .unwrap_or_default() + } + + pub fn get_search_index(self: &Workspace) -> Vec { + // self.workspace.metadata().search_index + vec![] + } + + pub fn set_search_index(self: &Workspace, fields: Vec) -> bool { + // self.workspace + // .set_search_index(fields) + // .expect("failed to set search index") + false + } +} diff --git a/libs/jwst-core-rpc/Cargo.toml b/libs/jwst-core-rpc/Cargo.toml new file mode 100644 index 000000000..ce2a2be49 --- /dev/null +++ b/libs/jwst-core-rpc/Cargo.toml @@ -0,0 +1,56 @@ +[package] +name = "jwst-core-rpc" +version = "0.1.0" +edition = "2021" +authors = ["DarkSky "] +license = "AGPL-3.0-only" + +[features] +default = ["websocket"] +websocket = ["axum", "tokio-tungstenite", "url"] +webrtc = ["bytes", "reqwest", "webrtcrs"] + +[dependencies] +anyhow = "1.0.70" +assert-json-diff = "2.0.2" +async-trait = "0.1.68" +byteorder = "1.4.3" +chrono = "0.4.26" +futures = "0.3.28" +lru_time_cache = "0.11.11" +nanoid = "0.4.0" +nom = "7.1.3" +rand = "0.8.5" +serde = "1.0.183" +serde_json = "1.0.104" +thiserror = "1.0.40" +tokio = { version = "1.27.0", features = [ + "macros", + "rt-multi-thread", + "signal", +] } + +# ======== websocket dependencies ======== +axum = { version = "0.6.16", features = ["ws"], optional = true } +tokio-tungstenite = { version = "0.20.0", features = [ + "rustls-tls-webpki-roots", +], optional = true } +url = { version = "2.3.1", optional = true } + +# ======== webrtc dependencies ======== +bytes = { version = "1.4", optional = true } +reqwest = { version = "0.11.18", default-features = false, features = [ + "json", + "rustls-tls", +], optional = true } +webrtcrs = { package = "webrtc", version = "0.8.0", optional = true } + +# ======= workspace dependencies ======= +jwst-codec = { workspace = true } +jwst-core = { workspace = true } +jwst-core-storage = { workspace = true } + +[dev-dependencies] +indicatif = "0.17.3" +jwst-logger = { path = "../jwst-logger" } +tempfile = "3.4.0" diff --git a/libs/jwst-core-rpc/src/broadcast.rs b/libs/jwst-core-rpc/src/broadcast.rs new file mode 100644 index 000000000..906f09c7e --- /dev/null +++ b/libs/jwst-core-rpc/src/broadcast.rs @@ -0,0 +1,109 @@ +use super::*; +use jwst_codec::{ + encode_update_as_message, encode_update_with_guid, write_sync_message, SyncMessage, +}; +use jwst_core::Workspace; +use lru_time_cache::LruCache; +use std::{collections::HashMap, sync::Mutex}; +use tokio::sync::{broadcast::Sender, RwLock}; + +#[derive(Clone)] +pub enum BroadcastType { + BroadcastAwareness(Vec), + BroadcastContent(Vec), + BroadcastRawContent(Vec), + CloseUser(String), + CloseAll, +} + +type Broadcast = Sender; +pub type BroadcastChannels = RwLock>; + +pub async fn subscribe(workspace: &mut Workspace, identifier: String, sender: Broadcast) { + { + let sender = sender.clone(); + let workspace_id = workspace.id(); + + let dedup_cache = Arc::new(Mutex::new(LruCache::with_expiry_duration_and_capacity( + Duration::from_micros(100), + 128, + ))); + + workspace + .on_awareness_update(move |awareness, e| { + let mut buffer = Vec::new(); + if let Err(e) = write_sync_message( + &mut buffer, + &SyncMessage::Awareness(e.get_updated(awareness.get_states())), + ) { + error!("failed to write awareness update: {}", e); + return; + } + + let mut dedup_cache = dedup_cache.lock().unwrap_or_else(|e| e.into_inner()); + if !dedup_cache.contains_key(&buffer) { + if sender + .send(BroadcastType::BroadcastAwareness(buffer.clone())) + .is_err() + { + debug!("broadcast channel {workspace_id} has been closed",) + } + dedup_cache.insert(buffer, ()); + } + }) + .await; + } + { + let sender = sender.clone(); + let workspace_id = workspace.id(); + workspace.doc().subscribe(move |update| { + trace!("workspace {} changed: {}bytes", workspace_id, update.len()); + + match encode_update_with_guid(update.to_vec(), workspace_id.clone()) + .and_then(|update| encode_update_as_message(update.clone()).map(|u| (update, u))) + { + Ok((broadcast_update, sendable_update)) => { + if sender + .send(BroadcastType::BroadcastRawContent(broadcast_update)) + .is_err() + { + debug!("broadcast channel {workspace_id} has been closed",) + } + + if sender + .send(BroadcastType::BroadcastContent(sendable_update)) + .is_err() + { + debug!("broadcast channel {workspace_id} has been closed",) + } + } + Err(e) => { + debug!("failed to encode update: {}", e); + } + } + }); + }; + + let workspace_id = workspace.id(); + tokio::spawn(async move { + let mut rx = sender.subscribe(); + loop { + tokio::select! { + Ok(msg) = rx.recv()=> { + match msg { + BroadcastType::CloseUser(user) if user == identifier => break, + BroadcastType::CloseAll => break, + _ => {} + } + }, + _ = sleep(Duration::from_millis(100)) => { + let count = sender.receiver_count(); + if count < 1 { + break; + } + } + } + } + debug!("broadcast channel {workspace_id} has been closed"); + }); +} diff --git a/libs/jwst-core-rpc/src/client/mod.rs b/libs/jwst-core-rpc/src/client/mod.rs new file mode 100644 index 000000000..d02ff2a5a --- /dev/null +++ b/libs/jwst-core-rpc/src/client/mod.rs @@ -0,0 +1,98 @@ +#[cfg(feature = "webrtc")] +mod webrtc; +#[cfg(feature = "websocket")] +mod websocket; + +use super::*; +use chrono::Utc; +use std::sync::Mutex; +use tokio::{runtime::Runtime, task::JoinHandle}; + +#[cfg(feature = "webrtc")] +pub use webrtc::start_webrtc_client_sync; +#[cfg(feature = "websocket")] +pub use websocket::start_websocket_client_sync; + +#[derive(Clone, Default)] +pub struct CachedLastSynced { + synced: Arc>>, + _threads: Arc>>>, +} + +impl CachedLastSynced { + pub fn add_receiver_wait_first_update(&self, rt: Arc, recv: Receiver) { + self.add_receiver(rt, recv); + while self.synced.lock().unwrap().is_empty() { + std::thread::sleep(Duration::from_millis(100)); + } + } + + pub fn add_receiver(&self, rt: Arc, mut recv: Receiver) { + let synced = self.synced.clone(); + rt.spawn(async move { + while let Some(last_synced) = recv.recv().await { + let mut synced = synced.lock().unwrap(); + if synced.is_empty() || *synced.last().unwrap() != last_synced { + synced.push(last_synced); + } + } + }); + } + + pub fn pop(&self) -> Vec { + let mut synced = self.synced.lock().unwrap(); + let ret = synced.clone(); + synced.clear(); + ret + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::thread::spawn; + use tokio::sync::mpsc::channel; + + #[test] + fn test_synced() { + let synced = CachedLastSynced::default(); + let (tx, rx) = channel::(1); + let rt = Arc::new(Runtime::new().unwrap()); + + synced.add_receiver(rt.clone(), rx); + { + let tx = tx.clone(); + rt.block_on(async { + tx.send(1).await.unwrap(); + tx.send(2).await.unwrap(); + tx.send(3).await.unwrap(); + sleep(Duration::from_millis(100)).await; + }); + } + { + let synced = synced.clone(); + spawn(move || { + assert_eq!(synced.pop(), vec![1, 2, 3]); + }) + .join() + .unwrap(); + } + + { + let tx = tx.clone(); + rt.block_on(async { + tx.send(4).await.unwrap(); + tx.send(5).await.unwrap(); + sleep(Duration::from_millis(100)).await; + }); + } + { + let synced = synced.clone(); + spawn(move || { + assert_eq!(synced.pop(), vec![4, 5]); + }) + .join() + .unwrap(); + } + } +} diff --git a/libs/jwst-core-rpc/src/client/webrtc.rs b/libs/jwst-core-rpc/src/client/webrtc.rs new file mode 100644 index 000000000..facf1f52a --- /dev/null +++ b/libs/jwst-core-rpc/src/client/webrtc.rs @@ -0,0 +1,96 @@ +use super::*; +use nanoid::nanoid; +use reqwest::Client; +use std::sync::RwLock; +use tokio::{runtime::Runtime, sync::mpsc::channel}; + +async fn webrtc_connection(remote: &str) -> (Sender, Receiver>) { + warn!("webrtc_connection start"); + let (offer, pc, tx, rx, mut s) = webrtc_datachannel_client_begin().await; + let client = Client::new(); + + match client.post(remote).json(&offer).send().await { + Ok(res) => { + webrtc_datachannel_client_commit( + res.json::().await.unwrap(), + pc, + ) + .await; + s.recv().await.ok(); + warn!("client already connected"); + } + Err(e) => { + error!("failed to http post: {}", e); + } + } + + (tx, rx) +} + +pub fn start_webrtc_client_sync( + rt: Arc, + context: Arc + Send + Sync + 'static>, + sync_state: Arc>, + remote: String, + workspace_id: String, +) -> CachedLastSynced { + debug!("spawn sync thread"); + let (last_synced_tx, last_synced_rx) = channel::(128); + + let runtime = rt.clone(); + std::thread::spawn(move || { + runtime.block_on(async move { + let workspace = match context.get_workspace(&workspace_id).await { + Ok(workspace) => workspace, + Err(e) => { + error!("failed to create workspace: {:?}", e); + return; + } + }; + if !workspace.is_empty() { + info!("Workspace not empty, starting async remote connection"); + last_synced_tx + .send(Utc::now().timestamp_millis()) + .await + .unwrap(); + } else { + info!("Workspace empty, starting sync remote connection"); + } + + loop { + let (tx, rx) = webrtc_connection(&remote).await; + *sync_state.write().unwrap() = SyncState::Connected; + + let ret = { + let identifier = nanoid!(); + let workspace_id = workspace_id.clone(); + let last_synced_tx = last_synced_tx.clone(); + handle_connector(context.clone(), workspace_id, identifier, move || { + (tx, rx, last_synced_tx.clone()) + }) + .await + }; + + { + last_synced_tx.send(0).await.unwrap(); + let mut state = sync_state.write().unwrap(); + if ret { + debug!("sync thread finished"); + *state = SyncState::Finished; + } else { + *state = + SyncState::Error("Remote sync connection disconnected".to_string()); + } + } + + warn!("remote sync connection disconnected, try again in 2 seconds"); + sleep(Duration::from_secs(2)).await; + } + }); + }); + + let timeline = CachedLastSynced::default(); + timeline.add_receiver(rt, last_synced_rx); + + timeline +} diff --git a/libs/jwst-core-rpc/src/client/websocket.rs b/libs/jwst-core-rpc/src/client/websocket.rs new file mode 100644 index 000000000..a50424498 --- /dev/null +++ b/libs/jwst-core-rpc/src/client/websocket.rs @@ -0,0 +1,103 @@ +use super::{types::JwstRpcResult, *}; +use nanoid::nanoid; +use std::sync::RwLock; +use tokio::{net::TcpStream, runtime::Runtime, sync::mpsc::channel}; +use tokio_tungstenite::{ + connect_async, + tungstenite::{client::IntoClientRequest, http::HeaderValue}, + MaybeTlsStream, WebSocketStream, +}; +use url::Url; + +type Socket = WebSocketStream>; + +async fn prepare_connection(remote: &str) -> JwstRpcResult { + debug!("generate remote config"); + let uri = Url::parse(remote)?; + + let mut req = uri.into_client_request()?; + req.headers_mut() + .append("Sec-WebSocket-Protocol", HeaderValue::from_static("AFFiNE")); + + debug!("connect to remote: {}", req.uri()); + Ok(connect_async(req).await?.0) +} + +pub fn start_websocket_client_sync( + rt: Arc, + context: Arc + Send + Sync + 'static>, + sync_state: Arc>, + remote: String, + workspace_id: String, +) -> CachedLastSynced { + debug!("spawn sync thread"); + let (last_synced_tx, last_synced_rx) = channel::(128); + + let runtime = rt.clone(); + runtime.spawn(async move { + debug!("start sync thread"); + let workspace = match context.get_workspace(&workspace_id).await { + Ok(workspace) => workspace, + Err(e) => { + warn!("failed to create workspace: {:?}", e); + return; + } + }; + if !workspace.is_empty() { + info!("Workspace not empty, starting async remote connection"); + last_synced_tx + .send(Utc::now().timestamp_millis()) + .await + .unwrap(); + } else { + info!("Workspace empty, starting sync remote connection"); + } + + loop { + let socket = match prepare_connection(&remote).await { + Ok(socket) => socket, + Err(e) => { + warn!("failed to connect to remote, try again in 2 seconds: {}", e); + sleep(Duration::from_secs(2)).await; + continue; + } + }; + *sync_state.write().unwrap() = SyncState::Connected; + + let ret = { + let identifier = nanoid!(); + let workspace_id = workspace_id.clone(); + let last_synced_tx = last_synced_tx.clone(); + handle_connector( + context.clone(), + workspace_id.clone(), + identifier, + move || { + let (tx, rx) = tungstenite_socket_connector(socket, &workspace_id); + (tx, rx, last_synced_tx) + }, + ) + .await + }; + + { + last_synced_tx.send(0).await.unwrap(); + let mut state = sync_state.write().unwrap(); + if ret { + debug!("sync thread finished"); + *state = SyncState::Finished; + } else { + *state = SyncState::Error("remote sync connection disconnected".to_string()); + } + } + + warn!("Remote sync connection disconnected, try again in 2 seconds"); + sleep(Duration::from_secs(2)).await; + } + }); + + let timeline = CachedLastSynced::default(); + timeline.add_receiver_wait_first_update(rt, last_synced_rx); + + timeline +} diff --git a/libs/jwst-core-rpc/src/connector/axum_socket.rs b/libs/jwst-core-rpc/src/connector/axum_socket.rs new file mode 100644 index 000000000..3aac137b3 --- /dev/null +++ b/libs/jwst-core-rpc/src/connector/axum_socket.rs @@ -0,0 +1,84 @@ +use super::*; +use axum::extract::ws::{Message as WebSocketMessage, WebSocket}; +use futures::{sink::SinkExt, stream::StreamExt}; +use tokio::sync::mpsc::{channel, Receiver, Sender}; +use tokio_tungstenite::tungstenite::Error as SocketError; + +impl From for WebSocketMessage { + fn from(value: Message) -> Self { + match value { + Message::Binary(data) => WebSocketMessage::Binary(data), + Message::Close => WebSocketMessage::Close(None), + Message::Ping => WebSocketMessage::Ping(vec![]), + } + } +} + +pub fn axum_socket_connector( + socket: WebSocket, + workspace_id: &str, +) -> (Sender, Receiver>, Sender) { + let (mut socket_tx, mut socket_rx) = socket.split(); + + // send to remote pipeline + let (local_sender, mut local_receiver) = channel::(100); + { + // socket send thread + let workspace_id = workspace_id.to_owned(); + tokio::spawn(async move { + while let Some(msg) = local_receiver.recv().await { + if let Err(e) = socket_tx.send(msg.into()).await { + let error = e.to_string(); + if !e.into_inner().downcast::().map_or_else( + |_| false, + |e| { + matches!( + e.as_ref(), + SocketError::ConnectionClosed | SocketError::AlreadyClosed + ) + }, + ) { + error!("socket send error: {}", error); + } + break; + } + } + info!("socket send final: {}", workspace_id); + }); + } + + let (remote_sender, remote_receiver) = channel::>(512); + { + // socket recv thread + let workspace_id = workspace_id.to_owned(); + tokio::spawn(async move { + while let Some(msg) = socket_rx.next().await { + if let Ok(WebSocketMessage::Binary(binary)) = msg { + trace!("recv from remote: {}bytes", binary.len()); + if remote_sender.send(binary).await.is_err() { + // pipeline was closed + break; + } + } + } + info!("socket recv final: {}", workspace_id); + }); + } + + let (first_init_tx, mut first_init_rx) = channel::(10); + { + // init notify thread + let workspace_id = workspace_id.to_owned(); + tokio::spawn(async move { + while let Some(time) = first_init_rx.recv().await { + if time > 0 { + info!("socket sync success: {}", workspace_id); + } else { + error!("socket sync failed: {}", workspace_id); + } + } + }); + } + + (local_sender, remote_receiver, first_init_tx) +} diff --git a/libs/jwst-core-rpc/src/connector/memory.rs b/libs/jwst-core-rpc/src/connector/memory.rs new file mode 100644 index 000000000..337780a11 --- /dev/null +++ b/libs/jwst-core-rpc/src/connector/memory.rs @@ -0,0 +1,125 @@ +use super::*; +use jwst_codec::{ + decode_update_with_guid, encode_update_as_message, Doc, DocMessage, SyncMessage, + SyncMessageScanner, +}; +use std::{ + sync::atomic::{AtomicBool, Ordering}, + thread::JoinHandle as StdJoinHandler, +}; +use tokio::{runtime::Runtime, sync::mpsc::channel, task::JoinHandle as TokioJoinHandler}; + +// just for test +pub fn memory_connector( + rt: Arc, + doc: Doc, +) -> ( + Sender, + Receiver>, + TokioJoinHandler<()>, + StdJoinHandler<()>, +) { + // recv from remote pipeline + let (remote_sender, remote_receiver) = channel::>(512); + // send to remote pipeline + let (local_sender, mut local_receiver) = channel::(100); + let id = rand::random::(); + + // recv thread + let recv_handler = { + debug!("init memory recv thread"); + let finish = Arc::new(AtomicBool::new(false)); + + { + let finish = finish.clone(); + doc.subscribe(move |update| { + debug!("send change: {}", update.len()); + + match encode_update_as_message(update.to_vec()) { + Ok(buffer) => { + if rt.block_on(remote_sender.send(buffer)).is_err() { + // pipeline was closed + finish.store(true, Ordering::Release); + } + debug!("send change: {} end", update.len()); + } + Err(e) => { + error!("write sync message error: {}", e); + } + } + }); + } + + { + let local_sender = local_sender.clone(); + let doc = doc.clone(); + std::thread::spawn(move || { + while let Ok(false) | Err(false) = finish + .compare_exchange_weak(true, false, Ordering::Acquire, Ordering::Acquire) + .or_else(|_| Ok(local_sender.is_closed())) + { + std::thread::sleep(Duration::from_millis(100)); + } + doc.unsubscribe_all(); + debug!("recv final: {}", id); + }) + } + }; + + // send thread + let send_handler = { + debug!("init memory send thread"); + tokio::spawn(async move { + while let Some(msg) = local_receiver.recv().await { + match msg { + Message::Binary(data) => { + info!("msg:binary"); + let mut doc = doc.clone(); + tokio::task::spawn_blocking(move || { + trace!("recv change: {:?}", data.len()); + for update in SyncMessageScanner::new(&data).filter_map(|m| { + m.ok().and_then(|m| { + if let SyncMessage::Doc(DocMessage::Update(update)) = m { + Some(update) + } else { + None + } + }) + }) { + match decode_update_with_guid(update.clone()) { + Ok((_, update1)) => { + if let Err(e) = doc.apply_update_from_binary(update1) { + error!( + "failed to decode update1: {}, update: {:?}", + e, update + ); + } + } + Err(e) => { + error!( + "failed to decode update2: {}, update: {:?}", + e, update + ); + } + } + } + + trace!("recv change: {} end", data.len()); + }); + } + Message::Close => { + info!("msg:close"); + break; + } + Message::Ping => { + info!("msg:ping"); + continue; + } + } + } + debug!("send final: {}", id); + }) + }; + + (local_sender, remote_receiver, send_handler, recv_handler) +} diff --git a/libs/jwst-core-rpc/src/connector/mod.rs b/libs/jwst-core-rpc/src/connector/mod.rs new file mode 100644 index 000000000..ca5db388c --- /dev/null +++ b/libs/jwst-core-rpc/src/connector/mod.rs @@ -0,0 +1,21 @@ +#[cfg(feature = "websocket")] +mod axum_socket; +mod memory; +#[cfg(feature = "websocket")] +mod tungstenite_socket; +#[cfg(feature = "webrtc")] +mod webrtc; + +#[cfg(feature = "websocket")] +pub use axum_socket::axum_socket_connector; +pub use memory::memory_connector; +#[cfg(feature = "websocket")] +pub use tungstenite_socket::tungstenite_socket_connector; +#[cfg(feature = "webrtc")] +pub use webrtc::webrtc_datachannel_client_begin; +#[cfg(feature = "webrtc")] +pub use webrtc::webrtc_datachannel_client_commit; +#[cfg(feature = "webrtc")] +pub use webrtc::webrtc_datachannel_server_connector; + +use super::*; diff --git a/libs/jwst-core-rpc/src/connector/tungstenite_socket.rs b/libs/jwst-core-rpc/src/connector/tungstenite_socket.rs new file mode 100644 index 000000000..25b2d6ab6 --- /dev/null +++ b/libs/jwst-core-rpc/src/connector/tungstenite_socket.rs @@ -0,0 +1,77 @@ +use super::*; +use futures::{sink::SinkExt, stream::StreamExt}; +use tokio::{ + net::TcpStream, + sync::mpsc::{channel, Receiver, Sender}, +}; +use tokio_tungstenite::{ + tungstenite::{Error as SocketError, Message as WebSocketMessage}, + MaybeTlsStream, WebSocketStream, +}; + +type WebSocket = WebSocketStream>; + +impl From for WebSocketMessage { + fn from(value: Message) -> Self { + match value { + Message::Binary(data) => WebSocketMessage::Binary(data), + Message::Close => WebSocketMessage::Close(None), + Message::Ping => WebSocketMessage::Ping(vec![]), + } + } +} + +pub fn tungstenite_socket_connector( + socket: WebSocket, + workspace_id: &str, +) -> (Sender, Receiver>) { + let (mut socket_tx, mut socket_rx) = socket.split(); + + // send to remote pipeline + let (local_sender, mut local_receiver) = channel::(100); + { + // socket send thread + let workspace_id = workspace_id.to_owned(); + tokio::spawn(async move { + let mut retry = 5; + while let Some(msg) = local_receiver.recv().await { + if let Err(e) = socket_tx.send(msg.into()).await { + let error = e.to_string(); + if matches!( + e, + SocketError::ConnectionClosed | SocketError::AlreadyClosed + ) || retry == 0 + { + break; + } else { + retry -= 1; + error!("socket send error: {}", error); + } + } else { + retry = 5; + } + } + info!("socket send final: {}", workspace_id); + }); + } + + let (remote_sender, remote_receiver) = channel::>(512); + { + // socket recv thread + let workspace_id = workspace_id.to_owned(); + tokio::spawn(async move { + while let Some(msg) = socket_rx.next().await { + if let Ok(WebSocketMessage::Binary(binary)) = msg { + trace!("recv from remote: {}bytes", binary.len()); + if remote_sender.send(binary).await.is_err() { + // pipeline was closed + break; + } + } + } + info!("socket recv final: {}", workspace_id); + }); + } + + (local_sender, remote_receiver) +} diff --git a/libs/jwst-core-rpc/src/connector/webrtc.rs b/libs/jwst-core-rpc/src/connector/webrtc.rs new file mode 100644 index 000000000..8bde98118 --- /dev/null +++ b/libs/jwst-core-rpc/src/connector/webrtc.rs @@ -0,0 +1,186 @@ +use super::Message; +use jwst_core::{debug, error, info, trace, warn}; + +use bytes::Bytes; +use std::sync::Arc; +use tokio::sync::mpsc::{channel, Receiver, Sender}; +use webrtcrs::{ + api::APIBuilder, + data_channel::{ + data_channel_init::RTCDataChannelInit, data_channel_message::DataChannelMessage, + OnMessageHdlrFn, + }, + peer_connection::{ + configuration::RTCConfiguration, peer_connection_state::RTCPeerConnectionState, + sdp::session_description::RTCSessionDescription, RTCPeerConnection, + }, +}; + +const DATA_CHANNEL_ID: u16 = 42; +const DATA_CHANNEL_LABEL: &str = "affine"; + +fn create_on_message_handler(channel: Sender>) -> OnMessageHdlrFn { + Box::new(move |message: DataChannelMessage| { + let channel = channel.clone(); + Box::pin(async move { + let m = message.data.to_vec(); + trace!("WebRTC Recv: {:?}", m.clone()); + channel.send(m).await.unwrap(); + }) + }) +} + +async fn new_peer_connection() -> ( + Arc, + Sender, + Receiver>, + tokio::sync::broadcast::Receiver, +) { + let api = APIBuilder::new().build(); + let peer_connection = Arc::new( + api.new_peer_connection(RTCConfiguration { + ..Default::default() + }) + .await + .unwrap(), + ); + + peer_connection.on_peer_connection_state_change(Box::new(move |s: RTCPeerConnectionState| { + warn!("Client: Peer Connection State has changed: {s}"); + if s == RTCPeerConnectionState::Failed { + error!("Peer Connection has gone to failed exiting"); + } + Box::pin(async {}) + })); + + let data_channel = peer_connection + .create_data_channel( + DATA_CHANNEL_LABEL, + Some(RTCDataChannelInit { + ordered: Some(true), + negotiated: Some(DATA_CHANNEL_ID), + ..Default::default() + }), + ) + .await + .unwrap(); + + let d0 = Arc::clone(&data_channel); + let (local_sender, mut local_receiver) = channel::(100); + + let (signal_tx, mut signal_rx) = tokio::sync::broadcast::channel::(1); + let signal_tx2 = signal_tx.subscribe(); + let sender_handle = tokio::spawn(async move { + signal_rx.recv().await.ok(); + while let Some(msg) = local_receiver.recv().await { + match msg { + Message::Binary(data) => { + trace!("WebRTC Send: {:?}", data.clone()); + d0.send(&Bytes::copy_from_slice(data.as_slice())) + .await + .unwrap(); + } + Message::Close => info!("Close"), + Message::Ping => info!("Ping"), + } + } + }); + + data_channel.on_open(Box::new(move || { + debug!("Data channel opened"); + Box::pin(async move { + signal_tx.send(true).ok(); + }) + })); + + data_channel.on_close(Box::new(move || { + debug!("Data channel closed"); + sender_handle.abort(); + Box::pin(async move {}) + })); + + let (remote_sender, remote_receiver) = channel::>(512); + data_channel.on_message(create_on_message_handler(remote_sender)); + + (peer_connection, local_sender, remote_receiver, signal_tx2) +} + +pub async fn webrtc_datachannel_client_begin() -> ( + RTCSessionDescription, + Arc, + Sender, + Receiver>, + tokio::sync::broadcast::Receiver, +) { + let (peer_connection, tx, rx, s) = new_peer_connection().await; + let offer = peer_connection.create_offer(None).await.unwrap(); + + // Create channel that is blocked until ICE Gathering is complete + let mut gather_complete = peer_connection.gathering_complete_promise().await; + + // Sets the LocalDescription, and starts our UDP listeners + match peer_connection.set_local_description(offer).await { + Ok(_) => {} + Err(e) => { + error!("set local description failed: {}", e); + } + } + + // Block until ICE Gathering is complete, disabling trickle ICE + // we do this because we only can exchange one signaling message + // in a production application you should exchange ICE Candidates via OnICECandidate + let _ = gather_complete.recv().await; + + let local_desc = peer_connection.local_description().await.unwrap(); + (local_desc, peer_connection, tx, rx, s) +} + +pub async fn webrtc_datachannel_client_commit( + answer: RTCSessionDescription, + peer_connection: Arc, +) { + match peer_connection.set_remote_description(answer).await { + Ok(_) => {} + Err(e) => { + error!("set remote description failed: {}", e); + } + } +} + +pub async fn webrtc_datachannel_server_connector( + offer: RTCSessionDescription, +) -> ( + RTCSessionDescription, + Sender, + Receiver>, + tokio::sync::broadcast::Receiver, +) { + let (peer_connection, tx, rx, s) = new_peer_connection().await; + + match peer_connection.set_remote_description(offer).await { + Ok(_) => {} + Err(e) => { + error!("set remote description failed: {}", e); + } + } + + let answer = peer_connection.create_answer(None).await.unwrap(); + + // Create channel that is blocked until ICE Gathering is complete + let mut gather_complete = peer_connection.gathering_complete_promise().await; + + match peer_connection.set_local_description(answer).await { + Ok(_) => {} + Err(e) => { + error!("set local description failed: {}", e); + } + } + + // Block until ICE Gathering is complete, disabling trickle ICE + // we do this because we only can exchange one signaling message + // in a production application you should exchange ICE Candidates via OnICECandidate + let _ = gather_complete.recv().await; + + let local_desc = peer_connection.local_description().await.unwrap(); + (local_desc, tx, rx, s) +} diff --git a/libs/jwst-core-rpc/src/context.rs b/libs/jwst-core-rpc/src/context.rs new file mode 100644 index 000000000..6e08c443b --- /dev/null +++ b/libs/jwst-core-rpc/src/context.rs @@ -0,0 +1,218 @@ +use std::collections::HashMap; + +use super::{ + broadcast::{subscribe, BroadcastChannels, BroadcastType}, + *, +}; +use async_trait::async_trait; +use chrono::Utc; +use jwst_codec::{CrdtReader, RawDecoder}; +use jwst_core::{DocStorage, Workspace}; +use jwst_core_storage::{JwstStorage, JwstStorageResult}; +use tokio::sync::{ + broadcast::{ + channel as broadcast, error::RecvError, Receiver as BroadcastReceiver, + Sender as BroadcastSender, + }, + mpsc::{Receiver as MpscReceiver, Sender as MpscSender}, + Mutex, +}; + +#[async_trait] +pub trait RpcContextImpl<'a> { + fn get_storage(&self) -> &JwstStorage; + fn get_channel(&self) -> &BroadcastChannels; + + async fn get_workspace(&self, id: &str) -> JwstStorageResult { + self.get_storage().create_workspace(id).await + } + + async fn join_server_broadcast(&self, id: &str) -> BroadcastReceiver> { + let id = id.into(); + match self.get_storage().docs().remote().write().await.entry(id) { + Entry::Occupied(tx) => tx.get().subscribe(), + Entry::Vacant(v) => { + let (tx, rx) = broadcast(100); + v.insert(tx); + rx + } + } + } + + async fn join_broadcast( + &self, + workspace: &mut Workspace, + identifier: String, + last_synced: Sender, + ) -> BroadcastSender { + let id = workspace.id(); + info!("join_broadcast, {:?}", workspace.id()); + // broadcast channel + let broadcast_tx = match self.get_channel().write().await.entry(id.clone()) { + Entry::Occupied(tx) => tx.get().clone(), + Entry::Vacant(v) => { + let (tx, _) = broadcast(10240); + v.insert(tx.clone()); + tx.clone() + } + }; + + // Listen to changes of the local workspace, encode changes in awareness and Doc, and broadcast them. + // It returns the 'broadcast_rx' object to receive the content that was sent + subscribe(workspace, identifier.clone(), broadcast_tx.clone()).await; + + // save update thread + self.save_update(&id, identifier, broadcast_tx.subscribe(), last_synced) + .await; + + // returns the 'broadcast_tx' which can be subscribed later, to receive local workspace changes + broadcast_tx + } + + async fn save_update( + &self, + id: &str, + identifier: String, + mut broadcast: BroadcastReceiver, + last_synced: Sender, + ) { + let docs = self.get_storage().docs().clone(); + let id = id.to_string(); + + tokio::spawn(async move { + trace!("save update thread {id}-{identifier} started"); + let updates = Arc::new(Mutex::new(HashMap::>>::new())); + + let handler = { + let id = id.clone(); + let updates = updates.clone(); + tokio::spawn(async move { + loop { + match broadcast.recv().await { + Ok(data) => match data { + BroadcastType::BroadcastRawContent(update) => { + trace!("receive raw update: {}", update.len()); + let mut decoder = RawDecoder::new(update); + if let Ok(guid) = decoder.read_var_string() { + match updates.lock().await.entry(guid) { + Entry::Occupied(mut updates) => { + updates.get_mut().push(decoder.drain()); + } + Entry::Vacant(v) => { + v.insert(vec![decoder.drain()]); + } + }; + }; + } + BroadcastType::CloseUser(user) if user == identifier => break, + BroadcastType::CloseAll => break, + _ => {} + }, + Err(RecvError::Lagged(num)) => { + debug!("save update thread {id}-{identifier} lagged: {num}"); + } + Err(RecvError::Closed) => { + debug!("save update thread {id}-{identifier} closed"); + break; + } + } + } + }) + }; + + loop { + { + let mut updates = updates.lock().await; + if !updates.is_empty() { + for (guid, updates) in updates.drain() { + debug!("save {} updates from {guid}", updates.len()); + + for update in updates { + if let Err(e) = + docs.update_doc(id.clone(), guid.clone(), &update).await + { + error!("failed to save update of {}: {:?}", id, e); + } + } + } + last_synced + .send(Utc::now().timestamp_millis()) + .await + .unwrap(); + } else if handler.is_finished() { + break; + } + } + sleep(Duration::from_secs(1)).await; + } + }); + } + + async fn apply_change( + &self, + id: &str, + identifier: &str, + local_tx: MpscSender, + mut remote_rx: MpscReceiver>, + last_synced: Sender, + ) { + // collect messages from remote + let identifier = identifier.to_owned(); + let id = id.to_string(); + let mut workspace = self + .get_storage() + .get_workspace(&id) + .await + .expect("workspace not found"); + tokio::spawn(async move { + trace!("apply update thread {id}-{identifier} started"); + let mut updates = Vec::>::new(); + + loop { + tokio::select! { + binary = remote_rx.recv() => { + if let Some(binary) = binary { + if binary == [0, 2, 2, 0, 0] || binary == [1, 1, 0] { + // skip empty update + continue; + } + trace!("apply_change: recv binary: {:?}", binary.len()); + updates.push(binary); + } else { + break; + } + }, + _ = sleep(Duration::from_millis(100)) => { + if !updates.is_empty() { + debug!("apply {} updates for {id}", updates.len()); + + let updates = updates.drain(..).collect::>(); + let updates_len = updates.len(); + let ts = Instant::now(); + let message = workspace.sync_messages(updates).await; + if ts.elapsed().as_micros() > 50 { + debug!( + "apply {updates_len} remote update cost: {}ms", + ts.elapsed().as_micros(), + ); + } + + for reply in message { + trace!("send pipeline message by {identifier:?}: {}", reply.len()); + if local_tx.send(Message::Binary(reply.clone())).await.is_err() { + // pipeline was closed + break; + } + } + + last_synced + .send(Utc::now().timestamp_millis()) + .await + .unwrap(); + } + } + } + } + }); + } +} diff --git a/libs/jwst-core-rpc/src/handler.rs b/libs/jwst-core-rpc/src/handler.rs new file mode 100644 index 000000000..b9691ce9b --- /dev/null +++ b/libs/jwst-core-rpc/src/handler.rs @@ -0,0 +1,481 @@ +use super::{BroadcastType, Message, RpcContextImpl}; +use chrono::Utc; +use jwst_core::{debug, error, info, trace, warn}; +use std::{sync::Arc, time::Instant}; +use tokio::{ + sync::mpsc::{Receiver, Sender}, + time::{sleep, Duration}, +}; + +pub async fn handle_connector( + context: Arc + Send + Sync + 'static>, + workspace_id: String, + identifier: String, + get_channel: impl FnOnce() -> (Sender, Receiver>, Sender), +) -> bool { + info!("{} collaborate with workspace {}", identifier, workspace_id); + + // An abstraction of the established socket connection. Use tx to broadcast and rx to receive. + let (tx, rx, last_synced) = get_channel(); + + let mut ws = context + .get_workspace(&workspace_id) + .await + .expect("failed to get workspace"); + + // Continuously receive information from the remote socket, apply it to the local workspace, and + // send the encoded updates back to the remote end through the socket. + context + .apply_change( + &workspace_id, + &identifier, + tx.clone(), + rx, + last_synced.clone(), + ) + .await; + + // Both of broadcast_update and server_update are sent to the remote socket through 'tx' + // The 'broadcast_update' is the receiver for updates to the awareness and Doc of the local workspace. + // It uses channel, which is owned by the server itself and is stored in the server's memory (not persisted)." + let broadcast_tx = context + .join_broadcast(&mut ws, identifier.clone(), last_synced.clone()) + .await; + let mut broadcast_rx = broadcast_tx.subscribe(); + // Obtaining the receiver corresponding to DocAutoStorage in storage. The sender is used in the + // doc::write_update(). The remote used is the one belonging to DocAutoStorage and is owned by + // the server itself, stored in the server's memory (not persisted). + let mut server_rx = context.join_server_broadcast(&workspace_id).await; + + // Send initialization message. + match ws.sync_init_message().await { + Ok(init_data) => { + if tx.send(Message::Binary(init_data)).await.is_err() { + warn!("failed to send init message: {}", identifier); + // client disconnected + if let Err(e) = tx.send(Message::Close).await { + error!("failed to send close event: {}", e); + } + last_synced.send(0).await.unwrap(); + return false; + } + } + Err(e) => { + warn!("failed to generate {} init message: {}", identifier, e); + if let Err(e) = tx.send(Message::Close).await { + error!("failed to send close event: {}", e); + } + last_synced.send(0).await.unwrap(); + return false; + } + } + + last_synced + .send(Utc::now().timestamp_millis()) + .await + .unwrap(); + + 'sync: loop { + tokio::select! { + Ok(msg) = server_rx.recv()=> { + let ts = Instant::now(); + trace!("recv from server update: {:?}", msg); + if tx.send(Message::Binary(msg.clone())).await.is_err() { + // pipeline was closed + break 'sync; + } + if ts.elapsed().as_micros() > 100 { + debug!("process server update cost: {}ms", ts.elapsed().as_micros()); + } + }, + Ok(msg) = broadcast_rx.recv()=> { + let ts = Instant::now(); + match msg { + BroadcastType::BroadcastAwareness(data) => { + let ts = Instant::now(); + trace!( + "recv awareness update from broadcast: {:?}bytes", + data.len() + ); + if tx.send(Message::Binary(data.clone())).await.is_err() { + // pipeline was closed + break 'sync; + } + if ts.elapsed().as_micros() > 100 { + debug!( + "process broadcast awareness cost: {}ms", + ts.elapsed().as_micros() + ); + } + } + BroadcastType::BroadcastContent(data) => { + let ts = Instant::now(); + trace!("recv content update from broadcast: {:?}bytes", data.len()); + if tx.send(Message::Binary(data.clone())).await.is_err() { + // pipeline was closed + break 'sync; + } + if ts.elapsed().as_micros() > 100 { + debug!( + "process broadcast content cost: {}ms", + ts.elapsed().as_micros() + ); + } + } + BroadcastType::CloseUser(user) if user == identifier => { + let ts = Instant::now(); + if tx.send(Message::Close).await.is_err() { + // pipeline was closed + break 'sync; + } + if ts.elapsed().as_micros() > 100 { + debug!("process close user cost: {}ms", ts.elapsed().as_micros()); + } + + break; + } + BroadcastType::CloseAll => { + let ts = Instant::now(); + if tx.send(Message::Close).await.is_err() { + // pipeline was closed + break 'sync; + } + if ts.elapsed().as_micros() > 100 { + debug!("process close all cost: {}ms", ts.elapsed().as_micros()); + } + + break 'sync; + } + _ => {} + } + + if ts.elapsed().as_micros() > 100 { + debug!("process broadcast cost: {}ms", ts.elapsed().as_micros()); + } + }, + _ = sleep(Duration::from_secs(1)) => { + if tx.is_closed() || tx.send(Message::Ping).await.is_err() { + break 'sync; + } + } + } + } + + // make a final store + context + .get_storage() + .full_migrate(workspace_id.clone(), None, false) + .await; + let _ = broadcast_tx.send(BroadcastType::CloseUser(identifier.clone())); + info!( + "{} stop collaborate with workspace {}", + identifier, workspace_id + ); + true +} + +#[cfg(test)] +mod test { + use super::{ + super::{connect_memory_workspace, MinimumServerContext}, + *, + }; + #[cfg(feature = "webrtc")] + use crate::{ + webrtc_datachannel_client_begin, webrtc_datachannel_client_commit, + webrtc_datachannel_server_connector, + }; + use indicatif::{MultiProgress, ProgressBar, ProgressIterator, ProgressStyle}; + use jwst_core::JwstResult; + use std::sync::atomic::{AtomicU64, Ordering}; + + #[tokio::test] + #[ignore = "skip in ci"] + #[cfg(feature = "webrtc")] + async fn webrtc_datachannel_connector_test() { + let (offer, pc, tx1, mut rx1, _) = webrtc_datachannel_client_begin().await; + let (answer, tx2, mut rx2, _) = webrtc_datachannel_server_connector(offer).await; + webrtc_datachannel_client_commit(answer, pc).await; + + let data_a_1 = String::from("data_a_1"); + let data_a_2 = String::from("data_a_2"); + let data_a_3 = String::from("data_a_3"); + + let data_b_1 = String::from("data_b_1"); + let data_b_2 = String::from("data_b_2"); + let data_b_3 = String::from("data_b_3"); + + tx1.send(Message::Binary(data_a_1.clone().into_bytes())) + .await + .unwrap(); + tx1.send(Message::Binary(data_a_2.clone().into_bytes())) + .await + .unwrap(); + + tx2.send(Message::Binary(data_b_1.clone().into_bytes())) + .await + .unwrap(); + tx2.send(Message::Binary(data_b_2.clone().into_bytes())) + .await + .unwrap(); + + tx1.send(Message::Binary(data_a_3.clone().into_bytes())) + .await + .unwrap(); + tx2.send(Message::Binary(data_b_3.clone().into_bytes())) + .await + .unwrap(); + + if let Some(message) = rx2.recv().await { + assert_eq!(String::from_utf8(message).ok().unwrap(), data_a_1); + } + if let Some(message) = rx2.recv().await { + assert_eq!(String::from_utf8(message).ok().unwrap(), data_a_2); + } + + if let Some(message) = rx1.recv().await { + assert_eq!(String::from_utf8(message).ok().unwrap(), data_b_1); + } + if let Some(message) = rx1.recv().await { + assert_eq!(String::from_utf8(message).ok().unwrap(), data_b_2); + } + + if let Some(message) = rx2.recv().await { + assert_eq!(String::from_utf8(message).ok().unwrap(), data_a_3); + } + if let Some(message) = rx1.recv().await { + assert_eq!(String::from_utf8(message).ok().unwrap(), data_b_3); + } + } + + #[tokio::test] + #[ignore = "unstable, skip in ci and wait for the refactoring of the sync logic"] + async fn sync_test() -> JwstResult<()> { + let workspace_id = format!("test{}", rand::random::()); + + let (server, mut ws, init_state) = + MinimumServerContext::new_with_workspace(&workspace_id).await; + + let (mut doc1, _, _, _, _) = + connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; + let (doc2, tx2, tx_handler, rx_handler, rt) = + connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; + + // close connection after doc1 is broadcasted + doc2.subscribe(move |_| { + rt.block_on(async { + tx2.send(Message::Close).await.unwrap(); + }); + }); + + // collect the update from yrs's editing + let update = { + let mut ws = + jwst_core::Workspace::from_binary(doc1.encode_update_v1().unwrap(), &workspace_id) + .unwrap(); + + let mut space = ws.get_space("space").unwrap(); + let mut block1 = space.create("block1", "flavour1").unwrap(); + block1.set("key1", "val1").unwrap(); + + ws.doc().encode_update_v1().unwrap() + }; + // apply update with jwst-codec + doc1.apply_update_from_binary(update).unwrap(); + + // await the task to make sure the doc1 is broadcasted before check doc2 + tx_handler.await.unwrap(); + rx_handler.join().unwrap(); + + // collect the update from jwst-codec and check the result + { + let mut ws = + jwst_core::Workspace::from_binary(doc2.encode_update_v1().unwrap(), &workspace_id) + .unwrap(); + + let space = ws.get_space("space").unwrap(); + let block1 = space.get("block1").unwrap(); + + assert_eq!(block1.flavour(), "flavour1"); + assert_eq!(block1.get("key1").unwrap().to_string(), "val1"); + } + + { + let space = ws.get_space("space").unwrap(); + let block1 = space.get("block1").unwrap(); + + assert_eq!(block1.flavour(), "flavour1"); + assert_eq!(block1.get("key1").unwrap().to_string(), "val1"); + } + + Ok(()) + } + + #[ignore = "somewhat slow, only natively tested"] + #[test] + fn sync_test_cycle() -> JwstResult<()> { + jwst_logger::init_logger("jwst-rpc"); + + for _ in 0..1000 { + sync_test()?; + } + Ok(()) + } + + async fn single_sync_stress_test(mp: &MultiProgress) -> JwstResult<()> { + // jwst_logger::init_logger("jwst-rpc"); + + let style = ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", + ) + .unwrap() + .progress_chars("##-"); + + let workspace_id = format!("test{}", rand::random::()); + + let (server, mut ws, init_state) = + MinimumServerContext::new_with_workspace(&workspace_id).await; + + let mut jobs = vec![]; + + let collaborator_pb = mp.add(ProgressBar::new(10)); + collaborator_pb.set_style(style.clone()); + collaborator_pb.set_message("collaborators"); + let collaborator = Arc::new(AtomicU64::new(0)); + + let pb = mp.add(ProgressBar::new(1000)); + pb.set_style(style.clone()); + pb.set_message("writing"); + for i in (0..1000).progress_with(pb) { + let collaborator = collaborator.clone(); + let collaborator_pb = collaborator_pb.clone(); + let init_state = init_state.clone(); + let server = server.clone(); + let mut ws = ws.clone(); + + collaborator.fetch_add(1, Ordering::Relaxed); + collaborator_pb.set_position(collaborator.load(Ordering::Relaxed)); + let (doc, doc_tx, tx_handler, rx_handler, _rt) = + connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; + let mut doc = + jwst_core::Workspace::from_binary(doc.encode_update_v1().unwrap(), &workspace_id) + .unwrap(); + + let handler = std::thread::spawn(move || { + // close connection after doc1 is broadcasted + let block_id = format!("block{}", i); + { + let block_id = block_id.clone(); + let doc_tx = doc_tx.clone(); + doc.doc().subscribe(move |_u| { + // TODO: support changed block record + // let block_changed = t + // .changed_parent_types() + // .iter() + // .filter_map(|ptr| { + // let value: yrs::types::Value = (*ptr).into(); + // value.to_ymap() + // }) + // .flat_map(|map| map.keys(t).map(|k| k.to_string()).collect::>()) + // .any(|key| key == block_id); + let block_changed = false; + + if block_changed { + if let Err(e) = futures::executor::block_on(doc_tx.send(Message::Close)) + { + error!("send close message failed: {}", e); + } + } + }); + } + + { + let mut space = doc.get_space("space").unwrap(); + let mut block = space + .create(block_id.clone(), format!("flavour{}", i)) + .unwrap(); + block + .set(&format!("key{}", i), format!("val{}", i)) + .unwrap(); + } + + // await the task to make sure the doc1 is broadcasted before check doc2 + futures::executor::block_on(tx_handler).unwrap(); + rx_handler.join().unwrap(); + + { + let space = ws.get_space("space").unwrap(); + let block1 = space.get(format!("block{}", i)).unwrap(); + + assert_eq!(block1.flavour(), format!("flavour{}", i)); + assert_eq!( + block1.get(&format!("key{}", i)).unwrap().to_string(), + format!("val{}", i) + ); + } + + collaborator.fetch_sub(1, Ordering::Relaxed); + collaborator_pb.set_position(collaborator.load(Ordering::Relaxed)); + }); + jobs.push(handler); + } + + let pb = mp.add(ProgressBar::new(jobs.len().try_into().unwrap())); + pb.set_style(style.clone()); + pb.set_message("joining"); + for handler in jobs.into_iter().progress_with(pb) { + if let Err(e) = handler.join() { + panic!("{:?}", e.as_ref()); + } + } + + info!("check the workspace"); + + let pb = mp.add(ProgressBar::new(1000)); + pb.set_style(style.clone()); + pb.set_message("final checking"); + + { + let space = ws.get_space("space").unwrap(); + for i in (0..1000).progress_with(pb) { + let block1 = space.get(format!("block{}", i)).unwrap(); + + assert_eq!(block1.flavour(), format!("flavour{}", i)); + assert_eq!( + block1.get(&format!("key{}", i)).unwrap().to_string(), + format!("val{}", i) + ); + } + } + + Ok(()) + } + + #[ignore = "somewhat slow, only natively tested"] + #[tokio::test(flavor = "multi_thread")] + async fn sync_stress_test() -> JwstResult<()> { + let mp = MultiProgress::new(); + single_sync_stress_test(&mp).await + } + + #[ignore = "somewhat slow, only natively tested"] + #[tokio::test(flavor = "multi_thread")] + async fn sync_stress_test_cycle() -> JwstResult<()> { + // jwst_logger::init_logger("jwst-rpc"); + + let mp = MultiProgress::new(); + let style = ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", + ) + .unwrap() + .progress_chars("##-"); + + let pb = mp.add(ProgressBar::new(1000)); + pb.set_style(style.clone()); + pb.set_message("cycle stress test"); + for _ in (0..1000).progress_with(pb.clone()) { + single_sync_stress_test(&mp).await?; + } + Ok(()) + } +} diff --git a/libs/jwst-core-rpc/src/lib.rs b/libs/jwst-core-rpc/src/lib.rs new file mode 100644 index 000000000..a282d37a4 --- /dev/null +++ b/libs/jwst-core-rpc/src/lib.rs @@ -0,0 +1,53 @@ +#[forbid(unsafe_code)] +mod broadcast; +mod client; +mod connector; +mod context; +mod handler; +mod types; +mod utils; + +#[cfg(feature = "webrtc")] +pub use client::start_webrtc_client_sync; +#[cfg(feature = "websocket")] +pub use client::start_websocket_client_sync; +#[cfg(feature = "webrtc")] +pub use connector::webrtc_datachannel_client_begin; +#[cfg(feature = "webrtc")] +pub use connector::webrtc_datachannel_client_commit; +#[cfg(feature = "webrtc")] +pub use connector::webrtc_datachannel_server_connector; +#[cfg(feature = "websocket")] +pub use connector::{axum_socket_connector, tungstenite_socket_connector}; +#[cfg(feature = "webrtc")] +pub use webrtcrs::peer_connection::sdp::session_description::RTCSessionDescription; + +pub use broadcast::{BroadcastChannels, BroadcastType}; +pub use client::CachedLastSynced; +pub use connector::memory_connector; +pub use context::RpcContextImpl; +pub use handler::handle_connector; +pub use utils::{connect_memory_workspace, MinimumServerContext}; + +use jwst_core::{debug, error, info, trace, warn}; +use std::{collections::hash_map::Entry, sync::Arc, time::Instant}; +use tokio::{ + sync::mpsc::{Receiver, Sender}, + time::{sleep, Duration}, +}; + +#[derive(Clone, PartialEq, Eq, Debug, Default)] +pub enum SyncState { + #[default] + Offline, + Connected, + Finished, + Error(String), +} + +#[derive(Debug)] +pub enum Message { + Binary(Vec), + Close, + Ping, +} diff --git a/libs/jwst-core-rpc/src/types.rs b/libs/jwst-core-rpc/src/types.rs new file mode 100644 index 000000000..177f1f7bb --- /dev/null +++ b/libs/jwst-core-rpc/src/types.rs @@ -0,0 +1,22 @@ +use thiserror::Error; +#[cfg(feature = "websocket")] +use tokio_tungstenite::tungstenite; + +#[derive(Debug, Error)] +pub enum JwstRpcError { + #[cfg(feature = "websocket")] + #[error("failed to connect websocket: {0}")] + WebsocketConnect(#[from] tungstenite::Error), + #[error("jwst error")] + Jwst(#[from] jwst_core::JwstError), + #[allow(dead_code)] + #[error("failed to encode sync message")] + ProtocolEncode(std::io::Error), + #[error("failed to decode sync message")] + ProtocolDecode(#[from] nom::Err>), + #[cfg(feature = "websocket")] + #[error("failed to parse url")] + UrlParse(#[from] url::ParseError), +} + +pub type JwstRpcResult = Result; diff --git a/libs/jwst-core-rpc/src/utils/memory_workspace.rs b/libs/jwst-core-rpc/src/utils/memory_workspace.rs new file mode 100644 index 000000000..a19a771d9 --- /dev/null +++ b/libs/jwst-core-rpc/src/utils/memory_workspace.rs @@ -0,0 +1,56 @@ +use super::*; +use jwst_codec::Doc; +use nanoid::nanoid; +use std::thread::JoinHandle as StdJoinHandler; +use tokio::{runtime::Runtime, sync::mpsc::channel, task::JoinHandle as TokioJoinHandler}; + +pub async fn connect_memory_workspace( + server: Arc, + init_state: &[u8], + id: &str, +) -> ( + Doc, + Sender, + TokioJoinHandler<()>, + StdJoinHandler<()>, + Arc, +) { + let rt = Arc::new(tokio::runtime::Runtime::new().unwrap()); + + let mut doc = Doc::default(); + doc.apply_update_from_binary(init_state.to_vec()).unwrap(); + + let (tx, rx, tx_handler, rx_handler) = memory_connector(rt.clone(), doc.clone()); + { + let (last_synced_tx, mut last_synced_rx) = channel::(128); + let tx = tx.clone(); + let workspace_id = id.to_string(); + { + let rt = rt.clone(); + std::thread::spawn(move || { + rt.block_on(handle_connector( + server, + workspace_id, + nanoid!(), + move || (tx, rx, last_synced_tx), + )); + }); + } + + let success = last_synced_rx.recv().await; + + if success.unwrap_or(0) > 0 { + info!("{id} first init success"); + } else { + error!("{id} first init failed"); + } + + rt.spawn(async move { + while let Some(last_synced) = last_synced_rx.recv().await { + info!("last synced: {}", last_synced); + } + }); + } + + (doc, tx, tx_handler, rx_handler, rt) +} diff --git a/libs/jwst-core-rpc/src/utils/mod.rs b/libs/jwst-core-rpc/src/utils/mod.rs new file mode 100644 index 000000000..92ef9fcf6 --- /dev/null +++ b/libs/jwst-core-rpc/src/utils/mod.rs @@ -0,0 +1,7 @@ +mod memory_workspace; +mod server_context; + +use super::*; + +pub use memory_workspace::connect_memory_workspace; +pub use server_context::MinimumServerContext; diff --git a/libs/jwst-core-rpc/src/utils/server_context.rs b/libs/jwst-core-rpc/src/utils/server_context.rs new file mode 100644 index 000000000..386846bde --- /dev/null +++ b/libs/jwst-core-rpc/src/utils/server_context.rs @@ -0,0 +1,75 @@ +use super::*; +use jwst_codec::StateVector; +use jwst_core::{DocStorage, Workspace}; +use jwst_core_storage::{BlobStorageType, JwstStorage}; +use std::{collections::HashMap, time::Duration}; +use tokio::{sync::RwLock, time::sleep}; + +pub struct MinimumServerContext { + channel: BroadcastChannels, + storage: JwstStorage, +} + +// just for test +impl MinimumServerContext { + pub async fn new() -> Arc { + let storage = 'connect: loop { + let mut retry = 3; + match JwstStorage::new_with_migration( + &std::env::var("DATABASE_URL") + .map(|url| format!("{url}_binary")) + .unwrap_or("sqlite::memory:".into()), + BlobStorageType::DB, + ) + .await + { + Ok(storage) => break 'connect Ok(storage), + Err(e) => { + retry -= 1; + if retry > 0 { + error!("failed to connect database: {}", e); + sleep(Duration::from_secs(1)).await; + } else { + break 'connect Err(e); + } + } + } + } + .unwrap(); + + Arc::new(Self { + channel: RwLock::new(HashMap::new()), + storage, + }) + } + + pub async fn new_with_workspace( + workspace_id: &str, + ) -> (Arc, Workspace, Vec) { + let server = Self::new().await; + server + .get_storage() + .docs() + .delete_workspace(workspace_id) + .await + .unwrap(); + let ws = server.get_workspace(workspace_id).await.unwrap(); + + let init_state = ws + .doc() + .encode_state_as_update_v1(&StateVector::default()) + .expect("encode_state_as_update_v1 failed"); + + (server, ws, init_state) + } +} + +impl RpcContextImpl<'_> for MinimumServerContext { + fn get_storage(&self) -> &JwstStorage { + &self.storage + } + + fn get_channel(&self) -> &BroadcastChannels { + &self.channel + } +} diff --git a/libs/jwst-core-storage/Cargo.toml b/libs/jwst-core-storage/Cargo.toml new file mode 100644 index 000000000..1119ece02 --- /dev/null +++ b/libs/jwst-core-storage/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "jwst-core-storage" +version = "0.1.0" +edition = "2021" +authors = ["DarkSky "] +license = "AGPL-3.0-only" + +[features] +default = ["sqlite"] +mysql = ["sea-orm/sqlx-mysql"] +postgres = ["sea-orm/sqlx-postgres"] +sqlite = ["sea-orm/sqlx-sqlite"] + +[dependencies] +anyhow = "1.0.70" +async-trait = "0.1.68" +bytes = "1.4.0" +chrono = { version = "0.4.24", features = ["serde"] } +futures = "0.3.28" +governor = "0.5.1" +image = { version = "0.24.6", features = ["webp-encoder"] } +path-ext = "0.1.0" +sha2 = "0.10.6" +sea-orm = { version = "0.12.2", features = ["runtime-tokio-rustls", "macros"] } +sea-orm-migration = "0.12.2" +thiserror = "1.0.40" +tokio = { version = "1.27.0", features = ["fs", "macros", "sync"] } +tokio-util = { version = "0.7.7", features = ["io"] } +url = "2.3.1" +opendal = { version = "0.38.0", default-features = false, features = [ + "rustls", + "services-s3", +] } +dotenvy = "0.15.7" + +# ======= workspace dependencies ======= +jwst-core = { workspace = true } +jwst-codec = { workspace = true } +jwst-logger = { workspace = true } +jwst-core-storage-migration = { path = "./src/migration" } + +[dev-dependencies] +rand = "0.8.5" diff --git a/libs/jwst-core-storage/src/entities/blobs.rs b/libs/jwst-core-storage/src/entities/blobs.rs new file mode 100644 index 000000000..4470efff4 --- /dev/null +++ b/libs/jwst-core-storage/src/entities/blobs.rs @@ -0,0 +1,21 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "blobs")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub workspace_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub hash: String, + #[sea_orm(column_type = "Binary(BlobSize::Blob(None))")] + pub blob: Vec, + pub length: i64, + pub created_at: DateTimeWithTimeZone, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/jwst-core-storage/src/entities/bucket_blobs.rs b/libs/jwst-core-storage/src/entities/bucket_blobs.rs new file mode 100644 index 000000000..8b92b05e1 --- /dev/null +++ b/libs/jwst-core-storage/src/entities/bucket_blobs.rs @@ -0,0 +1,19 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "bucket_blobs")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub workspace_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub hash: String, + pub length: i64, + pub created_at: DateTimeWithTimeZone, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/jwst-core-storage/src/entities/diff_log.rs b/libs/jwst-core-storage/src/entities/diff_log.rs new file mode 100644 index 000000000..f3e56c374 --- /dev/null +++ b/libs/jwst-core-storage/src/entities/diff_log.rs @@ -0,0 +1,18 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "diff_log")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub workspace: String, + pub timestamp: DateTimeWithTimeZone, + pub log: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/jwst-core-storage/src/entities/docs.rs b/libs/jwst-core-storage/src/entities/docs.rs new file mode 100644 index 000000000..2492bd5b5 --- /dev/null +++ b/libs/jwst-core-storage/src/entities/docs.rs @@ -0,0 +1,21 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "docs")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub workspace_id: String, + pub created_at: DateTimeWithTimeZone, + #[sea_orm(column_type = "Binary(BlobSize::Blob(None))")] + pub blob: Vec, + pub guid: String, + pub is_workspace: bool, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/jwst-core-storage/src/entities/mod.rs b/libs/jwst-core-storage/src/entities/mod.rs new file mode 100644 index 000000000..2425a9d77 --- /dev/null +++ b/libs/jwst-core-storage/src/entities/mod.rs @@ -0,0 +1,9 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +pub mod prelude; + +pub mod blobs; +pub mod bucket_blobs; +pub mod diff_log; +pub mod docs; +pub mod optimized_blobs; diff --git a/libs/jwst-core-storage/src/entities/optimized_blobs.rs b/libs/jwst-core-storage/src/entities/optimized_blobs.rs new file mode 100644 index 000000000..a1b581888 --- /dev/null +++ b/libs/jwst-core-storage/src/entities/optimized_blobs.rs @@ -0,0 +1,23 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "optimized_blobs")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub workspace_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub hash: String, + #[sea_orm(column_type = "Binary(BlobSize::Blob(None))")] + pub blob: Vec, + pub length: i64, + pub created_at: DateTimeWithTimeZone, + #[sea_orm(primary_key, auto_increment = false)] + pub params: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/jwst-core-storage/src/entities/prelude.rs b/libs/jwst-core-storage/src/entities/prelude.rs new file mode 100644 index 000000000..5bdda8eb3 --- /dev/null +++ b/libs/jwst-core-storage/src/entities/prelude.rs @@ -0,0 +1,7 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +pub use super::blobs::Entity as Blobs; +pub use super::bucket_blobs::Entity as BucketBlobs; +pub use super::diff_log::Entity as DiffLog; +pub use super::docs::Entity as Docs; +pub use super::optimized_blobs::Entity as OptimizedBlobs; diff --git a/libs/jwst-core-storage/src/lib.rs b/libs/jwst-core-storage/src/lib.rs new file mode 100644 index 000000000..553a0c957 --- /dev/null +++ b/libs/jwst-core-storage/src/lib.rs @@ -0,0 +1,39 @@ +#[forbid(unsafe_code)] +mod entities; +mod rate_limiter; +mod storage; +mod types; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use futures::{Future, Stream}; +use jwst_core::{DocStorage, JwstResult, Workspace}; +use jwst_logger::{debug, error, info, trace, warn}; +use path_ext::PathExt; +use rate_limiter::{get_bucket, is_sqlite, Bucket}; +use sea_orm::{prelude::*, ConnectOptions, Database, DbErr, QuerySelect, Set}; +use std::{path::PathBuf, sync::Arc, time::Duration}; + +pub use storage::blobs::{BlobStorageType, MixedBucketDBParam}; +pub use storage::JwstStorage; +pub use types::{JwstStorageError, JwstStorageResult}; + +#[inline] +async fn create_connection( + database: &str, + single_thread: bool, +) -> JwstStorageResult { + let connection = Database::connect( + ConnectOptions::from(database) + .max_connections(if single_thread { 1 } else { 50 }) + .min_connections(if single_thread { 1 } else { 10 }) + .acquire_timeout(Duration::from_secs(5)) + .connect_timeout(Duration::from_secs(5)) + .idle_timeout(Duration::from_secs(5)) + .max_lifetime(Duration::from_secs(30)) + .to_owned(), + ) + .await?; + + Ok(connection) +} diff --git a/libs/jwst-core-storage/src/migration/Cargo.toml b/libs/jwst-core-storage/src/migration/Cargo.toml new file mode 100644 index 000000000..38591c4a4 --- /dev/null +++ b/libs/jwst-core-storage/src/migration/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "jwst-core-storage-migration" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +name = "jwst_storage_migration" +path = "src/lib.rs" + +[dependencies] +tokio = { version = "^1", features = ["macros", "rt-multi-thread"] } + +[dependencies.sea-orm-migration] +version = "0.12.2" +features = ["runtime-tokio-rustls", "sqlx-postgres"] diff --git a/libs/jwst-core-storage/src/migration/README.md b/libs/jwst-core-storage/src/migration/README.md new file mode 100644 index 000000000..b3ea53eb4 --- /dev/null +++ b/libs/jwst-core-storage/src/migration/README.md @@ -0,0 +1,41 @@ +# Running Migrator CLI + +- Generate a new migration file + ```sh + cargo run -- migrate generate MIGRATION_NAME + ``` +- Apply all pending migrations + ```sh + cargo run + ``` + ```sh + cargo run -- up + ``` +- Apply first 10 pending migrations + ```sh + cargo run -- up -n 10 + ``` +- Rollback last applied migrations + ```sh + cargo run -- down + ``` +- Rollback last 10 applied migrations + ```sh + cargo run -- down -n 10 + ``` +- Drop all tables from the database, then reapply all migrations + ```sh + cargo run -- fresh + ``` +- Rollback all applied migrations, then reapply all migrations + ```sh + cargo run -- refresh + ``` +- Rollback all applied migrations + ```sh + cargo run -- reset + ``` +- Check the status of all migrations + ```sh + cargo run -- status + ``` diff --git a/libs/jwst-core-storage/src/migration/src/lib.rs b/libs/jwst-core-storage/src/migration/src/lib.rs new file mode 100644 index 000000000..b2b885e82 --- /dev/null +++ b/libs/jwst-core-storage/src/migration/src/lib.rs @@ -0,0 +1,25 @@ +pub use sea_orm_migration::prelude::*; + +mod m20220101_000001_initial_blob_table; +mod m20220101_000002_initial_doc_table; +mod m20230321_000001_blob_optimized_table; +mod m20230614_000001_initial_bucket_blob_table; +mod m20230626_023319_doc_guid; +mod m20230814_061223_initial_diff_log_table; +mod schema; + +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![ + Box::new(m20220101_000001_initial_blob_table::Migration), + Box::new(m20220101_000002_initial_doc_table::Migration), + Box::new(m20230321_000001_blob_optimized_table::Migration), + Box::new(m20230614_000001_initial_bucket_blob_table::Migration), + Box::new(m20230626_023319_doc_guid::Migration), + Box::new(m20230814_061223_initial_diff_log_table::Migration), + ] + } +} diff --git a/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs b/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs new file mode 100644 index 000000000..1f7377334 --- /dev/null +++ b/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs @@ -0,0 +1,57 @@ +use super::schema::Blobs; +use sea_orm_migration::prelude::*; + +pub struct Migration; + +impl MigrationName for Migration { + fn name(&self) -> &str { + "m20220101_000001_initial_blob_table" + } +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + // Define how to apply this migration: Create the Bakery table. + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(Blobs::Table) + .col(ColumnDef::new(Blobs::Workspace).string().not_null()) + .col(ColumnDef::new(Blobs::Hash).string().not_null()) + .col(ColumnDef::new(Blobs::Blob).binary().not_null()) + .col(ColumnDef::new(Blobs::Length).big_integer().not_null()) + .col( + ColumnDef::new(Blobs::Timestamp) + .timestamp_with_time_zone() + .not_null(), + ) + .primary_key(Index::create().col(Blobs::Workspace).col(Blobs::Hash)) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .name("blobs_list") + .table(Blobs::Table) + .col(Blobs::Workspace) + .to_owned(), + ) + .await?; + + Ok(()) + } + + // Define how to rollback this migration: Drop the Bakery table. + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_index(Index::drop().name("blobs_list").to_owned()) + .await?; + manager + .drop_table(Table::drop().table(Blobs::Table).to_owned()) + .await?; + Ok(()) + } +} diff --git a/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs b/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs new file mode 100644 index 000000000..8b51e52cb --- /dev/null +++ b/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs @@ -0,0 +1,60 @@ +use super::schema::Docs; +use sea_orm_migration::prelude::*; + +pub struct Migration; + +impl MigrationName for Migration { + fn name(&self) -> &str { + "m20220101_000001_initial_doc_table" + } +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + // Define how to apply this migration: Create the Bakery table. + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(Docs::Table) + .col( + ColumnDef::new(Docs::Id) + .integer() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(Docs::Workspace).string().not_null()) + .col( + ColumnDef::new(Docs::Timestamp) + .timestamp_with_time_zone() + .not_null(), + ) + .col(ColumnDef::new(Docs::Blob).binary().not_null()) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .name("workspaces_update") + .table(Docs::Table) + .col(Docs::Workspace) + .to_owned(), + ) + .await?; + + Ok(()) + } + + // Define how to rollback this migration: Drop the Bakery table. + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_index(Index::drop().name("workspaces_update").to_owned()) + .await?; + manager + .drop_table(Table::drop().table(Docs::Table).to_owned()) + .await?; + Ok(()) + } +} diff --git a/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs b/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs new file mode 100644 index 000000000..5e296501f --- /dev/null +++ b/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs @@ -0,0 +1,71 @@ +use super::schema::OptimizedBlobs; +use sea_orm_migration::prelude::*; + +pub struct Migration; + +impl MigrationName for Migration { + fn name(&self) -> &str { + "m20230321_000001_blob_optimized_table" + } +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + // Define how to apply this migration: Create the Bakery table. + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(OptimizedBlobs::Table) + .col( + ColumnDef::new(OptimizedBlobs::Workspace) + .string() + .not_null(), + ) + .col(ColumnDef::new(OptimizedBlobs::Hash).string().not_null()) + .col(ColumnDef::new(OptimizedBlobs::Blob).binary().not_null()) + .col( + ColumnDef::new(OptimizedBlobs::Length) + .big_integer() + .not_null(), + ) + .col( + ColumnDef::new(OptimizedBlobs::Timestamp) + .timestamp_with_time_zone() + .not_null(), + ) + .col(ColumnDef::new(OptimizedBlobs::Params).string().not_null()) + .primary_key( + Index::create() + .col(OptimizedBlobs::Workspace) + .col(OptimizedBlobs::Hash) + .col(OptimizedBlobs::Params), + ) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .name("blobs_optimized_list") + .table(OptimizedBlobs::Table) + .col(OptimizedBlobs::Workspace) + .to_owned(), + ) + .await?; + + Ok(()) + } + + // Define how to rollback this migration: Drop the Bakery table. + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_index(Index::drop().name("blobs_optimized_list").to_owned()) + .await?; + manager + .drop_table(Table::drop().table(OptimizedBlobs::Table).to_owned()) + .await?; + Ok(()) + } +} diff --git a/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs b/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs new file mode 100644 index 000000000..7f79985c9 --- /dev/null +++ b/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs @@ -0,0 +1,60 @@ +use super::schema::BucketBlobs; +use sea_orm_migration::prelude::*; + +pub struct Migration; + +impl MigrationName for Migration { + fn name(&self) -> &str { + "m20230614_000001_initial_bucket_blob_table" + } +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + // Define how to apply this migration: Create the Bakery table. + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(BucketBlobs::Table) + .col(ColumnDef::new(BucketBlobs::Workspace).string().not_null()) + .col(ColumnDef::new(BucketBlobs::Hash).string().not_null()) + .col(ColumnDef::new(BucketBlobs::Length).big_integer().not_null()) + .col( + ColumnDef::new(BucketBlobs::Timestamp) + .timestamp_with_time_zone() + .not_null(), + ) + .primary_key( + Index::create() + .col(BucketBlobs::Workspace) + .col(BucketBlobs::Hash), + ) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .name("bucket_blobs_list") + .table(BucketBlobs::Table) + .col(BucketBlobs::Workspace) + .to_owned(), + ) + .await?; + + Ok(()) + } + + // Define how to rollback this migration: Drop the Bakery table. + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_index(Index::drop().name("bucket_blobs_list").to_owned()) + .await?; + manager + .drop_table(Table::drop().table(BucketBlobs::Table).to_owned()) + .await?; + Ok(()) + } +} diff --git a/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs b/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs new file mode 100644 index 000000000..01ec2c326 --- /dev/null +++ b/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs @@ -0,0 +1,58 @@ +use sea_orm_migration::prelude::*; + +use crate::schema::Docs; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Docs::Table) + .add_column(ColumnDef::new(Docs::Guid).string().not_null()) + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(Docs::Table) + .add_column( + ColumnDef::new(Docs::IsWorkspace) + .boolean() + .not_null() + .default(true), + ) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .table(Docs::Table) + .name("docs_guid") + .col(Docs::Guid) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_index(Index::drop().name("docs_guid").to_owned()) + .await?; + + manager + .alter_table( + Table::alter() + .table(Docs::Table) + .drop_column(Docs::Guid) + .drop_column(Docs::IsWorkspace) + .to_owned(), + ) + .await + } +} diff --git a/libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs b/libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs new file mode 100644 index 000000000..d45c9b2cc --- /dev/null +++ b/libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs @@ -0,0 +1,47 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(DiffLog::Table) + .if_not_exists() + .col( + ColumnDef::new(DiffLog::Id) + .integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(DiffLog::Workspace).string().not_null()) + .col( + ColumnDef::new(DiffLog::Timestamp) + .timestamp_with_time_zone() + .not_null(), + ) + .col(ColumnDef::new(DiffLog::Log).string().not_null()) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(DiffLog::Table).to_owned()) + .await + } +} + +#[derive(DeriveIden)] +enum DiffLog { + Table, + Id, + Workspace, + Timestamp, + Log, +} diff --git a/libs/jwst-core-storage/src/migration/src/main.rs b/libs/jwst-core-storage/src/migration/src/main.rs new file mode 100644 index 000000000..632e03b8f --- /dev/null +++ b/libs/jwst-core-storage/src/migration/src/main.rs @@ -0,0 +1,6 @@ +use sea_orm_migration::prelude::*; + +#[tokio::main] +async fn main() { + cli::run_cli(jwst_storage_migration::Migrator).await; +} diff --git a/libs/jwst-core-storage/src/migration/src/schema.rs b/libs/jwst-core-storage/src/migration/src/schema.rs new file mode 100644 index 000000000..b3d6eeb33 --- /dev/null +++ b/libs/jwst-core-storage/src/migration/src/schema.rs @@ -0,0 +1,50 @@ +use sea_orm_migration::prelude::*; + +#[derive(Iden)] +pub enum Blobs { + Table, + #[iden = "workspace_id"] + Workspace, + Hash, + Blob, + Length, + #[iden = "created_at"] + Timestamp, +} + +#[derive(Iden)] +pub enum Docs { + Table, + Id, + #[iden = "workspace_id"] + Workspace, + Guid, + IsWorkspace, + #[iden = "created_at"] + Timestamp, + Blob, +} + +#[derive(Iden)] +pub enum OptimizedBlobs { + Table, + #[iden = "workspace_id"] + Workspace, + Hash, + Blob, + Length, + #[iden = "created_at"] + Timestamp, + Params, +} + +#[derive(Iden)] +pub enum BucketBlobs { + Table, + #[iden = "workspace_id"] + Workspace, + Hash, + Length, + #[iden = "created_at"] + Timestamp, +} diff --git a/libs/jwst-core-storage/src/rate_limiter.rs b/libs/jwst-core-storage/src/rate_limiter.rs new file mode 100644 index 000000000..334775159 --- /dev/null +++ b/libs/jwst-core-storage/src/rate_limiter.rs @@ -0,0 +1,81 @@ +use governor::{ + clock::{QuantaClock, QuantaInstant}, + middleware::NoOpMiddleware, + state::{InMemoryState, NotKeyed}, + Quota, RateLimiter, +}; +use std::{num::NonZeroU32, sync::Arc}; +use tokio::sync::{OwnedSemaphorePermit, RwLock, RwLockReadGuard, RwLockWriteGuard, Semaphore}; +use url::Url; + +pub enum BucketLocker<'a> { + Semaphore(OwnedSemaphorePermit), + ReadLock(RwLockReadGuard<'a, ()>), + WriteLock(RwLockWriteGuard<'a, ()>), +} + +enum BucketLock { + Semaphore(Arc), + RwLock(Arc>), +} + +pub struct Bucket { + bucket: Arc>>, + lock: BucketLock, +} + +impl Bucket { + fn new(bucket_size: u32, semaphore_size: usize) -> Self { + let bucket_size = + NonZeroU32::new(bucket_size).unwrap_or(unsafe { NonZeroU32::new_unchecked(1) }); + + Self { + bucket: Arc::new(RateLimiter::direct( + Quota::per_second(bucket_size).allow_burst(bucket_size), + )), + lock: if semaphore_size > 1 { + BucketLock::Semaphore(Arc::new(Semaphore::new(semaphore_size))) + } else { + // use for sqlite + BucketLock::RwLock(Arc::default()) + }, + } + } + + pub async fn read(&self) -> BucketLocker { + self.bucket.until_ready().await; + match &self.lock { + BucketLock::RwLock(lock) => BucketLocker::ReadLock(lock.read().await), + BucketLock::Semaphore(semaphore) => { + BucketLocker::Semaphore(semaphore.clone().acquire_owned().await.unwrap()) + } + } + } + + pub async fn write(&self) -> BucketLocker { + self.bucket.until_ready().await; + match &self.lock { + BucketLock::RwLock(lock) => BucketLocker::WriteLock(lock.write().await), + BucketLock::Semaphore(semaphore) => { + BucketLocker::Semaphore(semaphore.clone().acquire_owned().await.unwrap()) + } + } + } +} + +#[inline] +pub fn is_sqlite(database: &str) -> bool { + Url::parse(database) + .map(|u| u.scheme() == "sqlite") + .unwrap_or(false) +} + +#[inline] +pub fn get_bucket(single_thread: bool) -> Arc { + // sqlite only support 1 writer at a time + // we use semaphore to limit the number of concurrent writers + Arc::new(Bucket::new( + if single_thread { 10 } else { 25 }, + if single_thread { 1 } else { 5 }, + )) +} diff --git a/libs/jwst-core-storage/src/storage/blobs/bucket_local_db.rs b/libs/jwst-core-storage/src/storage/blobs/bucket_local_db.rs new file mode 100644 index 000000000..3601a4fc4 --- /dev/null +++ b/libs/jwst-core-storage/src/storage/blobs/bucket_local_db.rs @@ -0,0 +1,397 @@ +use super::utils::calculate_hash; +use super::utils::get_hash; +use super::*; +use crate::rate_limiter::Bucket; +use crate::JwstStorageError; +use bytes::Bytes; +use futures::Stream; +use jwst_core::{BlobMetadata, BlobStorage, BucketBlobStorage, JwstResult}; +use jwst_storage_migration::Migrator; +use opendal::services::S3; +use opendal::Operator; +use sea_orm::{DatabaseConnection, EntityTrait}; +use sea_orm_migration::MigratorTrait; + +use std::collections::HashMap; +use std::sync::Arc; + +pub(super) type BucketBlobModel = ::Model; +type BucketBlobActiveModel = entities::bucket_blobs::ActiveModel; +type BucketBlobColumn = ::Column; + +#[derive(Clone)] +pub struct BlobBucketDBStorage { + bucket: Arc, + pub(super) pool: DatabaseConnection, + pub(super) bucket_storage: BucketStorage, +} + +impl AsRef for BlobBucketDBStorage { + fn as_ref(&self) -> &DatabaseConnection { + &self.pool + } +} + +impl BlobBucketDBStorage { + pub async fn init_with_pool( + pool: DatabaseConnection, + bucket: Arc, + bucket_storage: Option, + ) -> JwstStorageResult { + Migrator::up(&pool, None).await?; + Ok(Self { + bucket, + pool, + bucket_storage: bucket_storage.unwrap_or(BucketStorage::new()?), + }) + } + + #[allow(unused)] + pub async fn init_pool( + database: &str, + bucket_storage: Option, + ) -> JwstStorageResult { + let is_sqlite = is_sqlite(database); + let pool = create_connection(database, is_sqlite).await?; + + Self::init_with_pool(pool, get_bucket(is_sqlite), bucket_storage).await + } + + #[allow(unused)] + async fn all(&self, workspace: &str) -> Result, DbErr> { + BucketBlobs::find() + .filter(BucketBlobColumn::WorkspaceId.eq(workspace)) + .all(&self.pool) + .await + } + + async fn keys(&self, workspace: &str) -> Result, DbErr> { + BucketBlobs::find() + .filter(BucketBlobColumn::WorkspaceId.eq(workspace)) + .column_as(BucketBlobColumn::Hash, "hash") + .all(&self.pool) + .await + .map(|r| r.into_iter().map(|f| f.hash).collect()) + } + + #[allow(unused)] + async fn count(&self, workspace: &str) -> Result { + BucketBlobs::find() + .filter(BucketBlobColumn::WorkspaceId.eq(workspace)) + .count(&self.pool) + .await + } + + async fn exists(&self, workspace: &str, hash: &str) -> Result { + BucketBlobs::find_by_id((workspace.into(), hash.into())) + .count(&self.pool) + .await + .map(|c| c > 0) + } + + pub(super) async fn metadata( + &self, + workspace: &str, + hash: &str, + ) -> JwstBlobResult { + BucketBlobs::find_by_id((workspace.into(), hash.into())) + .select_only() + .column_as(BucketBlobColumn::Length, "size") + .column_as(BucketBlobColumn::CreatedAt, "created_at") + .into_model::() + .one(&self.pool) + .await + .map_err(|e| e.into()) + .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) + } + + async fn get_blobs_size(&self, workspace: &str) -> Result, DbErr> { + BucketBlobs::find() + .filter(BucketBlobColumn::WorkspaceId.eq(workspace)) + .column_as(BucketBlobColumn::Length, "size") + .column_as(BucketBlobColumn::CreatedAt, "created_at") + .into_model::() + .all(&self.pool) + .await + .map(|r| r.into_iter().map(|f| f.size).reduce(|a, b| a + b)) + } + + async fn insert(&self, workspace: &str, hash: &str, blob: &[u8]) -> Result<(), DbErr> { + if !self.exists(workspace, hash).await? { + BucketBlobs::insert(BucketBlobActiveModel { + workspace_id: Set(workspace.into()), + hash: Set(hash.into()), + length: Set(blob.len().try_into().unwrap()), + created_at: Set(Utc::now().into()), + }) + .exec(&self.pool) + .await?; + } + + Ok(()) + } + + async fn delete(&self, workspace: &str, hash: &str) -> Result { + BucketBlobs::delete_by_id((workspace.into(), hash.into())) + .exec(&self.pool) + .await + .map(|r| r.rows_affected == 1) + } + + async fn drop(&self, workspace: &str) -> Result<(), DbErr> { + BucketBlobs::delete_many() + .filter(BucketBlobColumn::WorkspaceId.eq(workspace)) + .exec(&self.pool) + .await?; + + Ok(()) + } +} + +#[derive(Clone)] +pub struct BucketStorage { + pub(super) op: Operator, +} + +impl BucketStorage { + #[allow(unused)] + pub fn new() -> JwstStorageResult { + let access_key = dotenvy::var("BUCKET_ACCESS_TOKEN")?; + let secret_access_key = dotenvy::var("BUCKET_SECRET_TOKEN")?; + let endpoint = dotenvy::var("BUCKET_ENDPOINT")?; + let bucket = dotenvy::var("BUCKET_NAME"); + let root = dotenvy::var("BUCKET_ROOT"); + + let mut builder = S3::default(); + + builder.bucket(bucket.unwrap_or("__default_bucket__".to_string()).as_str()); + builder.root(root.unwrap_or("__default_root__".to_string()).as_str()); + builder.endpoint(endpoint.as_str()); + builder.access_key_id(access_key.as_str()); + builder.secret_access_key(secret_access_key.as_str()); + + Ok(Self { + op: Operator::new(builder)?.finish(), + }) + } +} + +#[async_trait] +impl BucketBlobStorage for BucketStorage { + async fn get_blob( + &self, + workspace: Option, + id: String, + ) -> JwstResult, JwstStorageError> { + let workspace = get_workspace(workspace); + let key = build_key(workspace, id); + let bs = self.op.read(&key).await?; + + Ok(bs) + } + + async fn put_blob( + &self, + workspace: Option, + hash: String, + blob: Vec, + ) -> JwstResult<(), JwstStorageError> { + let workspace = get_workspace(workspace); + let key = build_key(workspace, hash); + let _ = self.op.write(&key, blob).await?; + + Ok(()) + } + + async fn delete_blob( + &self, + workspace: Option, + id: String, + ) -> JwstResult { + let workspace = get_workspace(workspace); + let key = build_key(workspace, id); + + match self.op.delete(&key).await { + Ok(_) => Ok(true), + Err(e) => Err(JwstStorageError::from(e)), + } + } + + async fn delete_workspace(&self, workspace_id: String) -> JwstResult<(), JwstStorageError> { + self.op.remove_all(&workspace_id).await?; + + Ok(()) + } +} + +#[async_trait] +impl BlobStorage for BlobBucketDBStorage { + async fn list_blobs( + &self, + workspace: Option, + ) -> JwstResult, JwstStorageError> { + let _lock = self.bucket.read().await; + let workspace = workspace.unwrap_or("__default__".into()); + if let Ok(keys) = self.keys(&workspace).await { + return Ok(keys); + } + + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + + async fn check_blob( + &self, + workspace: Option, + id: String, + ) -> JwstResult { + let _lock = self.bucket.read().await; + let workspace = workspace.unwrap_or("__default__".into()); + if let Ok(exists) = self.exists(&workspace, &id).await { + return Ok(exists); + } + + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + + async fn get_blob( + &self, + workspace: Option, + id: String, + _params: Option>, + ) -> JwstResult, JwstStorageError> { + self.bucket_storage.get_blob(workspace, id).await + } + + async fn get_metadata( + &self, + workspace: Option, + id: String, + _params: Option>, + ) -> JwstResult { + let _lock = self.bucket.read().await; + let workspace = get_workspace(workspace); + if let Ok(metadata) = self.metadata(&workspace, &id).await { + Ok(metadata.into()) + } else { + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + } + + async fn put_blob_stream( + &self, + workspace: Option, + stream: impl Stream + Send, + ) -> JwstResult { + let (hash, blob) = get_hash(stream).await; + self.bucket_storage + .put_blob(workspace.clone(), hash.clone(), blob.clone()) + .await?; + let _lock = self.bucket.write().await; + let workspace = get_workspace(workspace); + + if self.insert(&workspace, &hash, &blob).await.is_ok() { + Ok(hash) + } else { + self.bucket_storage + .delete_blob(Some(workspace.clone()), hash) + .await?; + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + } + + async fn put_blob( + &self, + workspace: Option, + blob: Vec, + ) -> JwstResult { + let hash = calculate_hash(&blob); + self.bucket_storage + .put_blob(workspace.clone(), hash.clone(), blob.clone()) + .await?; + + let _lock = self.bucket.write().await; + let workspace = get_workspace(workspace); + + if self.insert(&workspace, &hash, &blob).await.is_ok() { + Ok(hash) + } else { + self.bucket_storage + .delete_blob(Some(workspace.clone()), hash) + .await?; + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + } + + async fn delete_blob( + &self, + workspace: Option, + id: String, + ) -> JwstResult { + self.bucket_storage + .delete_blob(workspace.clone(), id.clone()) + .await?; + let _lock = self.bucket.write().await; + let workspace = get_workspace(workspace); + if let Ok(success) = self.delete(&workspace, &id).await { + Ok(success) + } else { + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + } + + async fn delete_workspace(&self, workspace_id: String) -> JwstResult<(), JwstStorageError> { + self.bucket_storage + .delete_workspace(workspace_id.clone()) + .await?; + let _lock = self.bucket.write().await; + if self.drop(&workspace_id).await.is_ok() { + Ok(()) + } else { + Err(JwstStorageError::WorkspaceNotFound(workspace_id)) + } + } + + async fn get_blobs_size(&self, workspace_id: String) -> JwstResult { + let _lock = self.bucket.read().await; + let size = self.get_blobs_size(&workspace_id).await?; + return Ok(size.unwrap_or(0)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::blobs::utils::BucketStorageBuilder; + + #[tokio::test] + #[ignore = "need to config bucket auth"] + async fn test_init_bucket_storage() { + let bucket_storage = BucketStorageBuilder::new() + .endpoint("ENDPOINT") + .access_key("ACCESS_KEY") + .secret_access_key("SECRET_ACCESS_KEY") + .bucket("__default_bucket__") + .root("__default_root__") + .build() + .unwrap(); + + BlobBucketDBStorage::init_pool("sqlite::memory:", Some(bucket_storage)) + .await + .unwrap(); + } +} + +/// get_workspace will get the workspace name from the input. +/// +/// If the input is None, it will return the default workspace name. +fn get_workspace(workspace: Option) -> String { + match workspace { + Some(w) => w, + None => "__default__".into(), + } +} + +/// build_key will build the request key for the bucket storage. +fn build_key(workspace: String, id: String) -> String { + format!("{}/{}", workspace, id) +} diff --git a/libs/jwst-core-storage/src/storage/blobs/local_db.rs b/libs/jwst-core-storage/src/storage/blobs/local_db.rs new file mode 100644 index 000000000..b61caf557 --- /dev/null +++ b/libs/jwst-core-storage/src/storage/blobs/local_db.rs @@ -0,0 +1,306 @@ +use super::{utils::get_hash, *}; +use crate::types::JwstStorageResult; +use jwst_core::{Base64Engine, URL_SAFE_ENGINE}; + +use sha2::{Digest, Sha256}; +pub(super) type BlobModel = ::Model; +type BlobActiveModel = super::entities::blobs::ActiveModel; +type BlobColumn = ::Column; + +#[derive(Clone)] +pub struct BlobDBStorage { + bucket: Arc, + pub(super) pool: DatabaseConnection, +} + +impl AsRef for BlobDBStorage { + fn as_ref(&self) -> &DatabaseConnection { + &self.pool + } +} + +impl BlobDBStorage { + pub async fn init_with_pool( + pool: DatabaseConnection, + bucket: Arc, + ) -> JwstStorageResult { + Ok(Self { bucket, pool }) + } + + pub async fn init_pool(database: &str) -> JwstStorageResult { + let is_sqlite = is_sqlite(database); + let pool = create_connection(database, is_sqlite).await?; + + Self::init_with_pool(pool, get_bucket(is_sqlite)).await + } + + #[allow(unused)] + async fn all(&self, workspace: &str) -> Result, DbErr> { + Blobs::find() + .filter(BlobColumn::WorkspaceId.eq(workspace)) + .all(&self.pool) + .await + } + + async fn keys(&self, workspace: &str) -> Result, DbErr> { + Blobs::find() + .filter(BlobColumn::WorkspaceId.eq(workspace)) + .column(BlobColumn::Hash) + .all(&self.pool) + .await + .map(|r| r.into_iter().map(|f| f.hash).collect()) + } + + #[allow(unused)] + async fn count(&self, workspace: &str) -> Result { + Blobs::find() + .filter(BlobColumn::WorkspaceId.eq(workspace)) + .count(&self.pool) + .await + } + + async fn exists(&self, workspace: &str, hash: &str) -> Result { + Blobs::find_by_id((workspace.into(), hash.into())) + .count(&self.pool) + .await + .map(|c| c > 0) + } + + pub(super) async fn metadata( + &self, + workspace: &str, + hash: &str, + ) -> JwstBlobResult { + Blobs::find_by_id((workspace.into(), hash.into())) + .select_only() + .column_as(BlobColumn::Length, "size") + .column_as(BlobColumn::CreatedAt, "created_at") + .into_model::() + .one(&self.pool) + .await + .map_err(|e| e.into()) + .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) + } + + pub(super) async fn get_blobs_size(&self, workspace: &str) -> Result, DbErr> { + Blobs::find() + .filter(BlobColumn::WorkspaceId.eq(workspace)) + .column_as(BlobColumn::Length, "size") + .column_as(BlobColumn::CreatedAt, "created_at") + .into_model::() + .all(&self.pool) + .await + .map(|r| r.into_iter().map(|f| f.size).reduce(|a, b| a + b)) + } + + async fn insert(&self, workspace: &str, hash: &str, blob: &[u8]) -> Result<(), DbErr> { + if !self.exists(workspace, hash).await? { + Blobs::insert(BlobActiveModel { + workspace_id: Set(workspace.into()), + hash: Set(hash.into()), + blob: Set(blob.into()), + length: Set(blob.len().try_into().unwrap()), + created_at: Set(Utc::now().into()), + }) + .exec(&self.pool) + .await?; + } + + Ok(()) + } + + pub(super) async fn get(&self, workspace: &str, hash: &str) -> JwstBlobResult { + Blobs::find_by_id((workspace.into(), hash.into())) + .one(&self.pool) + .await + .map_err(|e| e.into()) + .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) + } + + async fn delete(&self, workspace: &str, hash: &str) -> Result { + Blobs::delete_by_id((workspace.into(), hash.into())) + .exec(&self.pool) + .await + .map(|r| r.rows_affected == 1) + } + + async fn drop(&self, workspace: &str) -> Result<(), DbErr> { + Blobs::delete_many() + .filter(BlobColumn::WorkspaceId.eq(workspace)) + .exec(&self.pool) + .await?; + + Ok(()) + } +} + +#[async_trait] +impl BlobStorage for BlobDBStorage { + async fn list_blobs(&self, workspace: Option) -> JwstStorageResult> { + let _lock = self.bucket.read().await; + let workspace = workspace.unwrap_or("__default__".into()); + if let Ok(keys) = self.keys(&workspace).await { + return Ok(keys); + } + + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + + async fn check_blob(&self, workspace: Option, id: String) -> JwstStorageResult { + let _lock = self.bucket.read().await; + let workspace = workspace.unwrap_or("__default__".into()); + if let Ok(exists) = self.exists(&workspace, &id).await { + return Ok(exists); + } + + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + + async fn get_blob( + &self, + workspace: Option, + id: String, + _params: Option>, + ) -> JwstStorageResult> { + let _lock = self.bucket.read().await; + let workspace = workspace.unwrap_or("__default__".into()); + if let Ok(blob) = self.get(&workspace, &id).await { + return Ok(blob.blob); + } + + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + + async fn get_metadata( + &self, + workspace: Option, + id: String, + _params: Option>, + ) -> JwstStorageResult { + let _lock = self.bucket.read().await; + let workspace = workspace.unwrap_or("__default__".into()); + if let Ok(metadata) = self.metadata(&workspace, &id).await { + Ok(metadata.into()) + } else { + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + } + + async fn put_blob_stream( + &self, + workspace: Option, + stream: impl Stream + Send, + ) -> JwstStorageResult { + let _lock = self.bucket.write().await; + let workspace = workspace.unwrap_or("__default__".into()); + + let (hash, blob) = get_hash(stream).await; + + if self.insert(&workspace, &hash, &blob).await.is_ok() { + Ok(hash) + } else { + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + } + + async fn put_blob( + &self, + workspace: Option, + blob: Vec, + ) -> JwstStorageResult { + let _lock = self.bucket.write().await; + let workspace = workspace.unwrap_or("__default__".into()); + let mut hasher = Sha256::new(); + + hasher.update(&blob); + let hash = URL_SAFE_ENGINE.encode(hasher.finalize()); + + if self.insert(&workspace, &hash, &blob).await.is_ok() { + Ok(hash) + } else { + Err(JwstStorageError::WorkspaceNotFound(workspace)) + } + } + + async fn delete_blob( + &self, + workspace_id: Option, + id: String, + ) -> JwstStorageResult { + let _lock = self.bucket.write().await; + let workspace_id = workspace_id.unwrap_or("__default__".into()); + if let Ok(success) = self.delete(&workspace_id, &id).await { + Ok(success) + } else { + Err(JwstStorageError::WorkspaceNotFound(workspace_id)) + } + } + + async fn delete_workspace(&self, workspace_id: String) -> JwstStorageResult<()> { + let _lock = self.bucket.write().await; + if self.drop(&workspace_id).await.is_ok() { + Ok(()) + } else { + Err(JwstStorageError::WorkspaceNotFound(workspace_id)) + } + } + + async fn get_blobs_size(&self, workspace_id: String) -> JwstStorageResult { + let _lock = self.bucket.read().await; + let size = self.get_blobs_size(&workspace_id).await?; + return Ok(size.unwrap_or(0)); + } +} + +#[cfg(test)] +pub async fn blobs_storage_test(pool: &BlobDBStorage) -> anyhow::Result<()> { + // empty table + assert_eq!(pool.count("basic").await?, 0); + + // first insert + pool.insert("basic", "test", &[1, 2, 3, 4]).await?; + assert_eq!(pool.count("basic").await?, 1); + + let all = pool.all("basic").await?; + assert_eq!( + all, + vec![BlobModel { + workspace_id: "basic".into(), + hash: "test".into(), + blob: vec![1, 2, 3, 4], + length: 4, + created_at: all.get(0).unwrap().created_at + }] + ); + assert_eq!(pool.count("basic").await?, 1); + assert_eq!(pool.keys("basic").await?, vec!["test"]); + + pool.drop("basic").await?; + assert_eq!(pool.count("basic").await?, 0); + assert_eq!(pool.keys("basic").await?, Vec::::new()); + + pool.insert("basic", "test1", &[1, 2, 3, 4]).await?; + + let all = pool.all("basic").await?; + assert_eq!( + all, + vec![BlobModel { + workspace_id: "basic".into(), + hash: "test1".into(), + blob: vec![1, 2, 3, 4], + length: 4, + created_at: all.get(0).unwrap().created_at + }] + ); + assert_eq!(pool.count("basic").await?, 1); + assert_eq!(pool.keys("basic").await?, vec!["test1"]); + + let metadata = pool.metadata("basic", "test1").await?; + + assert_eq!(metadata.size, 4); + assert!((metadata.created_at.timestamp() - Utc::now().timestamp()).abs() < 2); + + pool.drop("basic").await?; + + Ok(()) +} diff --git a/libs/jwst-core-storage/src/storage/blobs/mod.rs b/libs/jwst-core-storage/src/storage/blobs/mod.rs new file mode 100644 index 000000000..20a49d212 --- /dev/null +++ b/libs/jwst-core-storage/src/storage/blobs/mod.rs @@ -0,0 +1,669 @@ +mod bucket_local_db; +mod local_db; +mod utils; + +#[cfg(test)] +pub use local_db::blobs_storage_test; + +use super::{entities::prelude::*, *}; +use bytes::Bytes; +use image::ImageError; +use jwst_core::{BlobMetadata, BlobStorage}; +use local_db::BlobDBStorage; +use thiserror::Error; +use tokio::task::JoinError; +use utils::{ImageParams, InternalBlobMetadata}; + +pub use bucket_local_db::BlobBucketDBStorage; +pub use utils::BucketStorageBuilder; + +#[derive(Debug, Error)] +pub enum JwstBlobError { + #[error("blob not found: {0}")] + BlobNotFound(String), + #[error("database error")] + Database(#[from] DbErr), + #[error("failed to optimize image")] + Image(#[from] ImageError), + #[error("failed to optimize image")] + ImageThread(#[from] JoinError), + #[error("optimize params error: {0:?}")] + ImageParams(HashMap), +} +pub type JwstBlobResult = Result; + +pub(super) type OptimizedBlobModel = ::Model; +type OptimizedBlobActiveModel = super::entities::optimized_blobs::ActiveModel; +type OptimizedBlobColumn = ::Column; + +#[derive(Clone)] +pub struct BlobAutoStorage { + pub(super) db: Arc, + pool: DatabaseConnection, +} + +pub enum JwstBlobStorage { + DB(BlobAutoStorage), + MixedBucketDB(BlobBucketDBStorage), +} + +pub enum BlobStorageType { + DB, + MixedBucketDB(MixedBucketDBParam), +} + +pub struct MixedBucketDBParam { + pub(crate) access_key: String, + pub(crate) secret_access_key: String, + pub(crate) endpoint: String, + pub(crate) bucket: Option, + pub(crate) root: Option, +} + +impl MixedBucketDBParam { + pub fn new_from_env() -> JwstResult { + Ok(MixedBucketDBParam { + access_key: dotenvy::var("BUCKET_ACCESS_TOKEN")?, + secret_access_key: dotenvy::var("BUCKET_SECRET_TOKEN")?, + endpoint: dotenvy::var("BUCKET_ENDPOINT")?, + bucket: dotenvy::var("BUCKET_NAME").ok(), + root: dotenvy::var("BUCKET_ROOT").ok(), + }) + } + + pub fn new( + access_key: String, + secret_access_key: String, + endpoint: String, + bucket: Option, + root: Option, + ) -> Self { + MixedBucketDBParam { + access_key, + secret_access_key, + endpoint, + bucket, + root, + } + } +} + +impl BlobAutoStorage { + pub async fn init_with_pool( + pool: DatabaseConnection, + bucket: Arc, + ) -> JwstStorageResult { + let db = Arc::new(BlobDBStorage::init_with_pool(pool, bucket).await?); + let pool = db.pool.clone(); + Ok(Self { db, pool }) + } + + pub async fn init_pool(database: &str) -> JwstStorageResult { + let db = Arc::new(BlobDBStorage::init_pool(database).await?); + let pool = db.pool.clone(); + Ok(Self { db, pool }) + } + + async fn exists(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { + Ok( + OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) + .count(&self.pool) + .await + .map(|c| c > 0)?, + ) + } + + async fn insert( + &self, + table: &str, + hash: &str, + params: &str, + blob: &[u8], + ) -> JwstBlobResult<()> { + if !self.exists(table, hash, params).await? { + OptimizedBlobs::insert(OptimizedBlobActiveModel { + workspace_id: Set(table.into()), + hash: Set(hash.into()), + blob: Set(blob.into()), + length: Set(blob.len().try_into().unwrap()), + params: Set(params.into()), + created_at: Set(Utc::now().into()), + }) + .exec(&self.pool) + .await?; + } + + Ok(()) + } + + async fn get( + &self, + table: &str, + hash: &str, + params: &str, + ) -> JwstBlobResult { + OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) + .one(&self.pool) + .await + .map_err(|e| e.into()) + .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) + } + + async fn metadata( + &self, + table: &str, + hash: &str, + params: &str, + ) -> JwstBlobResult { + OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) + .select_only() + .column_as(OptimizedBlobColumn::Length, "size") + .column_as(OptimizedBlobColumn::CreatedAt, "created_at") + .into_model::() + .one(&self.pool) + .await + .map_err(|e| e.into()) + .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) + } + + async fn get_metadata_auto( + &self, + workspace: Option, + id: String, + params: Option>, + ) -> JwstBlobResult { + let workspace_id = workspace.as_deref().unwrap_or("__default__"); + if let Some(params) = params { + if let Ok(params) = ImageParams::try_from(¶ms) { + let params_token = params.to_string(); + if self.exists(workspace_id, &id, ¶ms_token).await? { + let metadata = self.metadata(workspace_id, &id, ¶ms_token).await?; + Ok(BlobMetadata { + content_type: format!("image/{}", params.format()), + ..metadata.into() + }) + } else { + self.db.metadata(workspace_id, &id).await.map(Into::into) + } + } else { + Err(JwstBlobError::ImageParams(params)) + } + } else { + self.db.metadata(workspace_id, &id).await.map(Into::into) + } + } + + async fn get_auto( + &self, + workspace: Option, + id: String, + params: Option>, + ) -> JwstBlobResult> { + let workspace_id = workspace.as_deref().unwrap_or("__default__"); + if let Some(params) = params { + if let Ok(params) = ImageParams::try_from(¶ms) { + let params_token = params.to_string(); + if let Ok(blob) = self.get(workspace_id, &id, ¶ms_token).await { + info!( + "exists optimized image: {} {} {}, {}bytes", + workspace_id, + id, + params_token, + blob.blob.len() + ); + Ok(blob.blob) + } else { + // TODO: need ddos mitigation + let blob = self.db.get(workspace_id, &id).await?; + let blob_len = blob.blob.len(); + let image = + tokio::task::spawn_blocking(move || params.optimize_image(&blob.blob)) + .await??; + self.insert(workspace_id, &id, ¶ms_token, &image) + .await?; + info!( + "optimized image: {} {} {}, {}bytes -> {}bytes", + workspace_id, + id, + params_token, + blob_len, + image.len() + ); + Ok(image) + } + } else { + Err(JwstBlobError::ImageParams(params)) + } + } else { + self.db.get(workspace_id, &id).await.map(|m| m.blob) + } + } + + async fn delete(&self, table: &str, hash: &str) -> JwstBlobResult { + Ok(OptimizedBlobs::delete_many() + .filter( + OptimizedBlobColumn::WorkspaceId + .eq(table) + .and(OptimizedBlobColumn::Hash.eq(hash)), + ) + .exec(&self.pool) + .await + .map(|r| r.rows_affected)?) + } + + async fn drop(&self, table: &str) -> Result<(), DbErr> { + OptimizedBlobs::delete_many() + .filter(OptimizedBlobColumn::WorkspaceId.eq(table)) + .exec(&self.pool) + .await?; + + Ok(()) + } +} + +#[async_trait] +impl BlobStorage for BlobAutoStorage { + async fn list_blobs(&self, workspace: Option) -> JwstStorageResult> { + self.db.list_blobs(workspace).await + } + + async fn check_blob(&self, workspace: Option, id: String) -> JwstStorageResult { + self.db.check_blob(workspace, id).await + } + + async fn get_blob( + &self, + workspace: Option, + id: String, + params: Option>, + ) -> JwstStorageResult> { + let blob = self.get_auto(workspace, id, params).await?; + Ok(blob) + } + + async fn get_metadata( + &self, + workspace: Option, + id: String, + params: Option>, + ) -> JwstStorageResult { + let metadata = self.get_metadata_auto(workspace, id, params).await?; + Ok(metadata) + } + + async fn put_blob_stream( + &self, + workspace: Option, + stream: impl Stream + Send, + ) -> JwstStorageResult { + self.db.put_blob_stream(workspace, stream).await + } + + async fn put_blob( + &self, + workspace: Option, + blob: Vec, + ) -> JwstStorageResult { + self.db.put_blob(workspace, blob).await + } + + async fn delete_blob( + &self, + workspace_id: Option, + id: String, + ) -> JwstStorageResult { + // delete origin blobs + let success = self + .db + .delete_blob(workspace_id.clone(), id.clone()) + .await?; + if success { + // delete optimized blobs + let workspace_id = workspace_id.unwrap_or("__default__".into()); + self.delete(&workspace_id, &id).await?; + } + Ok(success) + } + + async fn delete_workspace(&self, workspace_id: String) -> JwstStorageResult<()> { + // delete origin blobs + self.db.delete_workspace(workspace_id.clone()).await?; + + // delete optimized blobs + self.drop(&workspace_id).await?; + + Ok(()) + } + + async fn get_blobs_size(&self, workspace_id: String) -> JwstStorageResult { + let size = self.db.get_blobs_size(&workspace_id).await?; + + return Ok(size.unwrap_or(0)); + } +} + +#[async_trait] +impl BlobStorage for JwstBlobStorage { + async fn list_blobs( + &self, + workspace: Option, + ) -> JwstResult, JwstStorageError> { + match self { + JwstBlobStorage::DB(db) => db.list_blobs(workspace).await, + JwstBlobStorage::MixedBucketDB(db) => db.list_blobs(workspace).await, + } + } + + async fn check_blob( + &self, + workspace: Option, + id: String, + ) -> JwstResult { + match self { + JwstBlobStorage::DB(db) => db.check_blob(workspace, id).await, + JwstBlobStorage::MixedBucketDB(db) => db.check_blob(workspace, id).await, + } + } + + async fn get_blob( + &self, + workspace: Option, + id: String, + params: Option>, + ) -> JwstResult, JwstStorageError> { + match self { + JwstBlobStorage::DB(db) => db.get_blob(workspace, id, params).await, + JwstBlobStorage::MixedBucketDB(db) => db.get_blob(workspace, id, params).await, + } + } + + async fn get_metadata( + &self, + workspace: Option, + id: String, + params: Option>, + ) -> JwstResult { + match self { + JwstBlobStorage::DB(db) => db.get_metadata(workspace, id, params).await, + JwstBlobStorage::MixedBucketDB(db) => db.get_metadata(workspace, id, params).await, + } + } + + async fn put_blob_stream( + &self, + workspace: Option, + stream: impl Stream + Send, + ) -> JwstResult { + match self { + JwstBlobStorage::DB(db) => db.put_blob_stream(workspace, stream).await, + JwstBlobStorage::MixedBucketDB(db) => db.put_blob_stream(workspace, stream).await, + } + } + + async fn put_blob( + &self, + workspace: Option, + blob: Vec, + ) -> JwstResult { + match self { + JwstBlobStorage::DB(db) => db.put_blob(workspace, blob).await, + JwstBlobStorage::MixedBucketDB(db) => db.put_blob(workspace, blob).await, + } + } + + async fn delete_blob( + &self, + workspace: Option, + id: String, + ) -> JwstResult { + match self { + JwstBlobStorage::DB(db) => db.delete_blob(workspace, id).await, + JwstBlobStorage::MixedBucketDB(db) => db.delete_blob(workspace, id).await, + } + } + + async fn delete_workspace(&self, workspace_id: String) -> JwstResult<(), JwstStorageError> { + match self { + JwstBlobStorage::DB(db) => db.delete_workspace(workspace_id).await, + JwstBlobStorage::MixedBucketDB(db) => db.delete_workspace(workspace_id).await, + } + } + + async fn get_blobs_size(&self, workspace_id: String) -> JwstResult { + match self { + JwstBlobStorage::DB(db) => db.get_blobs_size(workspace_id).await, + JwstBlobStorage::MixedBucketDB(db) => db.get_blobs_size(workspace_id).await, + } + } +} + +impl JwstBlobStorage { + pub fn get_blob_db(&self) -> Option> { + match self { + JwstBlobStorage::DB(db) => Some(db.db.clone()), + JwstBlobStorage::MixedBucketDB(_) => None, + } + } + + pub fn get_mixed_bucket_db(&self) -> Option { + match self { + JwstBlobStorage::DB(_) => None, + JwstBlobStorage::MixedBucketDB(db) => Some(db.clone()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::FutureExt; + use image::{DynamicImage, ImageOutputFormat}; + use std::io::Cursor; + + #[tokio::test] + async fn test_blob_auto_storage() { + let storage = BlobAutoStorage::init_pool("sqlite::memory:").await.unwrap(); + Migrator::up(&storage.pool, None).await.unwrap(); + + let blob = Vec::from_iter((0..100).map(|_| rand::random())); + + let stream = async { Bytes::from(blob.clone()) }.into_stream(); + let hash1 = storage + .put_blob_stream(Some("blob".into()), stream) + .await + .unwrap(); + + // check origin blob result + assert_eq!( + storage + .get_blob(Some("blob".into()), hash1.clone(), None) + .await + .unwrap(), + blob + ); + assert_eq!( + storage + .get_metadata(Some("blob".into()), hash1.clone(), None) + .await + .unwrap() + .size as usize, + blob.len() + ); + + // optimize must failed if blob not supported + assert!(storage + .get_blob( + Some("blob".into()), + hash1.clone(), + Some(HashMap::from([("format".into(), "jpeg".into())])) + ) + .await + .is_err()); + + // generate image + let image = { + let mut image = Cursor::new(vec![]); + DynamicImage::new_rgba8(32, 32) + .write_to(&mut image, ImageOutputFormat::Png) + .unwrap(); + image.into_inner() + }; + let stream = async { Bytes::from(image.clone()) }.into_stream(); + let hash2 = storage + .put_blob_stream(Some("blob".into()), stream) + .await + .unwrap(); + + // check origin blob result + assert_eq!( + storage + .get_blob(Some("blob".into()), hash2.clone(), None) + .await + .unwrap(), + image + ); + assert_eq!( + storage + .get_metadata(Some("blob".into()), hash2.clone(), None) + .await + .unwrap() + .size as usize, + image.len() + ); + + // check optimized jpeg result + let jpeg_params = HashMap::from([("format".into(), "jpeg".into())]); + let jpeg = storage + .get_blob( + Some("blob".into()), + hash2.clone(), + Some(jpeg_params.clone()), + ) + .await + .unwrap(); + + assert!(jpeg.starts_with(&[0xff, 0xd8, 0xff])); + assert_eq!( + storage + .get_metadata(Some("blob".into()), hash2.clone(), Some(jpeg_params)) + .await + .unwrap() + .size as usize, + jpeg.len() + ); + + // check optimized webp result + let webp_params = HashMap::from([("format".into(), "webp".into())]); + let webp = storage + .get_blob( + Some("blob".into()), + hash2.clone(), + Some(webp_params.clone()), + ) + .await + .unwrap(); + + assert!(webp.starts_with(b"RIFF")); + assert_eq!( + storage + .get_metadata( + Some("blob".into()), + hash2.clone(), + Some(webp_params.clone()) + ) + .await + .unwrap() + .size as usize, + webp.len() + ); + + // optimize must failed if image params error + assert!(storage + .get_blob( + Some("blob".into()), + hash2.clone(), + Some(HashMap::from([("format".into(), "error_value".into()),])) + ) + .await + .is_err()); + assert!(storage + .get_blob( + Some("blob".into()), + hash2.clone(), + Some(HashMap::from([ + ("format".into(), "webp".into()), + ("size".into(), "error_value".into()) + ])) + ) + .await + .is_err()); + assert!(storage + .get_blob( + Some("blob".into()), + hash2.clone(), + Some(HashMap::from([ + ("format".into(), "webp".into()), + ("width".into(), "111".into()) + ])) + ) + .await + .is_err()); + assert!(storage + .get_blob( + Some("blob".into()), + hash2.clone(), + Some(HashMap::from([ + ("format".into(), "webp".into()), + ("height".into(), "111".into()) + ])) + ) + .await + .is_err()); + + assert_eq!( + storage.get_blobs_size("blob".into()).await.unwrap() as usize, + 100 + image.len() + ); + + assert!(storage + .delete_blob(Some("blob".into()), hash2.clone()) + .await + .unwrap()); + assert_eq!( + storage + .check_blob(Some("blob".into()), hash2.clone()) + .await + .unwrap(), + false + ); + assert!(storage + .get_blob(Some("blob".into()), hash2.clone(), None) + .await + .is_err()); + assert!(storage + .get_metadata(Some("blob".into()), hash2.clone(), None) + .await + .is_err()); + assert!(storage + .get_metadata(Some("blob".into()), hash2.clone(), Some(webp_params)) + .await + .is_err()); + + assert_eq!( + storage.get_blobs_size("blob".into()).await.unwrap() as usize, + 100 + ); + + assert_eq!( + storage.list_blobs(Some("blob".into())).await.unwrap(), + vec![hash1] + ); + assert_eq!( + storage + .list_blobs(Some("not_exists_workspace".into())) + .await + .unwrap(), + Vec::::new() + ); + } +} diff --git a/libs/jwst-core-storage/src/storage/blobs/utils.rs b/libs/jwst-core-storage/src/storage/blobs/utils.rs new file mode 100644 index 000000000..572e95d2a --- /dev/null +++ b/libs/jwst-core-storage/src/storage/blobs/utils.rs @@ -0,0 +1,252 @@ +use crate::storage::blobs::bucket_local_db::BucketStorage; +use crate::storage::blobs::MixedBucketDBParam; +use crate::{JwstStorageError, JwstStorageResult}; +use bytes::Bytes; +use chrono::{DateTime, Utc}; +use futures::{ + stream::{iter, StreamExt}, + Stream, +}; +use image::{load_from_memory, ImageOutputFormat, ImageResult}; +use jwst_core::{Base64Engine, BlobMetadata, URL_SAFE_ENGINE}; +use opendal::services::S3; +use opendal::Operator; +use sea_orm::FromQueryResult; +use sha2::{Digest, Sha256}; +use std::{collections::HashMap, io::Cursor}; + +enum ImageFormat { + Jpeg, + WebP, +} + +pub struct ImageParams { + format: ImageFormat, + width: Option, + height: Option, +} + +impl ImageParams { + #[inline] + fn check_size(w: Option, h: Option) -> bool { + if let Some(w) = w { + if w % 320 != 0 || w > 1920 { + return false; + } + } + if let Some(h) = h { + if h % 180 != 0 || h > 1080 { + return false; + } + } + true + } + + pub(super) fn format(&self) -> String { + match self.format { + ImageFormat::Jpeg => "jpeg".to_string(), + ImageFormat::WebP => "webp".to_string(), + } + } + + fn output_format(&self) -> ImageOutputFormat { + match self.format { + ImageFormat::Jpeg => ImageOutputFormat::Jpeg(80), + ImageFormat::WebP => ImageOutputFormat::WebP, + } + } + + pub fn optimize_image(&self, data: &[u8]) -> ImageResult> { + let mut buffer = Cursor::new(vec![]); + let image = load_from_memory(data)?; + image.write_to(&mut buffer, self.output_format())?; + Ok(buffer.into_inner()) + } +} + +impl TryFrom<&HashMap> for ImageParams { + type Error = (); + + fn try_from(value: &HashMap) -> Result { + let mut format = None; + let mut width = None; + let mut height = None; + for (key, value) in value { + match key.as_str() { + "format" => { + format = match value.as_str() { + "jpeg" => Some(ImageFormat::Jpeg), + "webp" => Some(ImageFormat::WebP), + _ => return Err(()), + } + } + "width" => width = value.parse().ok(), + "height" => height = value.parse().ok(), + _ => return Err(()), + } + } + + if let Some(format) = format { + if Self::check_size(width, height) { + return Ok(Self { + format, + width, + height, + }); + } + } + Err(()) + } +} + +impl ToString for ImageParams { + fn to_string(&self) -> String { + let mut params = String::new(); + + params.push_str(&format!("format={}", self.format())); + if let Some(width) = &self.width { + params.push_str(&format!("width={}", width)); + } + if let Some(height) = &self.height { + params.push_str(&format!("height={}", height)); + } + params + } +} + +pub async fn get_hash(stream: impl Stream + Send) -> (String, Vec) { + let mut hasher = Sha256::new(); + + let buffer = stream + .flat_map(|buffer| { + hasher.update(&buffer); + iter(buffer) + }) + .collect() + .await; + + let hash = URL_SAFE_ENGINE.encode(hasher.finalize()); + (hash, buffer) +} + +/// Calculate sha256 hash for given blob +pub fn calculate_hash(blob: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(blob); + URL_SAFE_ENGINE.encode(hasher.finalize()) +} + +#[derive(FromQueryResult)] +pub(super) struct InternalBlobMetadata { + pub(super) size: i64, + pub(super) created_at: DateTime, +} + +impl From for BlobMetadata { + fn from(val: InternalBlobMetadata) -> Self { + BlobMetadata { + content_type: "application/octet-stream".into(), + last_modified: val.created_at.naive_local(), + size: val.size, + } + } +} + +impl TryFrom> for BucketStorage { + type Error = JwstStorageError; + + fn try_from(map: HashMap) -> Result { + let mut builder = BucketStorageBuilder::new(); + let access_token = map.get("BUCKET_ACCESS_TOKEN"); + let secret_access_key = map.get("BUCKET_SECRET_TOKEN"); + let endpoint = map.get("BUCKET_ENDPOINT"); + let bucket = map.get("BUCKET_NAME"); + let root = map.get("BUCKET_ROOT"); + + if let Some(access_token) = access_token { + builder = builder.access_key(access_token); + } + if let Some(secret_access_key) = secret_access_key { + builder = builder.secret_access_key(secret_access_key); + } + if let Some(endpoint) = endpoint { + builder = builder.endpoint(endpoint); + } + if let Some(bucket) = bucket { + builder = builder.bucket(bucket); + } + if let Some(root) = root { + builder = builder.root(root); + } + + builder.build() + } +} + +impl TryFrom for BucketStorage { + type Error = JwstStorageError; + + fn try_from(value: MixedBucketDBParam) -> Result { + let mut builder = BucketStorageBuilder::new(); + builder = builder.access_key(&value.access_key); + builder = builder.secret_access_key(&value.secret_access_key); + builder = builder.endpoint(&value.endpoint); + builder = builder.bucket(&value.bucket.unwrap_or("__default_bucket__".to_string())); + builder = builder.root(&value.root.unwrap_or("__default_root__".to_string())); + builder.build() + } +} + +#[derive(Default)] +pub struct BucketStorageBuilder { + access_key: String, + secret_access_key: String, + endpoint: String, + bucket: String, + root: String, +} + +impl BucketStorageBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn access_key(mut self, access_key: &str) -> Self { + self.access_key = access_key.to_string(); + self + } + + pub fn secret_access_key(mut self, secret_access_key: &str) -> Self { + self.secret_access_key = secret_access_key.to_string(); + self + } + + pub fn endpoint(mut self, endpoint: &str) -> Self { + self.endpoint = endpoint.to_string(); + self + } + + pub fn bucket(mut self, bucket: &str) -> Self { + self.bucket = bucket.to_string(); + self + } + + pub fn root(mut self, root: &str) -> Self { + self.root = root.to_string(); + self + } + + pub fn build(self) -> JwstStorageResult { + let mut builder = S3::default(); + + builder.bucket(self.bucket.as_str()); + builder.root(self.root.as_str()); + builder.endpoint(self.endpoint.as_str()); + builder.access_key_id(self.access_key.as_str()); + builder.secret_access_key(self.secret_access_key.as_str()); + + Ok(BucketStorage { + op: Operator::new(builder)?.finish(), + }) + } +} diff --git a/libs/jwst-core-storage/src/storage/difflog.rs b/libs/jwst-core-storage/src/storage/difflog.rs new file mode 100644 index 000000000..80cdbd32d --- /dev/null +++ b/libs/jwst-core-storage/src/storage/difflog.rs @@ -0,0 +1,51 @@ +use super::{entities::prelude::*, *}; +use crate::types::JwstStorageResult; + +// type DiffLogModel = ::Model; +type DiffLogActiveModel = super::entities::diff_log::ActiveModel; +// type DiffLogColumn = ::Column; + +pub struct DiffLogRecord { + bucket: Arc, + pool: DatabaseConnection, +} + +impl DiffLogRecord { + pub async fn init_with_pool( + pool: DatabaseConnection, + bucket: Arc, + ) -> JwstStorageResult { + Ok(Self { bucket, pool }) + } + + pub async fn init_pool(database: &str) -> JwstStorageResult { + let is_sqlite = is_sqlite(database); + let pool = create_connection(database, is_sqlite).await?; + + Self::init_with_pool(pool, get_bucket(is_sqlite)).await + } + + pub async fn insert( + &self, + workspace: String, + ts: DateTime, + log: String, + ) -> JwstStorageResult<()> { + let _lock = self.bucket.write().await; + DiffLog::insert(DiffLogActiveModel { + workspace: Set(workspace), + timestamp: Set(ts.into()), + log: Set(log), + ..Default::default() + }) + .exec(&self.pool) + .await?; + Ok(()) + } + + pub async fn count(&self) -> JwstStorageResult { + let _lock = self.bucket.read().await; + let count = DiffLog::find().count(&self.pool).await?; + Ok(count) + } +} diff --git a/libs/jwst-core-storage/src/storage/docs/database.rs b/libs/jwst-core-storage/src/storage/docs/database.rs new file mode 100644 index 000000000..babb376ce --- /dev/null +++ b/libs/jwst-core-storage/src/storage/docs/database.rs @@ -0,0 +1,634 @@ +use super::{entities::prelude::*, *}; +use crate::types::JwstStorageResult; +use jwst_codec::{encode_update_as_message, CrdtReader, Doc, DocOptions, RawDecoder, StateVector}; +use jwst_core::{DocStorage, Workspace}; +use sea_orm::Condition; +use std::collections::hash_map::Entry; + +const MAX_TRIM_UPDATE_LIMIT: u64 = 500; + +type DocsModel = ::Model; +type DocsActiveModel = super::entities::docs::ActiveModel; +type DocsColumn = ::Column; + +pub struct DocDBStorage { + bucket: Arc, + pub(super) pool: DatabaseConnection, + workspaces: RwLock>, // memory cache + remote: RwLock>>>, +} + +impl DocDBStorage { + pub async fn init_with_pool( + pool: DatabaseConnection, + bucket: Arc, + ) -> JwstStorageResult { + Ok(Self { + bucket, + pool, + workspaces: RwLock::new(HashMap::new()), + remote: RwLock::new(HashMap::new()), + }) + } + + pub async fn init_pool(database: &str) -> JwstStorageResult { + let is_sqlite = is_sqlite(database); + let pool = create_connection(database, is_sqlite).await?; + + Self::init_with_pool(pool, get_bucket(is_sqlite)).await + } + + pub fn remote(&self) -> &RwLock>>> { + &self.remote + } + + /// warn: records of the same workspace may belong to different doc + #[allow(unused)] + async fn workspace_all(conn: &C, workspace: &str) -> JwstStorageResult> + where + C: ConnectionTrait, + { + trace!("start scan all records of workspace: {workspace}"); + let models = Docs::find() + .filter(DocsColumn::WorkspaceId.eq(workspace)) + .all(conn) + .await?; + trace!( + "end scan all records of workspace: {workspace}, {}", + models.len() + ); + Ok(models) + } + + async fn doc_all(conn: &C, guid: &str) -> JwstStorageResult> + where + C: ConnectionTrait, + { + trace!("start scan all records with guid: {guid}"); + let models = Docs::find() + .filter(DocsColumn::Guid.eq(guid)) + .all(conn) + .await?; + trace!("end scan all: {guid}, {}", models.len()); + Ok(models) + } + + async fn count(conn: &C, guid: &str) -> JwstStorageResult + where + C: ConnectionTrait, + { + trace!("start count: {guid}"); + let count = Docs::find() + .filter(DocsColumn::Guid.eq(guid)) + .count(conn) + .await?; + trace!("end count: {guid}, {count}"); + Ok(count) + } + + async fn workspace_count(conn: &C, workspace: &str) -> JwstStorageResult + where + C: ConnectionTrait, + { + trace!("start count: {workspace}"); + let count = Docs::find() + .filter(DocsColumn::WorkspaceId.eq(workspace)) + .count(conn) + .await?; + trace!("end count: {workspace}, {count}"); + Ok(count) + } + + async fn workspace_guid(conn: &C, workspace: &str) -> JwstStorageResult> + where + C: ConnectionTrait, + { + let record = Docs::find() + .filter( + Condition::all() + .add(DocsColumn::WorkspaceId.eq(workspace)) + .add(DocsColumn::IsWorkspace.eq(true)), + ) + .one(conn) + .await?; + + Ok(record.map(|r| r.guid)) + } + + async fn is_workspace(conn: &C, guid: &str) -> JwstStorageResult + where + C: ConnectionTrait, + { + let record = Docs::find() + .filter(DocsColumn::Guid.eq(guid)) + .one(conn) + .await?; + + Ok(record.map_or(false, |r| r.is_workspace)) + } + + async fn insert(conn: &C, workspace: &str, guid: &str, blob: &[u8]) -> JwstStorageResult<()> + where + C: ConnectionTrait, + { + let workspace_guid = Self::workspace_guid(conn, workspace).await?; + trace!("start insert: {workspace}"); + Docs::insert(DocsActiveModel { + workspace_id: Set(workspace.into()), + guid: Set(guid.into()), + blob: Set(blob.into()), + created_at: Set(Utc::now().into()), + // if not workspace guid found, the current insertion should be workspace init data. + is_workspace: Set(workspace_guid.map_or(true, |g| g == guid)), + ..Default::default() + }) + .exec(conn) + .await?; + trace!("end insert: {workspace}"); + Ok(()) + } + + async fn replace_with( + conn: &C, + workspace: &str, + guid: &str, + blob: Vec, + ) -> JwstStorageResult<()> + where + C: ConnectionTrait, + { + trace!("start replace: {guid}"); + + let is_workspace = Self::is_workspace(conn, guid).await?; + + Docs::delete_many() + .filter(DocsColumn::Guid.eq(guid)) + .exec(conn) + .await?; + Docs::insert(DocsActiveModel { + workspace_id: Set(workspace.into()), + guid: Set(guid.into()), + blob: Set(blob), + is_workspace: Set(is_workspace), + created_at: Set(Utc::now().into()), + ..Default::default() + }) + .exec(conn) + .await?; + trace!("end replace: {workspace}"); + Ok(()) + } + + pub async fn delete(conn: &C, guid: &str) -> JwstStorageResult<()> + where + C: ConnectionTrait, + { + trace!("start drop: {guid}"); + Docs::delete_many() + .filter(DocsColumn::Guid.eq(guid)) + .exec(conn) + .await?; + trace!("end drop: {guid}"); + Ok(()) + } + + pub async fn delete_workspace(conn: &C, workspace_id: &str) -> JwstStorageResult<()> + where + C: ConnectionTrait, + { + trace!("start drop workspace: {workspace_id}"); + Docs::delete_many() + .filter(DocsColumn::WorkspaceId.eq(workspace_id)) + .exec(conn) + .await?; + trace!("end drop workspace: {workspace_id}"); + Ok(()) + } + + async fn update( + &self, + conn: &C, + workspace: &str, + guid: &str, + blob: Vec, + ) -> JwstStorageResult<()> + where + C: ConnectionTrait, + { + trace!("start update: {guid}"); + let update_size = Self::count(conn, guid).await?; + if update_size > MAX_TRIM_UPDATE_LIMIT - 1 { + trace!("full migrate update: {guid}, {update_size}"); + let doc_records = Self::doc_all(conn, guid).await?; + + let data = tokio::task::spawn_blocking(move || utils::merge_doc_records(doc_records)) + .await + .map_err(JwstStorageError::DocMerge)??; + + Self::replace_with(conn, workspace, guid, data).await?; + } + + trace!("insert update: {guid}, {update_size}"); + Self::insert(conn, workspace, guid, &blob).await?; + trace!("end update: {guid}"); + + trace!("update {}bytes to {}", blob.len(), guid); + if let Entry::Occupied(remote) = self.remote.write().await.entry(guid.into()) { + let broadcast = &remote.get(); + if broadcast.send(encode_update_as_message(blob)?).is_err() { + // broadcast failures are not fatal errors, only warnings are required + warn!("send {guid} update to pipeline failed"); + } + } + trace!("end update broadcast: {guid}"); + + Ok(()) + } + + async fn full_migrate( + &self, + workspace: &str, + guid: &str, + blob: Vec, + ) -> JwstStorageResult<()> { + trace!("start full migrate: {guid}"); + Self::replace_with(&self.pool, workspace, guid, blob).await?; + trace!("end full migrate: {guid}"); + Ok(()) + } + + async fn init_workspace(conn: &C, workspace: &str) -> JwstStorageResult + where + C: ConnectionTrait, + { + trace!("start create doc in workspace: {workspace}"); + let all_data = Docs::find() + .filter( + Condition::all() + .add(DocsColumn::WorkspaceId.eq(workspace)) + .add(DocsColumn::IsWorkspace.eq(true)), + ) + .all(conn) + .await?; + + let ws = if all_data.is_empty() { + trace!("create workspace: {workspace}"); + // keep workspace root doc's guid the same as workspaceId + let doc = Doc::with_options(DocOptions { + guid: Some(workspace.into()), + ..Default::default() + }); + let ws = Workspace::from_doc(doc.clone(), workspace)?; + + let update = doc.encode_state_as_update_v1(&StateVector::default())?; + + Self::insert(conn, workspace, doc.guid(), &update).await?; + ws + } else { + trace!("migrate workspace: {workspace}"); + let doc = Doc::with_options(DocOptions { + guid: all_data.first().unwrap().guid.clone().into(), + ..Default::default() + }); + let doc = utils::migrate_update(all_data, doc)?; + Workspace::from_doc(doc, workspace)? + }; + + trace!("end create doc in workspace: {workspace}"); + + Ok(ws) + } +} + +#[async_trait] +impl DocStorage for DocDBStorage { + async fn detect_workspace(&self, workspace_id: &str) -> JwstStorageResult { + trace!("check workspace exists: get lock"); + let _lock = self.bucket.read().await; + + Ok(self.workspaces.read().await.contains_key(workspace_id) + || Self::workspace_count(&self.pool, workspace_id) + .await + .map(|c| c > 0)?) + } + + async fn get_or_create_workspace(&self, workspace_id: String) -> JwstStorageResult { + trace!("get workspace enter: {workspace_id}"); + match self.workspaces.write().await.entry(workspace_id.clone()) { + Entry::Occupied(ws) => { + trace!("get workspace cache: {workspace_id}"); + Ok(ws.get().clone()) + } + Entry::Vacant(v) => { + debug!("init workspace cache: get lock"); + let _lock = self.bucket.write().await; + info!("init workspace cache: {workspace_id}"); + let ws = Self::init_workspace(&self.pool, &workspace_id).await?; + + Ok(v.insert(ws).clone()) + } + } + } + + async fn flush_workspace( + &self, + workspace_id: String, + data: Vec, + ) -> JwstStorageResult { + trace!("create workspace: get lock"); + let _lock = self.bucket.write().await; + + let ws = Self::init_workspace(&self.pool, &workspace_id).await?; + + self.full_migrate(&workspace_id, ws.doc_guid(), data) + .await?; + + debug_assert_eq!( + Self::workspace_count(&self.pool, &workspace_id).await?, + 1u64 + ); + + Ok(ws) + } + + async fn delete_workspace(&self, workspace_id: &str) -> JwstStorageResult<()> { + debug!("delete workspace: get lock"); + let _lock = self.bucket.write().await; + + debug!("delete workspace cache: {workspace_id}"); + self.workspaces.write().await.remove(workspace_id); + DocDBStorage::delete_workspace(&self.pool, workspace_id).await?; + + Ok(()) + } + + async fn detect_doc(&self, guid: &str) -> JwstStorageResult { + trace!("check doc exists: get lock"); + let _lock = self.bucket.read().await; + + Ok(Self::count(&self.pool, guid).await? > 0) + } + + async fn get_doc(&self, guid: String) -> JwstStorageResult> { + let records = Docs::find() + .filter(DocsColumn::Guid.eq(guid)) + .all(&self.pool) + .await?; + + if records.is_empty() { + return Ok(None); + } + + let doc = utils::migrate_update(records, Doc::default())?; + + Ok(Some(doc)) + } + + async fn update_doc( + &self, + workspace_id: String, + guid: String, + data: &[u8], + ) -> JwstStorageResult<()> { + debug!("write_update: get lock"); + let _lock = self.bucket.write().await; + + trace!("write_update: {:?}", data); + self.update(&self.pool, &workspace_id, &guid, data.into()) + .await?; + + Ok(()) + } + + async fn update_doc_with_guid( + &self, + workspace_id: String, + data: &[u8], + ) -> JwstStorageResult<()> { + debug!("write_update: get lock"); + let _lock = self.bucket.write().await; + + trace!("write_update: {:?}", data); + let mut decoder = RawDecoder::new(data.to_vec()); + let guid = decoder.read_var_string()?; + + self.update(&self.pool, &workspace_id, &guid, decoder.drain()) + .await?; + + Ok(()) + } + + async fn delete_doc(&self, guid: &str) -> JwstStorageResult<()> { + trace!("start drop doc: {guid}"); + + Docs::delete_many() + .filter(DocsColumn::Guid.eq(guid)) + .exec(&self.pool) + .await?; + + trace!("end drop doc: {guid}"); + Ok(()) + } +} + +#[cfg(test)] +pub async fn docs_storage_test(pool: &DocDBStorage) -> anyhow::Result<()> { + let conn = &pool.pool; + + DocDBStorage::delete_workspace(conn, "basic").await?; + + // empty table + assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 0); + + // first insert + DocDBStorage::insert(conn, "basic", "1", &[1, 2, 3, 4]).await?; + DocDBStorage::insert(conn, "basic", "1", &[2, 2, 3, 4]).await?; + assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 2); + + // second insert + DocDBStorage::replace_with(conn, "basic", "1", vec![3, 2, 3, 4]).await?; + + let all = DocDBStorage::workspace_all(conn, "basic").await?; + assert_eq!( + all, + vec![DocsModel { + id: all.get(0).unwrap().id, + guid: all.get(0).unwrap().guid.clone(), + workspace_id: "basic".into(), + created_at: all.get(0).unwrap().created_at, + is_workspace: true, + blob: vec![3, 2, 3, 4] + }] + ); + assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 1); + + DocDBStorage::delete_workspace(conn, "basic").await?; + + DocDBStorage::insert(conn, "basic", "1", &[1, 2, 3, 4]).await?; + + let all = DocDBStorage::workspace_all(conn, "basic").await?; + assert_eq!( + all, + vec![DocsModel { + id: all.get(0).unwrap().id, + workspace_id: "basic".into(), + guid: all.get(0).unwrap().guid.clone(), + created_at: all.get(0).unwrap().created_at, + is_workspace: true, + blob: vec![1, 2, 3, 4] + }] + ); + assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 1); + + // no cache + { + pool.workspaces.write().await.clear(); + let workspace = DocDBStorage::init_workspace(conn, "basic").await?; + assert_eq!(workspace.doc_guid(), "1"); + } + + Ok(()) +} + +#[cfg(test)] +#[cfg(feature = "postgres")] +pub async fn full_migration_stress_test(pool: &DocDBStorage) -> anyhow::Result<()> { + let final_bytes: Vec = (0..1024 * 100).map(|_| rand::random::()).collect(); + for i in 0..=50 { + let random_bytes: Vec = if i == 50 { + final_bytes.clone() + } else { + (0..1024 * 100).map(|_| rand::random::()).collect() + }; + let (r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15) = tokio::join!( + pool.flush_workspace("full_migration_1".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_2".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_3".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_4".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_5".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_6".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_7".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_8".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_9".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_10".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_11".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_12".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_13".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_14".to_owned(), random_bytes.clone()), + pool.flush_workspace("full_migration_15".to_owned(), random_bytes.clone()) + ); + r1?; + r2?; + r3?; + r4?; + r5?; + r6?; + r7?; + r8?; + r9?; + r10?; + r11?; + r12?; + r13?; + r14?; + r15?; + } + + assert_eq!( + DocDBStorage::workspace_all(&pool.pool, "full_migration_1") + .await? + .into_iter() + .map(|d| d.blob) + .collect::>(), + vec![final_bytes.clone()] + ); + + assert_eq!( + DocDBStorage::workspace_all(&pool.pool, "full_migration_2") + .await? + .into_iter() + .map(|d| d.blob) + .collect::>(), + vec![final_bytes] + ); + + Ok(()) +} + +#[cfg(test)] +pub async fn docs_storage_partial_test(pool: &DocDBStorage) -> anyhow::Result<()> { + use tokio::sync::mpsc::channel; + + let conn = &pool.pool; + + DocDBStorage::delete_workspace(conn, "basic").await?; + + // empty table + assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 0); + + { + let mut ws = pool.get_or_create_workspace("basic".into()).await.unwrap(); + let guid = ws.doc_guid().to_string(); + let (tx, mut rx) = channel(100); + + ws.doc().subscribe(move |update| { + futures::executor::block_on(async { + tx.send(update.to_vec()).await.unwrap(); + }); + }); + + { + let mut space = ws.get_space("test").unwrap(); + let mut block = space.create("block1", "text").unwrap(); + block.set("test1", "value1").unwrap(); + } + + { + let space = ws.get_space("test").unwrap(); + let mut block = space.get("block1").unwrap(); + block.set("test2", "value2").unwrap(); + } + + { + let mut space = ws.get_space("test").unwrap(); + let mut block = space.create("block2", "block2").unwrap(); + block.set("test3", "value3").unwrap(); + } + + loop { + tokio::select! { + Some(update) = rx.recv() => { + info!("recv: {}", update.len()); + pool.update_doc("basic".into(), guid.clone(), &update) + .await + .unwrap(); + } + _ = tokio::time::sleep(std::time::Duration::from_millis(5000)) => { + break; + } + } + } + + assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 2); + } + + // clear memory cache + pool.workspaces.write().await.clear(); + + { + // memory cache empty, retrieve data from db + let mut ws = pool.get_or_create_workspace("basic".into()).await.unwrap(); + let space = ws.get_space("test").unwrap(); + + let block = space.get("block1").unwrap(); + assert_eq!(block.flavour(), "text"); + assert_eq!(block.get("test1"), Some("value1".into())); + assert_eq!(block.get("test2"), Some("value2".into())); + + let block = space.get("block2").unwrap(); + assert_eq!(block.flavour(), "block2"); + assert_eq!(block.get("test3"), Some("value3".into())); + } + + Ok(()) +} diff --git a/libs/jwst-core-storage/src/storage/docs/mod.rs b/libs/jwst-core-storage/src/storage/docs/mod.rs new file mode 100644 index 000000000..55320e30a --- /dev/null +++ b/libs/jwst-core-storage/src/storage/docs/mod.rs @@ -0,0 +1,148 @@ +mod database; +mod utils; + +use std::ops::Deref; + +use super::*; +use database::DocDBStorage; +use tokio::sync::{broadcast::Sender, RwLock}; + +#[cfg(test)] +#[cfg(feature = "postgres")] +pub(super) use database::full_migration_stress_test; +#[cfg(test)] +pub(super) use database::{docs_storage_partial_test, docs_storage_test}; + +#[derive(Clone)] +pub struct SharedDocDBStorage(pub(super) Arc); + +impl SharedDocDBStorage { + pub async fn init_with_pool( + pool: DatabaseConnection, + bucket: Arc, + ) -> JwstStorageResult { + Ok(Self(Arc::new( + DocDBStorage::init_with_pool(pool, bucket).await?, + ))) + } + + pub async fn init_pool(database: &str) -> JwstStorageResult { + Ok(Self(Arc::new(DocDBStorage::init_pool(database).await?))) + } + + pub fn remote(&self) -> &RwLock>>> { + self.0.remote() + } +} + +impl Deref for SharedDocDBStorage { + type Target = DocDBStorage; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[cfg(test)] +mod test { + use super::{error, info, DocStorage, SharedDocDBStorage}; + use crate::{JwstStorageError, JwstStorageResult}; + use jwst_storage_migration::Migrator; + use rand::random; + use sea_orm_migration::MigratorTrait; + use std::collections::HashSet; + use tokio::task::JoinSet; + + async fn create_workspace_stress_test( + storage: SharedDocDBStorage, + range: usize, + ) -> JwstStorageResult<()> { + let mut join_set = JoinSet::new(); + let mut set = HashSet::new(); + + for _ in 0..range { + let id = random::().to_string(); + set.insert(id.clone()); + let storage = storage.clone(); + + join_set.spawn(async move { + info!("create workspace: {}", id); + let workspace = storage.get_or_create_workspace(id.clone()).await?; + info!("create workspace finish: {}", id); + assert_eq!(workspace.id(), id); + Ok::<_, JwstStorageError>(()) + }); + } + + while let Some(ret) = join_set.join_next().await { + if let Err(e) = ret.map_err(JwstStorageError::DocMerge)? { + error!("failed to execute creator: {e}"); + } + } + + for (i, id) in set.iter().enumerate() { + let storage = storage.clone(); + info!("check {i}: {id}"); + let id = id.clone(); + tokio::spawn(async move { + info!("get workspace: {}", id); + let workspace = storage.get_or_create_workspace(id.clone()).await?; + assert_eq!(workspace.id(), id); + Ok::<_, JwstStorageError>(()) + }); + } + + while let Some(ret) = join_set.join_next().await { + if let Err(e) = ret.map_err(JwstStorageError::DocMerge)? { + error!("failed to execute: {e}"); + } + } + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn sqlite_create_workspace_stress_test_faster() -> anyhow::Result<()> { + jwst_logger::init_logger("jwst-storage"); + let storage = SharedDocDBStorage::init_pool("sqlite::memory:").await?; + Migrator::up(&storage.0.pool, None).await.unwrap(); + create_workspace_stress_test(storage.clone(), 100).await?; + + Ok(()) + } + + #[ignore = "for stress testing"] + #[tokio::test(flavor = "multi_thread")] + async fn sqlite_create_workspace_stress_test() -> anyhow::Result<()> { + jwst_logger::init_logger("jwst-storage"); + let storage = SharedDocDBStorage::init_pool("sqlite::memory:").await?; + create_workspace_stress_test(storage.clone(), 10000).await?; + + Ok(()) + } + + #[ignore = "for stress testing"] + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn postgres_create_workspace_stress_test() -> anyhow::Result<()> { + jwst_logger::init_logger("jwst-storage"); + let storage = SharedDocDBStorage::init_pool( + "postgresql://affine:affine@localhost:5432/affine_binary", + ) + .await?; + create_workspace_stress_test(storage.clone(), 10000).await?; + + Ok(()) + } + + #[ignore = "for stress testing"] + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn mysql_create_workspace_stress_test() -> anyhow::Result<()> { + // jwst_logger::init_logger(); + let storage = + SharedDocDBStorage::init_pool("mysql://affine:affine@localhost:3306/affine_binary") + .await?; + create_workspace_stress_test(storage.clone(), 10000).await?; + + Ok(()) + } +} diff --git a/libs/jwst-core-storage/src/storage/docs/utils.rs b/libs/jwst-core-storage/src/storage/docs/utils.rs new file mode 100644 index 000000000..851dfb926 --- /dev/null +++ b/libs/jwst-core-storage/src/storage/docs/utils.rs @@ -0,0 +1,26 @@ +use super::{entities::prelude::*, types::JwstStorageResult, *}; +use jwst_codec::{Doc, StateVector}; + +// apply all updates to the given doc +pub fn migrate_update( + update_records: Vec<::Model>, + mut doc: Doc, +) -> JwstResult { + for record in update_records { + let id = record.created_at; + if let Err(e) = doc.apply_update_from_binary(record.blob) { + warn!("update {} merge failed, skip it: {:?}", id, e); + } + } + + Ok(doc) +} + +pub fn merge_doc_records( + update_records: Vec<::Model>, +) -> JwstStorageResult> { + let doc = migrate_update(update_records, Doc::default())?; + let state_vector = doc.encode_state_as_update_v1(&StateVector::default())?; + + Ok(state_vector) +} diff --git a/libs/jwst-core-storage/src/storage/mod.rs b/libs/jwst-core-storage/src/storage/mod.rs new file mode 100644 index 000000000..8972588ef --- /dev/null +++ b/libs/jwst-core-storage/src/storage/mod.rs @@ -0,0 +1,262 @@ +pub(crate) mod blobs; +mod difflog; +mod docs; +mod test; + +use self::difflog::DiffLogRecord; + +use super::*; +use crate::storage::blobs::{BlobBucketDBStorage, BlobStorageType, JwstBlobStorage}; +use crate::types::JwstStorageError; +use blobs::BlobAutoStorage; +use docs::SharedDocDBStorage; +use jwst_storage_migration::{Migrator, MigratorTrait}; +use std::{collections::HashMap, time::Instant}; +use tokio::sync::Mutex; + +pub struct JwstStorage { + pool: DatabaseConnection, + blobs: JwstBlobStorage, + docs: SharedDocDBStorage, + difflog: DiffLogRecord, + last_migrate: Mutex>, +} + +impl JwstStorage { + pub async fn new( + database: &str, + blob_storage_type: BlobStorageType, + ) -> JwstStorageResult { + let is_sqlite = is_sqlite(database); + let pool = create_connection(database, is_sqlite).await?; + let bucket = get_bucket(is_sqlite); + + if is_sqlite { + pool.execute_unprepared("PRAGMA journal_mode=WAL;") + .await + .unwrap(); + } + + let blobs = match blob_storage_type { + BlobStorageType::DB => JwstBlobStorage::DB( + BlobAutoStorage::init_with_pool(pool.clone(), bucket.clone()).await?, + ), + BlobStorageType::MixedBucketDB(param) => JwstBlobStorage::MixedBucketDB( + BlobBucketDBStorage::init_with_pool( + pool.clone(), + bucket.clone(), + Some(param.try_into()?), + ) + .await?, + ), + }; + let docs = SharedDocDBStorage::init_with_pool(pool.clone(), bucket.clone()).await?; + let difflog = DiffLogRecord::init_with_pool(pool.clone(), bucket).await?; + + Ok(Self { + pool, + blobs, + docs, + difflog, + last_migrate: Mutex::new(HashMap::new()), + }) + } + + pub async fn new_with_migration( + database: &str, + blob_storage_type: BlobStorageType, + ) -> JwstStorageResult { + let storage = Self::new(database, blob_storage_type).await?; + + storage.db_migrate().await?; + + Ok(storage) + } + + async fn db_migrate(&self) -> JwstStorageResult<()> { + Migrator::up(&self.pool, None).await?; + Ok(()) + } + + pub async fn new_with_sqlite( + file: &str, + blob_storage_type: BlobStorageType, + ) -> JwstStorageResult { + use std::fs::create_dir; + + let data = PathBuf::from("./data"); + if !data.exists() { + create_dir(&data).map_err(JwstStorageError::CreateDataFolder)?; + } + + Self::new_with_migration( + &format!( + "sqlite:{}?mode=rwc", + data.join(PathBuf::from(file).name_str()) + .with_extension("db") + .display() + ), + blob_storage_type, + ) + .await + } + + pub fn database(&self) -> String { + format!("{:?}", self.pool) + } + + pub fn blobs(&self) -> &JwstBlobStorage { + &self.blobs + } + + pub fn docs(&self) -> &SharedDocDBStorage { + &self.docs + } + + pub fn difflog(&self) -> &DiffLogRecord { + &self.difflog + } + + pub async fn with_pool(&self, func: F) -> JwstStorageResult + where + F: Fn(DatabaseConnection) -> Fut, + Fut: Future>, + { + func(self.pool.clone()).await + } + + pub async fn create_workspace(&self, workspace_id: S) -> JwstStorageResult + where + S: AsRef, + { + info!("create_workspace: {}", workspace_id.as_ref()); + + self.docs + .get_or_create_workspace(workspace_id.as_ref().into()) + .await + .map_err(|_err| { + JwstStorageError::Crud(format!( + "Failed to create workspace {}", + workspace_id.as_ref() + )) + }) + } + + pub async fn get_workspace(&self, workspace_id: S) -> JwstStorageResult + where + S: AsRef, + { + trace!("get_workspace: {}", workspace_id.as_ref()); + if self + .docs + .detect_workspace(workspace_id.as_ref()) + .await + .map_err(|_err| { + JwstStorageError::Crud(format!( + "failed to check workspace {}", + workspace_id.as_ref() + )) + })? + { + Ok(self + .docs + .get_or_create_workspace(workspace_id.as_ref().into()) + .await + .map_err(|_err| { + JwstStorageError::Crud(format!( + "failed to get workspace {}", + workspace_id.as_ref() + )) + })?) + } else { + Err(JwstStorageError::WorkspaceNotFound( + workspace_id.as_ref().into(), + )) + } + } + + pub async fn full_migrate( + &self, + workspace_id: String, + update: Option>, + force: bool, + ) -> bool { + let mut map = self.last_migrate.lock().await; + let ts = map.entry(workspace_id.clone()).or_insert(Instant::now()); + + if ts.elapsed().as_secs() > 5 || force { + debug!("full migrate: {workspace_id}"); + match self + .docs + .get_or_create_workspace(workspace_id.clone()) + .await + { + Ok(workspace) => { + let update = if let Some(update) = update { + if let Err(e) = self.docs.delete_workspace(&workspace_id).await { + error!("full_migrate write error: {}", e.to_string()); + return false; + }; + Some(update) + } else { + workspace.sync_migration().ok() + }; + + let Some(update) = update else { + error!("full migrate failed: wait transact timeout"); + return false; + }; + if let Err(e) = self + .docs + .flush_workspace(workspace_id.clone(), update) + .await + { + error!("db write error: {}", e.to_string()); + return false; + } + + *ts = Instant::now(); + + info!("full migrate final: {workspace_id}"); + true + } + Err(e) => { + warn!("workspace {workspace_id} not exists in cache: {e:?}"); + false + } + } + } else { + true + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::blobs::MixedBucketDBParam; + + #[tokio::test] + async fn test_sqlite_storage() { + let storage = JwstStorage::new_with_sqlite(":memory:", BlobStorageType::DB) + .await + .unwrap(); + assert_eq!(storage.database(), "SqlxSqlitePoolConnection"); + } + + #[tokio::test] + #[ignore = "need to config bucket auth"] + async fn test_bucket_storage() { + let bucket_params = MixedBucketDBParam { + access_key: dotenvy::var("BUCKET_ACCESS_TOKEN").unwrap().to_string(), + secret_access_key: dotenvy::var("BUCKET_SECRET_TOKEN").unwrap().to_string(), + endpoint: dotenvy::var("BUCKET_ENDPOINT").unwrap().to_string(), + bucket: Some(dotenvy::var("BUCKET_NAME").unwrap()), + root: Some(dotenvy::var("BUCKET_ROOT").unwrap()), + }; + let _storage = + JwstStorage::new_with_sqlite(":memory:", BlobStorageType::MixedBucketDB(bucket_params)) + .await + .unwrap(); + } +} diff --git a/libs/jwst-core-storage/src/storage/test.rs b/libs/jwst-core-storage/src/storage/test.rs new file mode 100644 index 000000000..33b6aab36 --- /dev/null +++ b/libs/jwst-core-storage/src/storage/test.rs @@ -0,0 +1,44 @@ +#[cfg(test)] +use super::{ + blobs::blobs_storage_test, + docs::{docs_storage_partial_test, docs_storage_test}, + *, +}; + +#[tokio::test] +async fn sqlite_storage_test() -> anyhow::Result<()> { + let storage = JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB).await?; + + info!("blobs_storage_test"); + blobs_storage_test(&storage.blobs().get_blob_db().unwrap()).await?; + info!("docs_storage_test"); + docs_storage_test(&storage.docs().0).await?; + info!("docs_storage_partial_test"); + docs_storage_partial_test(&storage.docs().0).await?; + + Ok(()) +} + +#[ignore = "need postgres server"] +#[cfg(feature = "postgres")] +#[tokio::test] +async fn postgres_storage_test() -> anyhow::Result<()> { + use test::docs::full_migration_stress_test; + + let db = "postgresql://affine:affine@localhost:5432/affine_binary"; + let storage = JwstStorage::new_with_migration(db, BlobStorageType::DB).await?; + let blob_db = storage.blobs().get_blob_db().unwrap(); + let (r1, r2, r3, r4) = tokio::join!( + blobs_storage_test(&blob_db), + docs_storage_test(&storage.docs().0), + docs_storage_partial_test(&storage.docs().0), + full_migration_stress_test(&storage.docs().0), + ); + + r1?; + r2?; + r3?; + r4?; + + Ok(()) +} diff --git a/libs/jwst-core-storage/src/types.rs b/libs/jwst-core-storage/src/types.rs new file mode 100644 index 000000000..81d4be76e --- /dev/null +++ b/libs/jwst-core-storage/src/types.rs @@ -0,0 +1,28 @@ +use jwst_codec::JwstCodecError; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum JwstStorageError { + #[error("failed to create data directory")] + CreateDataFolder(std::io::Error), + #[error("db manipulate error: {0}")] + Crud(String), + #[error("db error")] + Db(#[from] sea_orm::DbErr), + #[error("doc codec error")] + DocJwstCodec(#[from] JwstCodecError), + #[error("merge thread panic")] + DocMerge(tokio::task::JoinError), + #[error("workspace {0} not found")] + WorkspaceNotFound(String), + #[error("jwst error")] + Jwst(#[from] jwst_core::JwstError), + #[error("failed to process blob")] + JwstBlob(#[from] crate::storage::blobs::JwstBlobError), + #[error("bucket error")] + JwstBucket(#[from] opendal::Error), + #[error("env variables read error")] + DotEnvy(#[from] dotenvy::Error), +} + +pub type JwstStorageResult = Result; From 4d4fe0f7f41731b31381247f36024d5656205cc1 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 21 Aug 2023 16:52:49 +0800 Subject: [PATCH 02/49] chore: format codes --- apps/cloud/src/api/blobs.rs | 7 +- apps/cloud/src/api/collaboration.rs | 8 +- apps/cloud/src/api/common.rs | 10 +- apps/cloud/src/api/mod.rs | 13 +- apps/cloud/src/api/permissions.rs | 10 +- apps/cloud/src/api/user_channel.rs | 21 ++- apps/cloud/src/api/workspace.rs | 12 +- apps/cloud/src/application/blob_service.rs | 38 ++--- apps/cloud/src/config.rs | 3 +- apps/cloud/src/context.rs | 8 +- apps/cloud/src/layer.rs | 7 +- apps/cloud/src/main.rs | 3 +- apps/doc_merger/src/main.rs | 4 +- apps/keck/src/server/api/blobs.rs | 6 +- apps/keck/src/server/api/blocks/block.rs | 6 +- apps/keck/src/server/api/blocks/mod.rs | 7 +- apps/keck/src/server/api/blocks/schema.rs | 3 +- apps/keck/src/server/api/blocks/workspace.rs | 29 ++-- apps/keck/src/server/api/doc.rs | 5 +- apps/keck/src/server/api/mod.rs | 6 +- apps/keck/src/server/files.rs | 3 +- apps/keck/src/server/mod.rs | 10 +- apps/keck/src/server/subscribe.rs | 9 +- apps/keck/src/server/sync/blobs.rs | 13 +- apps/keck/src/server/sync/collaboration.rs | 22 ++- apps/keck/src/server/sync/mod.rs | 3 +- ...0230101_000002_create_google_user_table.rs | 3 +- ...0230101_000004_create_permissions_table.rs | 14 +- ...0230217_000001_update_permissions_table.rs | 6 +- libs/cloud-database/src/database.rs | 9 +- libs/cloud-database/src/entities/prelude.rs | 8 +- libs/cloud-database/src/lib.rs | 3 +- libs/cloud-database/src/model.rs | 6 +- libs/cloud-infra/src/auth.rs | 9 +- libs/cloud-infra/src/hosting/api_doc.rs | 3 +- libs/cloud-infra/src/hosting/files.rs | 3 +- libs/cloud-infra/src/hosting/mod.rs | 3 +- libs/cloud-infra/src/lib.rs | 3 +- libs/cloud-infra/src/mail.rs | 14 +- libs/jwst-binding/jwst-core-swift/build.rs | 3 +- .../jwst-swift-integrate/src/main.rs | 14 +- .../jwst-binding/jwst-core-swift/src/block.rs | 12 +- .../jwst-core-swift/src/dynamic_value.rs | 3 +- libs/jwst-binding/jwst-core-swift/src/lib.rs | 5 +- .../jwst-core-swift/src/storage.rs | 32 ++-- .../jwst-core-swift/src/workspace.rs | 10 +- libs/jwst-binding/jwst-jni/build.rs | 7 +- libs/jwst-binding/jwst-jni/src/block.rs | 3 +- libs/jwst-binding/jwst-jni/src/difflog.rs | 6 +- libs/jwst-binding/jwst-jni/src/lib.rs | 4 +- libs/jwst-binding/jwst-jni/src/storage.rs | 6 +- libs/jwst-binding/jwst-jni/src/workspace.rs | 8 +- .../jwst-swift-integrate/src/main.rs | 8 +- libs/jwst-binding/jwst-swift/src/block.rs | 6 +- libs/jwst-binding/jwst-swift/src/difflog.rs | 6 +- .../jwst-swift/src/dynamic_value.rs | 3 +- libs/jwst-binding/jwst-swift/src/lib.rs | 7 +- libs/jwst-binding/jwst-swift/src/storage.rs | 9 +- libs/jwst-binding/jwst-swift/src/workspace.rs | 12 +- .../src/doc_operation/yrs_op/array.rs | 6 +- .../src/doc_operation/yrs_op/map.rs | 6 +- .../src/doc_operation/yrs_op/mod.rs | 11 +- .../src/doc_operation/yrs_op/text.rs | 3 +- .../src/doc_operation/yrs_op/xml_element.rs | 3 +- .../src/doc_operation/yrs_op/xml_fragment.rs | 3 +- .../src/doc_operation/yrs_op/xml_text.rs | 3 +- .../benches/array_ops_benchmarks.rs | 3 +- libs/jwst-codec/benches/codec_benchmarks.rs | 6 +- libs/jwst-codec/benches/map_ops_benchmarks.rs | 3 +- .../jwst-codec/benches/text_ops_benchmarks.rs | 3 +- libs/jwst-codec/benches/update_benchmarks.rs | 4 +- libs/jwst-codec/benches/utils/files.rs | 3 +- libs/jwst-codec/src/codec/buffer.rs | 9 +- libs/jwst-codec/src/codec/integer.rs | 9 +- libs/jwst-codec/src/codec/string.rs | 9 +- libs/jwst-codec/src/doc/awareness.rs | 3 +- libs/jwst-codec/src/doc/codec/any.rs | 9 +- libs/jwst-codec/src/doc/codec/content.rs | 6 +- libs/jwst-codec/src/doc/codec/delete_set.rs | 5 +- libs/jwst-codec/src/doc/codec/io/codec_v1.rs | 3 +- libs/jwst-codec/src/doc/codec/io/reader.rs | 6 +- libs/jwst-codec/src/doc/codec/io/writer.rs | 6 +- libs/jwst-codec/src/doc/codec/item.rs | 3 +- libs/jwst-codec/src/doc/codec/refs.rs | 3 +- libs/jwst-codec/src/doc/codec/update.rs | 38 +++-- libs/jwst-codec/src/doc/codec/utils/items.rs | 3 +- libs/jwst-codec/src/doc/common/range.rs | 3 +- libs/jwst-codec/src/doc/common/somr.rs | 9 +- libs/jwst-codec/src/doc/common/state.rs | 3 +- libs/jwst-codec/src/doc/document.rs | 3 +- libs/jwst-codec/src/doc/mod.rs | 4 +- libs/jwst-codec/src/doc/publisher.rs | 3 +- libs/jwst-codec/src/doc/store.rs | 20 ++- libs/jwst-codec/src/doc/types/array.rs | 3 +- libs/jwst-codec/src/doc/types/list/mod.rs | 1 - .../src/doc/types/list/search_marker.rs | 27 +-- libs/jwst-codec/src/doc/types/map.rs | 8 +- libs/jwst-codec/src/doc/types/mod.rs | 12 +- libs/jwst-codec/src/doc/types/text.rs | 10 +- libs/jwst-codec/src/doc/utils.rs | 3 +- libs/jwst-codec/src/lib.rs | 7 +- libs/jwst-codec/src/protocol/awareness.rs | 3 +- libs/jwst-codec/src/protocol/doc.rs | 5 +- libs/jwst-codec/src/protocol/mod.rs | 17 +- libs/jwst-codec/src/protocol/scanner.rs | 3 +- libs/jwst-codec/src/protocol/sync.rs | 6 +- libs/jwst-codec/src/protocol/utils.rs | 3 +- libs/jwst-codec/src/sync.rs | 20 +-- libs/jwst-core-rpc/src/broadcast.rs | 20 +-- libs/jwst-core-rpc/src/client/mod.rs | 11 +- libs/jwst-core-rpc/src/client/webrtc.rs | 20 +-- libs/jwst-core-rpc/src/client/websocket.rs | 24 +-- .../src/connector/axum_socket.rs | 10 +- libs/jwst-core-rpc/src/connector/memory.rs | 19 +-- .../src/connector/tungstenite_socket.rs | 14 +- libs/jwst-core-rpc/src/connector/webrtc.rs | 35 ++-- libs/jwst-core-rpc/src/context.rs | 31 ++-- libs/jwst-core-rpc/src/handler.rs | 133 ++++++--------- libs/jwst-core-rpc/src/lib.rs | 17 +- .../src/utils/memory_workspace.rs | 15 +- libs/jwst-core-rpc/src/utils/mod.rs | 4 +- .../jwst-core-rpc/src/utils/server_context.rs | 10 +- .../jwst-core-storage/src/entities/prelude.rs | 9 +- libs/jwst-core-storage/src/lib.rs | 15 +- .../m20220101_000001_initial_blob_table.rs | 17 +- .../src/m20220101_000002_initial_doc_table.rs | 20 +-- .../m20230321_000001_blob_optimized_table.rs | 15 +- ...230614_000001_initial_bucket_blob_table.rs | 9 +- .../src/m20230626_023319_doc_guid.rs | 11 +- ...m20230814_061223_initial_diff_log_table.rs | 10 +- libs/jwst-core-storage/src/rate_limiter.rs | 10 +- .../src/storage/blobs/bucket_local_db.rs | 79 +++------ .../src/storage/blobs/local_db.rs | 29 +--- .../src/storage/blobs/mod.rs | 154 ++++-------------- .../src/storage/blobs/utils.rs | 20 +-- libs/jwst-core-storage/src/storage/difflog.rs | 12 +- .../src/storage/docs/database.rs | 107 +++--------- .../jwst-core-storage/src/storage/docs/mod.rs | 39 ++--- .../src/storage/docs/utils.rs | 12 +- libs/jwst-core-storage/src/storage/mod.rs | 100 +++--------- libs/jwst-core/src/block/convert.rs | 3 +- libs/jwst-core/src/block/mod.rs | 6 +- libs/jwst-core/src/lib.rs | 3 +- libs/jwst-core/src/space/convert.rs | 3 +- libs/jwst-core/src/space/mod.rs | 6 +- libs/jwst-core/src/types/blob.rs | 6 +- libs/jwst-core/src/types/mod.rs | 4 +- libs/jwst-core/src/workspace/metadata/meta.rs | 9 +- libs/jwst-core/src/workspace/metadata/mod.rs | 4 +- .../jwst-core/src/workspace/metadata/pages.rs | 9 +- libs/jwst-core/src/workspace/mod.rs | 4 +- libs/jwst-core/src/workspace/observe.rs | 3 +- libs/jwst-core/src/workspace/sync.rs | 3 +- libs/jwst-core/src/workspace/workspace.rs | 46 +++--- libs/jwst-logger/src/formatter.rs | 3 +- libs/jwst-logger/src/lib.rs | 3 +- libs/jwst-logger/src/logger.rs | 10 +- libs/jwst-rpc/src/broadcast.rs | 6 +- libs/jwst-rpc/src/client/mod.rs | 11 +- libs/jwst-rpc/src/client/webrtc.rs | 6 +- libs/jwst-rpc/src/client/websocket.rs | 6 +- libs/jwst-rpc/src/connector/axum_socket.rs | 3 +- libs/jwst-rpc/src/connector/memory.rs | 6 +- .../src/connector/tungstenite_socket.rs | 3 +- libs/jwst-rpc/src/connector/webrtc.rs | 13 +- libs/jwst-rpc/src/context.rs | 18 +- libs/jwst-rpc/src/handler.rs | 38 +++-- libs/jwst-rpc/src/lib.rs | 17 +- libs/jwst-rpc/src/utils/memory_workspace.rs | 6 +- libs/jwst-rpc/src/utils/mod.rs | 4 +- libs/jwst-rpc/src/utils/server_context.rs | 6 +- libs/jwst-storage/src/entities/prelude.rs | 9 +- libs/jwst-storage/src/lib.rs | 10 +- .../m20220101_000001_initial_blob_table.rs | 3 +- .../src/m20220101_000002_initial_doc_table.rs | 3 +- .../m20230321_000001_blob_optimized_table.rs | 3 +- ...230614_000001_initial_bucket_blob_table.rs | 3 +- libs/jwst-storage/src/rate_limiter.rs | 3 +- .../src/storage/blobs/bucket_local_db.rs | 17 +- .../src/storage/blobs/local_db.rs | 6 +- libs/jwst-storage/src/storage/blobs/mod.rs | 17 +- libs/jwst-storage/src/storage/blobs/utils.rs | 14 +- .../jwst-storage/src/storage/docs/database.rs | 8 +- libs/jwst-storage/src/storage/docs/mod.rs | 16 +- libs/jwst-storage/src/storage/docs/utils.rs | 4 +- libs/jwst-storage/src/storage/mod.rs | 13 +- libs/jwst/src/block/convert.rs | 3 +- libs/jwst/src/block/mod.rs | 15 +- libs/jwst/src/history/raw.rs | 3 +- libs/jwst/src/lib.rs | 3 +- libs/jwst/src/space/convert.rs | 6 +- libs/jwst/src/space/mod.rs | 9 +- libs/jwst/src/space/transaction.rs | 5 +- libs/jwst/src/types/blob.rs | 6 +- libs/jwst/src/types/mod.rs | 4 +- libs/jwst/src/utils.rs | 3 +- libs/jwst/src/workspace/block_observer.rs | 4 +- libs/jwst/src/workspace/metadata/meta.rs | 9 +- libs/jwst/src/workspace/metadata/mod.rs | 4 +- libs/jwst/src/workspace/metadata/pages.rs | 7 +- libs/jwst/src/workspace/mod.rs | 4 +- libs/jwst/src/workspace/observe.rs | 8 +- .../src/workspace/plugins/indexing/indexer.rs | 87 ++++++---- .../src/workspace/plugins/indexing/mod.rs | 6 +- .../workspace/plugins/indexing/register.rs | 4 +- libs/jwst/src/workspace/plugins/mod.rs | 11 +- libs/jwst/src/workspace/plugins/plugin.rs | 26 +-- libs/jwst/src/workspace/sync.rs | 8 +- libs/jwst/src/workspace/transaction.rs | 9 +- libs/jwst/src/workspace/workspace.rs | 37 +++-- 210 files changed, 1209 insertions(+), 1356 deletions(-) diff --git a/apps/cloud/src/api/blobs.rs b/apps/cloud/src/api/blobs.rs index 340e2bee1..f87f4aa8d 100644 --- a/apps/cloud/src/api/blobs.rs +++ b/apps/cloud/src/api/blobs.rs @@ -1,4 +1,5 @@ -use crate::context::Context; +use std::sync::Arc; + use axum::{ extract::{BodyStream, Path}, headers::ContentLength, @@ -6,10 +7,10 @@ use axum::{ response::{IntoResponse, Response}, Extension, TypedHeader, }; - use cloud_database::Claims; use jwst_logger::{info, instrument, tracing}; -use std::sync::Arc; + +use crate::context::Context; /// Get `blob` by workspace_id and hash. /// - Return 200 and `blob`. diff --git a/apps/cloud/src/api/collaboration.rs b/apps/cloud/src/api/collaboration.rs index 20d9e9016..6a8ec26c6 100644 --- a/apps/cloud/src/api/collaboration.rs +++ b/apps/cloud/src/api/collaboration.rs @@ -1,5 +1,5 @@ -use super::*; -use crate::infrastructure::auth::get_claim_from_token; +use std::sync::Arc; + use axum::{ extract::{ws::WebSocketUpgrade, Path}, http::StatusCode, @@ -8,7 +8,9 @@ use axum::{ use futures_util::FutureExt; use jwst_rpc::{axum_socket_connector, handle_connector}; use serde::{Deserialize, Serialize}; -use std::sync::Arc; + +use super::*; +use crate::infrastructure::auth::get_claim_from_token; #[derive(Serialize)] pub struct WebSocketAuthentication { diff --git a/apps/cloud/src/api/common.rs b/apps/cloud/src/api/common.rs index 918e7b898..edc22c009 100644 --- a/apps/cloud/src/api/common.rs +++ b/apps/cloud/src/api/common.rs @@ -22,14 +22,16 @@ pub async fn health_check() -> Response { #[cfg(test)] mod test { + use std::sync::Arc; + + use axum::Extension; + use axum_test_helper::TestClient; + use cloud_database::CloudDatabase; + use super::{ super::{make_rest_route, Context}, *, }; - use axum::Extension; - use axum_test_helper::TestClient; - use cloud_database::CloudDatabase; - use std::sync::Arc; #[tokio::test] async fn test_health_check() { diff --git a/apps/cloud/src/api/mod.rs b/apps/cloud/src/api/mod.rs index 92582066b..98d9c29bd 100644 --- a/apps/cloud/src/api/mod.rs +++ b/apps/cloud/src/api/mod.rs @@ -6,9 +6,8 @@ mod common; mod user_channel; mod workspace; -pub use collaboration::make_ws_route; +use std::sync::Arc; -use crate::{context::Context, infrastructure::error_status::ErrorStatus, layer::make_firebase_auth_layer}; use axum::{ extract::Query, response::{IntoResponse, Response}, @@ -17,12 +16,14 @@ use axum::{ }; use chrono::Utc; use cloud_database::{Claims, MakeToken, RefreshToken, User, UserQuery, UserToken}; +pub use collaboration::make_ws_route; use jwst_logger::{error, info, instrument, tracing}; use lib0::any::Any; -use std::sync::Arc; pub use user_channel::*; use utoipa::OpenApi; +use crate::{context::Context, infrastructure::error_status::ErrorStatus, layer::make_firebase_auth_layer}; + #[derive(OpenApi)] #[openapi( paths( @@ -256,12 +257,14 @@ pub async fn make_token(Extension(ctx): Extension>, Json(payload): #[cfg(test)] mod test { - use super::*; + use std::sync::Arc; + use axum::http::StatusCode; use axum_test_helper::TestClient; use cloud_database::{CloudDatabase, CreateUser}; use serde_json::json; - use std::sync::Arc; + + use super::*; #[tokio::test] async fn test_query_user() { diff --git a/apps/cloud/src/api/permissions.rs b/apps/cloud/src/api/permissions.rs index 388113a6b..98b4ee16d 100644 --- a/apps/cloud/src/api/permissions.rs +++ b/apps/cloud/src/api/permissions.rs @@ -1,4 +1,5 @@ -use crate::{context::Context, infrastructure::error_status::ErrorStatus}; +use std::{io::Cursor, sync::Arc}; + use axum::{ extract::Path, http::{ @@ -13,8 +14,8 @@ use image::ImageOutputFormat; use jwst::{error, BlobStorage}; use jwst_logger::{info, instrument, tracing}; use lettre::message::Mailbox; -use std::io::Cursor; -use std::sync::Arc; + +use crate::{context::Context, infrastructure::error_status::ErrorStatus}; /// Get workspace's `Members` /// - Return 200 ok and `Members`. /// - Return 400 if request parameter error. @@ -411,7 +412,6 @@ pub async fn remove_user( #[cfg(test)] mod test { - use super::{super::make_rest_route, *}; use axum::body::Body; use axum_test_helper::TestClient; use bytes::Bytes; @@ -419,6 +419,8 @@ mod test { use futures::stream; use serde_json::json; + use super::{super::make_rest_route, *}; + #[tokio::test] async fn test_get_member() { let pool = CloudDatabase::init_pool("sqlite::memory:").await.unwrap(); diff --git a/apps/cloud/src/api/user_channel.rs b/apps/cloud/src/api/user_channel.rs index 128770dee..253da8fa3 100644 --- a/apps/cloud/src/api/user_channel.rs +++ b/apps/cloud/src/api/user_channel.rs @@ -1,5 +1,8 @@ -use super::*; -use crate::infrastructure::auth::get_claim_from_token; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; + use axum::{ extract::ws::{Message, WebSocket, WebSocketUpgrade}, response::Response, @@ -8,14 +11,14 @@ use cloud_database::{WorkspaceDetail, WorkspaceWithPermission}; use futures::{sink::SinkExt, stream::StreamExt}; use jwst_logger::error; use nanoid::nanoid; -use serde::Deserialize; -use serde::Serialize; -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, +use serde::{Deserialize, Serialize}; +use tokio::sync::{ + mpsc::{channel, Sender}, + RwLock, }; -use tokio::sync::mpsc::Sender; -use tokio::sync::{mpsc::channel, RwLock}; + +use super::*; +use crate::infrastructure::auth::get_claim_from_token; #[derive(Deserialize)] pub struct Param { diff --git a/apps/cloud/src/api/workspace.rs b/apps/cloud/src/api/workspace.rs index 2a5d69209..d31008627 100644 --- a/apps/cloud/src/api/workspace.rs +++ b/apps/cloud/src/api/workspace.rs @@ -1,4 +1,5 @@ -use crate::{context::Context, infrastructure::error_status::ErrorStatus}; +use std::sync::Arc; + use axum::{ extract::{BodyStream, Path}, headers::ContentLength, @@ -11,7 +12,8 @@ use futures::{future, StreamExt}; use jwst::{error, BlobStorage}; use jwst_logger::{info, instrument, tracing}; use jwst_storage::JwstStorageError; -use std::sync::Arc; + +use crate::{context::Context, infrastructure::error_status::ErrorStatus}; impl Context { #[instrument(skip(self, stream))] @@ -549,14 +551,16 @@ pub async fn search_workspace( #[cfg(test)] mod test { - use super::super::{make_rest_route, Context}; + use std::sync::Arc; + use axum::{body::Body, http::StatusCode, Extension}; use axum_test_helper::TestClient; use bytes::Bytes; use cloud_database::CloudDatabase; use futures::stream; use serde_json::json; - use std::sync::Arc; + + use super::super::{make_rest_route, Context}; #[tokio::test] async fn test_create_workspace() { diff --git a/apps/cloud/src/application/blob_service.rs b/apps/cloud/src/application/blob_service.rs index ae5ad6fb4..f4744faa0 100644 --- a/apps/cloud/src/application/blob_service.rs +++ b/apps/cloud/src/application/blob_service.rs @@ -1,6 +1,5 @@ -use crate::{ - context::Context, infrastructure::auth::get_claim_from_headers, infrastructure::error_status::ErrorStatus, -}; +use std::{collections::HashMap, path::PathBuf, sync::Arc}; + use axum::{ extract::BodyStream, headers::ContentLength, @@ -12,13 +11,15 @@ use axum::{ }; use chrono::{DateTime, Utc}; use futures::{future, StreamExt}; -use jwst::error; -use jwst::BlobStorage; +use jwst::{error, BlobStorage}; use jwst_logger::{info, instrument, tracing}; use mime::APPLICATION_OCTET_STREAM; use serde::Serialize; -use std::sync::Arc; -use std::{collections::HashMap, path::PathBuf}; + +use crate::{ + context::Context, + infrastructure::{auth::get_claim_from_headers, error_status::ErrorStatus}, +}; #[derive(Serialize)] pub struct Usage { @@ -218,12 +219,7 @@ impl BlobService { return Ok((header, None)); }; - let Ok(file) = ctx - .storage - .blobs() - .get_blob(workspace, id, params.clone()) - .await - else { + let Ok(file) = ctx.storage.blobs().get_blob(workspace, id, params.clone()).await else { return Err(ErrorStatus::NotFound); }; @@ -286,19 +282,21 @@ impl BlobService { #[cfg(test)] mod test { - use super::*; - use crate::infrastructure::auth::get_claim_from_token; - use crate::{api::make_rest_route, context::Context}; - use axum::extract::BodyStream; - use axum::extract::FromRequest; - use axum::http::Request; - use axum::{body::Body, http::StatusCode, Extension}; + use axum::{ + body::Body, + extract::{BodyStream, FromRequest}, + http::{Request, StatusCode}, + Extension, + }; use axum_test_helper::TestClient; use bytes::Bytes; use cloud_database::CloudDatabase; use futures::stream; use serde_json::json; + use super::*; + use crate::{api::make_rest_route, context::Context, infrastructure::auth::get_claim_from_token}; + async fn test_init() -> Arc { let pool = CloudDatabase::init_pool("sqlite::memory:").await.unwrap(); let context = Context::new_test_client(pool).await; diff --git a/apps/cloud/src/config.rs b/apps/cloud/src/config.rs index 75b792244..0c32eb0b3 100644 --- a/apps/cloud/src/config.rs +++ b/apps/cloud/src/config.rs @@ -1,6 +1,7 @@ -use chrono::Duration; use std::env; +use chrono::Duration; + /// A config of `Context`. #[derive(Clone, Debug)] pub struct Config { diff --git a/apps/cloud/src/context.rs b/apps/cloud/src/context.rs index c87709c52..59a737032 100644 --- a/apps/cloud/src/context.rs +++ b/apps/cloud/src/context.rs @@ -1,15 +1,17 @@ -use super::{api::UserChannel, config::Config, utils::create_debug_collaboration_workspace}; -use crate::application::blob_service::BlobService; +use std::{collections::HashMap, path::Path}; + use cloud_database::CloudDatabase; use cloud_infra::{FirebaseContext, KeyContext, MailContext}; use jwst::SearchResults; use jwst_logger::{error, info, warn}; use jwst_rpc::{BroadcastChannels, BroadcastType, RpcContextImpl}; use jwst_storage::{BlobStorageType, JwstStorage, MixedBucketDBParam}; -use std::{collections::HashMap, path::Path}; use tempfile::{tempdir, TempDir}; use tokio::sync::{Mutex, RwLock}; +use super::{api::UserChannel, config::Config, utils::create_debug_collaboration_workspace}; +use crate::application::blob_service::BlobService; + pub struct Context { pub key: KeyContext, pub firebase: Mutex, diff --git a/apps/cloud/src/layer.rs b/apps/cloud/src/layer.rs index 36cdc64c7..41d542064 100644 --- a/apps/cloud/src/layer.rs +++ b/apps/cloud/src/layer.rs @@ -1,6 +1,5 @@ -use crate::infrastructure::auth::get_claim_from_headers; +use std::sync::Arc; -use super::{infrastructure::error_status::ErrorStatus, *}; use axum::{ body::Body, http::{Request, Response}, @@ -13,13 +12,15 @@ use futures_util::future::BoxFuture; use http_body::combinators::UnsyncBoxBody; use jsonwebtoken::DecodingKey; use nanoid::nanoid; -use std::sync::Arc; use tower_http::{ auth::{AsyncAuthorizeRequest, AsyncRequireAuthorizationLayer}, request_id::{MakeRequestId, PropagateRequestIdLayer, RequestId, SetRequestIdLayer}, trace::TraceLayer, }; +use super::{infrastructure::error_status::ErrorStatus, *}; +use crate::infrastructure::auth::get_claim_from_headers; + #[derive(Clone)] pub struct Auth { decoding_key: DecodingKey, diff --git a/apps/cloud/src/main.rs b/apps/cloud/src/main.rs index e6e6bb18a..d5bdc3d07 100644 --- a/apps/cloud/src/main.rs +++ b/apps/cloud/src/main.rs @@ -7,9 +7,10 @@ mod infrastructure; mod layer; mod utils; +use std::{net::SocketAddr, sync::Arc}; + use axum::{http::Method, Extension, Router, Server}; use jwst_logger::{error, info, info_span, init_logger}; -use std::{net::SocketAddr, sync::Arc}; use tower_http::cors::{Any, CorsLayer}; #[global_allocator] diff --git a/apps/doc_merger/src/main.rs b/apps/doc_merger/src/main.rs index 1c48f7f09..1a254c95d 100644 --- a/apps/doc_merger/src/main.rs +++ b/apps/doc_merger/src/main.rs @@ -1,12 +1,12 @@ -use jwst_codec::Doc; use std::{ fs::{read, write}, io::{Error, ErrorKind}, path::PathBuf, }; -use yrs::{updates::decoder::Decode, ReadTxn, StateVector, Transact, Update}; use clap::Parser; +use jwst_codec::Doc; +use yrs::{updates::decoder::Decode, ReadTxn, StateVector, Transact, Update}; /// ybinary merger #[derive(Parser, Debug)] diff --git a/apps/keck/src/server/api/blobs.rs b/apps/keck/src/server/api/blobs.rs index 5c942dd97..3672421c1 100644 --- a/apps/keck/src/server/api/blobs.rs +++ b/apps/keck/src/server/api/blobs.rs @@ -1,9 +1,10 @@ -use super::*; use axum::{extract::BodyStream, response::Response, routing::post}; use futures::{future, StreamExt}; use jwst::BlobStorage; use utoipa::ToSchema; +use super::*; + #[derive(Serialize, ToSchema)] pub struct BlobStatus { id: Option, @@ -174,12 +175,13 @@ pub fn blobs_apis(router: Router) -> Router { #[cfg(test)] mod tests { - use super::*; use axum::body::{Body, Bytes}; use axum_test_helper::TestClient; use futures::stream; use jwst_storage::BlobStorageType; + use super::*; + #[tokio::test] async fn test_blobs_apis() { let ctx = Context::new( diff --git a/apps/keck/src/server/api/blocks/block.rs b/apps/keck/src/server/api/blocks/block.rs index 539893610..2b8e97a6c 100644 --- a/apps/keck/src/server/api/blocks/block.rs +++ b/apps/keck/src/server/api/blocks/block.rs @@ -1,9 +1,10 @@ -use super::*; use axum::{extract::Query, response::Response}; use jwst::{constants, DocStorage}; use lib0::any::Any; use serde_json::Value as JsonValue; +use super::*; + /// Get a `Block` by id /// - Return 200 and `Block`'s data if `Block` is exists. /// - Return 404 Not Found if `Workspace` or `Block` not exists. @@ -131,7 +132,8 @@ pub async fn set_block( /// Get exists `Blocks` in certain `Workspace` by flavour /// - Return 200 Ok and `Blocks`'s data if `Blocks` is exists. -/// - Return 404 Not Found if `Workspace` not exists or 500 Internal Server Error when transaction init fails. +/// - Return 404 Not Found if `Workspace` not exists or 500 Internal Server +/// Error when transaction init fails. #[utoipa::path( get, tag = "Blocks", diff --git a/apps/keck/src/server/api/blocks/mod.rs b/apps/keck/src/server/api/blocks/mod.rs index a07be64f3..9f7d22d29 100644 --- a/apps/keck/src/server/api/blocks/mod.rs +++ b/apps/keck/src/server/api/blocks/mod.rs @@ -3,14 +3,14 @@ pub mod schema; pub mod workspace; pub use block::{delete_block, get_block, get_block_history, insert_block_children, remove_block_children, set_block}; +use schema::InsertChildren; +pub use schema::SubscribeWorkspace; pub use workspace::{ delete_workspace, get_workspace, history_workspace, history_workspace_clients, set_workspace, subscribe_workspace, workspace_client, }; use super::*; -use schema::InsertChildren; -pub use schema::SubscribeWorkspace; fn block_apis(router: Router) -> Router { let block_operation = Router::new() @@ -56,10 +56,11 @@ pub fn blocks_apis(router: Router) -> Router { #[cfg(test)] mod tests { - use super::*; use axum_test_helper::TestClient; use serde_json::{from_str, json, to_string, Value}; + use super::*; + #[tokio::test] async fn test_doc_apis() { let client = TestClient::new(doc_apis(Router::new())); diff --git a/apps/keck/src/server/api/blocks/schema.rs b/apps/keck/src/server/api/blocks/schema.rs index 1f86888f8..c6507c771 100644 --- a/apps/keck/src/server/api/blocks/schema.rs +++ b/apps/keck/src/server/api/blocks/schema.rs @@ -1,5 +1,6 @@ -use serde::{Deserialize, Serialize}; use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; use utoipa::ToSchema; #[derive(Default, Deserialize, PartialEq, Debug, ToSchema)] diff --git a/apps/keck/src/server/api/blocks/workspace.rs b/apps/keck/src/server/api/blocks/workspace.rs index 78c94b5f8..462ee9369 100644 --- a/apps/keck/src/server/api/blocks/workspace.rs +++ b/apps/keck/src/server/api/blocks/workspace.rs @@ -1,5 +1,3 @@ -use super::*; -use crate::server::api::blocks::SubscribeWorkspace; use axum::{ extract::{Path, Query}, http::header, @@ -8,6 +6,9 @@ use axum::{ use jwst::{parse_history, parse_history_client, DocStorage}; use utoipa::IntoParams; +use super::*; +use crate::server::api::blocks::SubscribeWorkspace; + /// Get a exists `Workspace` by id /// - Return 200 Ok and `Workspace`'s data if `Workspace` is exists. /// - Return 404 Not Found if `Workspace` not exists. @@ -34,7 +35,8 @@ pub async fn get_workspace(Extension(context): Extension>, Path(wor } /// Create a `Workspace` by id -/// - Return 200 Ok and `Workspace`'s data if init success or `Workspace` is exists. +/// - Return 200 Ok and `Workspace`'s data if init success or `Workspace` is +/// exists. /// - Return 500 Internal Server Error if init failed. #[utoipa::path( post, @@ -89,8 +91,10 @@ pub async fn delete_workspace(Extension(context): Extension>, Path( /// Get current client id of server /// -/// When the server initializes or get the `Workspace`, a `Client` will be created. This `Client` will not be destroyed until the server restarts. -/// Therefore, the `Client ID` in the history generated by modifying `Block` through HTTP API will remain unchanged until the server restarts. +/// When the server initializes or get the `Workspace`, a `Client` will be +/// created. This `Client` will not be destroyed until the server restarts. +/// Therefore, the `Client ID` in the history generated by modifying `Block` +/// through HTTP API will remain unchanged until the server restarts. /// /// This interface return the client id that server will used. #[utoipa::path( @@ -265,15 +269,20 @@ pub async fn get_workspace_block( /// Get all client ids of the `Workspace` /// -/// This interface returns all `Client IDs` that includes history in the `Workspace` +/// This interface returns all `Client IDs` that includes history in the +/// `Workspace` /// /// Every client write something into a `Workspace` will has a unique id. /// /// For example: -/// - A user writes a new `Block` to a `Workspace` through `Client` on the front end, which will generate a series of histories. -/// A `Client ID` contained in these histories will be randomly generated by the `Client` and will remain unchanged until the Client instance is destroyed -/// - When the server initializes or get the `Workspace`, a `Client` will be created. This `Client` will not be destroyed until the server restarts. -/// Therefore, the `Client ID` in the history generated by modifying `Block` through HTTP API will remain unchanged until the server restarts. +/// - A user writes a new `Block` to a `Workspace` through `Client` on the +/// front end, which will generate a series of histories. A `Client ID` +/// contained in these histories will be randomly generated by the `Client` +/// and will remain unchanged until the Client instance is destroyed +/// - When the server initializes or get the `Workspace`, a `Client` will be +/// created. This `Client` will not be destroyed until the server restarts. +/// Therefore, the `Client ID` in the history generated by modifying `Block` +/// through HTTP API will remain unchanged until the server restarts. #[utoipa::path( get, tag = "Workspace", diff --git a/apps/keck/src/server/api/doc.rs b/apps/keck/src/server/api/doc.rs index 2e806eec6..45d9d48df 100644 --- a/apps/keck/src/server/api/doc.rs +++ b/apps/keck/src/server/api/doc.rs @@ -1,10 +1,11 @@ +use cloud_infra::with_api_doc; +use utoipa::OpenApi; + use super::{ blobs, blocks::{block, schema, workspace}, *, }; -use cloud_infra::with_api_doc; -use utoipa::OpenApi; #[derive(OpenApi)] #[openapi( diff --git a/apps/keck/src/server/api/mod.rs b/apps/keck/src/server/api/mod.rs index 90c398900..a770acb39 100644 --- a/apps/keck/src/server/api/mod.rs +++ b/apps/keck/src/server/api/mod.rs @@ -4,7 +4,8 @@ mod blobs; mod blocks; mod doc; -use super::*; +use std::collections::HashMap; + use anyhow::Context as AnyhowContext; use axum::Router; #[cfg(feature = "api")] @@ -17,9 +18,10 @@ use axum::{ use doc::doc_apis; use jwst_rpc::{BroadcastChannels, RpcContextImpl}; use jwst_storage::{BlobStorageType, JwstStorage, JwstStorageResult, MixedBucketDBParam}; -use std::collections::HashMap; use tokio::sync::RwLock; +use super::*; + #[derive(Deserialize)] #[cfg_attr(feature = "api", derive(utoipa::IntoParams))] pub struct Pagination { diff --git a/apps/keck/src/server/files.rs b/apps/keck/src/server/files.rs index cf01e445c..31e230547 100644 --- a/apps/keck/src/server/files.rs +++ b/apps/keck/src/server/files.rs @@ -1,4 +1,3 @@ -use super::*; use axum::{ body::BoxBody, http::{Response, Uri}, @@ -7,6 +6,8 @@ use axum::{ }; use cloud_infra::{fetch_static_response, rust_embed, RustEmbed}; +use super::*; + #[derive(RustEmbed)] #[folder = "../homepage/out/"] #[include = "*"] diff --git a/apps/keck/src/server/mod.rs b/apps/keck/src/server/mod.rs index f3d7b84da..4d8ba28f8 100644 --- a/apps/keck/src/server/mod.rs +++ b/apps/keck/src/server/mod.rs @@ -4,18 +4,14 @@ mod subscribe; mod sync; mod utils; +use std::{collections::HashMap, net::SocketAddr, sync::Arc, thread::sleep}; + use api::Context; use axum::{http::Method, Extension, Router, Server}; use jwst::Workspace; -use std::{ - collections::HashMap, - thread::sleep, - {net::SocketAddr, sync::Arc}, -}; +pub use subscribe::*; use tokio::{runtime, signal, sync::RwLock}; use tower_http::cors::{Any, CorsLayer}; - -pub use subscribe::*; pub use utils::*; async fn shutdown_signal() { diff --git a/apps/keck/src/server/subscribe.rs b/apps/keck/src/server/subscribe.rs index 037e77f7d..766d48e0c 100644 --- a/apps/keck/src/server/subscribe.rs +++ b/apps/keck/src/server/subscribe.rs @@ -1,10 +1,7 @@ +use std::{collections::HashMap, sync::Arc, thread, time::Duration}; + use reqwest::Client; -use std::collections::HashMap; -use std::sync::Arc; -use std::thread; -use std::time::Duration; -use tokio::runtime::Runtime; -use tokio::sync::RwLock; +use tokio::{runtime::Runtime, sync::RwLock}; use super::*; use crate::server::WorkspaceChangedBlocks; diff --git a/apps/keck/src/server/sync/blobs.rs b/apps/keck/src/server/sync/blobs.rs index f44852fb3..7d66a2bf5 100644 --- a/apps/keck/src/server/sync/blobs.rs +++ b/apps/keck/src/server/sync/blobs.rs @@ -1,4 +1,5 @@ -use super::*; +use std::sync::Arc; + use axum::{ extract::{BodyStream, Path}, headers::ContentLength, @@ -12,9 +13,10 @@ use axum::{ use futures::{future, StreamExt}; use jwst::BlobStorage; use jwst_rpc::RpcContextImpl; -use std::sync::Arc; use time::{format_description::well_known::Rfc2822, OffsetDateTime}; +use super::*; + #[derive(Serialize)] struct BlobStatus { exists: bool, @@ -71,12 +73,7 @@ impl Context { return header.into_response(); }; - let Ok(file) = self - .get_storage() - .blobs() - .get_blob(workspace, id, None) - .await - else { + let Ok(file) = self.get_storage().blobs().get_blob(workspace, id, None).await else { return StatusCode::NOT_FOUND.into_response(); }; diff --git a/apps/keck/src/server/sync/collaboration.rs b/apps/keck/src/server/sync/collaboration.rs index 2266b0e76..2dbb5e097 100644 --- a/apps/keck/src/server/sync/collaboration.rs +++ b/apps/keck/src/server/sync/collaboration.rs @@ -1,4 +1,5 @@ -use super::*; +use std::sync::Arc; + #[cfg(feature = "websocket")] use axum::extract; use axum::{ @@ -11,10 +12,11 @@ use jwst_rpc::{axum_socket_connector, handle_connector}; #[cfg(feature = "websocket")] use jwst_rpc::{webrtc_datachannel_server_connector, RTCSessionDescription}; use serde::Serialize; -use std::sync::Arc; #[cfg(feature = "websocket")] use tokio::sync::mpsc::channel; +use super::*; + #[derive(Serialize)] pub struct WebSocketAuthentication { protocol: String, @@ -77,8 +79,15 @@ pub async fn webrtc_handler( #[cfg(test)] mod test { - use jwst::DocStorage; - use jwst::{Block, Workspace}; + use std::{ + ffi::c_int, + io::{BufRead, BufReader}, + process::{Child, Command, Stdio}, + string::String, + sync::Arc, + }; + + use jwst::{Block, DocStorage, Workspace}; use jwst_logger::info; #[cfg(feature = "websocket")] use jwst_rpc::start_webrtc_client_sync; @@ -86,11 +95,6 @@ mod test { use jwst_storage::{BlobStorageType, JwstStorage}; use libc::{kill, SIGTERM}; use rand::{thread_rng, Rng}; - use std::ffi::c_int; - use std::io::{BufRead, BufReader}; - use std::process::{Child, Command, Stdio}; - use std::string::String; - use std::sync::Arc; use tokio::runtime::Runtime; struct TestContext { diff --git a/apps/keck/src/server/sync/mod.rs b/apps/keck/src/server/sync/mod.rs index 78927a968..92cd59598 100644 --- a/apps/keck/src/server/sync/mod.rs +++ b/apps/keck/src/server/sync/mod.rs @@ -1,9 +1,10 @@ mod blobs; mod collaboration; -use super::*; use axum::routing::{get, post, put}; +use super::*; + pub fn sync_handler(router: Router) -> Router { let router = if cfg!(feature = "api") { router diff --git a/libs/cloud-database/migration/src/m20230101_000002_create_google_user_table.rs b/libs/cloud-database/migration/src/m20230101_000002_create_google_user_table.rs index 1a5c1947f..3554d9a5e 100644 --- a/libs/cloud-database/migration/src/m20230101_000002_create_google_user_table.rs +++ b/libs/cloud-database/migration/src/m20230101_000002_create_google_user_table.rs @@ -1,6 +1,7 @@ -use super::m20220101_000001_create_user_table::Users; use sea_orm_migration::prelude::*; +use super::m20220101_000001_create_user_table::Users; + #[derive(DeriveMigrationName)] pub struct Migration; diff --git a/libs/cloud-database/migration/src/m20230101_000004_create_permissions_table.rs b/libs/cloud-database/migration/src/m20230101_000004_create_permissions_table.rs index 8920a938b..447bd4b72 100644 --- a/libs/cloud-database/migration/src/m20230101_000004_create_permissions_table.rs +++ b/libs/cloud-database/migration/src/m20230101_000004_create_permissions_table.rs @@ -1,7 +1,7 @@ -use crate::m20220101_000001_create_user_table::Users; +use sea_orm_migration::prelude::*; use super::m20230101_000003_create_workspaces_table::Workspaces; -use sea_orm_migration::prelude::*; +use crate::m20220101_000001_create_user_table::Users; #[derive(DeriveMigrationName)] pub struct Migration; @@ -84,9 +84,9 @@ pub enum Permissions { UserEmail, // TEXT, Type, // SMALLINT NOT NULL, Accepted, // BOOL DEFAULT False, - CreatedAt, // TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - // FOREIGN KEY(workspace_id) REFERENCES workspaces(id), - // FOREIGN KEY(user_id) REFERENCES users(id), - // UNIQUE (workspace_id, user_id), - // UNIQUE (workspace_id, user_email) + CreatedAt, /* TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + * FOREIGN KEY(workspace_id) REFERENCES workspaces(id), + * FOREIGN KEY(user_id) REFERENCES users(id), + * UNIQUE (workspace_id, user_id), + * UNIQUE (workspace_id, user_email) */ } diff --git a/libs/cloud-database/migration/src/m20230217_000001_update_permissions_table.rs b/libs/cloud-database/migration/src/m20230217_000001_update_permissions_table.rs index 6a6eb18a0..c497c5bc4 100644 --- a/libs/cloud-database/migration/src/m20230217_000001_update_permissions_table.rs +++ b/libs/cloud-database/migration/src/m20230217_000001_update_permissions_table.rs @@ -1,8 +1,6 @@ -use crate::m20220101_000001_create_user_table::Users; -use crate::m20230101_000004_create_permissions_table::Permissions; +use sea_orm_migration::{prelude::*, sea_orm::query::*}; -use sea_orm_migration::prelude::*; -use sea_orm_migration::sea_orm::query::*; +use crate::{m20220101_000001_create_user_table::Users, m20230101_000004_create_permissions_table::Permissions}; #[derive(DeriveMigrationName)] pub struct Migration; diff --git a/libs/cloud-database/src/database.rs b/libs/cloud-database/src/database.rs index a5928359f..faab0012d 100644 --- a/libs/cloud-database/src/database.rs +++ b/libs/cloud-database/src/database.rs @@ -1,3 +1,8 @@ +use affine_cloud_migration::{Expr, JoinType, Migrator, MigratorTrait, Query}; +use jwst_logger::{info, instrument, tracing}; +use nanoid::nanoid; +use sea_orm::{prelude::*, ConnectionTrait, Database, DatabaseTransaction, QuerySelect, Set, TransactionTrait}; + use super::{ model::{ CreateUser, FirebaseClaims, Member, MemberResult, PermissionType, RefreshToken, UpdateWorkspace, User, @@ -5,10 +10,6 @@ use super::{ }, *, }; -use affine_cloud_migration::{Expr, JoinType, Migrator, MigratorTrait, Query}; -use jwst_logger::{info, instrument, tracing}; -use nanoid::nanoid; -use sea_orm::{prelude::*, ConnectionTrait, Database, DatabaseTransaction, QuerySelect, Set, TransactionTrait}; // #[derive(FromRow)] // struct PermissionQuery { diff --git a/libs/cloud-database/src/entities/prelude.rs b/libs/cloud-database/src/entities/prelude.rs index 5dae92ece..f1eef61f0 100644 --- a/libs/cloud-database/src/entities/prelude.rs +++ b/libs/cloud-database/src/entities/prelude.rs @@ -1,6 +1,6 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.7 -pub use super::google_users::Entity as GoogleUsers; -pub use super::permissions::Entity as Permissions; -pub use super::users::Entity as Users; -pub use super::workspaces::Entity as Workspaces; +pub use super::{ + google_users::Entity as GoogleUsers, permissions::Entity as Permissions, users::Entity as Users, + workspaces::Entity as Workspaces, +}; diff --git a/libs/cloud-database/src/lib.rs b/libs/cloud-database/src/lib.rs index 043fe2848..0bec7099e 100644 --- a/libs/cloud-database/src/lib.rs +++ b/libs/cloud-database/src/lib.rs @@ -4,9 +4,8 @@ mod entities; mod model; pub use database::CloudDatabase; -pub use model::*; - use entities::prelude::*; +pub use model::*; use sea_orm::EntityTrait; type UsersModel = ::Model; diff --git a/libs/cloud-database/src/model.rs b/libs/cloud-database/src/model.rs index 45e33d01f..e3397e6ee 100644 --- a/libs/cloud-database/src/model.rs +++ b/libs/cloud-database/src/model.rs @@ -1,8 +1,10 @@ // use super::*; // use sqlx::{postgres::PgRow, FromRow, Result, Row}; -use chrono::naive::serde::{ts_milliseconds, ts_seconds}; -use chrono::{DateTime, Utc}; +use chrono::{ + naive::serde::{ts_milliseconds, ts_seconds}, + DateTime, Utc, +}; use jwst_logger::error; use schemars::{JsonSchema, JsonSchema_repr}; use sea_orm::{FromQueryResult, TryGetable}; diff --git a/libs/cloud-infra/src/auth.rs b/libs/cloud-infra/src/auth.rs index ada96e006..c33281752 100644 --- a/libs/cloud-infra/src/auth.rs +++ b/libs/cloud-infra/src/auth.rs @@ -1,6 +1,6 @@ -use super::*; -use aes_gcm::aead::Aead; -use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; +use std::collections::HashMap; + +use aes_gcm::{aead::Aead, Aes256Gcm, KeyInit, Nonce}; use axum::headers::{CacheControl, HeaderMapExt}; use chrono::{Duration, NaiveDateTime, Utc}; use cloud_database::{Claims, FirebaseClaims}; @@ -12,10 +12,11 @@ use pem::{encode as encode_pem, Pem}; use rand::{thread_rng, Rng}; use reqwest::{Client, RequestBuilder}; use sha2::{Digest, Sha256}; -use std::collections::HashMap; use thiserror::Error; use x509_parser::prelude::{nom::Err as NomErr, parse_x509_pem, PEMError, X509Error}; +use super::*; + #[derive(Debug, Error)] pub enum KeyError { #[error("base64 error")] diff --git a/libs/cloud-infra/src/hosting/api_doc.rs b/libs/cloud-infra/src/hosting/api_doc.rs index 51382dde0..cc83d5348 100644 --- a/libs/cloud-infra/src/hosting/api_doc.rs +++ b/libs/cloud-infra/src/hosting/api_doc.rs @@ -1,5 +1,6 @@ -use axum::{extract::Path, http::StatusCode, response::IntoResponse, routing::get, Extension, Json, Router}; use std::{env, sync::Arc}; + +use axum::{extract::Path, http::StatusCode, response::IntoResponse, routing::get, Extension, Json, Router}; use utoipa::openapi::{License, OpenApi}; use utoipa_swagger_ui::{serve, Config, Url}; diff --git a/libs/cloud-infra/src/hosting/files.rs b/libs/cloud-infra/src/hosting/files.rs index a4ed29709..a65542759 100644 --- a/libs/cloud-infra/src/hosting/files.rs +++ b/libs/cloud-infra/src/hosting/files.rs @@ -1,6 +1,7 @@ -use super::{pages::*, *}; use url_escape::decode; +use super::{pages::*, *}; + pub async fn fetch_static_response(uri: Uri, sap: bool, fetcher: Option) -> impl IntoResponse { let path = uri.path().trim_start_matches('/'); diff --git a/libs/cloud-infra/src/hosting/mod.rs b/libs/cloud-infra/src/hosting/mod.rs index b457bebd2..494f66b09 100644 --- a/libs/cloud-infra/src/hosting/mod.rs +++ b/libs/cloud-infra/src/hosting/mod.rs @@ -2,7 +2,6 @@ mod api_doc; mod files; mod pages; -use super::*; use axum::{ body::{boxed, BoxBody, Full}, http::{header::CONTENT_TYPE, StatusCode, Uri}, @@ -10,6 +9,8 @@ use axum::{ }; use rust_embed::EmbeddedFile; +use super::*; + type StaticFileFetcher = fn(&str) -> Option; pub use api_doc::with_api_doc; diff --git a/libs/cloud-infra/src/lib.rs b/libs/cloud-infra/src/lib.rs index c68221f80..36db0faaa 100644 --- a/libs/cloud-infra/src/lib.rs +++ b/libs/cloud-infra/src/lib.rs @@ -5,8 +5,7 @@ mod hosting; mod mail; pub use auth::{FirebaseContext, KeyContext}; +use constants::*; pub use hosting::{fetch_static_response, rust_embed, with_api_doc, RustEmbed}; pub use mail::{Mail, MailContext}; - -use constants::*; use nanoid::nanoid; diff --git a/libs/cloud-infra/src/mail.rs b/libs/cloud-infra/src/mail.rs index c9ded0ddd..a58d5d449 100644 --- a/libs/cloud-infra/src/mail.rs +++ b/libs/cloud-infra/src/mail.rs @@ -1,21 +1,20 @@ -pub use lettre::transport::smtp::commands::Mail; - -use super::*; use chrono::prelude::*; use cloud_database::Claims; use handlebars::{Handlebars, RenderError}; use jwst::{warn, Base64Engine, WorkspaceMetadata, STANDARD_ENGINE}; +pub use lettre::transport::smtp::commands::Mail; use lettre::{ error::Error as MailConfigError, message::{Mailbox, MultiPart, SinglePart}, - transport::smtp::authentication::Credentials, - transport::smtp::Error as MailSmtpError, + transport::smtp::{authentication::Credentials, Error as MailSmtpError}, AsyncSmtpTransport, AsyncTransport, Message, Tokio1Executor, }; use serde::Serialize; use thiserror::Error; use url::Url; +use super::*; + #[derive(Debug, Error)] pub enum MailError { #[error("Mail client not initialized")] @@ -67,7 +66,10 @@ impl MailContext { if client.is_none() { warn!("!!! no mail account provided, email will not be sent !!!"); - warn!("!!! please set MAIL_ACCOUNT and MAIL_PASSWORD in .env file or environmental variable to enable email service !!!"); + warn!( + "!!! please set MAIL_ACCOUNT and MAIL_PASSWORD in .env file or environmental variable to enable email \ + service !!!" + ); } let mail_box = MAIL_FROM.parse().expect("should provide valid mail from"); diff --git a/libs/jwst-binding/jwst-core-swift/build.rs b/libs/jwst-binding/jwst-core-swift/build.rs index 1ea6edd07..cc99dbdef 100644 --- a/libs/jwst-binding/jwst-core-swift/build.rs +++ b/libs/jwst-binding/jwst-core-swift/build.rs @@ -9,6 +9,5 @@ fn main() { } println!("cargo:rerun-if-env-changed={XCODE_CONFIGURATION_ENV}"); - swift_bridge_build::parse_bridges(bridges) - .write_all_concatenated(out_dir, env!("CARGO_PKG_NAME")); + swift_bridge_build::parse_bridges(bridges).write_all_concatenated(out_dir, env!("CARGO_PKG_NAME")); } diff --git a/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs b/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs index 3733bc14c..37e70c411 100644 --- a/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs +++ b/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs @@ -1,8 +1,6 @@ -use std::collections::HashMap; -use std::path::PathBuf; -use std::process::Command; -use swift_bridge_build::ApplePlatform as Platform; -use swift_bridge_build::{create_package, CreatePackageConfig}; +use std::{collections::HashMap, path::PathBuf, process::Command}; + +use swift_bridge_build::{create_package, ApplePlatform as Platform, CreatePackageConfig}; fn main() { let common_commands = [ @@ -35,11 +33,7 @@ fn main() { }) .status() .expect("Failed to build jwst-core-swift"); - let dir = if cfg!(debug_assertions) { - "debug" - } else { - "release" - }; + let dir = if cfg!(debug_assertions) { "debug" } else { "release" }; create_package(CreatePackageConfig { bridge_dir: PathBuf::from("libs/jwst-binding/jwst-core-swift/generated"), paths: HashMap::from([ diff --git a/libs/jwst-binding/jwst-core-swift/src/block.rs b/libs/jwst-binding/jwst-core-swift/src/block.rs index 9e81cc7ba..d55e59d1e 100644 --- a/libs/jwst-binding/jwst-core-swift/src/block.rs +++ b/libs/jwst-binding/jwst-core-swift/src/block.rs @@ -1,6 +1,7 @@ -use super::*; use jwst_core::{Any, Block as JwstBlock}; +use super::*; + pub struct Block { pub(crate) block: JwstBlock, } @@ -22,9 +23,7 @@ impl Block { let mut block = self.block.clone(); let mut target_block = target_block.block.clone(); - block - .push_children(&mut target_block) - .expect("failed to push children"); + block.push_children(&mut target_block).expect("failed to push children"); } pub fn insert_children_at(&self, target_block: &Block, pos: u32) { @@ -64,10 +63,7 @@ impl Block { } pub fn exists_children(&self, block_id: &str) -> i32 { - self.block - .exists_children(block_id) - .map(|i| i as i32) - .unwrap_or(-1) + self.block.exists_children(block_id).map(|i| i as i32).unwrap_or(-1) } pub fn parent(&self) -> String { diff --git a/libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs b/libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs index 453fc0983..5e5f7f5be 100644 --- a/libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs +++ b/libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs @@ -1,6 +1,7 @@ -use jwst_core::Any; use std::collections::HashMap; +use jwst_core::Any; + pub type DynamicValueMap = HashMap; pub struct DynamicValue { diff --git a/libs/jwst-binding/jwst-core-swift/src/lib.rs b/libs/jwst-binding/jwst-core-swift/src/lib.rs index 9d96d84c3..8bd239c8c 100644 --- a/libs/jwst-binding/jwst-core-swift/src/lib.rs +++ b/libs/jwst-binding/jwst-core-swift/src/lib.rs @@ -5,11 +5,10 @@ mod workspace; pub use block::Block; pub use dynamic_value::{DynamicValue, DynamicValueMap}; -pub use storage::Storage; -pub use workspace::Workspace; - use jwst_core::{error, warn, JwstError}; use jwst_logger::init_logger_with; +pub use storage::Storage; +pub use workspace::Workspace; type JwstWorkSpaceResult = Result; diff --git a/libs/jwst-binding/jwst-core-swift/src/storage.rs b/libs/jwst-binding/jwst-core-swift/src/storage.rs index 3cfbac253..b48be4560 100644 --- a/libs/jwst-binding/jwst-core-swift/src/storage.rs +++ b/libs/jwst-binding/jwst-core-swift/src/storage.rs @@ -1,16 +1,16 @@ -use super::*; +use std::sync::{Arc, RwLock}; + use futures::TryFutureExt; -use jwst_core_rpc::{ - start_websocket_client_sync, BroadcastChannels, CachedLastSynced, RpcContextImpl, SyncState, -}; +use jwst_core_rpc::{start_websocket_client_sync, BroadcastChannels, CachedLastSynced, RpcContextImpl, SyncState}; use jwst_core_storage::{BlobStorageType, JwstStorage as AutoStorage, JwstStorageResult}; use nanoid::nanoid; -use std::sync::{Arc, RwLock}; use tokio::{ runtime::{Builder, Runtime}, sync::mpsc::channel, }; +use super::*; + #[derive(Clone)] pub struct Storage { storage: Arc, @@ -35,15 +35,8 @@ impl Storage { let storage = rt .block_on( - AutoStorage::new_with_migration( - &format!("sqlite:{path}?mode=rwc"), - BlobStorageType::DB, - ) - .or_else(|e| { - warn!( - "Failed to open storage, falling back to memory storage: {}", - e - ); + AutoStorage::new_with_migration(&format!("sqlite:{path}?mode=rwc"), BlobStorageType::DB).or_else(|e| { + warn!("Failed to open storage, falling back to memory storage: {}", e); AutoStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) }), ) @@ -160,9 +153,10 @@ impl RpcContextImpl<'_> for Storage { #[cfg(test)] mod tests { - use crate::{Storage, Workspace}; use tokio::runtime::Runtime; + use crate::{Storage, Workspace}; + #[test] #[ignore = "need manually start collaboration server"] fn collaboration_test() { @@ -175,8 +169,7 @@ mod tests { let resp = get_block_from_server(workspace_id.to_string(), block.id().to_string()); assert!(!resp.is_empty()); - let prop_extractor = - r#"("prop:bool_prop":true)|("prop:float_prop":1\.0)|("sys:children":\["2"\])"#; + let prop_extractor = r#"("prop:bool_prop":true)|("prop:float_prop":1\.0)|("sys:children":\["2"\])"#; let re = regex::Regex::new(prop_extractor).unwrap(); assert_eq!(re.find_iter(resp.as_str()).count(), 3); } @@ -200,10 +193,7 @@ mod tests { rt.block_on(async { let client = reqwest::Client::builder().no_proxy().build().unwrap(); let resp = client - .get(format!( - "http://localhost:3000/api/block/{}/{}", - workspace_id, block_id - )) + .get(format!("http://localhost:3000/api/block/{}/{}", workspace_id, block_id)) .send() .await .unwrap(); diff --git a/libs/jwst-binding/jwst-core-swift/src/workspace.rs b/libs/jwst-binding/jwst-core-swift/src/workspace.rs index b2e87c4e9..2bd254592 100644 --- a/libs/jwst-binding/jwst-core-swift/src/workspace.rs +++ b/libs/jwst-binding/jwst-core-swift/src/workspace.rs @@ -1,6 +1,7 @@ -use super::*; use jwst_core::Workspace as JwstWorkspace; +use super::*; + pub struct Workspace { pub(crate) workspace: JwstWorkspace, } @@ -51,12 +52,7 @@ impl Workspace { workspace .get_blocks() - .map(|s| { - s.get_blocks_by_flavour(flavour) - .into_iter() - .map(Block::new) - .collect() - }) + .map(|s| s.get_blocks_by_flavour(flavour).into_iter().map(Block::new).collect()) .unwrap_or_default() } diff --git a/libs/jwst-binding/jwst-jni/build.rs b/libs/jwst-binding/jwst-jni/build.rs index 4daa34d14..ed9ee699e 100644 --- a/libs/jwst-binding/jwst-jni/build.rs +++ b/libs/jwst-binding/jwst-jni/build.rs @@ -1,7 +1,7 @@ +use std::{env, fs, path::Path}; + use flapigen::{JavaConfig, LanguageConfig}; use rifgen::{Generator, Language, TypeCases}; -use std::path::Path; -use std::{env, fs}; fn main() { let out_dir = env::var("OUT_DIR").unwrap(); @@ -104,7 +104,8 @@ foreign_class!( fs::write(in_src, template).unwrap(); - //delete the lib folder then create it again to prevent obsolete files from staying + //delete the lib folder then create it again to prevent obsolete files from + // staying let java_folder = Path::new("android").join("src/main/java/com/toeverything/jwst/lib"); if java_folder.exists() { fs::remove_dir_all(&java_folder).unwrap(); diff --git a/libs/jwst-binding/jwst-jni/src/block.rs b/libs/jwst-binding/jwst-jni/src/block.rs index e74668d90..599c74427 100644 --- a/libs/jwst-binding/jwst-jni/src/block.rs +++ b/libs/jwst-binding/jwst-jni/src/block.rs @@ -1,6 +1,7 @@ -use super::{generate_interface, JwstBlock, WorkspaceTransaction}; use lib0::any::Any; +use super::{generate_interface, JwstBlock, WorkspaceTransaction}; + pub struct Block { pub(crate) block: JwstBlock, diff --git a/libs/jwst-binding/jwst-jni/src/difflog.rs b/libs/jwst-binding/jwst-jni/src/difflog.rs index 34dee3f84..b33fe6e57 100644 --- a/libs/jwst-binding/jwst-jni/src/difflog.rs +++ b/libs/jwst-binding/jwst-jni/src/difflog.rs @@ -1,14 +1,16 @@ -use super::*; +use std::{sync::Arc, time::Duration}; + use chrono::{DateTime, Utc}; use jwst_storage::JwstStorage; use serde::Serialize; -use std::{sync::Arc, time::Duration}; use tokio::{ runtime::Runtime, sync::{mpsc::Receiver, Mutex}, time::sleep, }; +use super::*; + #[derive(Clone, Debug, Serialize)] pub struct Log { content: String, diff --git a/libs/jwst-binding/jwst-jni/src/lib.rs b/libs/jwst-binding/jwst-jni/src/lib.rs index c3d12671a..fc62fbc99 100644 --- a/libs/jwst-binding/jwst-jni/src/lib.rs +++ b/libs/jwst-binding/jwst-jni/src/lib.rs @@ -6,8 +6,6 @@ mod storage; mod transaction; mod workspace; -pub use crate::java_glue::*; - use block::Block; use block_observer::BlockObserver; use difflog::{CachedDiffLog, Log}; @@ -19,3 +17,5 @@ use rifgen::rifgen_attr::*; use storage::JwstStorage; use transaction::{OnWorkspaceTransaction, WorkspaceTransaction}; use workspace::Workspace; + +pub use crate::java_glue::*; diff --git a/libs/jwst-binding/jwst-jni/src/storage.rs b/libs/jwst-binding/jwst-jni/src/storage.rs index 33701350f..2d55edc33 100644 --- a/libs/jwst-binding/jwst-jni/src/storage.rs +++ b/libs/jwst-binding/jwst-jni/src/storage.rs @@ -1,16 +1,18 @@ -use super::*; +use std::sync::{Arc, RwLock}; + use android_logger::Config; use futures::TryFutureExt; use jwst_rpc::{start_websocket_client_sync, BroadcastChannels, CachedLastSynced, RpcContextImpl, SyncState}; use jwst_storage::{BlobStorageType, JwstStorage as AutoStorage, JwstStorageResult}; use nanoid::nanoid; -use std::sync::{Arc, RwLock}; use tokio::{ runtime::{Builder, Runtime}, sync::mpsc::channel, }; use yrs::{ReadTxn, StateVector, Transact}; +use super::*; + #[derive(Clone)] pub struct JwstStorage { storage: Arc, diff --git a/libs/jwst-binding/jwst-jni/src/workspace.rs b/libs/jwst-binding/jwst-jni/src/workspace.rs index 6126b0300..b393cec5f 100644 --- a/libs/jwst-binding/jwst-jni/src/workspace.rs +++ b/libs/jwst-binding/jwst-jni/src/workspace.rs @@ -1,9 +1,11 @@ -use super::*; -use crate::block_observer::{BlockObserver, BlockObserverWrapper}; -use jwst_rpc::workspace_compare; use std::sync::Arc; + +use jwst_rpc::workspace_compare; use tokio::{runtime::Runtime, sync::mpsc::Sender}; +use super::*; +use crate::block_observer::{BlockObserver, BlockObserverWrapper}; + pub struct Workspace { pub(crate) workspace: JwstWorkspace, pub(crate) jwst_workspace: Option, diff --git a/libs/jwst-binding/jwst-swift/jwst-swift-integrate/src/main.rs b/libs/jwst-binding/jwst-swift/jwst-swift-integrate/src/main.rs index 4c152cdd4..f7b2f3d3c 100644 --- a/libs/jwst-binding/jwst-swift/jwst-swift-integrate/src/main.rs +++ b/libs/jwst-binding/jwst-swift/jwst-swift-integrate/src/main.rs @@ -1,8 +1,6 @@ -use std::collections::HashMap; -use std::path::PathBuf; -use std::process::Command; -use swift_bridge_build::ApplePlatform as Platform; -use swift_bridge_build::{create_package, CreatePackageConfig}; +use std::{collections::HashMap, path::PathBuf, process::Command}; + +use swift_bridge_build::{create_package, ApplePlatform as Platform, CreatePackageConfig}; fn main() { let common_commands = [ diff --git a/libs/jwst-binding/jwst-swift/src/block.rs b/libs/jwst-binding/jwst-swift/src/block.rs index 5c257bb50..c1f87e093 100644 --- a/libs/jwst-binding/jwst-swift/src/block.rs +++ b/libs/jwst-binding/jwst-swift/src/block.rs @@ -1,10 +1,12 @@ -use super::*; +use std::sync::Arc; + use jwst::{Block as JwstBlock, Workspace}; use jwst_rpc::workspace_compare; use lib0::any::Any; -use std::sync::Arc; use tokio::{runtime::Runtime, sync::mpsc::Sender}; +use super::*; + pub struct Block { pub workspace: Workspace, pub block: JwstBlock, diff --git a/libs/jwst-binding/jwst-swift/src/difflog.rs b/libs/jwst-binding/jwst-swift/src/difflog.rs index 34dee3f84..b33fe6e57 100644 --- a/libs/jwst-binding/jwst-swift/src/difflog.rs +++ b/libs/jwst-binding/jwst-swift/src/difflog.rs @@ -1,14 +1,16 @@ -use super::*; +use std::{sync::Arc, time::Duration}; + use chrono::{DateTime, Utc}; use jwst_storage::JwstStorage; use serde::Serialize; -use std::{sync::Arc, time::Duration}; use tokio::{ runtime::Runtime, sync::{mpsc::Receiver, Mutex}, time::sleep, }; +use super::*; + #[derive(Clone, Debug, Serialize)] pub struct Log { content: String, diff --git a/libs/jwst-binding/jwst-swift/src/dynamic_value.rs b/libs/jwst-binding/jwst-swift/src/dynamic_value.rs index 37b5e62f3..754ae9141 100644 --- a/libs/jwst-binding/jwst-swift/src/dynamic_value.rs +++ b/libs/jwst-binding/jwst-swift/src/dynamic_value.rs @@ -1,6 +1,7 @@ -use lib0::any::Any; use std::collections::HashMap; +use lib0::any::Any; + pub type DynamicValueMap = HashMap; pub struct DynamicValue { diff --git a/libs/jwst-binding/jwst-swift/src/lib.rs b/libs/jwst-binding/jwst-swift/src/lib.rs index 5c296d585..3011b8dca 100644 --- a/libs/jwst-binding/jwst-swift/src/lib.rs +++ b/libs/jwst-binding/jwst-swift/src/lib.rs @@ -5,13 +5,12 @@ mod storage; mod workspace; pub use block::Block; -pub use dynamic_value::{DynamicValue, DynamicValueMap}; -pub use storage::Storage; -pub use workspace::Workspace; - use difflog::{CachedDiffLog, Log}; +pub use dynamic_value::{DynamicValue, DynamicValueMap}; use jwst::{error, info, warn, JwstError}; use jwst_logger::init_logger_with; +pub use storage::Storage; +pub use workspace::Workspace; type JwstWorkSpaceResult = Result; diff --git a/libs/jwst-binding/jwst-swift/src/storage.rs b/libs/jwst-binding/jwst-swift/src/storage.rs index 07ad75dd6..f078b2ac7 100644 --- a/libs/jwst-binding/jwst-swift/src/storage.rs +++ b/libs/jwst-binding/jwst-swift/src/storage.rs @@ -1,15 +1,17 @@ -use super::*; +use std::sync::{Arc, RwLock}; + use futures::TryFutureExt; use jwst_rpc::{start_websocket_client_sync, BroadcastChannels, CachedLastSynced, RpcContextImpl, SyncState}; use jwst_storage::{BlobStorageType, JwstStorage as AutoStorage, JwstStorageResult}; use nanoid::nanoid; -use std::sync::{Arc, RwLock}; use tokio::{ runtime::{Builder, Runtime}, sync::mpsc::channel, }; use yrs::{ReadTxn, StateVector, Transact}; +use super::*; + #[derive(Clone)] pub struct Storage { storage: Arc, @@ -198,9 +200,10 @@ impl RpcContextImpl<'_> for Storage { #[cfg(test)] mod tests { - use crate::{Storage, Workspace}; use tokio::runtime::Runtime; + use crate::{Storage, Workspace}; + #[test] #[ignore = "need manually start collaboration server"] fn collaboration_test() { diff --git a/libs/jwst-binding/jwst-swift/src/workspace.rs b/libs/jwst-binding/jwst-swift/src/workspace.rs index f14a31dae..ff2f75906 100644 --- a/libs/jwst-binding/jwst-swift/src/workspace.rs +++ b/libs/jwst-binding/jwst-swift/src/workspace.rs @@ -1,12 +1,14 @@ -use super::*; +use std::sync::Arc; + use jwst::Workspace as JwstWorkspace; use jwst_rpc::workspace_compare; -use std::sync::Arc; use tokio::{ runtime::Runtime, sync::mpsc::{channel, Sender}, }; +use super::*; + pub struct Workspace { pub(crate) workspace: JwstWorkspace, pub(crate) jwst_workspace: Option, @@ -89,8 +91,10 @@ impl Workspace { .and_then(|mut b| b.create_ffi(block_id, flavour, created)) .expect("failed to create jwst block"); - // let ret = workspace_compare(&workspace, &jwst_workspace); - // sender.send(Log::new(workspace.id(), ret)).unwrap(); + // let ret = workspace_compare(&workspace, + // &jwst_workspace); + // sender.send(Log::new(workspace.id(), + // ret)).unwrap(); } jwst_workspace diff --git a/libs/jwst-codec-util/src/doc_operation/yrs_op/array.rs b/libs/jwst-codec-util/src/doc_operation/yrs_op/array.rs index 6447e2c3a..419484d95 100644 --- a/libs/jwst-codec-util/src/doc_operation/yrs_op/array.rs +++ b/libs/jwst-codec-util/src/doc_operation/yrs_op/array.rs @@ -1,6 +1,7 @@ -use super::*; use phf::phf_map; +use super::*; + fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { let array = match nest_input { YrsNestType::ArrayType(array) => array, @@ -93,9 +94,10 @@ pub fn yrs_create_array_from_nest_type( #[cfg(test)] mod tests { - use super::*; use yrs::Doc; + use super::*; + #[test] fn test_gen_array_ref_ops() { let doc = Doc::new(); diff --git a/libs/jwst-codec-util/src/doc_operation/yrs_op/map.rs b/libs/jwst-codec-util/src/doc_operation/yrs_op/map.rs index cf0f9c145..666c36273 100644 --- a/libs/jwst-codec-util/src/doc_operation/yrs_op/map.rs +++ b/libs/jwst-codec-util/src/doc_operation/yrs_op/map.rs @@ -1,6 +1,7 @@ -use super::*; use phf::phf_map; +use super::*; + fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { let map = match nest_input { YrsNestType::MapType(map) => map, @@ -99,9 +100,10 @@ pub fn yrs_create_map_from_nest_type( #[cfg(test)] mod tests { - use super::*; use yrs::Doc; + use super::*; + #[test] fn test_gen_map_ref_ops() { let doc = Doc::new(); diff --git a/libs/jwst-codec-util/src/doc_operation/yrs_op/mod.rs b/libs/jwst-codec-util/src/doc_operation/yrs_op/mod.rs index 6f1fb9424..749ea96f2 100644 --- a/libs/jwst-codec-util/src/doc_operation/yrs_op/mod.rs +++ b/libs/jwst-codec-util/src/doc_operation/yrs_op/mod.rs @@ -5,12 +5,7 @@ pub mod xml_element; pub mod xml_fragment; pub mod xml_text; -use super::*; use std::collections::HashMap; -use yrs::{ - Array, ArrayPrelim, ArrayRef, Doc, GetString, Map, MapPrelim, MapRef, Text, TextPrelim, TextRef, Transact, - XmlFragment, XmlTextPrelim, -}; use array::*; use map::*; @@ -18,6 +13,12 @@ use text::*; use xml_element::*; use xml_fragment::*; use xml_text::*; +use yrs::{ + Array, ArrayPrelim, ArrayRef, Doc, GetString, Map, MapPrelim, MapRef, Text, TextPrelim, TextRef, Transact, + XmlFragment, XmlTextPrelim, +}; + +use super::*; type TestOp = fn(doc: &Doc, nest_input: &YrsNestType, params: CRDTParam) -> (); type TestOps = phf::Map<&'static str, TestOp>; diff --git a/libs/jwst-codec-util/src/doc_operation/yrs_op/text.rs b/libs/jwst-codec-util/src/doc_operation/yrs_op/text.rs index 4336005b0..f79b0f50f 100644 --- a/libs/jwst-codec-util/src/doc_operation/yrs_op/text.rs +++ b/libs/jwst-codec-util/src/doc_operation/yrs_op/text.rs @@ -1,6 +1,7 @@ -use super::*; use phf::phf_map; +use super::*; + fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { let text = match nest_input { YrsNestType::TextType(text) => text, diff --git a/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_element.rs b/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_element.rs index 28bea961d..19cb7ed8f 100644 --- a/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_element.rs +++ b/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_element.rs @@ -1,6 +1,7 @@ -use super::*; use phf::phf_map; +use super::*; + fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { let xml_element = match nest_input { YrsNestType::XMLElementType(xml_element) => xml_element, diff --git a/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_fragment.rs b/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_fragment.rs index e4fb41635..24ead2c99 100644 --- a/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_fragment.rs +++ b/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_fragment.rs @@ -1,6 +1,7 @@ -use super::*; use phf::phf_map; +use super::*; + fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { let xml_fragment = match nest_input { YrsNestType::XMLFragmentType(xml_fragment) => xml_fragment, diff --git a/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_text.rs b/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_text.rs index 3c9581194..792946ae1 100644 --- a/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_text.rs +++ b/libs/jwst-codec-util/src/doc_operation/yrs_op/xml_text.rs @@ -1,6 +1,7 @@ -use super::*; use phf::phf_map; +use super::*; + fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { let xml_text = match nest_input { YrsNestType::XMLTextType(xml_text) => xml_text, diff --git a/libs/jwst-codec/benches/array_ops_benchmarks.rs b/libs/jwst-codec/benches/array_ops_benchmarks.rs index fede5e209..bcf15216e 100644 --- a/libs/jwst-codec/benches/array_ops_benchmarks.rs +++ b/libs/jwst-codec/benches/array_ops_benchmarks.rs @@ -1,6 +1,7 @@ +use std::time::Duration; + use criterion::{criterion_group, criterion_main, Criterion}; use rand::{Rng, SeedableRng}; -use std::time::Duration; fn operations(c: &mut Criterion) { let mut group = c.benchmark_group("ops/array"); diff --git a/libs/jwst-codec/benches/codec_benchmarks.rs b/libs/jwst-codec/benches/codec_benchmarks.rs index f0094d0f3..0c90fd3fc 100644 --- a/libs/jwst-codec/benches/codec_benchmarks.rs +++ b/libs/jwst-codec/benches/codec_benchmarks.rs @@ -1,7 +1,9 @@ use criterion::{criterion_group, criterion_main, Criterion, SamplingMode}; use jwst_codec::{read_var_i32, read_var_u64, write_var_i32, write_var_u64}; -use lib0::decoding::{Cursor, Read}; -use lib0::encoding::Write; +use lib0::{ + decoding::{Cursor, Read}, + encoding::Write, +}; const BENCHMARK_SIZE: u32 = 100000; diff --git a/libs/jwst-codec/benches/map_ops_benchmarks.rs b/libs/jwst-codec/benches/map_ops_benchmarks.rs index 48baefe44..85489ed15 100644 --- a/libs/jwst-codec/benches/map_ops_benchmarks.rs +++ b/libs/jwst-codec/benches/map_ops_benchmarks.rs @@ -1,6 +1,7 @@ -use criterion::{criterion_group, criterion_main, Criterion}; use std::time::Duration; +use criterion::{criterion_group, criterion_main, Criterion}; + fn operations(c: &mut Criterion) { let mut group = c.benchmark_group("ops/map"); group.measurement_time(Duration::from_secs(15)); diff --git a/libs/jwst-codec/benches/text_ops_benchmarks.rs b/libs/jwst-codec/benches/text_ops_benchmarks.rs index 792d49258..9cfab1de6 100644 --- a/libs/jwst-codec/benches/text_ops_benchmarks.rs +++ b/libs/jwst-codec/benches/text_ops_benchmarks.rs @@ -1,6 +1,7 @@ +use std::time::Duration; + use criterion::{criterion_group, criterion_main, Criterion}; use rand::{Rng, SeedableRng}; -use std::time::Duration; fn operations(c: &mut Criterion) { let mut group = c.benchmark_group("ops/text"); diff --git a/libs/jwst-codec/benches/update_benchmarks.rs b/libs/jwst-codec/benches/update_benchmarks.rs index 27f4141ad..935855b04 100644 --- a/libs/jwst-codec/benches/update_benchmarks.rs +++ b/libs/jwst-codec/benches/update_benchmarks.rs @@ -1,9 +1,9 @@ mod utils; -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use std::time::Duration; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use path_ext::PathExt; -use std::time::Duration; use utils::Files; fn update(c: &mut Criterion) { diff --git a/libs/jwst-codec/benches/utils/files.rs b/libs/jwst-codec/benches/utils/files.rs index 9f175e446..377cf9b75 100644 --- a/libs/jwst-codec/benches/utils/files.rs +++ b/libs/jwst-codec/benches/utils/files.rs @@ -1,9 +1,10 @@ -use path_ext::PathExt; use std::{ fs::{read, read_dir}, path::{Path, PathBuf}, }; +use path_ext::PathExt; + pub struct File { pub path: PathBuf, pub content: Vec, diff --git a/libs/jwst-codec/src/codec/buffer.rs b/libs/jwst-codec/src/codec/buffer.rs index efd5bfd88..ee296073d 100644 --- a/libs/jwst-codec/src/codec/buffer.rs +++ b/libs/jwst-codec/src/codec/buffer.rs @@ -1,7 +1,9 @@ -use super::*; -use nom::bytes::complete::take; use std::io::{Error, Write}; +use nom::bytes::complete::take; + +use super::*; + pub fn read_var_buffer(input: &[u8]) -> IResult<&[u8], &[u8]> { let (tail, len) = read_var_u64(input)?; let (tail, val) = take(len as usize)(tail)?; @@ -16,12 +18,13 @@ pub fn write_var_buffer(buffer: &mut W, data: &[u8]) -> Result<(), Err #[cfg(test)] mod tests { - use super::*; use nom::{ error::{Error, ErrorKind}, AsBytes, Err, }; + use super::*; + #[test] fn test_read_var_buffer() { // Test case 1: valid input, buffer length = 5 diff --git a/libs/jwst-codec/src/codec/integer.rs b/libs/jwst-codec/src/codec/integer.rs index 650a5f5e4..1a29dbd86 100644 --- a/libs/jwst-codec/src/codec/integer.rs +++ b/libs/jwst-codec/src/codec/integer.rs @@ -1,7 +1,9 @@ -use super::*; +use std::io::{Error, Write}; + use byteorder::WriteBytesExt; use nom::Needed; -use std::io::{Error, Write}; + +use super::*; pub fn read_var_u64(input: &[u8]) -> IResult<&[u8], u64> { // parse the first byte @@ -110,9 +112,10 @@ pub fn write_var_i32(buffer: &mut W, num: i32) -> Result<(), Error> { #[cfg(test)] mod tests { - use super::*; use lib0::encoding::Write; + use super::*; + fn test_var_uint_enc_dec(num: u64) { let mut buf1 = Vec::new(); write_var_u64(&mut buf1, num).unwrap(); diff --git a/libs/jwst-codec/src/codec/string.rs b/libs/jwst-codec/src/codec/string.rs index a9844d268..0d4620d6c 100644 --- a/libs/jwst-codec/src/codec/string.rs +++ b/libs/jwst-codec/src/codec/string.rs @@ -1,7 +1,9 @@ -use super::*; -use nom::combinator::map_res; use std::io::{Error, Write}; +use nom::combinator::map_res; + +use super::*; + pub fn read_var_string(input: &[u8]) -> IResult<&[u8], String> { map_res(read_var_buffer, |s| String::from_utf8(s.to_vec()))(input) } @@ -14,12 +16,13 @@ pub fn write_var_string>(buffer: &mut W, input: S) -> Re #[cfg(test)] mod tests { - use super::*; use nom::{ error::{Error, ErrorKind}, AsBytes, Err, }; + use super::*; + #[test] fn test_read_var_string() { // Test case 1: valid input, string length = 5 diff --git a/libs/jwst-codec/src/doc/awareness.rs b/libs/jwst-codec/src/doc/awareness.rs index 70fc3f24c..96cfc1040 100644 --- a/libs/jwst-codec/src/doc/awareness.rs +++ b/libs/jwst-codec/src/doc/awareness.rs @@ -1,6 +1,7 @@ +use std::{cmp::max, collections::hash_map::Entry}; + use super::*; use crate::sync::Arc; -use std::{cmp::max, collections::hash_map::Entry}; pub struct Awareness { awareness: AwarenessStates, diff --git a/libs/jwst-codec/src/doc/codec/any.rs b/libs/jwst-codec/src/doc/codec/any.rs index fd5d2d5b4..1ab9dc137 100644 --- a/libs/jwst-codec/src/doc/codec/any.rs +++ b/libs/jwst-codec/src/doc/codec/any.rs @@ -1,7 +1,9 @@ -use super::*; -use ordered_float::OrderedFloat; use std::{collections::HashMap, fmt, ops::RangeInclusive}; +use ordered_float::OrderedFloat; + +use super::*; + const MAX_JS_INT: i64 = 0x001F_FFFF_FFFF_FFFF; // The smallest int in js number. const MIN_JS_INT: i64 = -MAX_JS_INT; @@ -489,9 +491,10 @@ impl ToString for Any { #[cfg(test)] mod tests { - use super::*; use proptest::{collection::vec, prelude::*}; + use super::*; + #[test] fn test_any_codec() { let any = Any::Object( diff --git a/libs/jwst-codec/src/doc/codec/content.rs b/libs/jwst-codec/src/doc/codec/content.rs index 8d85408d0..9e9ed0c8b 100644 --- a/libs/jwst-codec/src/doc/codec/content.rs +++ b/libs/jwst-codec/src/doc/codec/content.rs @@ -1,6 +1,7 @@ +use std::ops::Deref; + use super::*; use crate::sync::RwLock; -use std::ops::Deref; #[derive(Clone)] #[cfg_attr(test, derive(proptest_derive::Arbitrary))] @@ -282,9 +283,10 @@ impl Content { #[cfg(test)] mod tests { - use super::*; use proptest::{collection::vec, prelude::*}; + use super::*; + fn content_round_trip(content: &Content) -> JwstCodecResult { let mut writer = RawEncoder::default(); writer.write_u8(content.get_info())?; diff --git a/libs/jwst-codec/src/doc/codec/delete_set.rs b/libs/jwst-codec/src/doc/codec/delete_set.rs index 5822ad977..865d90393 100644 --- a/libs/jwst-codec/src/doc/codec/delete_set.rs +++ b/libs/jwst-codec/src/doc/codec/delete_set.rs @@ -1,10 +1,11 @@ -use super::*; -use crate::doc::OrderRange; use std::{ collections::{hash_map::Entry, HashMap}, ops::{Deref, DerefMut, Range}, }; +use super::*; +use crate::doc::OrderRange; + impl CrdtRead for Range { fn read(decoder: &mut R) -> JwstCodecResult { let clock = decoder.read_var_u64()?; diff --git a/libs/jwst-codec/src/doc/codec/io/codec_v1.rs b/libs/jwst-codec/src/doc/codec/io/codec_v1.rs index 640e1b658..9bfe5abe8 100644 --- a/libs/jwst-codec/src/doc/codec/io/codec_v1.rs +++ b/libs/jwst-codec/src/doc/codec/io/codec_v1.rs @@ -1,6 +1,7 @@ -use super::*; use std::io::Cursor; +use super::*; + // compatible with ydoc v1 #[derive(Clone)] pub struct RawDecoder { diff --git a/libs/jwst-codec/src/doc/codec/io/reader.rs b/libs/jwst-codec/src/doc/codec/io/reader.rs index c642ad684..b5dc0d0e5 100644 --- a/libs/jwst-codec/src/doc/codec/io/reader.rs +++ b/libs/jwst-codec/src/doc/codec/io/reader.rs @@ -1,7 +1,9 @@ -use super::*; -use byteorder::{BigEndian, ReadBytesExt}; use std::io::{Cursor, Error}; +use byteorder::{BigEndian, ReadBytesExt}; + +use super::*; + #[inline] fn read_with_cursor(buffer: &mut Cursor>, f: F) -> JwstCodecResult where diff --git a/libs/jwst-codec/src/doc/codec/io/writer.rs b/libs/jwst-codec/src/doc/codec/io/writer.rs index e31a18272..6740ad825 100644 --- a/libs/jwst-codec/src/doc/codec/io/writer.rs +++ b/libs/jwst-codec/src/doc/codec/io/writer.rs @@ -1,7 +1,9 @@ -use super::*; -use byteorder::{BigEndian, WriteBytesExt}; use std::io::{Cursor, Error, Write}; +use byteorder::{BigEndian, WriteBytesExt}; + +use super::*; + #[inline] fn map_io_error(e: Error) -> JwstCodecError { JwstCodecError::InvalidWriteBuffer(e.to_string()) diff --git a/libs/jwst-codec/src/doc/codec/item.rs b/libs/jwst-codec/src/doc/codec/item.rs index c7f73523b..26e28ad1d 100644 --- a/libs/jwst-codec/src/doc/codec/item.rs +++ b/libs/jwst-codec/src/doc/codec/item.rs @@ -446,9 +446,10 @@ impl Item { #[cfg(test)] mod tests { - use super::*; use proptest::{collection::vec, prelude::*}; + use super::*; + fn item_round_trip(item: &mut Item) -> JwstCodecResult { if !item.is_valid() { return Ok(()); diff --git a/libs/jwst-codec/src/doc/codec/refs.rs b/libs/jwst-codec/src/doc/codec/refs.rs index 2fa77d0f0..99304a596 100644 --- a/libs/jwst-codec/src/doc/codec/refs.rs +++ b/libs/jwst-codec/src/doc/codec/refs.rs @@ -267,9 +267,10 @@ impl From> for Somr { #[cfg(test)] mod tests { - use super::{utils::ItemBuilder, *}; use proptest::{collection::vec, prelude::*}; + use super::{utils::ItemBuilder, *}; + #[test] fn test_struct_info() { loom_model!({ diff --git a/libs/jwst-codec/src/doc/codec/update.rs b/libs/jwst-codec/src/doc/codec/update.rs index cc9ad6664..2848d17aa 100644 --- a/libs/jwst-codec/src/doc/codec/update.rs +++ b/libs/jwst-codec/src/doc/codec/update.rs @@ -1,7 +1,10 @@ +use std::{ + collections::{HashMap, VecDeque}, + ops::Range, +}; + use super::*; use crate::doc::StateVector; -use std::collections::{HashMap, VecDeque}; -use std::ops::Range; #[derive(Debug, Default, Clone)] pub struct Update { @@ -9,7 +12,8 @@ pub struct Update { pub(crate) delete_set: DeleteSet, /// all unapplicable items that we can't integrate into doc - /// any item with inconsistent id clock or missing dependency will be put here + /// any item with inconsistent id clock or missing dependency will be put + /// here pub(crate) pending_structs: HashMap>, /// missing state vector after applying updates pub(crate) missing_state: StateVector, @@ -132,7 +136,8 @@ impl Update { for structs in target.structs.values_mut() { structs.make_contiguous().sort_by_key(|s| s.id().clock); - // insert [Node::Skip] if structs[index].id().clock + structs[index].len() < structs[index + 1].id().clock + // insert [Node::Skip] if structs[index].id().clock + structs[index].len() < + // structs[index + 1].id().clock let mut index = 0; while index < structs.len() - 1 { let cur = &structs[index]; @@ -173,7 +178,8 @@ pub(crate) struct UpdateIterator<'a> { client_ids: Vec, /// current id of client of the updates we're processing cur_client_id: Option, - /// stack of previous iterating item with higher priority than updates in next iteration + /// stack of previous iterating item with higher priority than updates in + /// next iteration stack: Vec, } @@ -192,11 +198,12 @@ impl<'a> UpdateIterator<'a> { } } - /// iterate the client ids until we find the next client with left updates that can be consumed + /// iterate the client ids until we find the next client with left updates + /// that can be consumed /// /// note: - /// firstly we will check current client id as well to ensure current updates queue is not empty yet - /// + /// firstly we will check current client id as well to ensure current + /// updates queue is not empty yet fn next_client(&mut self) -> Option { while let Some(client_id) = self.cur_client_id { match self.update.structs.get(&client_id) { @@ -236,8 +243,8 @@ impl<'a> UpdateIterator<'a> { } } - /// tell if current update's dependencies(left, right, parent) has already been consumed and recorded - /// and return the client of them if not. + /// tell if current update's dependencies(left, right, parent) has already + /// been consumed and recorded and return the client of them if not. fn get_missing_dep(&self, struct_info: &Node) -> Option { if let Some(item) = struct_info.as_item().get() { let id = item.id; @@ -326,8 +333,8 @@ impl Iterator for UpdateIterator<'_> { } else { // we finally find the first applicable update let local_state = self.state.get(&id.client); - // we've already check the local state is greater or equal to current update's clock - // so offset here will never be negative + // we've already check the local state is greater or equal to current update's + // clock so offset here will never be negative let offset = local_state - id.clock; if offset == 0 || offset < cur_update.len() { self.state.set_max(id.client, id.clock + cur_update.len()); @@ -399,11 +406,12 @@ impl Iterator for DeleteSetIterator<'_> { #[cfg(test)] mod tests { - use crate::doc::common::OrderRange; + use std::{num::ParseIntError, path::PathBuf}; - use super::*; use serde::Deserialize; - use std::{num::ParseIntError, path::PathBuf}; + + use super::*; + use crate::doc::common::OrderRange; fn struct_item(id: (Client, Clock), len: usize) -> Node { Node::Item(Somr::new( diff --git a/libs/jwst-codec/src/doc/codec/utils/items.rs b/libs/jwst-codec/src/doc/codec/utils/items.rs index 6d8c30832..e16771074 100644 --- a/libs/jwst-codec/src/doc/codec/utils/items.rs +++ b/libs/jwst-codec/src/doc/codec/utils/items.rs @@ -1,5 +1,4 @@ -use super::item::item_flags; -use super::*; +use super::{item::item_flags, *}; use crate::sync::Arc; pub(crate) struct ItemBuilder { diff --git a/libs/jwst-codec/src/doc/common/range.rs b/libs/jwst-codec/src/doc/common/range.rs index f0c3ca3d7..1f9e1ea81 100644 --- a/libs/jwst-codec/src/doc/common/range.rs +++ b/libs/jwst-codec/src/doc/common/range.rs @@ -66,7 +66,8 @@ impl OrderRange { } /// Push new range to current one. - /// Range will be merged if overlap exists or turned into fragment if it's not continuous. + /// Range will be merged if overlap exists or turned into fragment if it's + /// not continuous. pub fn push(&mut self, range: Range) { match self { OrderRange::Range(r) => { diff --git a/libs/jwst-codec/src/doc/common/somr.rs b/libs/jwst-codec/src/doc/common/somr.rs index 3b6ddaea8..51b36a87d 100644 --- a/libs/jwst-codec/src/doc/common/somr.rs +++ b/libs/jwst-codec/src/doc/common/somr.rs @@ -12,7 +12,8 @@ fn is_dangling(ptr: NonNull) -> bool { ptr.as_ptr() as usize == DANGLING_PTR } -/// Heap data with single owner but multiple refs with dangling checking at runtime. +/// Heap data with single owner but multiple refs with dangling checking at +/// runtime. pub(crate) enum Somr { Owned(Owned), Ref(Ref), @@ -25,7 +26,8 @@ pub(crate) struct Ref(NonNull>); pub(crate) struct SomrInner { data: Option, - /// increase the size when we really meet the the secenerio with refs more then u16::MAX(65535) times + /// increase the size when we really meet the the secenerio with refs more + /// then u16::MAX(65535) times refs: AtomicU32, _marker: PhantomData>, } @@ -312,9 +314,8 @@ impl proptest::arbitrary::Arbitrary for Somr< #[cfg(test)] mod tests { - use crate::loom_model; - use super::*; + use crate::loom_model; #[test] fn basic_example() { diff --git a/libs/jwst-codec/src/doc/common/state.rs b/libs/jwst-codec/src/doc/common/state.rs index 4486ca97c..2db37c7d7 100644 --- a/libs/jwst-codec/src/doc/common/state.rs +++ b/libs/jwst-codec/src/doc/common/state.rs @@ -1,9 +1,10 @@ -use crate::{Client, Clock, CrdtRead, CrdtReader, CrdtWrite, CrdtWriter, Id, JwstCodecResult}; use std::{ collections::HashMap, ops::{Deref, DerefMut}, }; +use crate::{Client, Clock, CrdtRead, CrdtReader, CrdtWrite, CrdtWriter, Id, JwstCodecResult}; + #[derive(Default, Debug, PartialEq, Clone)] pub struct StateVector(HashMap); diff --git a/libs/jwst-codec/src/doc/document.rs b/libs/jwst-codec/src/doc/document.rs index 858c8b0d3..dddbb57f7 100644 --- a/libs/jwst-codec/src/doc/document.rs +++ b/libs/jwst-codec/src/doc/document.rs @@ -231,9 +231,10 @@ impl Doc { #[cfg(test)] mod tests { + use yrs::{types::ToJson, updates::decoder::Decode, Array, Map, Options, Transact}; + use super::*; use crate::sync::{AtomicU8, Ordering}; - use yrs::{types::ToJson, updates::decoder::Decode, Array, Map, Options, Transact}; #[test] #[cfg_attr(miri, ignore)] diff --git a/libs/jwst-codec/src/doc/mod.rs b/libs/jwst-codec/src/doc/mod.rs index ac901ebd7..ffcc6c2ff 100644 --- a/libs/jwst-codec/src/doc/mod.rs +++ b/libs/jwst-codec/src/doc/mod.rs @@ -7,8 +7,6 @@ mod store; mod types; mod utils; -use super::*; - pub use awareness::{Awareness, AwarenessEvent}; pub use codec::*; pub use common::*; @@ -16,3 +14,5 @@ pub use document::{Doc, DocOptions}; pub(crate) use store::DocStore; pub use types::*; pub use utils::*; + +use super::*; diff --git a/libs/jwst-codec/src/doc/publisher.rs b/libs/jwst-codec/src/doc/publisher.rs index ace33da5d..2292cfa9f 100644 --- a/libs/jwst-codec/src/doc/publisher.rs +++ b/libs/jwst-codec/src/doc/publisher.rs @@ -1,6 +1,7 @@ +use jwst_logger::{debug, trace}; + use super::{store::StoreRef, *}; use crate::sync::{Arc, AtomicBool, Mutex, Ordering, RwLock}; -use jwst_logger::{debug, trace}; pub type DocSubscriber = Box; diff --git a/libs/jwst-codec/src/doc/store.rs b/libs/jwst-codec/src/doc/store.rs index bdcc7c6d2..aa135bad0 100644 --- a/libs/jwst-codec/src/doc/store.rs +++ b/libs/jwst-codec/src/doc/store.rs @@ -1,14 +1,15 @@ -use super::*; -use crate::{ - doc::StateVector, - sync::{Arc, RwLock, RwLockWriteGuard, Weak}, -}; use std::{ collections::{hash_map::Entry, HashMap, HashSet, VecDeque}, mem, ops::Range, }; +use super::*; +use crate::{ + doc::StateVector, + sync::{Arc, RwLock, RwLockWriteGuard, Weak}, +}; + unsafe impl Send for DocStore {} unsafe impl Sync for DocStore {} @@ -291,7 +292,8 @@ impl DocStore { /// - split left if needed (insert in between a splitable item) /// - split right if needed (insert in between a splitable item) /// - recover parent to [Parent::Type] - /// - [Parent::String] for root level named type (e.g `doc.get_or_create_text("content")`) + /// - [Parent::String] for root level named type (e.g + /// `doc.get_or_create_text("content")`) /// - [Parent::Id] for type as item (e.g `doc.create_text()`) /// - [None] means borrow left.parent or right.parent pub fn repair(&mut self, item: &mut Item, store_ref: StoreRef) -> JwstCodecResult { @@ -677,7 +679,8 @@ impl DocStore { let mut update_structs: HashMap> = HashMap::new(); for (client, clock) in diff { - // We have made sure that the client is in the local state vector in diff_state_vectors() + // We have made sure that the client is in the local state vector in + // diff_state_vectors() if let Some(items) = map.get(&client) { if items.is_empty() { continue; @@ -900,7 +903,8 @@ mod tests { let left = doc_store.split_at_and_get_left((1, 1)).unwrap(); assert_eq!(left.len(), 2); // octo => oc_to - // s1 used to be (1, 4), but it actually ref of first item in store, so now it should be (1, 2) + // s1 used to be (1, 4), but it actually ref of first item in store, so now it + // should be (1, 2) assert_eq!(s1, left, "doc internal mutation should not modify the pointer"); let right = doc_store.split_at_and_get_right((1, 5)).unwrap(); assert_eq!(right.len(), 3); // base => b_ase diff --git a/libs/jwst-codec/src/doc/types/array.rs b/libs/jwst-codec/src/doc/types/array.rs index 5003c0c89..83859effa 100644 --- a/libs/jwst-codec/src/doc/types/array.rs +++ b/libs/jwst-codec/src/doc/types/array.rs @@ -83,9 +83,10 @@ impl serde::Serialize for Array { #[cfg(test)] mod tests { - use super::*; use yrs::{Array, Options, Text, Transact}; + use super::*; + #[test] fn test_yarray_insert() { let options = DocOptions { diff --git a/libs/jwst-codec/src/doc/types/list/mod.rs b/libs/jwst-codec/src/doc/types/list/mod.rs index 694b96342..15238eec0 100644 --- a/libs/jwst-codec/src/doc/types/list/mod.rs +++ b/libs/jwst-codec/src/doc/types/list/mod.rs @@ -39,7 +39,6 @@ impl ItemPosition { /// after: /// --------------------------------- /// ^left ^right - /// pub fn normalize(&mut self, store: &mut DocStore) -> JwstCodecResult { if self.offset > 0 { debug_assert!(self.left.is_some()); diff --git a/libs/jwst-codec/src/doc/types/list/search_marker.rs b/libs/jwst-codec/src/doc/types/list/search_marker.rs index 1b0ef7f4f..6466bbcc0 100644 --- a/libs/jwst-codec/src/doc/types/list/search_marker.rs +++ b/libs/jwst-codec/src/doc/types/list/search_marker.rs @@ -1,4 +1,3 @@ -use super::*; use std::{ cell::RefCell, cmp::max, @@ -6,6 +5,8 @@ use std::{ ops::{Deref, DerefMut}, }; +use super::*; + const MAX_SEARCH_MARKER: usize = 80; #[derive(Clone, Debug)] @@ -25,14 +26,18 @@ impl SearchMarker { } } -/// in yjs, a timestamp field is used to sort markers and the oldest marker is deleted once the limit is reached. -/// this was designed for optimization purposes for v8. In Rust, we can simply use a [VecDeque] and trust the compiler to optimize. -/// the [VecDeque] can naturally maintain the insertion order, allowing us to know which marker is the oldest without using an extra timestamp field. +/// in yjs, a timestamp field is used to sort markers and the oldest marker is +/// deleted once the limit is reached. this was designed for optimization +/// purposes for v8. In Rust, we can simply use a [VecDeque] and trust the +/// compiler to optimize. the [VecDeque] can naturally maintain the insertion +/// order, allowing us to know which marker is the oldest without using an extra +/// timestamp field. /// /// NOTE: /// A [MarkerList] is always belonging to a [YType], -/// which means whenever [MakerList] is used, we actually have a [YType] instance behind [RwLock] guard already, -/// so it's safe to make the list internal mutable. +/// which means whenever [MakerList] is used, we actually have a [YType] +/// instance behind [RwLock] guard already, so it's safe to make the list +/// internal mutable. #[derive(Debug)] pub struct MarkerList(RefCell>); @@ -159,9 +164,10 @@ impl MarkerList { } } - // we want to make sure that item_ptr can't be merged with left, because that would screw up everything - // in that case just return what we have (it is most likely the best marker anyway) - // iterate to left until item_ptr can't be merged with left + // we want to make sure that item_ptr can't be merged with left, because that + // would screw up everything in that case just return what we have + // (it is most likely the best marker anyway) iterate to left until + // item_ptr can't be merged with left while let Some(item) = item_ptr.clone().get() { let left_ref: ItemRef = item.left.clone().into(); if let Some(left) = left_ref.get() { @@ -202,11 +208,12 @@ impl MarkerList { #[cfg(test)] mod tests { - use super::*; use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha20Rng; use yrs::{Array, Options, Transact}; + use super::*; + #[test] fn test_marker_list() { let options = DocOptions { diff --git a/libs/jwst-codec/src/doc/types/map.rs b/libs/jwst-codec/src/doc/types/map.rs index 8c0a9231f..5044a485b 100644 --- a/libs/jwst-codec/src/doc/types/map.rs +++ b/libs/jwst-codec/src/doc/types/map.rs @@ -1,10 +1,12 @@ +use std::collections::HashMap; + use super::*; -use crate::sync::Arc; use crate::{ doc::{AsInner, Node, Parent, YTypeRef}, - impl_type, Content, JwstCodecResult, + impl_type, + sync::Arc, + Content, JwstCodecResult, }; -use std::collections::HashMap; impl_type!(Map); diff --git a/libs/jwst-codec/src/doc/types/mod.rs b/libs/jwst-codec/src/doc/types/mod.rs index e8b08bd12..6b12497f4 100644 --- a/libs/jwst-codec/src/doc/types/mod.rs +++ b/libs/jwst-codec/src/doc/types/mod.rs @@ -3,20 +3,20 @@ mod list; mod map; mod text; -use super::*; -use crate::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; -use list::*; use std::collections::{hash_map::Entry, HashMap}; pub use array::*; +use list::*; pub use map::*; pub use text::*; -use crate::{Item, JwstCodecError, JwstCodecResult}; - use super::{ store::{StoreRef, WeakStoreRef}, - Node, + Node, *, +}; +use crate::{ + sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}, + Item, JwstCodecError, JwstCodecResult, }; #[derive(Debug, Default)] diff --git a/libs/jwst-codec/src/doc/types/text.rs b/libs/jwst-codec/src/doc/types/text.rs index e4515f9b3..f8733eff6 100644 --- a/libs/jwst-codec/src/doc/types/text.rs +++ b/libs/jwst-codec/src/doc/types/text.rs @@ -1,6 +1,5 @@ -use crate::{impl_type, Content, JwstCodecResult}; - use super::list::ListType; +use crate::{impl_type, Content, JwstCodecResult}; impl_type!(Text); @@ -55,14 +54,15 @@ impl serde::Serialize for Text { #[cfg(test)] mod tests { + use rand::{Rng, SeedableRng}; + use rand_chacha::ChaCha20Rng; + use yrs::{Options, Text, Transact}; + use crate::{ loom_model, sync::{thread, Arc, AtomicUsize, Ordering}, Doc, DocOptions, }; - use rand::{Rng, SeedableRng}; - use rand_chacha::ChaCha20Rng; - use yrs::{Options, Text, Transact}; #[test] fn test_manipulate_text() { diff --git a/libs/jwst-codec/src/doc/utils.rs b/libs/jwst-codec/src/doc/utils.rs index a87146ed8..9f317cccf 100644 --- a/libs/jwst-codec/src/doc/utils.rs +++ b/libs/jwst-codec/src/doc/utils.rs @@ -1,6 +1,7 @@ -use super::*; use std::io::Write; +use super::*; + pub fn encode_update_with_guid>(update: Vec, guid: S) -> JwstCodecResult> { let mut encoder = RawEncoder::default(); encoder.write_var_string(guid)?; diff --git a/libs/jwst-codec/src/lib.rs b/libs/jwst-codec/src/lib.rs index 206d7c86f..c80f3c14d 100644 --- a/libs/jwst-codec/src/lib.rs +++ b/libs/jwst-codec/src/lib.rs @@ -11,13 +11,12 @@ pub use doc::{ CrdtWriter, Doc, DocOptions, Id, Map, RawDecoder, RawEncoder, StateVector, Text, Update, Value, }; pub(crate) use doc::{Content, Item}; -pub use protocol::{ - read_sync_message, write_sync_message, AwarenessState, AwarenessStates, DocMessage, SyncMessage, SyncMessageScanner, -}; - use jwst_logger::{debug, warn}; use nanoid::nanoid; use nom::IResult; +pub use protocol::{ + read_sync_message, write_sync_message, AwarenessState, AwarenessStates, DocMessage, SyncMessage, SyncMessageScanner, +}; use thiserror::Error; #[derive(Debug, Error, PartialEq)] diff --git a/libs/jwst-codec/src/protocol/awareness.rs b/libs/jwst-codec/src/protocol/awareness.rs index 36cdd4dc8..72201ef48 100644 --- a/libs/jwst-codec/src/protocol/awareness.rs +++ b/libs/jwst-codec/src/protocol/awareness.rs @@ -1,6 +1,7 @@ -use super::*; use nom::multi::count; +use super::*; + const NULL_STR: &str = "null"; #[derive(Debug, Clone, PartialEq)] diff --git a/libs/jwst-codec/src/protocol/doc.rs b/libs/jwst-codec/src/protocol/doc.rs index d06f2d16e..6a590170c 100644 --- a/libs/jwst-codec/src/protocol/doc.rs +++ b/libs/jwst-codec/src/protocol/doc.rs @@ -5,8 +5,9 @@ use super::*; #[cfg_attr(test, derive(proptest_derive::Arbitrary))] pub enum DocMessage { // state vector - // TODO: temporarily skipped in the test, because yrs decoding needs to ensure that the update in step1 is the correct state vector binary - // and any data can be included in our implementation (we will ensure the correctness of encoding and decoding in the subsequent decoding process) + // TODO: temporarily skipped in the test, because yrs decoding needs to ensure that the update in step1 is the + // correct state vector binary and any data can be included in our implementation (we will ensure the + // correctness of encoding and decoding in the subsequent decoding process) #[cfg_attr(test, proptest(skip))] Step1(Vec), // update diff --git a/libs/jwst-codec/src/protocol/mod.rs b/libs/jwst-codec/src/protocol/mod.rs index b651934e7..e4270fe18 100644 --- a/libs/jwst-codec/src/protocol/mod.rs +++ b/libs/jwst-codec/src/protocol/mod.rs @@ -5,18 +5,21 @@ mod sync; #[cfg(test)] mod utils; -pub use awareness::{AwarenessState, AwarenessStates}; -pub use doc::DocMessage; -pub use scanner::SyncMessageScanner; -pub use sync::{read_sync_message, write_sync_message, SyncMessage}; +use std::{ + collections::HashMap, + io::{Error as IoError, Write}, +}; -use super::*; use awareness::{read_awareness, write_awareness}; +pub use awareness::{AwarenessState, AwarenessStates}; +pub use doc::DocMessage; use doc::{read_doc_message, write_doc_message}; use jwst_logger::debug; use nom::{ error::{Error, ErrorKind}, IResult, }; -use std::collections::HashMap; -use std::io::{Error as IoError, Write}; +pub use scanner::SyncMessageScanner; +pub use sync::{read_sync_message, write_sync_message, SyncMessage}; + +use super::*; diff --git a/libs/jwst-codec/src/protocol/scanner.rs b/libs/jwst-codec/src/protocol/scanner.rs index e953fb289..cb51e227c 100644 --- a/libs/jwst-codec/src/protocol/scanner.rs +++ b/libs/jwst-codec/src/protocol/scanner.rs @@ -43,11 +43,12 @@ impl<'a> Iterator for SyncMessageScanner<'a> { #[cfg(test)] mod tests { - use super::{utils::to_sync_message, *}; use proptest::{collection::vec, prelude::*}; use y_sync::sync::MessageReader; use yrs::updates::decoder::DecoderV1; + use super::{utils::to_sync_message, *}; + proptest! { #[test] #[cfg_attr(miri, ignore)] diff --git a/libs/jwst-codec/src/protocol/sync.rs b/libs/jwst-codec/src/protocol/sync.rs index c04e9c52e..ec8a0450a 100644 --- a/libs/jwst-codec/src/protocol/sync.rs +++ b/libs/jwst-codec/src/protocol/sync.rs @@ -1,6 +1,7 @@ -use super::*; use byteorder::WriteBytesExt; +use super::*; + #[derive(Debug, Clone, PartialEq)] enum MessageType { Auth, @@ -164,13 +165,14 @@ mod tests { #[test] fn test_sync_message_compatibility() { - use super::utils::{to_sync_message, to_y_message}; use y_sync::sync::Message as YMessage; use yrs::updates::{ decoder::{Decode, DecoderV1}, encoder::{Encode, Encoder, EncoderV1}, }; + use super::utils::{to_sync_message, to_y_message}; + let messages = [ SyncMessage::Auth(Some("reason".to_string())), SyncMessage::Awareness(HashMap::from([(1, AwarenessState::new(1, "test".into()))])), diff --git a/libs/jwst-codec/src/protocol/utils.rs b/libs/jwst-codec/src/protocol/utils.rs index 516565b06..e1012c1f4 100644 --- a/libs/jwst-codec/src/protocol/utils.rs +++ b/libs/jwst-codec/src/protocol/utils.rs @@ -1,10 +1,11 @@ -use super::*; use y_sync::sync::Message as YMessage; use yrs::{ updates::{decoder::Decode, encoder::Encode}, StateVector, }; +use super::*; + pub fn to_sync_message(msg: YMessage) -> Option { match msg { YMessage::Auth(reason) => Some(SyncMessage::Auth(reason)), diff --git a/libs/jwst-codec/src/sync.rs b/libs/jwst-codec/src/sync.rs index 53cbee021..b97681df4 100644 --- a/libs/jwst-codec/src/sync.rs +++ b/libs/jwst-codec/src/sync.rs @@ -1,26 +1,24 @@ -#[cfg(loom)] -pub(crate) use loom::{ - sync::{ - atomic::{AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering}, - Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard, - }, - thread, -}; - #[cfg(not(loom))] pub(crate) use std::sync::{ atomic::{AtomicBool, AtomicU32, AtomicU8, Ordering}, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard, }; - pub use std::sync::{Arc, Weak}; - #[cfg(all(test, not(loom)))] pub(crate) use std::{ sync::{atomic::AtomicUsize, MutexGuard}, thread, }; +#[cfg(loom)] +pub(crate) use loom::{ + sync::{ + atomic::{AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering}, + Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard, + }, + thread, +}; + #[macro_export(local_inner_macros)] macro_rules! loom_model { ($test:block) => { diff --git a/libs/jwst-core-rpc/src/broadcast.rs b/libs/jwst-core-rpc/src/broadcast.rs index 906f09c7e..b6aa7c75c 100644 --- a/libs/jwst-core-rpc/src/broadcast.rs +++ b/libs/jwst-core-rpc/src/broadcast.rs @@ -1,12 +1,12 @@ -use super::*; -use jwst_codec::{ - encode_update_as_message, encode_update_with_guid, write_sync_message, SyncMessage, -}; +use std::{collections::HashMap, sync::Mutex}; + +use jwst_codec::{encode_update_as_message, encode_update_with_guid, write_sync_message, SyncMessage}; use jwst_core::Workspace; use lru_time_cache::LruCache; -use std::{collections::HashMap, sync::Mutex}; use tokio::sync::{broadcast::Sender, RwLock}; +use super::*; + #[derive(Clone)] pub enum BroadcastType { BroadcastAwareness(Vec), @@ -42,10 +42,7 @@ pub async fn subscribe(workspace: &mut Workspace, identifier: String, sender: Br let mut dedup_cache = dedup_cache.lock().unwrap_or_else(|e| e.into_inner()); if !dedup_cache.contains_key(&buffer) { - if sender - .send(BroadcastType::BroadcastAwareness(buffer.clone())) - .is_err() - { + if sender.send(BroadcastType::BroadcastAwareness(buffer.clone())).is_err() { debug!("broadcast channel {workspace_id} has been closed",) } dedup_cache.insert(buffer, ()); @@ -70,10 +67,7 @@ pub async fn subscribe(workspace: &mut Workspace, identifier: String, sender: Br debug!("broadcast channel {workspace_id} has been closed",) } - if sender - .send(BroadcastType::BroadcastContent(sendable_update)) - .is_err() - { + if sender.send(BroadcastType::BroadcastContent(sendable_update)).is_err() { debug!("broadcast channel {workspace_id} has been closed",) } } diff --git a/libs/jwst-core-rpc/src/client/mod.rs b/libs/jwst-core-rpc/src/client/mod.rs index d02ff2a5a..fb40f4f47 100644 --- a/libs/jwst-core-rpc/src/client/mod.rs +++ b/libs/jwst-core-rpc/src/client/mod.rs @@ -3,16 +3,17 @@ mod webrtc; #[cfg(feature = "websocket")] mod websocket; -use super::*; -use chrono::Utc; use std::sync::Mutex; -use tokio::{runtime::Runtime, task::JoinHandle}; +use chrono::Utc; +use tokio::{runtime::Runtime, task::JoinHandle}; #[cfg(feature = "webrtc")] pub use webrtc::start_webrtc_client_sync; #[cfg(feature = "websocket")] pub use websocket::start_websocket_client_sync; +use super::*; + #[derive(Clone, Default)] pub struct CachedLastSynced { synced: Arc>>, @@ -49,10 +50,12 @@ impl CachedLastSynced { #[cfg(test)] mod tests { - use super::*; use std::thread::spawn; + use tokio::sync::mpsc::channel; + use super::*; + #[test] fn test_synced() { let synced = CachedLastSynced::default(); diff --git a/libs/jwst-core-rpc/src/client/webrtc.rs b/libs/jwst-core-rpc/src/client/webrtc.rs index facf1f52a..872452205 100644 --- a/libs/jwst-core-rpc/src/client/webrtc.rs +++ b/libs/jwst-core-rpc/src/client/webrtc.rs @@ -1,9 +1,11 @@ -use super::*; +use std::sync::RwLock; + use nanoid::nanoid; use reqwest::Client; -use std::sync::RwLock; use tokio::{runtime::Runtime, sync::mpsc::channel}; +use super::*; + async fn webrtc_connection(remote: &str) -> (Sender, Receiver>) { warn!("webrtc_connection start"); let (offer, pc, tx, rx, mut s) = webrtc_datachannel_client_begin().await; @@ -11,11 +13,7 @@ async fn webrtc_connection(remote: &str) -> (Sender, Receiver>) match client.post(remote).json(&offer).send().await { Ok(res) => { - webrtc_datachannel_client_commit( - res.json::().await.unwrap(), - pc, - ) - .await; + webrtc_datachannel_client_commit(res.json::().await.unwrap(), pc).await; s.recv().await.ok(); warn!("client already connected"); } @@ -49,10 +47,7 @@ pub fn start_webrtc_client_sync( }; if !workspace.is_empty() { info!("Workspace not empty, starting async remote connection"); - last_synced_tx - .send(Utc::now().timestamp_millis()) - .await - .unwrap(); + last_synced_tx.send(Utc::now().timestamp_millis()).await.unwrap(); } else { info!("Workspace empty, starting sync remote connection"); } @@ -78,8 +73,7 @@ pub fn start_webrtc_client_sync( debug!("sync thread finished"); *state = SyncState::Finished; } else { - *state = - SyncState::Error("Remote sync connection disconnected".to_string()); + *state = SyncState::Error("Remote sync connection disconnected".to_string()); } } diff --git a/libs/jwst-core-rpc/src/client/websocket.rs b/libs/jwst-core-rpc/src/client/websocket.rs index a50424498..f7f9a7305 100644 --- a/libs/jwst-core-rpc/src/client/websocket.rs +++ b/libs/jwst-core-rpc/src/client/websocket.rs @@ -1,6 +1,6 @@ -use super::{types::JwstRpcResult, *}; -use nanoid::nanoid; use std::sync::RwLock; + +use nanoid::nanoid; use tokio::{net::TcpStream, runtime::Runtime, sync::mpsc::channel}; use tokio_tungstenite::{ connect_async, @@ -9,6 +9,8 @@ use tokio_tungstenite::{ }; use url::Url; +use super::{types::JwstRpcResult, *}; + type Socket = WebSocketStream>; async fn prepare_connection(remote: &str) -> JwstRpcResult { @@ -45,10 +47,7 @@ pub fn start_websocket_client_sync( }; if !workspace.is_empty() { info!("Workspace not empty, starting async remote connection"); - last_synced_tx - .send(Utc::now().timestamp_millis()) - .await - .unwrap(); + last_synced_tx.send(Utc::now().timestamp_millis()).await.unwrap(); } else { info!("Workspace empty, starting sync remote connection"); } @@ -68,15 +67,10 @@ pub fn start_websocket_client_sync( let identifier = nanoid!(); let workspace_id = workspace_id.clone(); let last_synced_tx = last_synced_tx.clone(); - handle_connector( - context.clone(), - workspace_id.clone(), - identifier, - move || { - let (tx, rx) = tungstenite_socket_connector(socket, &workspace_id); - (tx, rx, last_synced_tx) - }, - ) + handle_connector(context.clone(), workspace_id.clone(), identifier, move || { + let (tx, rx) = tungstenite_socket_connector(socket, &workspace_id); + (tx, rx, last_synced_tx) + }) .await }; diff --git a/libs/jwst-core-rpc/src/connector/axum_socket.rs b/libs/jwst-core-rpc/src/connector/axum_socket.rs index 3aac137b3..6cab797bf 100644 --- a/libs/jwst-core-rpc/src/connector/axum_socket.rs +++ b/libs/jwst-core-rpc/src/connector/axum_socket.rs @@ -1,9 +1,10 @@ -use super::*; use axum::extract::ws::{Message as WebSocketMessage, WebSocket}; use futures::{sink::SinkExt, stream::StreamExt}; use tokio::sync::mpsc::{channel, Receiver, Sender}; use tokio_tungstenite::tungstenite::Error as SocketError; +use super::*; + impl From for WebSocketMessage { fn from(value: Message) -> Self { match value { @@ -31,12 +32,7 @@ pub fn axum_socket_connector( let error = e.to_string(); if !e.into_inner().downcast::().map_or_else( |_| false, - |e| { - matches!( - e.as_ref(), - SocketError::ConnectionClosed | SocketError::AlreadyClosed - ) - }, + |e| matches!(e.as_ref(), SocketError::ConnectionClosed | SocketError::AlreadyClosed), ) { error!("socket send error: {}", error); } diff --git a/libs/jwst-core-rpc/src/connector/memory.rs b/libs/jwst-core-rpc/src/connector/memory.rs index 337780a11..6b666049f 100644 --- a/libs/jwst-core-rpc/src/connector/memory.rs +++ b/libs/jwst-core-rpc/src/connector/memory.rs @@ -1,14 +1,13 @@ -use super::*; -use jwst_codec::{ - decode_update_with_guid, encode_update_as_message, Doc, DocMessage, SyncMessage, - SyncMessageScanner, -}; use std::{ sync::atomic::{AtomicBool, Ordering}, thread::JoinHandle as StdJoinHandler, }; + +use jwst_codec::{decode_update_with_guid, encode_update_as_message, Doc, DocMessage, SyncMessage, SyncMessageScanner}; use tokio::{runtime::Runtime, sync::mpsc::channel, task::JoinHandle as TokioJoinHandler}; +use super::*; + // just for test pub fn memory_connector( rt: Arc, @@ -89,17 +88,11 @@ pub fn memory_connector( match decode_update_with_guid(update.clone()) { Ok((_, update1)) => { if let Err(e) = doc.apply_update_from_binary(update1) { - error!( - "failed to decode update1: {}, update: {:?}", - e, update - ); + error!("failed to decode update1: {}, update: {:?}", e, update); } } Err(e) => { - error!( - "failed to decode update2: {}, update: {:?}", - e, update - ); + error!("failed to decode update2: {}, update: {:?}", e, update); } } } diff --git a/libs/jwst-core-rpc/src/connector/tungstenite_socket.rs b/libs/jwst-core-rpc/src/connector/tungstenite_socket.rs index 25b2d6ab6..498a66ea2 100644 --- a/libs/jwst-core-rpc/src/connector/tungstenite_socket.rs +++ b/libs/jwst-core-rpc/src/connector/tungstenite_socket.rs @@ -1,4 +1,3 @@ -use super::*; use futures::{sink::SinkExt, stream::StreamExt}; use tokio::{ net::TcpStream, @@ -9,6 +8,8 @@ use tokio_tungstenite::{ MaybeTlsStream, WebSocketStream, }; +use super::*; + type WebSocket = WebSocketStream>; impl From for WebSocketMessage { @@ -21,10 +22,7 @@ impl From for WebSocketMessage { } } -pub fn tungstenite_socket_connector( - socket: WebSocket, - workspace_id: &str, -) -> (Sender, Receiver>) { +pub fn tungstenite_socket_connector(socket: WebSocket, workspace_id: &str) -> (Sender, Receiver>) { let (mut socket_tx, mut socket_rx) = socket.split(); // send to remote pipeline @@ -37,11 +35,7 @@ pub fn tungstenite_socket_connector( while let Some(msg) = local_receiver.recv().await { if let Err(e) = socket_tx.send(msg.into()).await { let error = e.to_string(); - if matches!( - e, - SocketError::ConnectionClosed | SocketError::AlreadyClosed - ) || retry == 0 - { + if matches!(e, SocketError::ConnectionClosed | SocketError::AlreadyClosed) || retry == 0 { break; } else { retry -= 1; diff --git a/libs/jwst-core-rpc/src/connector/webrtc.rs b/libs/jwst-core-rpc/src/connector/webrtc.rs index 8bde98118..5c49ea0ad 100644 --- a/libs/jwst-core-rpc/src/connector/webrtc.rs +++ b/libs/jwst-core-rpc/src/connector/webrtc.rs @@ -1,21 +1,19 @@ -use super::Message; -use jwst_core::{debug, error, info, trace, warn}; +use std::sync::Arc; use bytes::Bytes; -use std::sync::Arc; +use jwst_core::{debug, error, info, trace, warn}; use tokio::sync::mpsc::{channel, Receiver, Sender}; use webrtcrs::{ api::APIBuilder, - data_channel::{ - data_channel_init::RTCDataChannelInit, data_channel_message::DataChannelMessage, - OnMessageHdlrFn, - }, + data_channel::{data_channel_init::RTCDataChannelInit, data_channel_message::DataChannelMessage, OnMessageHdlrFn}, peer_connection::{ configuration::RTCConfiguration, peer_connection_state::RTCPeerConnectionState, sdp::session_description::RTCSessionDescription, RTCPeerConnection, }, }; +use super::Message; + const DATA_CHANNEL_ID: u16 = 42; const DATA_CHANNEL_LABEL: &str = "affine"; @@ -38,11 +36,9 @@ async fn new_peer_connection() -> ( ) { let api = APIBuilder::new().build(); let peer_connection = Arc::new( - api.new_peer_connection(RTCConfiguration { - ..Default::default() - }) - .await - .unwrap(), + api.new_peer_connection(RTCConfiguration { ..Default::default() }) + .await + .unwrap(), ); peer_connection.on_peer_connection_state_change(Box::new(move |s: RTCPeerConnectionState| { @@ -76,9 +72,7 @@ async fn new_peer_connection() -> ( match msg { Message::Binary(data) => { trace!("WebRTC Send: {:?}", data.clone()); - d0.send(&Bytes::copy_from_slice(data.as_slice())) - .await - .unwrap(); + d0.send(&Bytes::copy_from_slice(data.as_slice())).await.unwrap(); } Message::Close => info!("Close"), Message::Ping => info!("Ping"), @@ -128,17 +122,15 @@ pub async fn webrtc_datachannel_client_begin() -> ( // Block until ICE Gathering is complete, disabling trickle ICE // we do this because we only can exchange one signaling message - // in a production application you should exchange ICE Candidates via OnICECandidate + // in a production application you should exchange ICE Candidates via + // OnICECandidate let _ = gather_complete.recv().await; let local_desc = peer_connection.local_description().await.unwrap(); (local_desc, peer_connection, tx, rx, s) } -pub async fn webrtc_datachannel_client_commit( - answer: RTCSessionDescription, - peer_connection: Arc, -) { +pub async fn webrtc_datachannel_client_commit(answer: RTCSessionDescription, peer_connection: Arc) { match peer_connection.set_remote_description(answer).await { Ok(_) => {} Err(e) => { @@ -178,7 +170,8 @@ pub async fn webrtc_datachannel_server_connector( // Block until ICE Gathering is complete, disabling trickle ICE // we do this because we only can exchange one signaling message - // in a production application you should exchange ICE Candidates via OnICECandidate + // in a production application you should exchange ICE Candidates via + // OnICECandidate let _ = gather_complete.recv().await; let local_desc = peer_connection.local_description().await.unwrap(); diff --git a/libs/jwst-core-rpc/src/context.rs b/libs/jwst-core-rpc/src/context.rs index 6e08c443b..7d5444088 100644 --- a/libs/jwst-core-rpc/src/context.rs +++ b/libs/jwst-core-rpc/src/context.rs @@ -1,23 +1,21 @@ use std::collections::HashMap; -use super::{ - broadcast::{subscribe, BroadcastChannels, BroadcastType}, - *, -}; use async_trait::async_trait; use chrono::Utc; use jwst_codec::{CrdtReader, RawDecoder}; use jwst_core::{DocStorage, Workspace}; use jwst_core_storage::{JwstStorage, JwstStorageResult}; use tokio::sync::{ - broadcast::{ - channel as broadcast, error::RecvError, Receiver as BroadcastReceiver, - Sender as BroadcastSender, - }, + broadcast::{channel as broadcast, error::RecvError, Receiver as BroadcastReceiver, Sender as BroadcastSender}, mpsc::{Receiver as MpscReceiver, Sender as MpscSender}, Mutex, }; +use super::{ + broadcast::{subscribe, BroadcastChannels, BroadcastType}, + *, +}; + #[async_trait] pub trait RpcContextImpl<'a> { fn get_storage(&self) -> &JwstStorage; @@ -57,15 +55,17 @@ pub trait RpcContextImpl<'a> { } }; - // Listen to changes of the local workspace, encode changes in awareness and Doc, and broadcast them. - // It returns the 'broadcast_rx' object to receive the content that was sent + // Listen to changes of the local workspace, encode changes in awareness and + // Doc, and broadcast them. It returns the 'broadcast_rx' object to + // receive the content that was sent subscribe(workspace, identifier.clone(), broadcast_tx.clone()).await; // save update thread self.save_update(&id, identifier, broadcast_tx.subscribe(), last_synced) .await; - // returns the 'broadcast_tx' which can be subscribed later, to receive local workspace changes + // returns the 'broadcast_tx' which can be subscribed later, to receive local + // workspace changes broadcast_tx } @@ -128,17 +128,12 @@ pub trait RpcContextImpl<'a> { debug!("save {} updates from {guid}", updates.len()); for update in updates { - if let Err(e) = - docs.update_doc(id.clone(), guid.clone(), &update).await - { + if let Err(e) = docs.update_doc(id.clone(), guid.clone(), &update).await { error!("failed to save update of {}: {:?}", id, e); } } } - last_synced - .send(Utc::now().timestamp_millis()) - .await - .unwrap(); + last_synced.send(Utc::now().timestamp_millis()).await.unwrap(); } else if handler.is_finished() { break; } diff --git a/libs/jwst-core-rpc/src/handler.rs b/libs/jwst-core-rpc/src/handler.rs index b9691ce9b..0df9e1e18 100644 --- a/libs/jwst-core-rpc/src/handler.rs +++ b/libs/jwst-core-rpc/src/handler.rs @@ -1,12 +1,14 @@ -use super::{BroadcastType, Message, RpcContextImpl}; +use std::{sync::Arc, time::Instant}; + use chrono::Utc; use jwst_core::{debug, error, info, trace, warn}; -use std::{sync::Arc, time::Instant}; use tokio::{ sync::mpsc::{Receiver, Sender}, time::{sleep, Duration}, }; +use super::{BroadcastType, Message, RpcContextImpl}; + pub async fn handle_connector( context: Arc + Send + Sync + 'static>, workspace_id: String, @@ -15,7 +17,8 @@ pub async fn handle_connector( ) -> bool { info!("{} collaborate with workspace {}", identifier, workspace_id); - // An abstraction of the established socket connection. Use tx to broadcast and rx to receive. + // An abstraction of the established socket connection. Use tx to broadcast and + // rx to receive. let (tx, rx, last_synced) = get_channel(); let mut ws = context @@ -23,28 +26,25 @@ pub async fn handle_connector( .await .expect("failed to get workspace"); - // Continuously receive information from the remote socket, apply it to the local workspace, and - // send the encoded updates back to the remote end through the socket. + // Continuously receive information from the remote socket, apply it to the + // local workspace, and send the encoded updates back to the remote end + // through the socket. context - .apply_change( - &workspace_id, - &identifier, - tx.clone(), - rx, - last_synced.clone(), - ) + .apply_change(&workspace_id, &identifier, tx.clone(), rx, last_synced.clone()) .await; - // Both of broadcast_update and server_update are sent to the remote socket through 'tx' - // The 'broadcast_update' is the receiver for updates to the awareness and Doc of the local workspace. - // It uses channel, which is owned by the server itself and is stored in the server's memory (not persisted)." + // Both of broadcast_update and server_update are sent to the remote socket + // through 'tx' The 'broadcast_update' is the receiver for updates to the + // awareness and Doc of the local workspace. It uses channel, which is owned + // by the server itself and is stored in the server's memory (not persisted)." let broadcast_tx = context .join_broadcast(&mut ws, identifier.clone(), last_synced.clone()) .await; let mut broadcast_rx = broadcast_tx.subscribe(); - // Obtaining the receiver corresponding to DocAutoStorage in storage. The sender is used in the - // doc::write_update(). The remote used is the one belonging to DocAutoStorage and is owned by - // the server itself, stored in the server's memory (not persisted). + // Obtaining the receiver corresponding to DocAutoStorage in storage. The sender + // is used in the doc::write_update(). The remote used is the one belonging + // to DocAutoStorage and is owned by the server itself, stored in the + // server's memory (not persisted). let mut server_rx = context.join_server_broadcast(&workspace_id).await; // Send initialization message. @@ -70,10 +70,7 @@ pub async fn handle_connector( } } - last_synced - .send(Utc::now().timestamp_millis()) - .await - .unwrap(); + last_synced.send(Utc::now().timestamp_millis()).await.unwrap(); 'sync: loop { tokio::select! { @@ -167,27 +164,25 @@ pub async fn handle_connector( .full_migrate(workspace_id.clone(), None, false) .await; let _ = broadcast_tx.send(BroadcastType::CloseUser(identifier.clone())); - info!( - "{} stop collaborate with workspace {}", - identifier, workspace_id - ); + info!("{} stop collaborate with workspace {}", identifier, workspace_id); true } #[cfg(test)] mod test { + use std::sync::atomic::{AtomicU64, Ordering}; + + use indicatif::{MultiProgress, ProgressBar, ProgressIterator, ProgressStyle}; + use jwst_core::JwstResult; + use super::{ super::{connect_memory_workspace, MinimumServerContext}, *, }; #[cfg(feature = "webrtc")] use crate::{ - webrtc_datachannel_client_begin, webrtc_datachannel_client_commit, - webrtc_datachannel_server_connector, + webrtc_datachannel_client_begin, webrtc_datachannel_client_commit, webrtc_datachannel_server_connector, }; - use indicatif::{MultiProgress, ProgressBar, ProgressIterator, ProgressStyle}; - use jwst_core::JwstResult; - use std::sync::atomic::{AtomicU64, Ordering}; #[tokio::test] #[ignore = "skip in ci"] @@ -205,26 +200,14 @@ mod test { let data_b_2 = String::from("data_b_2"); let data_b_3 = String::from("data_b_3"); - tx1.send(Message::Binary(data_a_1.clone().into_bytes())) - .await - .unwrap(); - tx1.send(Message::Binary(data_a_2.clone().into_bytes())) - .await - .unwrap(); - - tx2.send(Message::Binary(data_b_1.clone().into_bytes())) - .await - .unwrap(); - tx2.send(Message::Binary(data_b_2.clone().into_bytes())) - .await - .unwrap(); - - tx1.send(Message::Binary(data_a_3.clone().into_bytes())) - .await - .unwrap(); - tx2.send(Message::Binary(data_b_3.clone().into_bytes())) - .await - .unwrap(); + tx1.send(Message::Binary(data_a_1.clone().into_bytes())).await.unwrap(); + tx1.send(Message::Binary(data_a_2.clone().into_bytes())).await.unwrap(); + + tx2.send(Message::Binary(data_b_1.clone().into_bytes())).await.unwrap(); + tx2.send(Message::Binary(data_b_2.clone().into_bytes())).await.unwrap(); + + tx1.send(Message::Binary(data_a_3.clone().into_bytes())).await.unwrap(); + tx2.send(Message::Binary(data_b_3.clone().into_bytes())).await.unwrap(); if let Some(message) = rx2.recv().await { assert_eq!(String::from_utf8(message).ok().unwrap(), data_a_1); @@ -253,11 +236,9 @@ mod test { async fn sync_test() -> JwstResult<()> { let workspace_id = format!("test{}", rand::random::()); - let (server, mut ws, init_state) = - MinimumServerContext::new_with_workspace(&workspace_id).await; + let (server, mut ws, init_state) = MinimumServerContext::new_with_workspace(&workspace_id).await; - let (mut doc1, _, _, _, _) = - connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; + let (mut doc1, _, _, _, _) = connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; let (doc2, tx2, tx_handler, rx_handler, rt) = connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; @@ -270,9 +251,7 @@ mod test { // collect the update from yrs's editing let update = { - let mut ws = - jwst_core::Workspace::from_binary(doc1.encode_update_v1().unwrap(), &workspace_id) - .unwrap(); + let mut ws = jwst_core::Workspace::from_binary(doc1.encode_update_v1().unwrap(), &workspace_id).unwrap(); let mut space = ws.get_space("space").unwrap(); let mut block1 = space.create("block1", "flavour1").unwrap(); @@ -289,9 +268,7 @@ mod test { // collect the update from jwst-codec and check the result { - let mut ws = - jwst_core::Workspace::from_binary(doc2.encode_update_v1().unwrap(), &workspace_id) - .unwrap(); + let mut ws = jwst_core::Workspace::from_binary(doc2.encode_update_v1().unwrap(), &workspace_id).unwrap(); let space = ws.get_space("space").unwrap(); let block1 = space.get("block1").unwrap(); @@ -325,16 +302,13 @@ mod test { async fn single_sync_stress_test(mp: &MultiProgress) -> JwstResult<()> { // jwst_logger::init_logger("jwst-rpc"); - let style = ProgressStyle::with_template( - "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", - ) - .unwrap() - .progress_chars("##-"); + let style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}") + .unwrap() + .progress_chars("##-"); let workspace_id = format!("test{}", rand::random::()); - let (server, mut ws, init_state) = - MinimumServerContext::new_with_workspace(&workspace_id).await; + let (server, mut ws, init_state) = MinimumServerContext::new_with_workspace(&workspace_id).await; let mut jobs = vec![]; @@ -357,9 +331,7 @@ mod test { collaborator_pb.set_position(collaborator.load(Ordering::Relaxed)); let (doc, doc_tx, tx_handler, rx_handler, _rt) = connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; - let mut doc = - jwst_core::Workspace::from_binary(doc.encode_update_v1().unwrap(), &workspace_id) - .unwrap(); + let mut doc = jwst_core::Workspace::from_binary(doc.encode_update_v1().unwrap(), &workspace_id).unwrap(); let handler = std::thread::spawn(move || { // close connection after doc1 is broadcasted @@ -381,8 +353,7 @@ mod test { let block_changed = false; if block_changed { - if let Err(e) = futures::executor::block_on(doc_tx.send(Message::Close)) - { + if let Err(e) = futures::executor::block_on(doc_tx.send(Message::Close)) { error!("send close message failed: {}", e); } } @@ -391,12 +362,8 @@ mod test { { let mut space = doc.get_space("space").unwrap(); - let mut block = space - .create(block_id.clone(), format!("flavour{}", i)) - .unwrap(); - block - .set(&format!("key{}", i), format!("val{}", i)) - .unwrap(); + let mut block = space.create(block_id.clone(), format!("flavour{}", i)).unwrap(); + block.set(&format!("key{}", i), format!("val{}", i)).unwrap(); } // await the task to make sure the doc1 is broadcasted before check doc2 @@ -464,11 +431,9 @@ mod test { // jwst_logger::init_logger("jwst-rpc"); let mp = MultiProgress::new(); - let style = ProgressStyle::with_template( - "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", - ) - .unwrap() - .progress_chars("##-"); + let style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}") + .unwrap() + .progress_chars("##-"); let pb = mp.add(ProgressBar::new(1000)); pb.set_style(style.clone()); diff --git a/libs/jwst-core-rpc/src/lib.rs b/libs/jwst-core-rpc/src/lib.rs index a282d37a4..5c3a884df 100644 --- a/libs/jwst-core-rpc/src/lib.rs +++ b/libs/jwst-core-rpc/src/lib.rs @@ -7,10 +7,15 @@ mod handler; mod types; mod utils; +use std::{collections::hash_map::Entry, sync::Arc, time::Instant}; + +pub use broadcast::{BroadcastChannels, BroadcastType}; #[cfg(feature = "webrtc")] pub use client::start_webrtc_client_sync; #[cfg(feature = "websocket")] pub use client::start_websocket_client_sync; +pub use client::CachedLastSynced; +pub use connector::memory_connector; #[cfg(feature = "webrtc")] pub use connector::webrtc_datachannel_client_begin; #[cfg(feature = "webrtc")] @@ -19,22 +24,16 @@ pub use connector::webrtc_datachannel_client_commit; pub use connector::webrtc_datachannel_server_connector; #[cfg(feature = "websocket")] pub use connector::{axum_socket_connector, tungstenite_socket_connector}; -#[cfg(feature = "webrtc")] -pub use webrtcrs::peer_connection::sdp::session_description::RTCSessionDescription; - -pub use broadcast::{BroadcastChannels, BroadcastType}; -pub use client::CachedLastSynced; -pub use connector::memory_connector; pub use context::RpcContextImpl; pub use handler::handle_connector; -pub use utils::{connect_memory_workspace, MinimumServerContext}; - use jwst_core::{debug, error, info, trace, warn}; -use std::{collections::hash_map::Entry, sync::Arc, time::Instant}; use tokio::{ sync::mpsc::{Receiver, Sender}, time::{sleep, Duration}, }; +pub use utils::{connect_memory_workspace, MinimumServerContext}; +#[cfg(feature = "webrtc")] +pub use webrtcrs::peer_connection::sdp::session_description::RTCSessionDescription; #[derive(Clone, PartialEq, Eq, Debug, Default)] pub enum SyncState { diff --git a/libs/jwst-core-rpc/src/utils/memory_workspace.rs b/libs/jwst-core-rpc/src/utils/memory_workspace.rs index a19a771d9..9e38031c9 100644 --- a/libs/jwst-core-rpc/src/utils/memory_workspace.rs +++ b/libs/jwst-core-rpc/src/utils/memory_workspace.rs @@ -1,9 +1,11 @@ -use super::*; +use std::thread::JoinHandle as StdJoinHandler; + use jwst_codec::Doc; use nanoid::nanoid; -use std::thread::JoinHandle as StdJoinHandler; use tokio::{runtime::Runtime, sync::mpsc::channel, task::JoinHandle as TokioJoinHandler}; +use super::*; + pub async fn connect_memory_workspace( server: Arc, init_state: &[u8], @@ -28,12 +30,9 @@ pub async fn connect_memory_workspace( { let rt = rt.clone(); std::thread::spawn(move || { - rt.block_on(handle_connector( - server, - workspace_id, - nanoid!(), - move || (tx, rx, last_synced_tx), - )); + rt.block_on(handle_connector(server, workspace_id, nanoid!(), move || { + (tx, rx, last_synced_tx) + })); }); } diff --git a/libs/jwst-core-rpc/src/utils/mod.rs b/libs/jwst-core-rpc/src/utils/mod.rs index 92ef9fcf6..90bd7951c 100644 --- a/libs/jwst-core-rpc/src/utils/mod.rs +++ b/libs/jwst-core-rpc/src/utils/mod.rs @@ -1,7 +1,7 @@ mod memory_workspace; mod server_context; -use super::*; - pub use memory_workspace::connect_memory_workspace; pub use server_context::MinimumServerContext; + +use super::*; diff --git a/libs/jwst-core-rpc/src/utils/server_context.rs b/libs/jwst-core-rpc/src/utils/server_context.rs index 386846bde..d2e114bcb 100644 --- a/libs/jwst-core-rpc/src/utils/server_context.rs +++ b/libs/jwst-core-rpc/src/utils/server_context.rs @@ -1,10 +1,12 @@ -use super::*; +use std::{collections::HashMap, time::Duration}; + use jwst_codec::StateVector; use jwst_core::{DocStorage, Workspace}; use jwst_core_storage::{BlobStorageType, JwstStorage}; -use std::{collections::HashMap, time::Duration}; use tokio::{sync::RwLock, time::sleep}; +use super::*; + pub struct MinimumServerContext { channel: BroadcastChannels, storage: JwstStorage, @@ -43,9 +45,7 @@ impl MinimumServerContext { }) } - pub async fn new_with_workspace( - workspace_id: &str, - ) -> (Arc, Workspace, Vec) { + pub async fn new_with_workspace(workspace_id: &str) -> (Arc, Workspace, Vec) { let server = Self::new().await; server .get_storage() diff --git a/libs/jwst-core-storage/src/entities/prelude.rs b/libs/jwst-core-storage/src/entities/prelude.rs index 5bdda8eb3..aaabca6ac 100644 --- a/libs/jwst-core-storage/src/entities/prelude.rs +++ b/libs/jwst-core-storage/src/entities/prelude.rs @@ -1,7 +1,6 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 -pub use super::blobs::Entity as Blobs; -pub use super::bucket_blobs::Entity as BucketBlobs; -pub use super::diff_log::Entity as DiffLog; -pub use super::docs::Entity as Docs; -pub use super::optimized_blobs::Entity as OptimizedBlobs; +pub use super::{ + blobs::Entity as Blobs, bucket_blobs::Entity as BucketBlobs, diff_log::Entity as DiffLog, docs::Entity as Docs, + optimized_blobs::Entity as OptimizedBlobs, +}; diff --git a/libs/jwst-core-storage/src/lib.rs b/libs/jwst-core-storage/src/lib.rs index 553a0c957..2612da8bf 100644 --- a/libs/jwst-core-storage/src/lib.rs +++ b/libs/jwst-core-storage/src/lib.rs @@ -4,6 +4,8 @@ mod rate_limiter; mod storage; mod types; +use std::{path::PathBuf, sync::Arc, time::Duration}; + use async_trait::async_trait; use chrono::{DateTime, Utc}; use futures::{Future, Stream}; @@ -12,17 +14,14 @@ use jwst_logger::{debug, error, info, trace, warn}; use path_ext::PathExt; use rate_limiter::{get_bucket, is_sqlite, Bucket}; use sea_orm::{prelude::*, ConnectOptions, Database, DbErr, QuerySelect, Set}; -use std::{path::PathBuf, sync::Arc, time::Duration}; - -pub use storage::blobs::{BlobStorageType, MixedBucketDBParam}; -pub use storage::JwstStorage; +pub use storage::{ + blobs::{BlobStorageType, MixedBucketDBParam}, + JwstStorage, +}; pub use types::{JwstStorageError, JwstStorageResult}; #[inline] -async fn create_connection( - database: &str, - single_thread: bool, -) -> JwstStorageResult { +async fn create_connection(database: &str, single_thread: bool) -> JwstStorageResult { let connection = Database::connect( ConnectOptions::from(database) .max_connections(if single_thread { 1 } else { 50 }) diff --git a/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs b/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs index 1f7377334..f253f6971 100644 --- a/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs +++ b/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs @@ -1,6 +1,7 @@ -use super::schema::Blobs; use sea_orm_migration::prelude::*; +use super::schema::Blobs; + pub struct Migration; impl MigrationName for Migration { @@ -21,11 +22,7 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Blobs::Hash).string().not_null()) .col(ColumnDef::new(Blobs::Blob).binary().not_null()) .col(ColumnDef::new(Blobs::Length).big_integer().not_null()) - .col( - ColumnDef::new(Blobs::Timestamp) - .timestamp_with_time_zone() - .not_null(), - ) + .col(ColumnDef::new(Blobs::Timestamp).timestamp_with_time_zone().not_null()) .primary_key(Index::create().col(Blobs::Workspace).col(Blobs::Hash)) .to_owned(), ) @@ -46,12 +43,8 @@ impl MigrationTrait for Migration { // Define how to rollback this migration: Drop the Bakery table. async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .drop_index(Index::drop().name("blobs_list").to_owned()) - .await?; - manager - .drop_table(Table::drop().table(Blobs::Table).to_owned()) - .await?; + manager.drop_index(Index::drop().name("blobs_list").to_owned()).await?; + manager.drop_table(Table::drop().table(Blobs::Table).to_owned()).await?; Ok(()) } } diff --git a/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs b/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs index 8b51e52cb..47edd0483 100644 --- a/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs +++ b/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs @@ -1,6 +1,7 @@ -use super::schema::Docs; use sea_orm_migration::prelude::*; +use super::schema::Docs; + pub struct Migration; impl MigrationName for Migration { @@ -17,18 +18,9 @@ impl MigrationTrait for Migration { .create_table( Table::create() .table(Docs::Table) - .col( - ColumnDef::new(Docs::Id) - .integer() - .auto_increment() - .primary_key(), - ) + .col(ColumnDef::new(Docs::Id).integer().auto_increment().primary_key()) .col(ColumnDef::new(Docs::Workspace).string().not_null()) - .col( - ColumnDef::new(Docs::Timestamp) - .timestamp_with_time_zone() - .not_null(), - ) + .col(ColumnDef::new(Docs::Timestamp).timestamp_with_time_zone().not_null()) .col(ColumnDef::new(Docs::Blob).binary().not_null()) .to_owned(), ) @@ -52,9 +44,7 @@ impl MigrationTrait for Migration { manager .drop_index(Index::drop().name("workspaces_update").to_owned()) .await?; - manager - .drop_table(Table::drop().table(Docs::Table).to_owned()) - .await?; + manager.drop_table(Table::drop().table(Docs::Table).to_owned()).await?; Ok(()) } } diff --git a/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs b/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs index 5e296501f..d89fab857 100644 --- a/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs +++ b/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs @@ -1,6 +1,7 @@ -use super::schema::OptimizedBlobs; use sea_orm_migration::prelude::*; +use super::schema::OptimizedBlobs; + pub struct Migration; impl MigrationName for Migration { @@ -17,18 +18,10 @@ impl MigrationTrait for Migration { .create_table( Table::create() .table(OptimizedBlobs::Table) - .col( - ColumnDef::new(OptimizedBlobs::Workspace) - .string() - .not_null(), - ) + .col(ColumnDef::new(OptimizedBlobs::Workspace).string().not_null()) .col(ColumnDef::new(OptimizedBlobs::Hash).string().not_null()) .col(ColumnDef::new(OptimizedBlobs::Blob).binary().not_null()) - .col( - ColumnDef::new(OptimizedBlobs::Length) - .big_integer() - .not_null(), - ) + .col(ColumnDef::new(OptimizedBlobs::Length).big_integer().not_null()) .col( ColumnDef::new(OptimizedBlobs::Timestamp) .timestamp_with_time_zone() diff --git a/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs b/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs index 7f79985c9..3f778d8cd 100644 --- a/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs +++ b/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs @@ -1,6 +1,7 @@ -use super::schema::BucketBlobs; use sea_orm_migration::prelude::*; +use super::schema::BucketBlobs; + pub struct Migration; impl MigrationName for Migration { @@ -25,11 +26,7 @@ impl MigrationTrait for Migration { .timestamp_with_time_zone() .not_null(), ) - .primary_key( - Index::create() - .col(BucketBlobs::Workspace) - .col(BucketBlobs::Hash), - ) + .primary_key(Index::create().col(BucketBlobs::Workspace).col(BucketBlobs::Hash)) .to_owned(), ) .await?; diff --git a/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs b/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs index 01ec2c326..bfe91e36b 100644 --- a/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs +++ b/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs @@ -20,12 +20,7 @@ impl MigrationTrait for Migration { .alter_table( Table::alter() .table(Docs::Table) - .add_column( - ColumnDef::new(Docs::IsWorkspace) - .boolean() - .not_null() - .default(true), - ) + .add_column(ColumnDef::new(Docs::IsWorkspace).boolean().not_null().default(true)) .to_owned(), ) .await?; @@ -41,9 +36,7 @@ impl MigrationTrait for Migration { } async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .drop_index(Index::drop().name("docs_guid").to_owned()) - .await?; + manager.drop_index(Index::drop().name("docs_guid").to_owned()).await?; manager .alter_table( diff --git a/libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs b/libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs index d45c9b2cc..8662469b8 100644 --- a/libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs +++ b/libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs @@ -19,11 +19,7 @@ impl MigrationTrait for Migration { .primary_key(), ) .col(ColumnDef::new(DiffLog::Workspace).string().not_null()) - .col( - ColumnDef::new(DiffLog::Timestamp) - .timestamp_with_time_zone() - .not_null(), - ) + .col(ColumnDef::new(DiffLog::Timestamp).timestamp_with_time_zone().not_null()) .col(ColumnDef::new(DiffLog::Log).string().not_null()) .to_owned(), ) @@ -31,9 +27,7 @@ impl MigrationTrait for Migration { } async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .drop_table(Table::drop().table(DiffLog::Table).to_owned()) - .await + manager.drop_table(Table::drop().table(DiffLog::Table).to_owned()).await } } diff --git a/libs/jwst-core-storage/src/rate_limiter.rs b/libs/jwst-core-storage/src/rate_limiter.rs index 334775159..2ba158ff4 100644 --- a/libs/jwst-core-storage/src/rate_limiter.rs +++ b/libs/jwst-core-storage/src/rate_limiter.rs @@ -1,10 +1,11 @@ +use std::{num::NonZeroU32, sync::Arc}; + use governor::{ clock::{QuantaClock, QuantaInstant}, middleware::NoOpMiddleware, state::{InMemoryState, NotKeyed}, Quota, RateLimiter, }; -use std::{num::NonZeroU32, sync::Arc}; use tokio::sync::{OwnedSemaphorePermit, RwLock, RwLockReadGuard, RwLockWriteGuard, Semaphore}; use url::Url; @@ -26,8 +27,7 @@ pub struct Bucket { impl Bucket { fn new(bucket_size: u32, semaphore_size: usize) -> Self { - let bucket_size = - NonZeroU32::new(bucket_size).unwrap_or(unsafe { NonZeroU32::new_unchecked(1) }); + let bucket_size = NonZeroU32::new(bucket_size).unwrap_or(unsafe { NonZeroU32::new_unchecked(1) }); Self { bucket: Arc::new(RateLimiter::direct( @@ -65,9 +65,7 @@ impl Bucket { #[inline] pub fn is_sqlite(database: &str) -> bool { - Url::parse(database) - .map(|u| u.scheme() == "sqlite") - .unwrap_or(false) + Url::parse(database).map(|u| u.scheme() == "sqlite").unwrap_or(false) } #[inline] diff --git a/libs/jwst-core-storage/src/storage/blobs/bucket_local_db.rs b/libs/jwst-core-storage/src/storage/blobs/bucket_local_db.rs index 3601a4fc4..1b19910d9 100644 --- a/libs/jwst-core-storage/src/storage/blobs/bucket_local_db.rs +++ b/libs/jwst-core-storage/src/storage/blobs/bucket_local_db.rs @@ -1,19 +1,18 @@ -use super::utils::calculate_hash; -use super::utils::get_hash; -use super::*; -use crate::rate_limiter::Bucket; -use crate::JwstStorageError; +use std::{collections::HashMap, sync::Arc}; + use bytes::Bytes; use futures::Stream; use jwst_core::{BlobMetadata, BlobStorage, BucketBlobStorage, JwstResult}; use jwst_storage_migration::Migrator; -use opendal::services::S3; -use opendal::Operator; +use opendal::{services::S3, Operator}; use sea_orm::{DatabaseConnection, EntityTrait}; use sea_orm_migration::MigratorTrait; -use std::collections::HashMap; -use std::sync::Arc; +use super::{ + utils::{calculate_hash, get_hash}, + *, +}; +use crate::{rate_limiter::Bucket, JwstStorageError}; pub(super) type BucketBlobModel = ::Model; type BucketBlobActiveModel = entities::bucket_blobs::ActiveModel; @@ -47,10 +46,7 @@ impl BlobBucketDBStorage { } #[allow(unused)] - pub async fn init_pool( - database: &str, - bucket_storage: Option, - ) -> JwstStorageResult { + pub async fn init_pool(database: &str, bucket_storage: Option) -> JwstStorageResult { let is_sqlite = is_sqlite(database); let pool = create_connection(database, is_sqlite).await?; @@ -89,11 +85,7 @@ impl BlobBucketDBStorage { .map(|c| c > 0) } - pub(super) async fn metadata( - &self, - workspace: &str, - hash: &str, - ) -> JwstBlobResult { + pub(super) async fn metadata(&self, workspace: &str, hash: &str) -> JwstBlobResult { BucketBlobs::find_by_id((workspace.into(), hash.into())) .select_only() .column_as(BucketBlobColumn::Length, "size") @@ -178,11 +170,7 @@ impl BucketStorage { #[async_trait] impl BucketBlobStorage for BucketStorage { - async fn get_blob( - &self, - workspace: Option, - id: String, - ) -> JwstResult, JwstStorageError> { + async fn get_blob(&self, workspace: Option, id: String) -> JwstResult, JwstStorageError> { let workspace = get_workspace(workspace); let key = build_key(workspace, id); let bs = self.op.read(&key).await?; @@ -203,11 +191,7 @@ impl BucketBlobStorage for BucketStorage { Ok(()) } - async fn delete_blob( - &self, - workspace: Option, - id: String, - ) -> JwstResult { + async fn delete_blob(&self, workspace: Option, id: String) -> JwstResult { let workspace = get_workspace(workspace); let key = build_key(workspace, id); @@ -226,10 +210,7 @@ impl BucketBlobStorage for BucketStorage { #[async_trait] impl BlobStorage for BlobBucketDBStorage { - async fn list_blobs( - &self, - workspace: Option, - ) -> JwstResult, JwstStorageError> { + async fn list_blobs(&self, workspace: Option) -> JwstResult, JwstStorageError> { let _lock = self.bucket.read().await; let workspace = workspace.unwrap_or("__default__".into()); if let Ok(keys) = self.keys(&workspace).await { @@ -239,11 +220,7 @@ impl BlobStorage for BlobBucketDBStorage { Err(JwstStorageError::WorkspaceNotFound(workspace)) } - async fn check_blob( - &self, - workspace: Option, - id: String, - ) -> JwstResult { + async fn check_blob(&self, workspace: Option, id: String) -> JwstResult { let _lock = self.bucket.read().await; let workspace = workspace.unwrap_or("__default__".into()); if let Ok(exists) = self.exists(&workspace, &id).await { @@ -292,18 +269,12 @@ impl BlobStorage for BlobBucketDBStorage { if self.insert(&workspace, &hash, &blob).await.is_ok() { Ok(hash) } else { - self.bucket_storage - .delete_blob(Some(workspace.clone()), hash) - .await?; + self.bucket_storage.delete_blob(Some(workspace.clone()), hash).await?; Err(JwstStorageError::WorkspaceNotFound(workspace)) } } - async fn put_blob( - &self, - workspace: Option, - blob: Vec, - ) -> JwstResult { + async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstResult { let hash = calculate_hash(&blob); self.bucket_storage .put_blob(workspace.clone(), hash.clone(), blob.clone()) @@ -315,21 +286,13 @@ impl BlobStorage for BlobBucketDBStorage { if self.insert(&workspace, &hash, &blob).await.is_ok() { Ok(hash) } else { - self.bucket_storage - .delete_blob(Some(workspace.clone()), hash) - .await?; + self.bucket_storage.delete_blob(Some(workspace.clone()), hash).await?; Err(JwstStorageError::WorkspaceNotFound(workspace)) } } - async fn delete_blob( - &self, - workspace: Option, - id: String, - ) -> JwstResult { - self.bucket_storage - .delete_blob(workspace.clone(), id.clone()) - .await?; + async fn delete_blob(&self, workspace: Option, id: String) -> JwstResult { + self.bucket_storage.delete_blob(workspace.clone(), id.clone()).await?; let _lock = self.bucket.write().await; let workspace = get_workspace(workspace); if let Ok(success) = self.delete(&workspace, &id).await { @@ -340,9 +303,7 @@ impl BlobStorage for BlobBucketDBStorage { } async fn delete_workspace(&self, workspace_id: String) -> JwstResult<(), JwstStorageError> { - self.bucket_storage - .delete_workspace(workspace_id.clone()) - .await?; + self.bucket_storage.delete_workspace(workspace_id.clone()).await?; let _lock = self.bucket.write().await; if self.drop(&workspace_id).await.is_ok() { Ok(()) diff --git a/libs/jwst-core-storage/src/storage/blobs/local_db.rs b/libs/jwst-core-storage/src/storage/blobs/local_db.rs index b61caf557..fd5b82308 100644 --- a/libs/jwst-core-storage/src/storage/blobs/local_db.rs +++ b/libs/jwst-core-storage/src/storage/blobs/local_db.rs @@ -1,8 +1,8 @@ -use super::{utils::get_hash, *}; -use crate::types::JwstStorageResult; use jwst_core::{Base64Engine, URL_SAFE_ENGINE}; - use sha2::{Digest, Sha256}; + +use super::{utils::get_hash, *}; +use crate::types::JwstStorageResult; pub(super) type BlobModel = ::Model; type BlobActiveModel = super::entities::blobs::ActiveModel; type BlobColumn = ::Column; @@ -20,10 +20,7 @@ impl AsRef for BlobDBStorage { } impl BlobDBStorage { - pub async fn init_with_pool( - pool: DatabaseConnection, - bucket: Arc, - ) -> JwstStorageResult { + pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { Ok(Self { bucket, pool }) } @@ -66,11 +63,7 @@ impl BlobDBStorage { .map(|c| c > 0) } - pub(super) async fn metadata( - &self, - workspace: &str, - hash: &str, - ) -> JwstBlobResult { + pub(super) async fn metadata(&self, workspace: &str, hash: &str) -> JwstBlobResult { Blobs::find_by_id((workspace.into(), hash.into())) .select_only() .column_as(BlobColumn::Length, "size") @@ -203,11 +196,7 @@ impl BlobStorage for BlobDBStorage { } } - async fn put_blob( - &self, - workspace: Option, - blob: Vec, - ) -> JwstStorageResult { + async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstStorageResult { let _lock = self.bucket.write().await; let workspace = workspace.unwrap_or("__default__".into()); let mut hasher = Sha256::new(); @@ -222,11 +211,7 @@ impl BlobStorage for BlobDBStorage { } } - async fn delete_blob( - &self, - workspace_id: Option, - id: String, - ) -> JwstStorageResult { + async fn delete_blob(&self, workspace_id: Option, id: String) -> JwstStorageResult { let _lock = self.bucket.write().await; let workspace_id = workspace_id.unwrap_or("__default__".into()); if let Ok(success) = self.delete(&workspace_id, &id).await { diff --git a/libs/jwst-core-storage/src/storage/blobs/mod.rs b/libs/jwst-core-storage/src/storage/blobs/mod.rs index 20a49d212..fb52037ed 100644 --- a/libs/jwst-core-storage/src/storage/blobs/mod.rs +++ b/libs/jwst-core-storage/src/storage/blobs/mod.rs @@ -2,20 +2,19 @@ mod bucket_local_db; mod local_db; mod utils; -#[cfg(test)] -pub use local_db::blobs_storage_test; - -use super::{entities::prelude::*, *}; +pub use bucket_local_db::BlobBucketDBStorage; use bytes::Bytes; use image::ImageError; use jwst_core::{BlobMetadata, BlobStorage}; +#[cfg(test)] +pub use local_db::blobs_storage_test; use local_db::BlobDBStorage; use thiserror::Error; use tokio::task::JoinError; +pub use utils::BucketStorageBuilder; use utils::{ImageParams, InternalBlobMetadata}; -pub use bucket_local_db::BlobBucketDBStorage; -pub use utils::BucketStorageBuilder; +use super::{entities::prelude::*, *}; #[derive(Debug, Error)] pub enum JwstBlobError { @@ -89,10 +88,7 @@ impl MixedBucketDBParam { } impl BlobAutoStorage { - pub async fn init_with_pool( - pool: DatabaseConnection, - bucket: Arc, - ) -> JwstStorageResult { + pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { let db = Arc::new(BlobDBStorage::init_with_pool(pool, bucket).await?); let pool = db.pool.clone(); Ok(Self { db, pool }) @@ -105,21 +101,13 @@ impl BlobAutoStorage { } async fn exists(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { - Ok( - OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) - .count(&self.pool) - .await - .map(|c| c > 0)?, - ) + Ok(OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) + .count(&self.pool) + .await + .map(|c| c > 0)?) } - async fn insert( - &self, - table: &str, - hash: &str, - params: &str, - blob: &[u8], - ) -> JwstBlobResult<()> { + async fn insert(&self, table: &str, hash: &str, params: &str, blob: &[u8]) -> JwstBlobResult<()> { if !self.exists(table, hash, params).await? { OptimizedBlobs::insert(OptimizedBlobActiveModel { workspace_id: Set(table.into()), @@ -136,12 +124,7 @@ impl BlobAutoStorage { Ok(()) } - async fn get( - &self, - table: &str, - hash: &str, - params: &str, - ) -> JwstBlobResult { + async fn get(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) .one(&self.pool) .await @@ -149,12 +132,7 @@ impl BlobAutoStorage { .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) } - async fn metadata( - &self, - table: &str, - hash: &str, - params: &str, - ) -> JwstBlobResult { + async fn metadata(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) .select_only() .column_as(OptimizedBlobColumn::Length, "size") @@ -216,11 +194,8 @@ impl BlobAutoStorage { // TODO: need ddos mitigation let blob = self.db.get(workspace_id, &id).await?; let blob_len = blob.blob.len(); - let image = - tokio::task::spawn_blocking(move || params.optimize_image(&blob.blob)) - .await??; - self.insert(workspace_id, &id, ¶ms_token, &image) - .await?; + let image = tokio::task::spawn_blocking(move || params.optimize_image(&blob.blob)).await??; + self.insert(workspace_id, &id, ¶ms_token, &image).await?; info!( "optimized image: {} {} {}, {}bytes -> {}bytes", workspace_id, @@ -299,24 +274,13 @@ impl BlobStorage for BlobAutoStorage { self.db.put_blob_stream(workspace, stream).await } - async fn put_blob( - &self, - workspace: Option, - blob: Vec, - ) -> JwstStorageResult { + async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstStorageResult { self.db.put_blob(workspace, blob).await } - async fn delete_blob( - &self, - workspace_id: Option, - id: String, - ) -> JwstStorageResult { + async fn delete_blob(&self, workspace_id: Option, id: String) -> JwstStorageResult { // delete origin blobs - let success = self - .db - .delete_blob(workspace_id.clone(), id.clone()) - .await?; + let success = self.db.delete_blob(workspace_id.clone(), id.clone()).await?; if success { // delete optimized blobs let workspace_id = workspace_id.unwrap_or("__default__".into()); @@ -344,21 +308,14 @@ impl BlobStorage for BlobAutoStorage { #[async_trait] impl BlobStorage for JwstBlobStorage { - async fn list_blobs( - &self, - workspace: Option, - ) -> JwstResult, JwstStorageError> { + async fn list_blobs(&self, workspace: Option) -> JwstResult, JwstStorageError> { match self { JwstBlobStorage::DB(db) => db.list_blobs(workspace).await, JwstBlobStorage::MixedBucketDB(db) => db.list_blobs(workspace).await, } } - async fn check_blob( - &self, - workspace: Option, - id: String, - ) -> JwstResult { + async fn check_blob(&self, workspace: Option, id: String) -> JwstResult { match self { JwstBlobStorage::DB(db) => db.check_blob(workspace, id).await, JwstBlobStorage::MixedBucketDB(db) => db.check_blob(workspace, id).await, @@ -400,22 +357,14 @@ impl BlobStorage for JwstBlobStorage { } } - async fn put_blob( - &self, - workspace: Option, - blob: Vec, - ) -> JwstResult { + async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstResult { match self { JwstBlobStorage::DB(db) => db.put_blob(workspace, blob).await, JwstBlobStorage::MixedBucketDB(db) => db.put_blob(workspace, blob).await, } } - async fn delete_blob( - &self, - workspace: Option, - id: String, - ) -> JwstResult { + async fn delete_blob(&self, workspace: Option, id: String) -> JwstResult { match self { JwstBlobStorage::DB(db) => db.delete_blob(workspace, id).await, JwstBlobStorage::MixedBucketDB(db) => db.delete_blob(workspace, id).await, @@ -455,10 +404,12 @@ impl JwstBlobStorage { #[cfg(test)] mod tests { - use super::*; + use std::io::Cursor; + use futures::FutureExt; use image::{DynamicImage, ImageOutputFormat}; - use std::io::Cursor; + + use super::*; #[tokio::test] async fn test_blob_auto_storage() { @@ -468,10 +419,7 @@ mod tests { let blob = Vec::from_iter((0..100).map(|_| rand::random())); let stream = async { Bytes::from(blob.clone()) }.into_stream(); - let hash1 = storage - .put_blob_stream(Some("blob".into()), stream) - .await - .unwrap(); + let hash1 = storage.put_blob_stream(Some("blob".into()), stream).await.unwrap(); // check origin blob result assert_eq!( @@ -509,10 +457,7 @@ mod tests { image.into_inner() }; let stream = async { Bytes::from(image.clone()) }.into_stream(); - let hash2 = storage - .put_blob_stream(Some("blob".into()), stream) - .await - .unwrap(); + let hash2 = storage.put_blob_stream(Some("blob".into()), stream).await.unwrap(); // check origin blob result assert_eq!( @@ -534,11 +479,7 @@ mod tests { // check optimized jpeg result let jpeg_params = HashMap::from([("format".into(), "jpeg".into())]); let jpeg = storage - .get_blob( - Some("blob".into()), - hash2.clone(), - Some(jpeg_params.clone()), - ) + .get_blob(Some("blob".into()), hash2.clone(), Some(jpeg_params.clone())) .await .unwrap(); @@ -555,22 +496,14 @@ mod tests { // check optimized webp result let webp_params = HashMap::from([("format".into(), "webp".into())]); let webp = storage - .get_blob( - Some("blob".into()), - hash2.clone(), - Some(webp_params.clone()), - ) + .get_blob(Some("blob".into()), hash2.clone(), Some(webp_params.clone())) .await .unwrap(); assert!(webp.starts_with(b"RIFF")); assert_eq!( storage - .get_metadata( - Some("blob".into()), - hash2.clone(), - Some(webp_params.clone()) - ) + .get_metadata(Some("blob".into()), hash2.clone(), Some(webp_params.clone())) .await .unwrap() .size as usize, @@ -625,15 +558,9 @@ mod tests { 100 + image.len() ); - assert!(storage - .delete_blob(Some("blob".into()), hash2.clone()) - .await - .unwrap()); + assert!(storage.delete_blob(Some("blob".into()), hash2.clone()).await.unwrap()); assert_eq!( - storage - .check_blob(Some("blob".into()), hash2.clone()) - .await - .unwrap(), + storage.check_blob(Some("blob".into()), hash2.clone()).await.unwrap(), false ); assert!(storage @@ -649,20 +576,11 @@ mod tests { .await .is_err()); - assert_eq!( - storage.get_blobs_size("blob".into()).await.unwrap() as usize, - 100 - ); + assert_eq!(storage.get_blobs_size("blob".into()).await.unwrap() as usize, 100); + assert_eq!(storage.list_blobs(Some("blob".into())).await.unwrap(), vec![hash1]); assert_eq!( - storage.list_blobs(Some("blob".into())).await.unwrap(), - vec![hash1] - ); - assert_eq!( - storage - .list_blobs(Some("not_exists_workspace".into())) - .await - .unwrap(), + storage.list_blobs(Some("not_exists_workspace".into())).await.unwrap(), Vec::::new() ); } diff --git a/libs/jwst-core-storage/src/storage/blobs/utils.rs b/libs/jwst-core-storage/src/storage/blobs/utils.rs index 572e95d2a..a967df3b9 100644 --- a/libs/jwst-core-storage/src/storage/blobs/utils.rs +++ b/libs/jwst-core-storage/src/storage/blobs/utils.rs @@ -1,6 +1,5 @@ -use crate::storage::blobs::bucket_local_db::BucketStorage; -use crate::storage::blobs::MixedBucketDBParam; -use crate::{JwstStorageError, JwstStorageResult}; +use std::{collections::HashMap, io::Cursor}; + use bytes::Bytes; use chrono::{DateTime, Utc}; use futures::{ @@ -9,11 +8,14 @@ use futures::{ }; use image::{load_from_memory, ImageOutputFormat, ImageResult}; use jwst_core::{Base64Engine, BlobMetadata, URL_SAFE_ENGINE}; -use opendal::services::S3; -use opendal::Operator; +use opendal::{services::S3, Operator}; use sea_orm::FromQueryResult; use sha2::{Digest, Sha256}; -use std::{collections::HashMap, io::Cursor}; + +use crate::{ + storage::blobs::{bucket_local_db::BucketStorage, MixedBucketDBParam}, + JwstStorageError, JwstStorageResult, +}; enum ImageFormat { Jpeg, @@ -88,11 +90,7 @@ impl TryFrom<&HashMap> for ImageParams { if let Some(format) = format { if Self::check_size(width, height) { - return Ok(Self { - format, - width, - height, - }); + return Ok(Self { format, width, height }); } } Err(()) diff --git a/libs/jwst-core-storage/src/storage/difflog.rs b/libs/jwst-core-storage/src/storage/difflog.rs index 80cdbd32d..4c7ec4bfd 100644 --- a/libs/jwst-core-storage/src/storage/difflog.rs +++ b/libs/jwst-core-storage/src/storage/difflog.rs @@ -11,10 +11,7 @@ pub struct DiffLogRecord { } impl DiffLogRecord { - pub async fn init_with_pool( - pool: DatabaseConnection, - bucket: Arc, - ) -> JwstStorageResult { + pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { Ok(Self { bucket, pool }) } @@ -25,12 +22,7 @@ impl DiffLogRecord { Self::init_with_pool(pool, get_bucket(is_sqlite)).await } - pub async fn insert( - &self, - workspace: String, - ts: DateTime, - log: String, - ) -> JwstStorageResult<()> { + pub async fn insert(&self, workspace: String, ts: DateTime, log: String) -> JwstStorageResult<()> { let _lock = self.bucket.write().await; DiffLog::insert(DiffLogActiveModel { workspace: Set(workspace), diff --git a/libs/jwst-core-storage/src/storage/docs/database.rs b/libs/jwst-core-storage/src/storage/docs/database.rs index babb376ce..c44f2fc02 100644 --- a/libs/jwst-core-storage/src/storage/docs/database.rs +++ b/libs/jwst-core-storage/src/storage/docs/database.rs @@ -1,9 +1,11 @@ -use super::{entities::prelude::*, *}; -use crate::types::JwstStorageResult; +use std::collections::hash_map::Entry; + use jwst_codec::{encode_update_as_message, CrdtReader, Doc, DocOptions, RawDecoder, StateVector}; use jwst_core::{DocStorage, Workspace}; use sea_orm::Condition; -use std::collections::hash_map::Entry; + +use super::{entities::prelude::*, *}; +use crate::types::JwstStorageResult; const MAX_TRIM_UPDATE_LIMIT: u64 = 500; @@ -19,10 +21,7 @@ pub struct DocDBStorage { } impl DocDBStorage { - pub async fn init_with_pool( - pool: DatabaseConnection, - bucket: Arc, - ) -> JwstStorageResult { + pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { Ok(Self { bucket, pool, @@ -53,10 +52,7 @@ impl DocDBStorage { .filter(DocsColumn::WorkspaceId.eq(workspace)) .all(conn) .await?; - trace!( - "end scan all records of workspace: {workspace}, {}", - models.len() - ); + trace!("end scan all records of workspace: {workspace}, {}", models.len()); Ok(models) } @@ -65,10 +61,7 @@ impl DocDBStorage { C: ConnectionTrait, { trace!("start scan all records with guid: {guid}"); - let models = Docs::find() - .filter(DocsColumn::Guid.eq(guid)) - .all(conn) - .await?; + let models = Docs::find().filter(DocsColumn::Guid.eq(guid)).all(conn).await?; trace!("end scan all: {guid}, {}", models.len()); Ok(models) } @@ -78,10 +71,7 @@ impl DocDBStorage { C: ConnectionTrait, { trace!("start count: {guid}"); - let count = Docs::find() - .filter(DocsColumn::Guid.eq(guid)) - .count(conn) - .await?; + let count = Docs::find().filter(DocsColumn::Guid.eq(guid)).count(conn).await?; trace!("end count: {guid}, {count}"); Ok(count) } @@ -119,10 +109,7 @@ impl DocDBStorage { where C: ConnectionTrait, { - let record = Docs::find() - .filter(DocsColumn::Guid.eq(guid)) - .one(conn) - .await?; + let record = Docs::find().filter(DocsColumn::Guid.eq(guid)).one(conn).await?; Ok(record.map_or(false, |r| r.is_workspace)) } @@ -148,12 +135,7 @@ impl DocDBStorage { Ok(()) } - async fn replace_with( - conn: &C, - workspace: &str, - guid: &str, - blob: Vec, - ) -> JwstStorageResult<()> + async fn replace_with(conn: &C, workspace: &str, guid: &str, blob: Vec) -> JwstStorageResult<()> where C: ConnectionTrait, { @@ -161,10 +143,7 @@ impl DocDBStorage { let is_workspace = Self::is_workspace(conn, guid).await?; - Docs::delete_many() - .filter(DocsColumn::Guid.eq(guid)) - .exec(conn) - .await?; + Docs::delete_many().filter(DocsColumn::Guid.eq(guid)).exec(conn).await?; Docs::insert(DocsActiveModel { workspace_id: Set(workspace.into()), guid: Set(guid.into()), @@ -184,10 +163,7 @@ impl DocDBStorage { C: ConnectionTrait, { trace!("start drop: {guid}"); - Docs::delete_many() - .filter(DocsColumn::Guid.eq(guid)) - .exec(conn) - .await?; + Docs::delete_many().filter(DocsColumn::Guid.eq(guid)).exec(conn).await?; trace!("end drop: {guid}"); Ok(()) } @@ -205,13 +181,7 @@ impl DocDBStorage { Ok(()) } - async fn update( - &self, - conn: &C, - workspace: &str, - guid: &str, - blob: Vec, - ) -> JwstStorageResult<()> + async fn update(&self, conn: &C, workspace: &str, guid: &str, blob: Vec) -> JwstStorageResult<()> where C: ConnectionTrait, { @@ -245,12 +215,7 @@ impl DocDBStorage { Ok(()) } - async fn full_migrate( - &self, - workspace: &str, - guid: &str, - blob: Vec, - ) -> JwstStorageResult<()> { + async fn full_migrate(&self, workspace: &str, guid: &str, blob: Vec) -> JwstStorageResult<()> { trace!("start full migrate: {guid}"); Self::replace_with(&self.pool, workspace, guid, blob).await?; trace!("end full migrate: {guid}"); @@ -307,9 +272,7 @@ impl DocStorage for DocDBStorage { let _lock = self.bucket.read().await; Ok(self.workspaces.read().await.contains_key(workspace_id) - || Self::workspace_count(&self.pool, workspace_id) - .await - .map(|c| c > 0)?) + || Self::workspace_count(&self.pool, workspace_id).await.map(|c| c > 0)?) } async fn get_or_create_workspace(&self, workspace_id: String) -> JwstStorageResult { @@ -330,23 +293,15 @@ impl DocStorage for DocDBStorage { } } - async fn flush_workspace( - &self, - workspace_id: String, - data: Vec, - ) -> JwstStorageResult { + async fn flush_workspace(&self, workspace_id: String, data: Vec) -> JwstStorageResult { trace!("create workspace: get lock"); let _lock = self.bucket.write().await; let ws = Self::init_workspace(&self.pool, &workspace_id).await?; - self.full_migrate(&workspace_id, ws.doc_guid(), data) - .await?; + self.full_migrate(&workspace_id, ws.doc_guid(), data).await?; - debug_assert_eq!( - Self::workspace_count(&self.pool, &workspace_id).await?, - 1u64 - ); + debug_assert_eq!(Self::workspace_count(&self.pool, &workspace_id).await?, 1u64); Ok(ws) } @@ -370,10 +325,7 @@ impl DocStorage for DocDBStorage { } async fn get_doc(&self, guid: String) -> JwstStorageResult> { - let records = Docs::find() - .filter(DocsColumn::Guid.eq(guid)) - .all(&self.pool) - .await?; + let records = Docs::find().filter(DocsColumn::Guid.eq(guid)).all(&self.pool).await?; if records.is_empty() { return Ok(None); @@ -384,27 +336,17 @@ impl DocStorage for DocDBStorage { Ok(Some(doc)) } - async fn update_doc( - &self, - workspace_id: String, - guid: String, - data: &[u8], - ) -> JwstStorageResult<()> { + async fn update_doc(&self, workspace_id: String, guid: String, data: &[u8]) -> JwstStorageResult<()> { debug!("write_update: get lock"); let _lock = self.bucket.write().await; trace!("write_update: {:?}", data); - self.update(&self.pool, &workspace_id, &guid, data.into()) - .await?; + self.update(&self.pool, &workspace_id, &guid, data.into()).await?; Ok(()) } - async fn update_doc_with_guid( - &self, - workspace_id: String, - data: &[u8], - ) -> JwstStorageResult<()> { + async fn update_doc_with_guid(&self, workspace_id: String, data: &[u8]) -> JwstStorageResult<()> { debug!("write_update: get lock"); let _lock = self.bucket.write().await; @@ -412,8 +354,7 @@ impl DocStorage for DocDBStorage { let mut decoder = RawDecoder::new(data.to_vec()); let guid = decoder.read_var_string()?; - self.update(&self.pool, &workspace_id, &guid, decoder.drain()) - .await?; + self.update(&self.pool, &workspace_id, &guid, decoder.drain()).await?; Ok(()) } diff --git a/libs/jwst-core-storage/src/storage/docs/mod.rs b/libs/jwst-core-storage/src/storage/docs/mod.rs index 55320e30a..50ac6a891 100644 --- a/libs/jwst-core-storage/src/storage/docs/mod.rs +++ b/libs/jwst-core-storage/src/storage/docs/mod.rs @@ -3,27 +3,22 @@ mod utils; use std::ops::Deref; -use super::*; -use database::DocDBStorage; -use tokio::sync::{broadcast::Sender, RwLock}; - #[cfg(test)] #[cfg(feature = "postgres")] pub(super) use database::full_migration_stress_test; +use database::DocDBStorage; #[cfg(test)] pub(super) use database::{docs_storage_partial_test, docs_storage_test}; +use tokio::sync::{broadcast::Sender, RwLock}; + +use super::*; #[derive(Clone)] pub struct SharedDocDBStorage(pub(super) Arc); impl SharedDocDBStorage { - pub async fn init_with_pool( - pool: DatabaseConnection, - bucket: Arc, - ) -> JwstStorageResult { - Ok(Self(Arc::new( - DocDBStorage::init_with_pool(pool, bucket).await?, - ))) + pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { + Ok(Self(Arc::new(DocDBStorage::init_with_pool(pool, bucket).await?))) } pub async fn init_pool(database: &str) -> JwstStorageResult { @@ -45,18 +40,17 @@ impl Deref for SharedDocDBStorage { #[cfg(test)] mod test { - use super::{error, info, DocStorage, SharedDocDBStorage}; - use crate::{JwstStorageError, JwstStorageResult}; + use std::collections::HashSet; + use jwst_storage_migration::Migrator; use rand::random; use sea_orm_migration::MigratorTrait; - use std::collections::HashSet; use tokio::task::JoinSet; - async fn create_workspace_stress_test( - storage: SharedDocDBStorage, - range: usize, - ) -> JwstStorageResult<()> { + use super::{error, info, DocStorage, SharedDocDBStorage}; + use crate::{JwstStorageError, JwstStorageResult}; + + async fn create_workspace_stress_test(storage: SharedDocDBStorage, range: usize) -> JwstStorageResult<()> { let mut join_set = JoinSet::new(); let mut set = HashSet::new(); @@ -125,10 +119,7 @@ mod test { #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn postgres_create_workspace_stress_test() -> anyhow::Result<()> { jwst_logger::init_logger("jwst-storage"); - let storage = SharedDocDBStorage::init_pool( - "postgresql://affine:affine@localhost:5432/affine_binary", - ) - .await?; + let storage = SharedDocDBStorage::init_pool("postgresql://affine:affine@localhost:5432/affine_binary").await?; create_workspace_stress_test(storage.clone(), 10000).await?; Ok(()) @@ -138,9 +129,7 @@ mod test { #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn mysql_create_workspace_stress_test() -> anyhow::Result<()> { // jwst_logger::init_logger(); - let storage = - SharedDocDBStorage::init_pool("mysql://affine:affine@localhost:3306/affine_binary") - .await?; + let storage = SharedDocDBStorage::init_pool("mysql://affine:affine@localhost:3306/affine_binary").await?; create_workspace_stress_test(storage.clone(), 10000).await?; Ok(()) diff --git a/libs/jwst-core-storage/src/storage/docs/utils.rs b/libs/jwst-core-storage/src/storage/docs/utils.rs index 851dfb926..6564e2225 100644 --- a/libs/jwst-core-storage/src/storage/docs/utils.rs +++ b/libs/jwst-core-storage/src/storage/docs/utils.rs @@ -1,11 +1,9 @@ -use super::{entities::prelude::*, types::JwstStorageResult, *}; use jwst_codec::{Doc, StateVector}; +use super::{entities::prelude::*, types::JwstStorageResult, *}; + // apply all updates to the given doc -pub fn migrate_update( - update_records: Vec<::Model>, - mut doc: Doc, -) -> JwstResult { +pub fn migrate_update(update_records: Vec<::Model>, mut doc: Doc) -> JwstResult { for record in update_records { let id = record.created_at; if let Err(e) = doc.apply_update_from_binary(record.blob) { @@ -16,9 +14,7 @@ pub fn migrate_update( Ok(doc) } -pub fn merge_doc_records( - update_records: Vec<::Model>, -) -> JwstStorageResult> { +pub fn merge_doc_records(update_records: Vec<::Model>) -> JwstStorageResult> { let doc = migrate_update(update_records, Doc::default())?; let state_vector = doc.encode_state_as_update_v1(&StateVector::default())?; diff --git a/libs/jwst-core-storage/src/storage/mod.rs b/libs/jwst-core-storage/src/storage/mod.rs index 8972588ef..8bc6fdb0f 100644 --- a/libs/jwst-core-storage/src/storage/mod.rs +++ b/libs/jwst-core-storage/src/storage/mod.rs @@ -3,17 +3,20 @@ mod difflog; mod docs; mod test; -use self::difflog::DiffLogRecord; +use std::{collections::HashMap, time::Instant}; -use super::*; -use crate::storage::blobs::{BlobBucketDBStorage, BlobStorageType, JwstBlobStorage}; -use crate::types::JwstStorageError; use blobs::BlobAutoStorage; use docs::SharedDocDBStorage; use jwst_storage_migration::{Migrator, MigratorTrait}; -use std::{collections::HashMap, time::Instant}; use tokio::sync::Mutex; +use self::difflog::DiffLogRecord; +use super::*; +use crate::{ + storage::blobs::{BlobBucketDBStorage, BlobStorageType, JwstBlobStorage}, + types::JwstStorageError, +}; + pub struct JwstStorage { pool: DatabaseConnection, blobs: JwstBlobStorage, @@ -23,31 +26,21 @@ pub struct JwstStorage { } impl JwstStorage { - pub async fn new( - database: &str, - blob_storage_type: BlobStorageType, - ) -> JwstStorageResult { + pub async fn new(database: &str, blob_storage_type: BlobStorageType) -> JwstStorageResult { let is_sqlite = is_sqlite(database); let pool = create_connection(database, is_sqlite).await?; let bucket = get_bucket(is_sqlite); if is_sqlite { - pool.execute_unprepared("PRAGMA journal_mode=WAL;") - .await - .unwrap(); + pool.execute_unprepared("PRAGMA journal_mode=WAL;").await.unwrap(); } let blobs = match blob_storage_type { - BlobStorageType::DB => JwstBlobStorage::DB( - BlobAutoStorage::init_with_pool(pool.clone(), bucket.clone()).await?, - ), + BlobStorageType::DB => { + JwstBlobStorage::DB(BlobAutoStorage::init_with_pool(pool.clone(), bucket.clone()).await?) + } BlobStorageType::MixedBucketDB(param) => JwstBlobStorage::MixedBucketDB( - BlobBucketDBStorage::init_with_pool( - pool.clone(), - bucket.clone(), - Some(param.try_into()?), - ) - .await?, + BlobBucketDBStorage::init_with_pool(pool.clone(), bucket.clone(), Some(param.try_into()?)).await?, ), }; let docs = SharedDocDBStorage::init_with_pool(pool.clone(), bucket.clone()).await?; @@ -62,10 +55,7 @@ impl JwstStorage { }) } - pub async fn new_with_migration( - database: &str, - blob_storage_type: BlobStorageType, - ) -> JwstStorageResult { + pub async fn new_with_migration(database: &str, blob_storage_type: BlobStorageType) -> JwstStorageResult { let storage = Self::new(database, blob_storage_type).await?; storage.db_migrate().await?; @@ -78,10 +68,7 @@ impl JwstStorage { Ok(()) } - pub async fn new_with_sqlite( - file: &str, - blob_storage_type: BlobStorageType, - ) -> JwstStorageResult { + pub async fn new_with_sqlite(file: &str, blob_storage_type: BlobStorageType) -> JwstStorageResult { use std::fs::create_dir; let data = PathBuf::from("./data"); @@ -92,9 +79,7 @@ impl JwstStorage { Self::new_with_migration( &format!( "sqlite:{}?mode=rwc", - data.join(PathBuf::from(file).name_str()) - .with_extension("db") - .display() + data.join(PathBuf::from(file).name_str()).with_extension("db").display() ), blob_storage_type, ) @@ -134,12 +119,7 @@ impl JwstStorage { self.docs .get_or_create_workspace(workspace_id.as_ref().into()) .await - .map_err(|_err| { - JwstStorageError::Crud(format!( - "Failed to create workspace {}", - workspace_id.as_ref() - )) - }) + .map_err(|_err| JwstStorageError::Crud(format!("Failed to create workspace {}", workspace_id.as_ref()))) } pub async fn get_workspace(&self, workspace_id: S) -> JwstStorageResult @@ -151,46 +131,25 @@ impl JwstStorage { .docs .detect_workspace(workspace_id.as_ref()) .await - .map_err(|_err| { - JwstStorageError::Crud(format!( - "failed to check workspace {}", - workspace_id.as_ref() - )) - })? + .map_err(|_err| JwstStorageError::Crud(format!("failed to check workspace {}", workspace_id.as_ref())))? { Ok(self .docs .get_or_create_workspace(workspace_id.as_ref().into()) .await - .map_err(|_err| { - JwstStorageError::Crud(format!( - "failed to get workspace {}", - workspace_id.as_ref() - )) - })?) + .map_err(|_err| JwstStorageError::Crud(format!("failed to get workspace {}", workspace_id.as_ref())))?) } else { - Err(JwstStorageError::WorkspaceNotFound( - workspace_id.as_ref().into(), - )) + Err(JwstStorageError::WorkspaceNotFound(workspace_id.as_ref().into())) } } - pub async fn full_migrate( - &self, - workspace_id: String, - update: Option>, - force: bool, - ) -> bool { + pub async fn full_migrate(&self, workspace_id: String, update: Option>, force: bool) -> bool { let mut map = self.last_migrate.lock().await; let ts = map.entry(workspace_id.clone()).or_insert(Instant::now()); if ts.elapsed().as_secs() > 5 || force { debug!("full migrate: {workspace_id}"); - match self - .docs - .get_or_create_workspace(workspace_id.clone()) - .await - { + match self.docs.get_or_create_workspace(workspace_id.clone()).await { Ok(workspace) => { let update = if let Some(update) = update { if let Err(e) = self.docs.delete_workspace(&workspace_id).await { @@ -206,11 +165,7 @@ impl JwstStorage { error!("full migrate failed: wait transact timeout"); return false; }; - if let Err(e) = self - .docs - .flush_workspace(workspace_id.clone(), update) - .await - { + if let Err(e) = self.docs.flush_workspace(workspace_id.clone(), update).await { error!("db write error: {}", e.to_string()); return false; } @@ -254,9 +209,8 @@ mod tests { bucket: Some(dotenvy::var("BUCKET_NAME").unwrap()), root: Some(dotenvy::var("BUCKET_ROOT").unwrap()), }; - let _storage = - JwstStorage::new_with_sqlite(":memory:", BlobStorageType::MixedBucketDB(bucket_params)) - .await - .unwrap(); + let _storage = JwstStorage::new_with_sqlite(":memory:", BlobStorageType::MixedBucketDB(bucket_params)) + .await + .unwrap(); } } diff --git a/libs/jwst-core/src/block/convert.rs b/libs/jwst-core/src/block/convert.rs index 2618e4681..be1349adc 100644 --- a/libs/jwst-core/src/block/convert.rs +++ b/libs/jwst-core/src/block/convert.rs @@ -227,9 +227,10 @@ impl Block { #[cfg(test)] mod tests { - use super::*; use jwst_codec::Update; + use super::*; + #[test] fn test_multiple_layer_space_clone() { let mut doc1 = Doc::default(); diff --git a/libs/jwst-core/src/block/mod.rs b/libs/jwst-core/src/block/mod.rs index 3b403f2a5..d1933bf17 100644 --- a/libs/jwst-core/src/block/mod.rs +++ b/libs/jwst-core/src/block/mod.rs @@ -1,9 +1,11 @@ mod convert; -use super::{constants::sys, *}; +use std::fmt; + use jwst_codec::{Any, Array, Doc, Map, Text, Value}; use serde::{Serialize, Serializer}; -use std::fmt; + +use super::{constants::sys, *}; #[derive(Clone)] pub struct Block { diff --git a/libs/jwst-core/src/lib.rs b/libs/jwst-core/src/lib.rs index d28de97d6..a6322bf69 100644 --- a/libs/jwst-core/src/lib.rs +++ b/libs/jwst-core/src/lib.rs @@ -9,10 +9,9 @@ pub mod constants; pub use block::Block; pub use history::{BlockHistory, HistoryOperation}; +pub use jwst_codec::Any; pub use space::Space; pub use tracing::{debug, error, info, log::LevelFilter, trace, warn}; pub use types::{BlobMetadata, BlobStorage, BucketBlobStorage, DocStorage, JwstError, JwstResult}; pub use utils::{Base64DecodeError, Base64Engine, STANDARD_ENGINE, URL_SAFE_ENGINE}; pub use workspace::{Workspace, WorkspaceMetadata}; - -pub use jwst_codec::Any; diff --git a/libs/jwst-core/src/space/convert.rs b/libs/jwst-core/src/space/convert.rs index f9aef4266..97c1ac6b1 100644 --- a/libs/jwst-core/src/space/convert.rs +++ b/libs/jwst-core/src/space/convert.rs @@ -1,7 +1,8 @@ -use super::*; use chrono::Utc; use jwst_codec::{Array, Map}; +use super::*; + impl Space { pub fn to_markdown(&self) -> Option { if let Some(title) = self.get_blocks_by_flavour("affine:page").first() { diff --git a/libs/jwst-core/src/space/mod.rs b/libs/jwst-core/src/space/mod.rs index f3d57beeb..dcafd9da1 100644 --- a/libs/jwst-core/src/space/mod.rs +++ b/libs/jwst-core/src/space/mod.rs @@ -1,9 +1,10 @@ mod convert; -use super::{block::MarkdownState, workspace::Pages, *}; use jwst_codec::{Any, Doc, Map}; use serde::{ser::SerializeMap, Serialize, Serializer}; +use super::{block::MarkdownState, workspace::Pages, *}; + // Workspace // / \ // Space ... Space @@ -181,9 +182,10 @@ impl Serialize for Space { #[cfg(test)] mod test { - use super::*; use jwst_codec::{StateVector, Update}; + use super::*; + #[test] fn doc_load_test() { let space_id = "space"; diff --git a/libs/jwst-core/src/types/blob.rs b/libs/jwst-core/src/types/blob.rs index b1df24d2c..8eec9f227 100644 --- a/libs/jwst-core/src/types/blob.rs +++ b/libs/jwst-core/src/types/blob.rs @@ -1,8 +1,10 @@ -use super::*; +use std::collections::HashMap; + use bytes::Bytes; use chrono::NaiveDateTime; use futures::Stream; -use std::collections::HashMap; + +use super::*; #[derive(Debug)] pub struct BlobMetadata { diff --git a/libs/jwst-core/src/types/mod.rs b/libs/jwst-core/src/types/mod.rs index 58ed23e26..f140d0674 100644 --- a/libs/jwst-core/src/types/mod.rs +++ b/libs/jwst-core/src/types/mod.rs @@ -2,9 +2,9 @@ mod blob; mod doc; mod error; -use super::Workspace; use async_trait::async_trait; - pub use blob::{BlobMetadata, BlobStorage, BucketBlobStorage}; pub use doc::DocStorage; pub use error::{JwstError, JwstResult}; + +use super::Workspace; diff --git a/libs/jwst-core/src/workspace/metadata/meta.rs b/libs/jwst-core/src/workspace/metadata/meta.rs index eaf74aa67..482bfc9e8 100644 --- a/libs/jwst-core/src/workspace/metadata/meta.rs +++ b/libs/jwst-core/src/workspace/metadata/meta.rs @@ -1,7 +1,9 @@ -use super::*; +use std::collections::HashMap; + use jwst_codec::{Any, Array, Map}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; + +use super::*; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct WorkspaceMetadata { @@ -66,9 +68,10 @@ impl Workspace { #[cfg(test)] mod tests { - use super::*; use jwst_codec::Doc; + use super::*; + #[test] fn test_workspace_metadata() { let doc = Doc::default(); diff --git a/libs/jwst-core/src/workspace/metadata/mod.rs b/libs/jwst-core/src/workspace/metadata/mod.rs index 231f4ff34..c2f841136 100644 --- a/libs/jwst-core/src/workspace/metadata/mod.rs +++ b/libs/jwst-core/src/workspace/metadata/mod.rs @@ -1,7 +1,7 @@ mod meta; mod pages; -use super::*; - pub use meta::WorkspaceMetadata; pub use pages::Pages; + +use super::*; diff --git a/libs/jwst-core/src/workspace/metadata/pages.rs b/libs/jwst-core/src/workspace/metadata/pages.rs index 9fbc10d76..7a80146b2 100644 --- a/libs/jwst-core/src/workspace/metadata/pages.rs +++ b/libs/jwst-core/src/workspace/metadata/pages.rs @@ -1,6 +1,7 @@ -use jwst_codec::{Any, Array, Map, Value}; use std::collections::HashMap; +use jwst_codec::{Any, Array, Map, Value}; + pub struct PageMeta { pub id: String, pub favorite: Option, @@ -148,9 +149,10 @@ impl Pages { #[cfg(test)] mod tests { - use super::*; use jwst_codec::Doc; + use super::*; + #[test] fn test_page_meta() { let doc = Doc::default(); @@ -180,7 +182,8 @@ mod tests { // let doc = Doc::default(); // doc.apply_update( // Update::from_ybinary1( - // include_bytes!("../../../fixtures/test_shared_page.bin").to_vec(), + // + // include_bytes!("../../../fixtures/test_shared_page.bin").to_vec(), // ) // .unwrap(), // ); diff --git a/libs/jwst-core/src/workspace/mod.rs b/libs/jwst-core/src/workspace/mod.rs index 3602bc1f9..8636c2560 100644 --- a/libs/jwst-core/src/workspace/mod.rs +++ b/libs/jwst-core/src/workspace/mod.rs @@ -4,7 +4,7 @@ mod spaces; mod sync; mod workspace; -use super::{constants, info, trace, warn, JwstResult, Space}; - pub use metadata::{Pages, WorkspaceMetadata}; pub use workspace::Workspace; + +use super::{constants, info, trace, warn, JwstResult, Space}; diff --git a/libs/jwst-core/src/workspace/observe.rs b/libs/jwst-core/src/workspace/observe.rs index 1bfd2cd42..431d20d58 100644 --- a/libs/jwst-core/src/workspace/observe.rs +++ b/libs/jwst-core/src/workspace/observe.rs @@ -1,6 +1,7 @@ -use super::*; use jwst_codec::{Awareness, AwarenessEvent}; +use super::*; + impl Workspace { pub async fn on_awareness_update(&mut self, f: impl Fn(&Awareness, AwarenessEvent) + Send + Sync + 'static) { self.awareness.write().await.on_update(f); diff --git a/libs/jwst-core/src/workspace/sync.rs b/libs/jwst-core/src/workspace/sync.rs index 8e0efa4cd..c3cef42f0 100644 --- a/libs/jwst-core/src/workspace/sync.rs +++ b/libs/jwst-core/src/workspace/sync.rs @@ -1,9 +1,10 @@ -use super::*; use jwst_codec::{ write_sync_message, CrdtRead, CrdtWrite, DocMessage, RawDecoder, RawEncoder, StateVector, SyncMessage, SyncMessageScanner, Update, }; +use super::*; + impl Workspace { pub fn sync_migration(&self) -> JwstResult> { Ok(self.doc.encode_state_as_update_v1(&StateVector::default())?) diff --git a/libs/jwst-core/src/workspace/workspace.rs b/libs/jwst-core/src/workspace/workspace.rs index 45622d070..006610466 100644 --- a/libs/jwst-core/src/workspace/workspace.rs +++ b/libs/jwst-core/src/workspace/workspace.rs @@ -1,9 +1,11 @@ -use super::*; +use std::sync::Arc; + use jwst_codec::{Awareness, Doc, Map, Update}; use serde::{ser::SerializeMap, Serialize, Serializer}; -use std::sync::Arc; use tokio::sync::RwLock; +use super::*; + pub struct Workspace { workspace_id: String, pub(super) awareness: Arc>, @@ -109,9 +111,10 @@ impl Clone for Workspace { #[cfg(test)] mod test { - use super::{super::super::Block, *}; use jwst_codec::StateVector; + use super::{super::super::Block, *}; + #[test] fn doc_load_test() { let doc = { @@ -324,11 +327,11 @@ mod test { // TODO: fix merge update // { - // let merged_update = merge_updates_v1(&[&update1, &update2]).unwrap(); - // let mut doc = Doc::default(); + // let merged_update = merge_updates_v1(&[&update1, + // &update2]).unwrap(); let mut doc = Doc::default(); // doc.apply_update( - // Update::from_ybinary1(merged_update.into_ybinary1().unwrap()).unwrap(), - // ) + // Update::from_ybinary1(merged_update.into_ybinary1(). + // unwrap()).unwrap(), ) // .unwrap(); // let mut ws = Workspace::from_doc(doc, "test").unwrap(); @@ -343,10 +346,11 @@ mod test { // let block = space.get("test").unwrap(); // let mut children = block.children(); // children.sort(); - // assert_eq!(children, vec!["test1".to_owned(), "test2".to_owned()]); - // // assert_eq!(block.get("test1").unwrap().to_string(), "test1"); - // // assert_eq!(block.get("test2").unwrap().to_string(), "test2"); - // } + // assert_eq!(children, vec!["test1".to_owned(), + // "test2".to_owned()]); // + // assert_eq!(block.get("test1").unwrap().to_string(), "test1"); + // // assert_eq!(block.get("test2").unwrap().to_string(), + // "test2"); } // } } @@ -380,8 +384,9 @@ mod test { { let mut space = ws.get_space("space").unwrap(); let mut _new_block = space.create("test2", "test2").unwrap(); - // TODO: fix merge update crash if we create same block in two doc then merge them - // let mut block = space.create("test", "test1").unwrap(); + // TODO: fix merge update crash if we create same block in two + // doc then merge them let mut block = + // space.create("test", "test1").unwrap(); // block.insert_children_at(&mut new_block, 0).unwrap(); } @@ -413,11 +418,11 @@ mod test { // TODO: fix merge update // { - // let merged_update = merge_updates_v1(&[&update1, &update2]).unwrap(); - // let mut doc = Doc::default(); + // let merged_update = merge_updates_v1(&[&update1, + // &update2]).unwrap(); let mut doc = Doc::default(); // doc.apply_update( - // Update::from_ybinary1(merged_update.into_ybinary1().unwrap()).unwrap(), - // ) + // Update::from_ybinary1(merged_update.into_ybinary1(). + // unwrap()).unwrap(), ) // .unwrap(); // let mut ws = Workspace::from_doc(doc, "test").unwrap(); @@ -432,9 +437,10 @@ mod test { // let block = space.get("test").unwrap(); // let mut children = block.children(); // children.sort(); - // assert_ne!(children, vec!["test1".to_owned(), "test2".to_owned()]); - // // assert_eq!(block.get( "test1").unwrap().to_string(), "test1"); - // // assert_eq!(block.get( "test2").unwrap().to_string(), "test2"); + // assert_ne!(children, vec!["test1".to_owned(), + // "test2".to_owned()]); // assert_eq!(block.get( + // "test1").unwrap().to_string(), "test1"); // + // assert_eq!(block.get( "test2").unwrap().to_string(), "test2"); // } // } } diff --git a/libs/jwst-logger/src/formatter.rs b/libs/jwst-logger/src/formatter.rs index 4e2f14688..09387f925 100644 --- a/libs/jwst-logger/src/formatter.rs +++ b/libs/jwst-logger/src/formatter.rs @@ -1,5 +1,6 @@ -use nu_ansi_term::{AnsiGenericString, Color}; use std::fmt::Result; + +use nu_ansi_term::{AnsiGenericString, Color}; use tracing::{Event, Level, Metadata, Subscriber}; use tracing_log::NormalizeEvent; use tracing_subscriber::{ diff --git a/libs/jwst-logger/src/lib.rs b/libs/jwst-logger/src/lib.rs index 2de45f6ab..06159cd37 100644 --- a/libs/jwst-logger/src/lib.rs +++ b/libs/jwst-logger/src/lib.rs @@ -2,10 +2,9 @@ mod formatter; mod logger; +use formatter::JWSTFormatter; pub use logger::{init_logger, init_logger_with}; pub use tracing::{ self, debug, debug_span, error, error_span, info, info_span, instrument, log::LevelFilter, trace, trace_span, warn, warn_span, Level, }; - -use formatter::JWSTFormatter; diff --git a/libs/jwst-logger/src/logger.rs b/libs/jwst-logger/src/logger.rs index 0883af159..9b8ad984f 100644 --- a/libs/jwst-logger/src/logger.rs +++ b/libs/jwst-logger/src/logger.rs @@ -1,9 +1,13 @@ -use super::*; -use std::io::{stderr, stdout}; -use std::sync::Once; +use std::{ + io::{stderr, stdout}, + sync::Once, +}; + use tracing::Level; use tracing_subscriber::{fmt, prelude::*, EnvFilter}; +use super::*; + static INIT: Once = Once::new(); /// Initialize a logger with the value of the given environment variable. diff --git a/libs/jwst-rpc/src/broadcast.rs b/libs/jwst-rpc/src/broadcast.rs index 27df095a6..29bf8a84e 100644 --- a/libs/jwst-rpc/src/broadcast.rs +++ b/libs/jwst-rpc/src/broadcast.rs @@ -1,10 +1,12 @@ -use super::*; +use std::{collections::HashMap, sync::Mutex}; + use jwst::Workspace; use jwst_codec::{encode_update_as_message, encode_update_with_guid, write_sync_message, SyncMessage}; use lru_time_cache::LruCache; -use std::{collections::HashMap, sync::Mutex}; use tokio::sync::{broadcast::Sender, RwLock}; +use super::*; + #[derive(Clone)] pub enum BroadcastType { BroadcastAwareness(Vec), diff --git a/libs/jwst-rpc/src/client/mod.rs b/libs/jwst-rpc/src/client/mod.rs index d02ff2a5a..fb40f4f47 100644 --- a/libs/jwst-rpc/src/client/mod.rs +++ b/libs/jwst-rpc/src/client/mod.rs @@ -3,16 +3,17 @@ mod webrtc; #[cfg(feature = "websocket")] mod websocket; -use super::*; -use chrono::Utc; use std::sync::Mutex; -use tokio::{runtime::Runtime, task::JoinHandle}; +use chrono::Utc; +use tokio::{runtime::Runtime, task::JoinHandle}; #[cfg(feature = "webrtc")] pub use webrtc::start_webrtc_client_sync; #[cfg(feature = "websocket")] pub use websocket::start_websocket_client_sync; +use super::*; + #[derive(Clone, Default)] pub struct CachedLastSynced { synced: Arc>>, @@ -49,10 +50,12 @@ impl CachedLastSynced { #[cfg(test)] mod tests { - use super::*; use std::thread::spawn; + use tokio::sync::mpsc::channel; + use super::*; + #[test] fn test_synced() { let synced = CachedLastSynced::default(); diff --git a/libs/jwst-rpc/src/client/webrtc.rs b/libs/jwst-rpc/src/client/webrtc.rs index 0747d621e..872452205 100644 --- a/libs/jwst-rpc/src/client/webrtc.rs +++ b/libs/jwst-rpc/src/client/webrtc.rs @@ -1,9 +1,11 @@ -use super::*; +use std::sync::RwLock; + use nanoid::nanoid; use reqwest::Client; -use std::sync::RwLock; use tokio::{runtime::Runtime, sync::mpsc::channel}; +use super::*; + async fn webrtc_connection(remote: &str) -> (Sender, Receiver>) { warn!("webrtc_connection start"); let (offer, pc, tx, rx, mut s) = webrtc_datachannel_client_begin().await; diff --git a/libs/jwst-rpc/src/client/websocket.rs b/libs/jwst-rpc/src/client/websocket.rs index cb219dde2..f7f9a7305 100644 --- a/libs/jwst-rpc/src/client/websocket.rs +++ b/libs/jwst-rpc/src/client/websocket.rs @@ -1,6 +1,6 @@ -use super::{types::JwstRpcResult, *}; -use nanoid::nanoid; use std::sync::RwLock; + +use nanoid::nanoid; use tokio::{net::TcpStream, runtime::Runtime, sync::mpsc::channel}; use tokio_tungstenite::{ connect_async, @@ -9,6 +9,8 @@ use tokio_tungstenite::{ }; use url::Url; +use super::{types::JwstRpcResult, *}; + type Socket = WebSocketStream>; async fn prepare_connection(remote: &str) -> JwstRpcResult { diff --git a/libs/jwst-rpc/src/connector/axum_socket.rs b/libs/jwst-rpc/src/connector/axum_socket.rs index e187f97d4..6cab797bf 100644 --- a/libs/jwst-rpc/src/connector/axum_socket.rs +++ b/libs/jwst-rpc/src/connector/axum_socket.rs @@ -1,9 +1,10 @@ -use super::*; use axum::extract::ws::{Message as WebSocketMessage, WebSocket}; use futures::{sink::SinkExt, stream::StreamExt}; use tokio::sync::mpsc::{channel, Receiver, Sender}; use tokio_tungstenite::tungstenite::Error as SocketError; +use super::*; + impl From for WebSocketMessage { fn from(value: Message) -> Self { match value { diff --git a/libs/jwst-rpc/src/connector/memory.rs b/libs/jwst-rpc/src/connector/memory.rs index 914df4930..6b666049f 100644 --- a/libs/jwst-rpc/src/connector/memory.rs +++ b/libs/jwst-rpc/src/connector/memory.rs @@ -1,11 +1,13 @@ -use super::*; -use jwst_codec::{decode_update_with_guid, encode_update_as_message, Doc, DocMessage, SyncMessage, SyncMessageScanner}; use std::{ sync::atomic::{AtomicBool, Ordering}, thread::JoinHandle as StdJoinHandler, }; + +use jwst_codec::{decode_update_with_guid, encode_update_as_message, Doc, DocMessage, SyncMessage, SyncMessageScanner}; use tokio::{runtime::Runtime, sync::mpsc::channel, task::JoinHandle as TokioJoinHandler}; +use super::*; + // just for test pub fn memory_connector( rt: Arc, diff --git a/libs/jwst-rpc/src/connector/tungstenite_socket.rs b/libs/jwst-rpc/src/connector/tungstenite_socket.rs index b9a97e6cb..498a66ea2 100644 --- a/libs/jwst-rpc/src/connector/tungstenite_socket.rs +++ b/libs/jwst-rpc/src/connector/tungstenite_socket.rs @@ -1,4 +1,3 @@ -use super::*; use futures::{sink::SinkExt, stream::StreamExt}; use tokio::{ net::TcpStream, @@ -9,6 +8,8 @@ use tokio_tungstenite::{ MaybeTlsStream, WebSocketStream, }; +use super::*; + type WebSocket = WebSocketStream>; impl From for WebSocketMessage { diff --git a/libs/jwst-rpc/src/connector/webrtc.rs b/libs/jwst-rpc/src/connector/webrtc.rs index ea712666d..839966fd7 100644 --- a/libs/jwst-rpc/src/connector/webrtc.rs +++ b/libs/jwst-rpc/src/connector/webrtc.rs @@ -1,8 +1,7 @@ -use super::Message; -use jwst::{debug, error, info, trace, warn}; +use std::sync::Arc; use bytes::Bytes; -use std::sync::Arc; +use jwst::{debug, error, info, trace, warn}; use tokio::sync::mpsc::{channel, Receiver, Sender}; use webrtcrs::{ api::APIBuilder, @@ -13,6 +12,8 @@ use webrtcrs::{ }, }; +use super::Message; + const DATA_CHANNEL_ID: u16 = 42; const DATA_CHANNEL_LABEL: &str = "affine"; @@ -121,7 +122,8 @@ pub async fn webrtc_datachannel_client_begin() -> ( // Block until ICE Gathering is complete, disabling trickle ICE // we do this because we only can exchange one signaling message - // in a production application you should exchange ICE Candidates via OnICECandidate + // in a production application you should exchange ICE Candidates via + // OnICECandidate let _ = gather_complete.recv().await; let local_desc = peer_connection.local_description().await.unwrap(); @@ -168,7 +170,8 @@ pub async fn webrtc_datachannel_server_connector( // Block until ICE Gathering is complete, disabling trickle ICE // we do this because we only can exchange one signaling message - // in a production application you should exchange ICE Candidates via OnICECandidate + // in a production application you should exchange ICE Candidates via + // OnICECandidate let _ = gather_complete.recv().await; let local_desc = peer_connection.local_description().await.unwrap(); diff --git a/libs/jwst-rpc/src/context.rs b/libs/jwst-rpc/src/context.rs index 6f8a320eb..b8a6d5422 100644 --- a/libs/jwst-rpc/src/context.rs +++ b/libs/jwst-rpc/src/context.rs @@ -3,10 +3,6 @@ use std::{ panic::{catch_unwind, AssertUnwindSafe}, }; -use super::{ - broadcast::{subscribe, BroadcastChannels, BroadcastType}, - *, -}; use async_trait::async_trait; use chrono::Utc; use jwst::{DocStorage, Workspace}; @@ -17,9 +13,13 @@ use tokio::sync::{ mpsc::{Receiver as MpscReceiver, Sender as MpscSender}, Mutex, }; - use yrs::merge_updates_v1; +use super::{ + broadcast::{subscribe, BroadcastChannels, BroadcastType}, + *, +}; + fn merge_updates(id: &str, updates: Vec>) -> Vec> { match catch_unwind(AssertUnwindSafe(move || { match merge_updates_v1(&updates.iter().map(std::ops::Deref::deref).collect::>()) { @@ -79,15 +79,17 @@ pub trait RpcContextImpl<'a> { } }; - // Listen to changes of the local workspace, encode changes in awareness and Doc, and broadcast them. - // It returns the 'broadcast_rx' object to receive the content that was sent + // Listen to changes of the local workspace, encode changes in awareness and + // Doc, and broadcast them. It returns the 'broadcast_rx' object to + // receive the content that was sent subscribe(workspace, identifier.clone(), broadcast_tx.clone()).await; // save update thread self.save_update(&id, identifier, broadcast_tx.subscribe(), last_synced) .await; - // returns the 'broadcast_tx' which can be subscribed later, to receive local workspace changes + // returns the 'broadcast_tx' which can be subscribed later, to receive local + // workspace changes broadcast_tx } diff --git a/libs/jwst-rpc/src/handler.rs b/libs/jwst-rpc/src/handler.rs index 076db7521..2c63afa3a 100644 --- a/libs/jwst-rpc/src/handler.rs +++ b/libs/jwst-rpc/src/handler.rs @@ -1,12 +1,14 @@ -use super::{BroadcastType, Message, RpcContextImpl}; +use std::{sync::Arc, time::Instant}; + use chrono::Utc; use jwst::{debug, error, info, trace, warn}; -use std::{sync::Arc, time::Instant}; use tokio::{ sync::mpsc::{Receiver, Sender}, time::{sleep, Duration}, }; +use super::{BroadcastType, Message, RpcContextImpl}; + pub async fn handle_connector( context: Arc + Send + Sync + 'static>, workspace_id: String, @@ -15,7 +17,8 @@ pub async fn handle_connector( ) -> bool { info!("{} collaborate with workspace {}", identifier, workspace_id); - // An abstraction of the established socket connection. Use tx to broadcast and rx to receive. + // An abstraction of the established socket connection. Use tx to broadcast and + // rx to receive. let (tx, rx, last_synced) = get_channel(); let mut ws = context @@ -23,22 +26,25 @@ pub async fn handle_connector( .await .expect("failed to get workspace"); - // Continuously receive information from the remote socket, apply it to the local workspace, and - // send the encoded updates back to the remote end through the socket. + // Continuously receive information from the remote socket, apply it to the + // local workspace, and send the encoded updates back to the remote end + // through the socket. context .apply_change(&workspace_id, &identifier, tx.clone(), rx, last_synced.clone()) .await; - // Both of broadcast_update and server_update are sent to the remote socket through 'tx' - // The 'broadcast_update' is the receiver for updates to the awareness and Doc of the local workspace. - // It uses channel, which is owned by the server itself and is stored in the server's memory (not persisted)." + // Both of broadcast_update and server_update are sent to the remote socket + // through 'tx' The 'broadcast_update' is the receiver for updates to the + // awareness and Doc of the local workspace. It uses channel, which is owned + // by the server itself and is stored in the server's memory (not persisted)." let broadcast_tx = context .join_broadcast(&mut ws, identifier.clone(), last_synced.clone()) .await; let mut broadcast_rx = broadcast_tx.subscribe(); - // Obtaining the receiver corresponding to DocAutoStorage in storage. The sender is used in the - // doc::write_update(). The remote used is the one belonging to DocAutoStorage and is owned by - // the server itself, stored in the server's memory (not persisted). + // Obtaining the receiver corresponding to DocAutoStorage in storage. The sender + // is used in the doc::write_update(). The remote used is the one belonging + // to DocAutoStorage and is owned by the server itself, stored in the + // server's memory (not persisted). let mut server_rx = context.join_server_broadcast(&workspace_id).await; // Send initialization message. @@ -164,6 +170,12 @@ pub async fn handle_connector( #[cfg(test)] mod test { + use std::sync::atomic::{AtomicU64, Ordering}; + + use indicatif::{MultiProgress, ProgressBar, ProgressIterator, ProgressStyle}; + use jwst::{JwstError, JwstResult}; + use yrs::Map; + use super::{ super::{connect_memory_workspace, MinimumServerContext}, *, @@ -172,10 +184,6 @@ mod test { use crate::{ webrtc_datachannel_client_begin, webrtc_datachannel_client_commit, webrtc_datachannel_server_connector, }; - use indicatif::{MultiProgress, ProgressBar, ProgressIterator, ProgressStyle}; - use jwst::{JwstError, JwstResult}; - use std::sync::atomic::{AtomicU64, Ordering}; - use yrs::Map; #[tokio::test] #[ignore = "skip in ci"] diff --git a/libs/jwst-rpc/src/lib.rs b/libs/jwst-rpc/src/lib.rs index bcd2ee67f..98a5ffdb9 100644 --- a/libs/jwst-rpc/src/lib.rs +++ b/libs/jwst-rpc/src/lib.rs @@ -7,10 +7,15 @@ mod handler; mod types; mod utils; +use std::{collections::hash_map::Entry, sync::Arc, time::Instant}; + +pub use broadcast::{BroadcastChannels, BroadcastType}; #[cfg(feature = "webrtc")] pub use client::start_webrtc_client_sync; #[cfg(feature = "websocket")] pub use client::start_websocket_client_sync; +pub use client::CachedLastSynced; +pub use connector::memory_connector; #[cfg(feature = "webrtc")] pub use connector::webrtc_datachannel_client_begin; #[cfg(feature = "webrtc")] @@ -19,22 +24,16 @@ pub use connector::webrtc_datachannel_client_commit; pub use connector::webrtc_datachannel_server_connector; #[cfg(feature = "websocket")] pub use connector::{axum_socket_connector, tungstenite_socket_connector}; -#[cfg(feature = "webrtc")] -pub use webrtcrs::peer_connection::sdp::session_description::RTCSessionDescription; - -pub use broadcast::{BroadcastChannels, BroadcastType}; -pub use client::CachedLastSynced; -pub use connector::memory_connector; pub use context::RpcContextImpl; pub use handler::handle_connector; -pub use utils::{connect_memory_workspace, workspace_compare, MinimumServerContext}; - use jwst::{debug, error, info, trace, warn}; -use std::{collections::hash_map::Entry, sync::Arc, time::Instant}; use tokio::{ sync::mpsc::{Receiver, Sender}, time::{sleep, Duration}, }; +pub use utils::{connect_memory_workspace, workspace_compare, MinimumServerContext}; +#[cfg(feature = "webrtc")] +pub use webrtcrs::peer_connection::sdp::session_description::RTCSessionDescription; #[derive(Clone, PartialEq, Eq, Debug, Default)] pub enum SyncState { diff --git a/libs/jwst-rpc/src/utils/memory_workspace.rs b/libs/jwst-rpc/src/utils/memory_workspace.rs index 4cec4a213..9e38031c9 100644 --- a/libs/jwst-rpc/src/utils/memory_workspace.rs +++ b/libs/jwst-rpc/src/utils/memory_workspace.rs @@ -1,9 +1,11 @@ -use super::*; +use std::thread::JoinHandle as StdJoinHandler; + use jwst_codec::Doc; use nanoid::nanoid; -use std::thread::JoinHandle as StdJoinHandler; use tokio::{runtime::Runtime, sync::mpsc::channel, task::JoinHandle as TokioJoinHandler}; +use super::*; + pub async fn connect_memory_workspace( server: Arc, init_state: &[u8], diff --git a/libs/jwst-rpc/src/utils/mod.rs b/libs/jwst-rpc/src/utils/mod.rs index eed7e172f..d7fb74de4 100644 --- a/libs/jwst-rpc/src/utils/mod.rs +++ b/libs/jwst-rpc/src/utils/mod.rs @@ -2,8 +2,8 @@ mod compare; mod memory_workspace; mod server_context; -use super::*; - pub use compare::workspace_compare; pub use memory_workspace::connect_memory_workspace; pub use server_context::MinimumServerContext; + +use super::*; diff --git a/libs/jwst-rpc/src/utils/server_context.rs b/libs/jwst-rpc/src/utils/server_context.rs index 2a71fab2b..f2d40edef 100644 --- a/libs/jwst-rpc/src/utils/server_context.rs +++ b/libs/jwst-rpc/src/utils/server_context.rs @@ -1,10 +1,12 @@ -use super::*; +use std::{collections::HashMap, time::Duration}; + use jwst::{DocStorage, Workspace}; use jwst_storage::{BlobStorageType, JwstStorage}; -use std::{collections::HashMap, time::Duration}; use tokio::{sync::RwLock, time::sleep}; use yrs::{ReadTxn, StateVector, Transact}; +use super::*; + pub struct MinimumServerContext { channel: BroadcastChannels, storage: JwstStorage, diff --git a/libs/jwst-storage/src/entities/prelude.rs b/libs/jwst-storage/src/entities/prelude.rs index 5bdda8eb3..aaabca6ac 100644 --- a/libs/jwst-storage/src/entities/prelude.rs +++ b/libs/jwst-storage/src/entities/prelude.rs @@ -1,7 +1,6 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 -pub use super::blobs::Entity as Blobs; -pub use super::bucket_blobs::Entity as BucketBlobs; -pub use super::diff_log::Entity as DiffLog; -pub use super::docs::Entity as Docs; -pub use super::optimized_blobs::Entity as OptimizedBlobs; +pub use super::{ + blobs::Entity as Blobs, bucket_blobs::Entity as BucketBlobs, diff_log::Entity as DiffLog, docs::Entity as Docs, + optimized_blobs::Entity as OptimizedBlobs, +}; diff --git a/libs/jwst-storage/src/lib.rs b/libs/jwst-storage/src/lib.rs index 8035ef98e..647d8b19d 100644 --- a/libs/jwst-storage/src/lib.rs +++ b/libs/jwst-storage/src/lib.rs @@ -4,6 +4,8 @@ mod rate_limiter; mod storage; mod types; +use std::{path::PathBuf, sync::Arc, time::Duration}; + use async_trait::async_trait; use chrono::{DateTime, Utc}; use futures::{Future, Stream}; @@ -12,10 +14,10 @@ use jwst_logger::{debug, error, info, trace, warn}; use path_ext::PathExt; use rate_limiter::{get_bucket, is_sqlite, Bucket}; use sea_orm::{prelude::*, ConnectOptions, Database, DbErr, QuerySelect, Set}; -use std::{path::PathBuf, sync::Arc, time::Duration}; - -pub use storage::blobs::{BlobStorageType, MixedBucketDBParam}; -pub use storage::JwstStorage; +pub use storage::{ + blobs::{BlobStorageType, MixedBucketDBParam}, + JwstStorage, +}; pub use types::{JwstStorageError, JwstStorageResult}; #[inline] diff --git a/libs/jwst-storage/src/migration/src/m20220101_000001_initial_blob_table.rs b/libs/jwst-storage/src/migration/src/m20220101_000001_initial_blob_table.rs index 05dd6e9ee..f253f6971 100644 --- a/libs/jwst-storage/src/migration/src/m20220101_000001_initial_blob_table.rs +++ b/libs/jwst-storage/src/migration/src/m20220101_000001_initial_blob_table.rs @@ -1,6 +1,7 @@ -use super::schema::Blobs; use sea_orm_migration::prelude::*; +use super::schema::Blobs; + pub struct Migration; impl MigrationName for Migration { diff --git a/libs/jwst-storage/src/migration/src/m20220101_000002_initial_doc_table.rs b/libs/jwst-storage/src/migration/src/m20220101_000002_initial_doc_table.rs index bbd4302b0..47edd0483 100644 --- a/libs/jwst-storage/src/migration/src/m20220101_000002_initial_doc_table.rs +++ b/libs/jwst-storage/src/migration/src/m20220101_000002_initial_doc_table.rs @@ -1,6 +1,7 @@ -use super::schema::Docs; use sea_orm_migration::prelude::*; +use super::schema::Docs; + pub struct Migration; impl MigrationName for Migration { diff --git a/libs/jwst-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs b/libs/jwst-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs index 4d82479c5..d89fab857 100644 --- a/libs/jwst-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs +++ b/libs/jwst-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs @@ -1,6 +1,7 @@ -use super::schema::OptimizedBlobs; use sea_orm_migration::prelude::*; +use super::schema::OptimizedBlobs; + pub struct Migration; impl MigrationName for Migration { diff --git a/libs/jwst-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs b/libs/jwst-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs index 8ec4ea4ca..3f778d8cd 100644 --- a/libs/jwst-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs +++ b/libs/jwst-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs @@ -1,6 +1,7 @@ -use super::schema::BucketBlobs; use sea_orm_migration::prelude::*; +use super::schema::BucketBlobs; + pub struct Migration; impl MigrationName for Migration { diff --git a/libs/jwst-storage/src/rate_limiter.rs b/libs/jwst-storage/src/rate_limiter.rs index bc03a1abb..2ba158ff4 100644 --- a/libs/jwst-storage/src/rate_limiter.rs +++ b/libs/jwst-storage/src/rate_limiter.rs @@ -1,10 +1,11 @@ +use std::{num::NonZeroU32, sync::Arc}; + use governor::{ clock::{QuantaClock, QuantaInstant}, middleware::NoOpMiddleware, state::{InMemoryState, NotKeyed}, Quota, RateLimiter, }; -use std::{num::NonZeroU32, sync::Arc}; use tokio::sync::{OwnedSemaphorePermit, RwLock, RwLockReadGuard, RwLockWriteGuard, Semaphore}; use url::Url; diff --git a/libs/jwst-storage/src/storage/blobs/bucket_local_db.rs b/libs/jwst-storage/src/storage/blobs/bucket_local_db.rs index 4c8f45e2c..4b3e0a84b 100644 --- a/libs/jwst-storage/src/storage/blobs/bucket_local_db.rs +++ b/libs/jwst-storage/src/storage/blobs/bucket_local_db.rs @@ -1,19 +1,18 @@ -use super::utils::calculate_hash; -use super::utils::get_hash; -use super::*; -use crate::rate_limiter::Bucket; -use crate::JwstStorageError; +use std::{collections::HashMap, sync::Arc}; + use bytes::Bytes; use futures::Stream; use jwst::{BlobMetadata, BlobStorage, BucketBlobStorage, JwstResult}; use jwst_storage_migration::Migrator; -use opendal::services::S3; -use opendal::Operator; +use opendal::{services::S3, Operator}; use sea_orm::{DatabaseConnection, EntityTrait}; use sea_orm_migration::MigratorTrait; -use std::collections::HashMap; -use std::sync::Arc; +use super::{ + utils::{calculate_hash, get_hash}, + *, +}; +use crate::{rate_limiter::Bucket, JwstStorageError}; pub(super) type BucketBlobModel = ::Model; type BucketBlobActiveModel = entities::bucket_blobs::ActiveModel; diff --git a/libs/jwst-storage/src/storage/blobs/local_db.rs b/libs/jwst-storage/src/storage/blobs/local_db.rs index 76ae7cfc4..49c4ef363 100644 --- a/libs/jwst-storage/src/storage/blobs/local_db.rs +++ b/libs/jwst-storage/src/storage/blobs/local_db.rs @@ -1,8 +1,8 @@ -use super::{utils::get_hash, *}; -use crate::types::JwstStorageResult; use jwst::{Base64Engine, URL_SAFE_ENGINE}; - use sha2::{Digest, Sha256}; + +use super::{utils::get_hash, *}; +use crate::types::JwstStorageResult; pub(super) type BlobModel = ::Model; type BlobActiveModel = super::entities::blobs::ActiveModel; type BlobColumn = ::Column; diff --git a/libs/jwst-storage/src/storage/blobs/mod.rs b/libs/jwst-storage/src/storage/blobs/mod.rs index 9274c81aa..90320bbc8 100644 --- a/libs/jwst-storage/src/storage/blobs/mod.rs +++ b/libs/jwst-storage/src/storage/blobs/mod.rs @@ -2,20 +2,19 @@ mod bucket_local_db; mod local_db; mod utils; -#[cfg(test)] -pub use local_db::blobs_storage_test; - -use super::{entities::prelude::*, *}; +pub use bucket_local_db::BlobBucketDBStorage; use bytes::Bytes; use image::ImageError; use jwst::{BlobMetadata, BlobStorage}; +#[cfg(test)] +pub use local_db::blobs_storage_test; use local_db::BlobDBStorage; use thiserror::Error; use tokio::task::JoinError; +pub use utils::BucketStorageBuilder; use utils::{ImageParams, InternalBlobMetadata}; -pub use bucket_local_db::BlobBucketDBStorage; -pub use utils::BucketStorageBuilder; +use super::{entities::prelude::*, *}; #[derive(Debug, Error)] pub enum JwstBlobError { @@ -405,10 +404,12 @@ impl JwstBlobStorage { #[cfg(test)] mod tests { - use super::*; + use std::io::Cursor; + use futures::FutureExt; use image::{DynamicImage, ImageOutputFormat}; - use std::io::Cursor; + + use super::*; #[tokio::test] async fn test_blob_auto_storage() { diff --git a/libs/jwst-storage/src/storage/blobs/utils.rs b/libs/jwst-storage/src/storage/blobs/utils.rs index 7243fe8f4..f47caab58 100644 --- a/libs/jwst-storage/src/storage/blobs/utils.rs +++ b/libs/jwst-storage/src/storage/blobs/utils.rs @@ -1,6 +1,5 @@ -use crate::storage::blobs::bucket_local_db::BucketStorage; -use crate::storage::blobs::MixedBucketDBParam; -use crate::{JwstStorageError, JwstStorageResult}; +use std::{collections::HashMap, io::Cursor}; + use bytes::Bytes; use chrono::{DateTime, Utc}; use futures::{ @@ -9,11 +8,14 @@ use futures::{ }; use image::{load_from_memory, ImageOutputFormat, ImageResult}; use jwst::{Base64Engine, BlobMetadata, URL_SAFE_ENGINE}; -use opendal::services::S3; -use opendal::Operator; +use opendal::{services::S3, Operator}; use sea_orm::FromQueryResult; use sha2::{Digest, Sha256}; -use std::{collections::HashMap, io::Cursor}; + +use crate::{ + storage::blobs::{bucket_local_db::BucketStorage, MixedBucketDBParam}, + JwstStorageError, JwstStorageResult, +}; enum ImageFormat { Jpeg, diff --git a/libs/jwst-storage/src/storage/docs/database.rs b/libs/jwst-storage/src/storage/docs/database.rs index 9738d5ec8..1122f28e8 100644 --- a/libs/jwst-storage/src/storage/docs/database.rs +++ b/libs/jwst-storage/src/storage/docs/database.rs @@ -1,11 +1,13 @@ -use super::{entities::prelude::*, *}; -use crate::types::JwstStorageResult; +use std::collections::hash_map::Entry; + use jwst::{sync_encode_update, DocStorage, Workspace}; use jwst_codec::{CrdtReader, RawDecoder}; use sea_orm::Condition; -use std::collections::hash_map::Entry; use yrs::{Doc, Options, ReadTxn, StateVector, Transact}; +use super::{entities::prelude::*, *}; +use crate::types::JwstStorageResult; + const MAX_TRIM_UPDATE_LIMIT: u64 = 500; type DocsModel = ::Model; diff --git a/libs/jwst-storage/src/storage/docs/mod.rs b/libs/jwst-storage/src/storage/docs/mod.rs index f816a2b7e..50ac6a891 100644 --- a/libs/jwst-storage/src/storage/docs/mod.rs +++ b/libs/jwst-storage/src/storage/docs/mod.rs @@ -3,15 +3,15 @@ mod utils; use std::ops::Deref; -use super::*; -use database::DocDBStorage; -use tokio::sync::{broadcast::Sender, RwLock}; - #[cfg(test)] #[cfg(feature = "postgres")] pub(super) use database::full_migration_stress_test; +use database::DocDBStorage; #[cfg(test)] pub(super) use database::{docs_storage_partial_test, docs_storage_test}; +use tokio::sync::{broadcast::Sender, RwLock}; + +use super::*; #[derive(Clone)] pub struct SharedDocDBStorage(pub(super) Arc); @@ -40,14 +40,16 @@ impl Deref for SharedDocDBStorage { #[cfg(test)] mod test { - use super::{error, info, DocStorage, SharedDocDBStorage}; - use crate::{JwstStorageError, JwstStorageResult}; + use std::collections::HashSet; + use jwst_storage_migration::Migrator; use rand::random; use sea_orm_migration::MigratorTrait; - use std::collections::HashSet; use tokio::task::JoinSet; + use super::{error, info, DocStorage, SharedDocDBStorage}; + use crate::{JwstStorageError, JwstStorageResult}; + async fn create_workspace_stress_test(storage: SharedDocDBStorage, range: usize) -> JwstStorageResult<()> { let mut join_set = JoinSet::new(); let mut set = HashSet::new(); diff --git a/libs/jwst-storage/src/storage/docs/utils.rs b/libs/jwst-storage/src/storage/docs/utils.rs index 0d54909b3..4a8148be5 100644 --- a/libs/jwst-storage/src/storage/docs/utils.rs +++ b/libs/jwst-storage/src/storage/docs/utils.rs @@ -1,7 +1,9 @@ -use super::{entities::prelude::*, types::JwstStorageResult, *}; use std::panic::{catch_unwind, AssertUnwindSafe}; + use yrs::{updates::decoder::Decode, Doc, ReadTxn, StateVector, Transact, Update}; +use super::{entities::prelude::*, types::JwstStorageResult, *}; + // apply all updates to the given doc pub fn migrate_update(update_records: Vec<::Model>, doc: Doc) -> JwstResult { { diff --git a/libs/jwst-storage/src/storage/mod.rs b/libs/jwst-storage/src/storage/mod.rs index e502f35c9..8bc6fdb0f 100644 --- a/libs/jwst-storage/src/storage/mod.rs +++ b/libs/jwst-storage/src/storage/mod.rs @@ -3,17 +3,20 @@ mod difflog; mod docs; mod test; -use self::difflog::DiffLogRecord; +use std::{collections::HashMap, time::Instant}; -use super::*; -use crate::storage::blobs::{BlobBucketDBStorage, BlobStorageType, JwstBlobStorage}; -use crate::types::JwstStorageError; use blobs::BlobAutoStorage; use docs::SharedDocDBStorage; use jwst_storage_migration::{Migrator, MigratorTrait}; -use std::{collections::HashMap, time::Instant}; use tokio::sync::Mutex; +use self::difflog::DiffLogRecord; +use super::*; +use crate::{ + storage::blobs::{BlobBucketDBStorage, BlobStorageType, JwstBlobStorage}, + types::JwstStorageError, +}; + pub struct JwstStorage { pool: DatabaseConnection, blobs: JwstBlobStorage, diff --git a/libs/jwst/src/block/convert.rs b/libs/jwst/src/block/convert.rs index ba0787a7e..ac8e76763 100644 --- a/libs/jwst/src/block/convert.rs +++ b/libs/jwst/src/block/convert.rs @@ -254,9 +254,10 @@ impl Block { #[cfg(test)] mod tests { - use super::*; use yrs::{updates::decoder::Decode, Update}; + use super::*; + #[test] fn test_multiple_layer_space_clone() { let doc1 = Doc::new(); diff --git a/libs/jwst/src/block/mod.rs b/libs/jwst/src/block/mod.rs index 75aba88e4..1edce7719 100644 --- a/libs/jwst/src/block/mod.rs +++ b/libs/jwst/src/block/mod.rs @@ -1,12 +1,14 @@ mod convert; -use super::{constants::sys, utils::JS_INT_RANGE, *}; +use std::{ + collections::HashMap, + fmt, + sync::{Arc, RwLock}, +}; + use lib0::any::Any; use serde::{Serialize, Serializer}; use serde_json::Value as JsonValue; -use std::collections::HashMap; -use std::fmt; -use std::sync::{Arc, RwLock}; use yrs::{ types::{ text::{Diff, YChange}, @@ -16,6 +18,8 @@ use yrs::{ Transact, TransactionMut, }; +use super::{constants::sys, utils::JS_INT_RANGE, *}; + #[derive(Clone)] pub struct Block { id: String, @@ -471,9 +475,10 @@ impl Serialize for Block { #[cfg(test)] mod test { - use super::*; use std::collections::HashMap; + use super::*; + #[test] fn init_block() { let workspace = Workspace::new("workspace"); diff --git a/libs/jwst/src/history/raw.rs b/libs/jwst/src/history/raw.rs index 5bc6f9691..9d5b14431 100644 --- a/libs/jwst/src/history/raw.rs +++ b/libs/jwst/src/history/raw.rs @@ -1,5 +1,6 @@ -use serde::Serialize; use std::collections::{HashMap, HashSet, VecDeque}; + +use serde::Serialize; use tracing::{debug, info}; use utoipa::ToSchema; use yrs::{ diff --git a/libs/jwst/src/lib.rs b/libs/jwst/src/lib.rs index 298bd10a9..300845735 100644 --- a/libs/jwst/src/lib.rs +++ b/libs/jwst/src/lib.rs @@ -13,8 +13,7 @@ pub use space::Space; pub use tracing::{debug, error, info, log::LevelFilter, trace, warn}; pub use types::{BlobMetadata, BlobStorage, BucketBlobStorage, DocStorage, JwstError, JwstResult}; pub use utils::{sync_encode_update, Base64DecodeError, Base64Engine, STANDARD_ENGINE, URL_SAFE_ENGINE}; -pub use workspace::BlockObserverConfig; -pub use workspace::{MapSubscription, Workspace, WorkspaceMetadata, WorkspaceTransaction}; +pub use workspace::{BlockObserverConfig, MapSubscription, Workspace, WorkspaceMetadata, WorkspaceTransaction}; #[cfg(feature = "workspace-search")] pub use workspace::{SearchResult, SearchResults}; diff --git a/libs/jwst/src/space/convert.rs b/libs/jwst/src/space/convert.rs index 198023eea..71cdd4d2c 100644 --- a/libs/jwst/src/space/convert.rs +++ b/libs/jwst/src/space/convert.rs @@ -1,9 +1,11 @@ -use super::*; +use std::collections::HashMap; + use chrono::Utc; use lib0::any::Any; -use std::collections::HashMap; use yrs::{types::ToJson, Array, ArrayPrelim, ArrayRef, Map, MapPrelim, TextPrelim}; +use super::*; + impl Space { pub fn to_markdown(&self, trx: &T) -> Option where diff --git a/libs/jwst/src/space/mod.rs b/libs/jwst/src/space/mod.rs index 6228cd0bb..5aeb47745 100644 --- a/libs/jwst/src/space/mod.rs +++ b/libs/jwst/src/space/mod.rs @@ -1,12 +1,14 @@ mod convert; mod transaction; -use super::{block::MarkdownState, workspace::Pages, *}; -use serde::{ser::SerializeMap, Serialize, Serializer}; use std::sync::Arc; + +use serde::{ser::SerializeMap, Serialize, Serializer}; use transaction::SpaceTransaction; use yrs::{Doc, Map, MapRef, ReadTxn, Transact, TransactionMut, WriteTxn}; +use super::{block::MarkdownState, workspace::Pages, *}; + // Workspace // / \ // Space ... Space @@ -233,10 +235,11 @@ impl Serialize for Space { #[cfg(test)] mod test { - use super::*; use tracing::info; use yrs::{types::ToJson, updates::decoder::Decode, ArrayPrelim, Doc, StateVector, Update}; + use super::*; + #[test] fn doc_load_test() { let space_id = "space"; diff --git a/libs/jwst/src/space/transaction.rs b/libs/jwst/src/space/transaction.rs index c19990f72..50f2ca730 100644 --- a/libs/jwst/src/space/transaction.rs +++ b/libs/jwst/src/space/transaction.rs @@ -1,8 +1,9 @@ -use super::*; -use crate::utils::JS_INT_RANGE; use lib0::any::Any; use yrs::{Map, TransactionMut}; +use super::*; +use crate::utils::JS_INT_RANGE; + pub struct SpaceTransaction<'a> { pub space: &'a Space, pub trx: TransactionMut<'a>, diff --git a/libs/jwst/src/types/blob.rs b/libs/jwst/src/types/blob.rs index b1df24d2c..8eec9f227 100644 --- a/libs/jwst/src/types/blob.rs +++ b/libs/jwst/src/types/blob.rs @@ -1,8 +1,10 @@ -use super::*; +use std::collections::HashMap; + use bytes::Bytes; use chrono::NaiveDateTime; use futures::Stream; -use std::collections::HashMap; + +use super::*; #[derive(Debug)] pub struct BlobMetadata { diff --git a/libs/jwst/src/types/mod.rs b/libs/jwst/src/types/mod.rs index 58ed23e26..f140d0674 100644 --- a/libs/jwst/src/types/mod.rs +++ b/libs/jwst/src/types/mod.rs @@ -2,9 +2,9 @@ mod blob; mod doc; mod error; -use super::Workspace; use async_trait::async_trait; - pub use blob::{BlobMetadata, BlobStorage, BucketBlobStorage}; pub use doc::DocStorage; pub use error::{JwstError, JwstResult}; + +use super::Workspace; diff --git a/libs/jwst/src/utils.rs b/libs/jwst/src/utils.rs index a5c5cd62f..52e7169ae 100644 --- a/libs/jwst/src/utils.rs +++ b/libs/jwst/src/utils.rs @@ -1,10 +1,11 @@ +use std::ops::RangeInclusive; + use base64::{ alphabet::{STANDARD, URL_SAFE}, engine::{general_purpose::PAD, GeneralPurpose}, }; pub use base64::{DecodeError as Base64DecodeError, Engine as Base64Engine}; use lib0::encoding::Write; -use std::ops::RangeInclusive; use yrs::updates::encoder::{Encoder, EncoderV1}; const MSG_SYNC: usize = 0; diff --git a/libs/jwst/src/workspace/block_observer.rs b/libs/jwst/src/workspace/block_observer.rs index 19583bffd..e7e537565 100644 --- a/libs/jwst/src/workspace/block_observer.rs +++ b/libs/jwst/src/workspace/block_observer.rs @@ -1,4 +1,3 @@ -use super::Workspace; use std::{ collections::HashSet, sync::{ @@ -12,6 +11,7 @@ use std::{ thread::JoinHandle, time::Duration, }; + use tokio::{ runtime::{self, Runtime}, sync::RwLock, @@ -19,6 +19,8 @@ use tokio::{ }; use tracing::debug; +use super::Workspace; + type CallbackFn = Arc) + Send + Sync>>>>>; pub struct BlockObserverConfig { pub(crate) workspace_id: String, diff --git a/libs/jwst/src/workspace/metadata/meta.rs b/libs/jwst/src/workspace/metadata/meta.rs index b1dd3a5d2..bad0117d5 100644 --- a/libs/jwst/src/workspace/metadata/meta.rs +++ b/libs/jwst/src/workspace/metadata/meta.rs @@ -1,10 +1,12 @@ -use super::*; +use std::collections::HashMap; + use lib0::any::Any; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use yrs::{ArrayPrelim, ArrayRef, Map, MapRef, ReadTxn, Transact, TransactionMut}; +use super::*; + #[derive(Debug, Clone, JsonSchema, Serialize, Deserialize, PartialEq)] pub struct WorkspaceMetadata { pub name: Option, @@ -57,9 +59,10 @@ impl Workspace { #[cfg(test)] mod tests { - use super::*; use yrs::Doc; + use super::*; + #[test] fn test_workspace_metadata() { let doc = Doc::new(); diff --git a/libs/jwst/src/workspace/metadata/mod.rs b/libs/jwst/src/workspace/metadata/mod.rs index 231f4ff34..c2f841136 100644 --- a/libs/jwst/src/workspace/metadata/mod.rs +++ b/libs/jwst/src/workspace/metadata/mod.rs @@ -1,7 +1,7 @@ mod meta; mod pages; -use super::*; - pub use meta::WorkspaceMetadata; pub use pages::Pages; + +use super::*; diff --git a/libs/jwst/src/workspace/metadata/pages.rs b/libs/jwst/src/workspace/metadata/pages.rs index 267223bb8..0f7e5c504 100644 --- a/libs/jwst/src/workspace/metadata/pages.rs +++ b/libs/jwst/src/workspace/metadata/pages.rs @@ -1,5 +1,6 @@ -use lib0::any::Any; use std::collections::HashMap; + +use lib0::any::Any; use yrs::{types::Value, Array, ArrayRef, Map, MapRef, ReadTxn}; pub struct PageMeta { @@ -143,10 +144,10 @@ impl Pages { #[cfg(test)] mod tests { - use crate::Workspace; + use yrs::{updates::decoder::Decode, ArrayPrelim, Doc, Transact, Update}; use super::*; - use yrs::{updates::decoder::Decode, ArrayPrelim, Doc, Transact, Update}; + use crate::Workspace; #[test] fn test_page_meta() { diff --git a/libs/jwst/src/workspace/mod.rs b/libs/jwst/src/workspace/mod.rs index f7653dcd5..7c5166ccc 100644 --- a/libs/jwst/src/workspace/mod.rs +++ b/libs/jwst/src/workspace/mod.rs @@ -6,11 +6,11 @@ mod sync; mod transaction; mod workspace; -use super::{constants, error, info, trace, warn, JwstError, JwstResult, Space}; - pub use block_observer::BlockObserverConfig; pub use metadata::{Pages, WorkspaceMetadata}; #[cfg(feature = "workspace-search")] pub use plugins::{SearchResult, SearchResults}; pub use transaction::WorkspaceTransaction; pub use workspace::{MapSubscription, Workspace}; + +use super::{constants, error, info, trace, warn, JwstError, JwstResult, Space}; diff --git a/libs/jwst/src/workspace/observe.rs b/libs/jwst/src/workspace/observe.rs index c937f46e6..5cf598d45 100644 --- a/libs/jwst/src/workspace/observe.rs +++ b/libs/jwst/src/workspace/observe.rs @@ -1,13 +1,15 @@ -use super::*; -use jwst_codec::{Awareness, AwarenessEvent}; -use nanoid::nanoid; use std::{ panic::{catch_unwind, AssertUnwindSafe}, thread::sleep, time::Duration, }; + +use jwst_codec::{Awareness, AwarenessEvent}; +use nanoid::nanoid; use yrs::{TransactionMut, UpdateEvent}; +use super::*; + impl Workspace { pub async fn on_awareness_update(&mut self, f: impl Fn(&Awareness, AwarenessEvent) + Send + Sync + 'static) { self.awareness.write().await.on_update(f); diff --git a/libs/jwst/src/workspace/plugins/indexing/indexer.rs b/libs/jwst/src/workspace/plugins/indexing/indexer.rs index 0860c0832..d1d4abfd5 100644 --- a/libs/jwst/src/workspace/plugins/indexing/indexer.rs +++ b/libs/jwst/src/workspace/plugins/indexing/indexer.rs @@ -1,12 +1,16 @@ -use super::{PluginImpl, Workspace}; +use std::{ + collections::HashMap, + rc::Rc, + sync::{atomic::AtomicU32, Arc}, +}; + use lib0::any::Any; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::rc::Rc; -use std::sync::{atomic::AtomicU32, Arc}; use tantivy::{collector::TopDocs, query::QueryParser, schema::*, Index, ReloadPolicy}; use utoipa::ToSchema; +use super::{PluginImpl, Workspace}; + #[derive(Debug, Serialize, Deserialize, ToSchema)] pub struct SearchResult { pub block_id: String, @@ -52,7 +56,8 @@ impl IndexingPluginImpl { let query = self.query_parser.parse_query(query.as_ref())?; let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?; // The actual documents still need to be retrieved from Tantivy’s store. - // Since the body field was not configured as stored, the document returned will only contain a title. + // Since the body field was not configured as stored, the document returned will + // only contain a title. if !top_docs.is_empty() { let block_id_field = self.schema.get_field("block_id").unwrap(); @@ -158,8 +163,8 @@ impl IndexingPluginImpl { writer.add_document(block_doc)?; } - // If .commit() returns correctly, then all of the documents that have been added - // are guaranteed to be persistently indexed. + // If .commit() returns correctly, then all of the documents that have been + // added are guaranteed to be persistently indexed. writer.commit()?; Ok(()) @@ -168,12 +173,11 @@ impl IndexingPluginImpl { #[cfg(test)] mod test { - use super::super::*; - use super::*; + use super::{super::*, *}; - // out of order for now, in the future, this can be made in order by sorting before - // we reduce to just the block ids. Then maybe we could first sort on score, then sort on - // block id. + // out of order for now, in the future, this can be made in order by sorting + // before we reduce to just the block ids. Then maybe we could first sort on + // score, then sort on block id. macro_rules! expect_result_ids { ($search_results:ident, $id_str_array:expr) => { let mut sorted_ids = $search_results.0.iter().map(|i| &i.block_id).collect::>(); @@ -230,10 +234,22 @@ mod test { d.set(&mut t.trx, "text", "Text D content ddd yyy").unwrap(); e.set(&mut t.trx, "title", "人民日报").unwrap(); - e.set(&mut t.trx,"text", "张华考上了北京大学；李萍进了中等技术学校；我在百货公司当售货员：我们都有光明的前途").unwrap(); - - f.set(&mut t.trx, "title", "美国首次成功在核聚变反应中实现“净能量增益”").unwrap(); - f.set(&mut t.trx, "text", "当地时间13日，美国能源部官员宣布，由美国政府资助的加州劳伦斯·利弗莫尔国家实验室（LLNL），首次成功在核聚变反应中实现“净能量增益”，即聚变反应产生的能量大于促发该反应的镭射能量。").unwrap(); + e.set( + &mut t.trx, + "text", + "张华考上了北京大学；李萍进了中等技术学校；我在百货公司当售货员：我们都有光明的前途", + ) + .unwrap(); + + f.set(&mut t.trx, "title", "美国首次成功在核聚变反应中实现“净能量增益”") + .unwrap(); + f.set( + &mut t.trx, + "text", + "当地时间13日，美国能源部官员宣布，由美国政府资助的加州劳伦斯·利弗莫尔国家实验室（LLNL），\ + 首次成功在核聚变反应中实现“净能量增益”，即聚变反应产生的能量大于促发该反应的镭射能量。", + ) + .unwrap(); // pushing blocks in block.push_children(&mut t.trx, &b).unwrap(); @@ -253,11 +269,13 @@ mod test { ] ); - // Question: Is this supposed to indicate that since this block is detached, then we should not be indexing it? - // For example, should we walk up the parent tree to check if each block is actually attached? + // Question: Is this supposed to indicate that since this block is detached, + // then we should not be indexing it? For example, should we walk up + // the parent tree to check if each block is actually attached? block.remove_children(&mut t.trx, &d).unwrap(); - println!("Blocks: {:#?}", space1.blocks); // shown if there is an issue running the test. + println!("Blocks: {:#?}", space1.blocks); // shown if there is an + // issue running the test. }); workspace.with_trx(|mut t| { @@ -278,17 +296,28 @@ mod test { b.set(&mut t.trx, "text", "Text B content bbb xxx").unwrap(); c.set(&mut t.trx, "title", "Title C content").unwrap(); - c.set(&mut t.trx, "text", "Text C content ccc xxx yyy") - .unwrap(); + c.set(&mut t.trx, "text", "Text C content ccc xxx yyy").unwrap(); d.set(&mut t.trx, "title", "Title D content").unwrap(); d.set(&mut t.trx, "text", "Text D content ddd yyy").unwrap(); e.set(&mut t.trx, "title", "人民日报").unwrap(); - e.set(&mut t.trx,"text", "张华考上了北京大学；李萍进了中等技术学校；我在百货公司当售货员：我们都有光明的前途").unwrap(); - - f.set(&mut t.trx, "title", "美国首次成功在核聚变反应中实现“净能量增益”").unwrap(); - f.set(&mut t.trx, "text", "当地时间13日，美国能源部官员宣布，由美国政府资助的加州劳伦斯·利弗莫尔国家实验室（LLNL），首次成功在核聚变反应中实现“净能量增益”，即聚变反应产生的能量大于促发该反应的镭射能量。").unwrap(); + e.set( + &mut t.trx, + "text", + "张华考上了北京大学；李萍进了中等技术学校；我在百货公司当售货员：我们都有光明的前途", + ) + .unwrap(); + + f.set(&mut t.trx, "title", "美国首次成功在核聚变反应中实现“净能量增益”") + .unwrap(); + f.set( + &mut t.trx, + "text", + "当地时间13日，美国能源部官员宣布，由美国政府资助的加州劳伦斯·利弗莫尔国家实验室（LLNL），\ + 首次成功在核聚变反应中实现“净能量增益”，即聚变反应产生的能量大于促发该反应的镭射能量。", + ) + .unwrap(); // pushing blocks in block.push_children(&mut t.trx, &b).unwrap(); @@ -308,11 +337,13 @@ mod test { ] ); - // Question: Is this supposed to indicate that since this block is detached, then we should not be indexing it? - // For example, should we walk up the parent tree to check if each block is actually attached? + // Question: Is this supposed to indicate that since this block is detached, + // then we should not be indexing it? For example, should we walk up + // the parent tree to check if each block is actually attached? block.remove_children(&mut t.trx, &d).unwrap(); - println!("Blocks: {:#?}", space2.blocks); // shown if there is an issue running the test. + println!("Blocks: {:#?}", space2.blocks); // shown if there is an + // issue running the test. }); workspace diff --git a/libs/jwst/src/workspace/plugins/indexing/mod.rs b/libs/jwst/src/workspace/plugins/indexing/mod.rs index 3fb32ed07..18f207dbd 100644 --- a/libs/jwst/src/workspace/plugins/indexing/mod.rs +++ b/libs/jwst/src/workspace/plugins/indexing/mod.rs @@ -2,8 +2,8 @@ mod indexer; mod register; mod tokenizer; -use super::{PluginImpl, PluginRegister, Workspace}; -use tokenizer::{tokenizers_register, GRAM_TOKENIZER}; - pub use indexer::{IndexingPluginImpl, SearchResult, SearchResults}; pub(super) use register::IndexingPluginRegister; +use tokenizer::{tokenizers_register, GRAM_TOKENIZER}; + +use super::{PluginImpl, PluginRegister, Workspace}; diff --git a/libs/jwst/src/workspace/plugins/indexing/register.rs b/libs/jwst/src/workspace/plugins/indexing/register.rs index 37f125ca2..35665f1b7 100644 --- a/libs/jwst/src/workspace/plugins/indexing/register.rs +++ b/libs/jwst/src/workspace/plugins/indexing/register.rs @@ -1,15 +1,17 @@ -use super::*; use std::{ path::PathBuf, rc::Rc, sync::{atomic::AtomicU32, Arc}, }; + use tantivy::{ query::QueryParser, schema::{IndexRecordOption, Schema, TextFieldIndexing, TextOptions, STORED, STRING}, Index, }; +use super::*; + #[derive(Debug)] enum IndexingStorageKind { /// Store index in memory (default) diff --git a/libs/jwst/src/workspace/plugins/mod.rs b/libs/jwst/src/workspace/plugins/mod.rs index 8a22c68f5..e82f8bb99 100644 --- a/libs/jwst/src/workspace/plugins/mod.rs +++ b/libs/jwst/src/workspace/plugins/mod.rs @@ -2,14 +2,13 @@ mod indexing; mod plugin; -use super::*; - #[cfg(feature = "workspace-search")] use indexing::IndexingPluginImpl; -pub(super) use plugin::{PluginImpl, PluginMap, PluginRegister}; - #[cfg(feature = "workspace-search")] pub use indexing::{SearchResult, SearchResults}; +pub(super) use plugin::{PluginImpl, PluginMap, PluginRegister}; + +use super::*; /// Setup a [WorkspacePlugin] and insert it into the [Workspace]. /// See [plugins]. @@ -35,8 +34,8 @@ pub(crate) fn setup_plugin(workspace: Workspace) -> Workspace { } impl Workspace { - /// Allow the plugin to run any necessary updates it could have flagged via observers. - /// See [plugins]. + /// Allow the plugin to run any necessary updates it could have flagged via + /// observers. See [plugins]. fn update_plugin(&self) -> Result<(), Box> { self.plugins.update_plugin::

(self) } diff --git a/libs/jwst/src/workspace/plugins/plugin.rs b/libs/jwst/src/workspace/plugins/plugin.rs index e5b9e5d96..667b350bc 100644 --- a/libs/jwst/src/workspace/plugins/plugin.rs +++ b/libs/jwst/src/workspace/plugins/plugin.rs @@ -1,9 +1,12 @@ -//! Plugins are an internal experimental interface for extending the [Workspace]. +//! Plugins are an internal experimental interface for extending the +//! [Workspace]. -use super::*; use std::sync::{Arc, RwLock}; + use type_map::TypeMap; +use super::*; + /// A configuration from which a [WorkspacePlugin] can be created from. pub(crate) trait PluginRegister { type Plugin: PluginImpl; @@ -13,13 +16,15 @@ pub(crate) trait PluginRegister { // -> Box; } -/// A workspace plugin which comes from a corresponding [WorkspacePluginConfig::setup]. -/// In that setup call, the plugin will have initial access to the whole [Workspace], -/// and will be able to add listeners to changes to blocks in the [Workspace]. +/// A workspace plugin which comes from a corresponding +/// [WorkspacePluginConfig::setup]. In that setup call, the plugin will have +/// initial access to the whole [Workspace], and will be able to add listeners +/// to changes to blocks in the [Workspace]. pub(crate) trait PluginImpl: 'static { /// IDEA 1/10: - /// This update is called sometime between when we know changes have been made to the workspace - /// and the time when we will get the plugin to query its data (e.g. search()) + /// This update is called sometime between when we know changes have been + /// made to the workspace and the time when we will get the plugin to + /// query its data (e.g. search()) fn on_update(&mut self, _ws: &Workspace) -> Result<(), Box> { // Default implementation for a WorkspacePlugin update does nothing. Ok(()) @@ -28,9 +33,10 @@ pub(crate) trait PluginImpl: 'static { #[derive(Default, Clone)] pub(crate) struct PluginMap { - /// We store plugins into the TypeMap, so that their ownership is tied to [Workspace]. - /// This enables us to properly manage lifetimes of observers which will subscribe - /// into events that the [Workspace] experiences, like block updates. + /// We store plugins into the TypeMap, so that their ownership is tied to + /// [Workspace]. This enables us to properly manage lifetimes of + /// observers which will subscribe into events that the [Workspace] + /// experiences, like block updates. map: Arc>, } diff --git a/libs/jwst/src/workspace/sync.rs b/libs/jwst/src/workspace/sync.rs index a267fd257..116cc7dea 100644 --- a/libs/jwst/src/workspace/sync.rs +++ b/libs/jwst/src/workspace/sync.rs @@ -1,16 +1,18 @@ -use super::*; -use crate::RETRY_NUM; -use jwst_codec::{write_sync_message, DocMessage, SyncMessage, SyncMessageScanner}; use std::{ panic::{catch_unwind, AssertUnwindSafe}, thread::sleep, time::Duration, }; + +use jwst_codec::{write_sync_message, DocMessage, SyncMessage, SyncMessageScanner}; use yrs::{ updates::{decoder::Decode, encoder::Encode}, ReadTxn, StateVector, Transact, Update, }; +use super::*; +use crate::RETRY_NUM; + impl Workspace { pub fn sync_migration(&self) -> JwstResult> { let mut retry = RETRY_NUM; diff --git a/libs/jwst/src/workspace/transaction.rs b/libs/jwst/src/workspace/transaction.rs index a51485f06..5285dfa21 100644 --- a/libs/jwst/src/workspace/transaction.rs +++ b/libs/jwst/src/workspace/transaction.rs @@ -1,10 +1,11 @@ -use super::*; -use crate::{utils::JS_INT_RANGE, RETRY_NUM}; +use std::{cmp::max, thread::sleep, time::Duration}; + use lib0::any::Any; -use std::cmp::max; -use std::{thread::sleep, time::Duration}; use yrs::{Map, ReadTxn, Transact, TransactionMut}; +use super::*; +use crate::{utils::JS_INT_RANGE, RETRY_NUM}; + pub struct WorkspaceTransaction<'a> { pub ws: &'a Workspace, pub trx: TransactionMut<'a>, diff --git a/libs/jwst/src/workspace/workspace.rs b/libs/jwst/src/workspace/workspace.rs index 18a7d2e6e..1d40cb4b9 100644 --- a/libs/jwst/src/workspace/workspace.rs +++ b/libs/jwst/src/workspace/workspace.rs @@ -1,11 +1,10 @@ -use super::{ - block_observer::BlockObserverConfig, - plugins::{setup_plugin, PluginMap}, +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, }; + use jwst_codec::Awareness; use serde::{ser::SerializeMap, Serialize, Serializer}; -use std::sync::Mutex; -use std::{collections::HashMap, sync::Arc}; use tokio::sync::RwLock; use yrs::{ types::{map::MapEvent, ToJson}, @@ -13,6 +12,11 @@ use yrs::{ Doc, Map, MapRef, Subscription, Transact, TransactionMut, Update, UpdateSubscription, }; +use super::{ + block_observer::BlockObserverConfig, + plugins::{setup_plugin, PluginMap}, +}; + pub type MapSubscription = Subscription>; pub struct Workspace { @@ -24,8 +28,9 @@ pub struct Workspace { pub(crate) updated: MapRef, pub(crate) metadata: MapRef, /// We store plugins so that their ownership is tied to [Workspace]. - /// This enables us to properly manage lifetimes of observers which will subscribe - /// into events that the [Workspace] experiences, like block updates. + /// This enables us to properly manage lifetimes of observers which will + /// subscribe into events that the [Workspace] experiences, like block + /// updates. /// /// Public just for the crate as we experiment with the plugins interface. /// See [super::plugins]. @@ -150,13 +155,13 @@ impl Clone for Workspace { #[cfg(test)] mod test { - use super::{super::super::Block, *}; - use std::collections::HashSet; - use std::thread::sleep; - use std::time::Duration; + use std::{collections::HashSet, thread::sleep, time::Duration}; + use tracing::info; use yrs::{updates::decoder::Decode, Doc, Map, ReadTxn, StateVector, Update}; + use super::{super::super::Block, *}; + #[test] fn doc_load_test() { let workspace = Workspace::new("test"); @@ -467,8 +472,9 @@ mod test { let mut children = block.children(&t.trx); children.sort(); assert_eq!(children, vec!["test1".to_owned(), "test2".to_owned()]); - // assert_eq!(block.get(&t.trx, "test1").unwrap().to_string(), "test1"); - // assert_eq!(block.get(&t.trx, "test2").unwrap().to_string(), "test2"); + // assert_eq!(block.get(&t.trx, "test1").unwrap().to_string(), + // "test1"); assert_eq!(block.get(&t.trx, + // "test2").unwrap().to_string(), "test2"); }); } } @@ -559,8 +565,9 @@ mod test { let mut children = block.children(&t.trx); children.sort(); assert_ne!(children, vec!["test1".to_owned(), "test2".to_owned()]); - // assert_eq!(block.get(&t.trx, "test1").unwrap().to_string(), "test1"); - // assert_eq!(block.get(&t.trx, "test2").unwrap().to_string(), "test2"); + // assert_eq!(block.get(&t.trx, "test1").unwrap().to_string(), + // "test1"); assert_eq!(block.get(&t.trx, + // "test2").unwrap().to_string(), "test2"); }); } } From b482de4b2454fe88b57e3975ed109f7af49e7789 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 21 Aug 2023 22:07:10 +0800 Subject: [PATCH 03/49] fix: dead lock in map --- .../RustXcframework.xcframework/Info.plist | 8 ++++---- .../{jwst-swift.h => jwst-core-swift.h} | 3 +-- .../Headers/module.modulemap | 2 +- .../{jwst-swift.h => jwst-core-swift.h} | 3 +-- .../ios-arm64/Headers/module.modulemap | 2 +- .../{jwst-swift.h => jwst-core-swift.h} | 3 +-- .../macos-arm64/Headers/module.modulemap | 2 +- .../Sources/OctoBase/jwst-swift.swift | 7 +------ libs/jwst-codec/src/doc/types/map.rs | 19 ++++++++++++++++--- 9 files changed, 27 insertions(+), 22 deletions(-) rename apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/{jwst-swift.h => jwst-core-swift.h} (98%) rename apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/{jwst-swift.h => jwst-core-swift.h} (98%) rename apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/{jwst-swift.h => jwst-core-swift.h} (98%) diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist index b3d1aee06..75a6e1fa6 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist @@ -24,7 +24,7 @@ HeadersPath Headers LibraryIdentifier - macos-arm64 + ios-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -32,13 +32,13 @@ arm64 SupportedPlatform - macos + ios HeadersPath Headers LibraryIdentifier - ios-arm64 + macos-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -46,7 +46,7 @@ arm64 SupportedPlatform - ios + macos CFBundlePackageType diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/jwst-swift.h b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/jwst-core-swift.h similarity index 98% rename from apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/jwst-swift.h rename to apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/jwst-core-swift.h index 9cd36238f..7fa5b3b6f 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/jwst-swift.h +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/jwst-core-swift.h @@ -95,7 +95,7 @@ bool __swift_bridge__$Block$is_bool(void* self, void* key); bool __swift_bridge__$Block$is_string(void* self, void* key); bool __swift_bridge__$Block$is_float(void* self, void* key); bool __swift_bridge__$Block$is_integer(void* self, void* key); -struct __private__OptionI64 __swift_bridge__$Block$get_bool(void* self, void* key); +struct __private__OptionBool __swift_bridge__$Block$get_bool(void* self, void* key); void* __swift_bridge__$Block$get_string(void* self, void* key); struct __private__OptionF64 __swift_bridge__$Block$get_float(void* self, void* key); struct __private__OptionI64 __swift_bridge__$Block$get_integer(void* self, void* key); @@ -114,7 +114,6 @@ void* __swift_bridge__$Workspace$search(void* self, void* query); void* __swift_bridge__$Workspace$get_blocks_by_flavour(void* self, struct RustStr flavour); void* __swift_bridge__$Workspace$get_search_index(void* self); bool __swift_bridge__$Workspace$set_search_index(void* self, void* fields); -void* __swift_bridge__$Workspace$compare(void* self); void* __swift_bridge__$Storage$new(void* path); void* __swift_bridge__$Storage$new_with_log_level(void* path, void* level); void* __swift_bridge__$Storage$error(void* self); diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/module.modulemap b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/module.modulemap index f09070708..49cb3768c 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/module.modulemap +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/module.modulemap @@ -1,5 +1,5 @@ module RustXcframework { header "SwiftBridgeCore.h" - header "jwst-swift.h" + header "jwst-core-swift.h" export * } diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/jwst-swift.h b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/jwst-core-swift.h similarity index 98% rename from apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/jwst-swift.h rename to apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/jwst-core-swift.h index 9cd36238f..7fa5b3b6f 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/jwst-swift.h +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/jwst-core-swift.h @@ -95,7 +95,7 @@ bool __swift_bridge__$Block$is_bool(void* self, void* key); bool __swift_bridge__$Block$is_string(void* self, void* key); bool __swift_bridge__$Block$is_float(void* self, void* key); bool __swift_bridge__$Block$is_integer(void* self, void* key); -struct __private__OptionI64 __swift_bridge__$Block$get_bool(void* self, void* key); +struct __private__OptionBool __swift_bridge__$Block$get_bool(void* self, void* key); void* __swift_bridge__$Block$get_string(void* self, void* key); struct __private__OptionF64 __swift_bridge__$Block$get_float(void* self, void* key); struct __private__OptionI64 __swift_bridge__$Block$get_integer(void* self, void* key); @@ -114,7 +114,6 @@ void* __swift_bridge__$Workspace$search(void* self, void* query); void* __swift_bridge__$Workspace$get_blocks_by_flavour(void* self, struct RustStr flavour); void* __swift_bridge__$Workspace$get_search_index(void* self); bool __swift_bridge__$Workspace$set_search_index(void* self, void* fields); -void* __swift_bridge__$Workspace$compare(void* self); void* __swift_bridge__$Storage$new(void* path); void* __swift_bridge__$Storage$new_with_log_level(void* path, void* level); void* __swift_bridge__$Storage$error(void* self); diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/module.modulemap b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/module.modulemap index f09070708..49cb3768c 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/module.modulemap +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/module.modulemap @@ -1,5 +1,5 @@ module RustXcframework { header "SwiftBridgeCore.h" - header "jwst-swift.h" + header "jwst-core-swift.h" export * } diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/jwst-swift.h b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/jwst-core-swift.h similarity index 98% rename from apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/jwst-swift.h rename to apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/jwst-core-swift.h index 9cd36238f..7fa5b3b6f 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/jwst-swift.h +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/jwst-core-swift.h @@ -95,7 +95,7 @@ bool __swift_bridge__$Block$is_bool(void* self, void* key); bool __swift_bridge__$Block$is_string(void* self, void* key); bool __swift_bridge__$Block$is_float(void* self, void* key); bool __swift_bridge__$Block$is_integer(void* self, void* key); -struct __private__OptionI64 __swift_bridge__$Block$get_bool(void* self, void* key); +struct __private__OptionBool __swift_bridge__$Block$get_bool(void* self, void* key); void* __swift_bridge__$Block$get_string(void* self, void* key); struct __private__OptionF64 __swift_bridge__$Block$get_float(void* self, void* key); struct __private__OptionI64 __swift_bridge__$Block$get_integer(void* self, void* key); @@ -114,7 +114,6 @@ void* __swift_bridge__$Workspace$search(void* self, void* query); void* __swift_bridge__$Workspace$get_blocks_by_flavour(void* self, struct RustStr flavour); void* __swift_bridge__$Workspace$get_search_index(void* self); bool __swift_bridge__$Workspace$set_search_index(void* self, void* fields); -void* __swift_bridge__$Workspace$compare(void* self); void* __swift_bridge__$Storage$new(void* path); void* __swift_bridge__$Storage$new_with_log_level(void* path, void* level); void* __swift_bridge__$Storage$error(void* self); diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/module.modulemap b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/module.modulemap index f09070708..49cb3768c 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/module.modulemap +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/module.modulemap @@ -1,5 +1,5 @@ module RustXcframework { header "SwiftBridgeCore.h" - header "jwst-swift.h" + header "jwst-core-swift.h" export * } diff --git a/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-swift.swift b/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-swift.swift index e5e400c90..0556654ee 100644 --- a/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-swift.swift +++ b/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-swift.swift @@ -120,7 +120,7 @@ extension BlockRef { __swift_bridge__$Block$is_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) } - public func get_bool(_ key: GenericIntoRustString) -> Optional { + public func get_bool(_ key: GenericIntoRustString) -> Optional { { let val = __swift_bridge__$Block$get_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() } @@ -371,11 +371,6 @@ public class WorkspaceRefMut: WorkspaceRef { super.init(ptr: ptr) } } -extension WorkspaceRefMut { - public func compare() -> Optional { - { let val = __swift_bridge__$Workspace$compare(ptr); if val != nil { return RustString(ptr: val!) } else { return nil } }() - } -} public class WorkspaceRef { var ptr: UnsafeMutableRawPointer diff --git a/libs/jwst-codec/src/doc/types/map.rs b/libs/jwst-codec/src/doc/types/map.rs index 5044a485b..111656370 100644 --- a/libs/jwst-codec/src/doc/types/map.rs +++ b/libs/jwst-codec/src/doc/types/map.rs @@ -12,13 +12,18 @@ impl_type!(Map); pub(crate) trait MapType: AsInner { fn insert(&mut self, key: impl AsRef, value: impl Into) -> JwstCodecResult { - let mut inner = self.as_inner().get().unwrap().write().unwrap(); + // when we apply update, we may get write store then get ytype, so we ensure + // that everywhere, the write store is fetched before the write ytype to + // avoid deadlocks + let inner = self.as_inner().get().unwrap().read().unwrap(); let left = inner.map.as_ref().and_then(|map| { map.get(key.as_ref()) .and_then(|struct_info| struct_info.left()) .map(|l| l.as_item()) }); if let Some(store) = inner.store.upgrade() { + drop(inner); + let mut store = store.write().unwrap(); let item = store.create_item( value.into(), @@ -27,6 +32,7 @@ pub(crate) trait MapType: AsInner { Some(Parent::Type(self.as_inner().clone())), Some(key.as_ref().into()), ); + let mut inner = self.as_inner().get().unwrap().write().unwrap(); store.integrate(Node::Item(item), 0, Some(&mut inner))?; } @@ -74,12 +80,19 @@ pub(crate) trait MapType: AsInner { } fn remove(&mut self, key: impl AsRef) -> bool { - let mut inner = self.as_inner().get().unwrap().write().unwrap(); + // when we apply update, we may get write store then get ytype, so we ensure + // that everywhere, the write store is fetched before the write ytype to + // avoid deadlocks + let inner = self.as_inner().get().unwrap().read().unwrap(); let node = inner.map.as_ref().and_then(|map| map.get(key.as_ref())); if let Some(store) = inner.store.upgrade() { - let mut store = store.write().unwrap(); if let Some(item) = ItemRef::from(node).get() { + drop(inner); + + let mut store = store.write().unwrap(); store.delete_set.add(item.id.client, item.id.clock, item.len()); + + let mut inner = self.as_inner().get().unwrap().write().unwrap(); DocStore::delete_item(item, Some(&mut inner)); return true; } From dc47ab7e413ce6b5ba5ec0e5542b1eda30c93ab5 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 21 Aug 2023 22:08:28 +0800 Subject: [PATCH 04/49] feat: keep storage thread alive --- libs/jwst-binding/jwst-core-swift/src/storage.rs | 2 +- libs/jwst-binding/jwst-core-swift/src/workspace.rs | 5 +++++ libs/jwst-core-rpc/src/broadcast.rs | 2 +- libs/jwst-core-rpc/src/context.rs | 1 + libs/jwst-core/src/workspace/metadata/pages.rs | 2 +- 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/libs/jwst-binding/jwst-core-swift/src/storage.rs b/libs/jwst-binding/jwst-core-swift/src/storage.rs index b48be4560..65ca7f8a2 100644 --- a/libs/jwst-binding/jwst-core-swift/src/storage.rs +++ b/libs/jwst-binding/jwst-core-swift/src/storage.rs @@ -130,7 +130,7 @@ impl Storage { ); } - Ok(Workspace { workspace }) + Ok(Workspace { workspace, rt }) } Err(e) => Err(e), } diff --git a/libs/jwst-binding/jwst-core-swift/src/workspace.rs b/libs/jwst-binding/jwst-core-swift/src/workspace.rs index 2bd254592..147450401 100644 --- a/libs/jwst-binding/jwst-core-swift/src/workspace.rs +++ b/libs/jwst-binding/jwst-core-swift/src/workspace.rs @@ -1,15 +1,20 @@ +use std::sync::Arc; + use jwst_core::Workspace as JwstWorkspace; +use tokio::runtime::Runtime; use super::*; pub struct Workspace { pub(crate) workspace: JwstWorkspace, + pub(crate) rt: Arc, } impl Workspace { pub fn new(id: String) -> Self { Self { workspace: JwstWorkspace::new(&id).unwrap(), + rt: Arc::new(Runtime::new().unwrap()), } } diff --git a/libs/jwst-core-rpc/src/broadcast.rs b/libs/jwst-core-rpc/src/broadcast.rs index b6aa7c75c..1767410e3 100644 --- a/libs/jwst-core-rpc/src/broadcast.rs +++ b/libs/jwst-core-rpc/src/broadcast.rs @@ -54,7 +54,7 @@ pub async fn subscribe(workspace: &mut Workspace, identifier: String, sender: Br let sender = sender.clone(); let workspace_id = workspace.id(); workspace.doc().subscribe(move |update| { - trace!("workspace {} changed: {}bytes", workspace_id, update.len()); + debug!("workspace {} changed: {}bytes", workspace_id, update.len()); match encode_update_with_guid(update.to_vec(), workspace_id.clone()) .and_then(|update| encode_update_as_message(update.clone()).map(|u| (update, u))) diff --git a/libs/jwst-core-rpc/src/context.rs b/libs/jwst-core-rpc/src/context.rs index 7d5444088..dd4a15f89 100644 --- a/libs/jwst-core-rpc/src/context.rs +++ b/libs/jwst-core-rpc/src/context.rs @@ -117,6 +117,7 @@ pub trait RpcContextImpl<'a> { } } } + trace!("save update thread {id}-{identifier} finished"); }) }; diff --git a/libs/jwst-core/src/workspace/metadata/pages.rs b/libs/jwst-core/src/workspace/metadata/pages.rs index 7a80146b2..b096ed57f 100644 --- a/libs/jwst-core/src/workspace/metadata/pages.rs +++ b/libs/jwst-core/src/workspace/metadata/pages.rs @@ -182,7 +182,7 @@ mod tests { // let doc = Doc::default(); // doc.apply_update( // Update::from_ybinary1( - // + // // include_bytes!("../../../fixtures/test_shared_page.bin").to_vec(), // ) // .unwrap(), From 61aa776f59d79e49b878ecc777bf7475b84e7935 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 21 Aug 2023 22:27:44 +0800 Subject: [PATCH 05/49] feat: merge update at startup --- libs/jwst-binding/jwst-core-swift/src/storage.rs | 5 ++++- libs/jwst-core-storage/src/storage/docs/database.rs | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/libs/jwst-binding/jwst-core-swift/src/storage.rs b/libs/jwst-binding/jwst-core-swift/src/storage.rs index 65ca7f8a2..41ec2cf2d 100644 --- a/libs/jwst-binding/jwst-core-swift/src/storage.rs +++ b/libs/jwst-binding/jwst-core-swift/src/storage.rs @@ -27,7 +27,10 @@ impl Storage { pub fn new_with_log_level(path: String, level: String) -> Self { init_logger_with( - &format!("{level},mio=off,hyper=off,rustls=off,tantivy=off,sqlx::query=off"), + &format!( + "{level},mio=off,hyper=off,rustls=off,tantivy=off,sqlx::query=off,tokio_tungstenite=off,\ + tungstenite=off" + ), false, ); diff --git a/libs/jwst-core-storage/src/storage/docs/database.rs b/libs/jwst-core-storage/src/storage/docs/database.rs index c44f2fc02..bb93e8dba 100644 --- a/libs/jwst-core-storage/src/storage/docs/database.rs +++ b/libs/jwst-core-storage/src/storage/docs/database.rs @@ -255,7 +255,16 @@ impl DocDBStorage { guid: all_data.first().unwrap().guid.clone().into(), ..Default::default() }); + + let can_merge = all_data.len() > 1; + let doc = utils::migrate_update(all_data, doc)?; + + if can_merge { + let update = doc.encode_state_as_update_v1(&StateVector::default())?; + Self::replace_with(conn, workspace, doc.guid(), update).await?; + } + Workspace::from_doc(doc, workspace)? }; From 5479def0b8811cabbec0f39fe187647c3f116486 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 21 Aug 2023 22:28:38 +0800 Subject: [PATCH 06/49] feat: stop dispatch update at migrate --- .../OctoBaseSwift/RustXcframework.xcframework/Info.plist | 8 ++++---- libs/jwst-core-storage/src/storage/docs/utils.rs | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist index 75a6e1fa6..b3d1aee06 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist @@ -24,7 +24,7 @@ HeadersPath Headers LibraryIdentifier - ios-arm64 + macos-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -32,13 +32,13 @@ arm64 SupportedPlatform - ios + macos HeadersPath Headers LibraryIdentifier - macos-arm64 + ios-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -46,7 +46,7 @@ arm64 SupportedPlatform - macos + ios CFBundlePackageType diff --git a/libs/jwst-core-storage/src/storage/docs/utils.rs b/libs/jwst-core-storage/src/storage/docs/utils.rs index 6564e2225..3a05346fe 100644 --- a/libs/jwst-core-storage/src/storage/docs/utils.rs +++ b/libs/jwst-core-storage/src/storage/docs/utils.rs @@ -4,12 +4,15 @@ use super::{entities::prelude::*, types::JwstStorageResult, *}; // apply all updates to the given doc pub fn migrate_update(update_records: Vec<::Model>, mut doc: Doc) -> JwstResult { + // stop update dispatch before apply updates + doc.publisher.stop(); for record in update_records { let id = record.created_at; if let Err(e) = doc.apply_update_from_binary(record.blob) { warn!("update {} merge failed, skip it: {:?}", id, e); } } + doc.publisher.start(); Ok(doc) } From 2b3684bd6d0ef36b7f5ecafcdc2071209c312ac3 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 21 Aug 2023 22:29:27 +0800 Subject: [PATCH 07/49] feat: return changed update after apply --- libs/jwst-codec/src/doc/document.rs | 5 +++-- libs/jwst-codec/src/doc/store.rs | 4 +++- libs/jwst-core/src/workspace/sync.rs | 23 +++++++++++++++-------- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/libs/jwst-codec/src/doc/document.rs b/libs/jwst-codec/src/doc/document.rs index dddbb57f7..e16c1a30c 100644 --- a/libs/jwst-codec/src/doc/document.rs +++ b/libs/jwst-codec/src/doc/document.rs @@ -94,9 +94,10 @@ impl Doc { Ok(()) } - pub fn apply_update(&mut self, mut update: Update) -> JwstCodecResult { + pub fn apply_update(&mut self, mut update: Update) -> JwstCodecResult { let mut store = self.store.write().unwrap(); let mut retry = false; + let before_state = store.get_state_vector(); loop { for (mut s, offset) in update.iter(store.get_state_vector()) { if let Node::Item(item) = &mut s { @@ -154,7 +155,7 @@ impl Doc { } } - Ok(()) + store.diff_state_vector(&before_state) } pub fn keys(&self) -> Vec { diff --git a/libs/jwst-codec/src/doc/store.rs b/libs/jwst-codec/src/doc/store.rs index aa135bad0..b0df855f5 100644 --- a/libs/jwst-codec/src/doc/store.rs +++ b/libs/jwst-codec/src/doc/store.rs @@ -561,7 +561,9 @@ impl DocStore { if item.parent_sub.is_none() && item.countable() { if let Some(parent) = parent { - parent.len -= item.len(); + if parent.len != 0 { + parent.len -= item.len(); + } } else if let Some(Parent::Type(ty)) = &item.parent { ty.get().unwrap().write().unwrap().len -= item.len(); } diff --git a/libs/jwst-core/src/workspace/sync.rs b/libs/jwst-core/src/workspace/sync.rs index c3cef42f0..48745e689 100644 --- a/libs/jwst-core/src/workspace/sync.rs +++ b/libs/jwst-core/src/workspace/sync.rs @@ -34,6 +34,7 @@ impl Workspace { let mut content = vec![]; for buffer in buffers { + trace!("sync message: {:?}", buffer); let (awareness_msg, content_msg): (Vec<_>, Vec<_>) = SyncMessageScanner::new(&buffer) .flatten() .partition(|msg| matches!(msg, SyncMessage::Awareness(_) | SyncMessage::AwarenessQuery)); @@ -98,15 +99,21 @@ impl Workspace { } DocMessage::Update(update) => { if let Ok(update) = Update::read(&mut RawDecoder::new(update)) { - if let Err(e) = doc.apply_update(update) { - warn!("failed to apply update: {:?}", e); + match doc.apply_update(update) { + Ok(update) => { + let mut encoder = RawEncoder::default(); + if let Err(e) = update.write(&mut encoder) { + warn!("failed to encode update: {:?}", e); + None + } else { + Some(SyncMessage::Doc(DocMessage::Update(encoder.into_inner()))) + } + } + Err(e) => { + warn!("failed to apply update: {:?}", e); + None + } } - - // TODO: let apply_update return changed state vector after apply - // then we can make generate a diff update and send to client - // trx.encode_update_v1() - // .map(|update| SyncMessage::Doc(DocMessage::Update(update))); - None } else { None } From a40e7ee60e8860a4efb49ea5945d95e637b40f1e Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 21 Aug 2023 22:56:22 +0800 Subject: [PATCH 08/49] chore: disable remote temporarily --- libs/jwst-binding/jwst-core-swift/src/storage.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/jwst-binding/jwst-core-swift/src/storage.rs b/libs/jwst-binding/jwst-core-swift/src/storage.rs index 41ec2cf2d..d0b725bcd 100644 --- a/libs/jwst-binding/jwst-core-swift/src/storage.rs +++ b/libs/jwst-binding/jwst-core-swift/src/storage.rs @@ -114,7 +114,8 @@ impl Storage { match workspace { Ok(mut workspace) => { - if is_offline { + // disable remote temporarily + if is_offline || true { let identifier = nanoid!(); let (last_synced_tx, last_synced_rx) = channel::(128); self.last_sync.add_receiver(rt.clone(), last_synced_rx); From 1c94c52094ed498e28222fb54902499b431c5bbb Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 21 Aug 2023 23:47:32 +0800 Subject: [PATCH 09/49] chore: merge swift project --- Cargo.lock | 34 +- Cargo.toml | 2 - .../RustXcframework.xcframework/Info.plist | 14 +- .../{jwst-core-swift.h => jwst-swift.h} | 0 .../Headers/module.modulemap | 2 +- .../{jwst-core-swift.h => jwst-swift.h} | 0 .../ios-arm64/Headers/module.modulemap | 2 +- .../{jwst-core-swift.h => jwst-swift.h} | 0 .../macos-arm64/Headers/module.modulemap | 2 +- libs/jwst-binding/jwst-core-swift/.gitignore | 2 - libs/jwst-binding/jwst-core-swift/Cargo.toml | 35 - libs/jwst-binding/jwst-core-swift/binding.h | 105 --- libs/jwst-binding/jwst-core-swift/build.rs | 13 - .../generated/SwiftBridgeCore.h | 148 --- .../generated/SwiftBridgeCore.swift | 881 ------------------ .../jwst-core-swift/jwst-core-swift.h | 128 --- .../jwst-core-swift/jwst-core-swift.swift | 646 ------------- .../jwst-swift-integrate/Cargo.toml | 9 - .../jwst-swift-integrate/src/main.rs | 56 -- .../jwst-binding/jwst-core-swift/src/block.rs | 182 ---- .../jwst-core-swift/src/dynamic_value.rs | 72 -- libs/jwst-binding/jwst-core-swift/src/lib.rs | 142 --- .../jwst-core-swift/src/storage.rs | 207 ---- .../jwst-core-swift/src/workspace.rs | 75 -- libs/jwst-binding/jwst-swift/Cargo.toml | 7 +- libs/jwst-binding/jwst-swift/src/block.rs | 766 ++------------- libs/jwst-binding/jwst-swift/src/difflog.rs | 64 -- .../jwst-swift/src/dynamic_value.rs | 15 +- libs/jwst-binding/jwst-swift/src/lib.rs | 8 +- libs/jwst-binding/jwst-swift/src/storage.rs | 61 +- libs/jwst-binding/jwst-swift/src/workspace.rs | 171 +--- 31 files changed, 163 insertions(+), 3686 deletions(-) rename apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/{jwst-core-swift.h => jwst-swift.h} (100%) rename apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/{jwst-core-swift.h => jwst-swift.h} (100%) rename apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/{jwst-core-swift.h => jwst-swift.h} (100%) delete mode 100644 libs/jwst-binding/jwst-core-swift/.gitignore delete mode 100644 libs/jwst-binding/jwst-core-swift/Cargo.toml delete mode 100644 libs/jwst-binding/jwst-core-swift/binding.h delete mode 100644 libs/jwst-binding/jwst-core-swift/build.rs delete mode 100644 libs/jwst-binding/jwst-core-swift/generated/SwiftBridgeCore.h delete mode 100644 libs/jwst-binding/jwst-core-swift/generated/SwiftBridgeCore.swift delete mode 100644 libs/jwst-binding/jwst-core-swift/generated/jwst-core-swift/jwst-core-swift.h delete mode 100644 libs/jwst-binding/jwst-core-swift/generated/jwst-core-swift/jwst-core-swift.swift delete mode 100644 libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/Cargo.toml delete mode 100644 libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs delete mode 100644 libs/jwst-binding/jwst-core-swift/src/block.rs delete mode 100644 libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs delete mode 100644 libs/jwst-binding/jwst-core-swift/src/lib.rs delete mode 100644 libs/jwst-binding/jwst-core-swift/src/storage.rs delete mode 100644 libs/jwst-binding/jwst-core-swift/src/workspace.rs delete mode 100644 libs/jwst-binding/jwst-swift/src/difflog.rs diff --git a/Cargo.lock b/Cargo.lock index 53362ac6c..9f8d88f8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2830,33 +2830,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "jwst-core-swift" -version = "0.1.0" -dependencies = [ - "chrono", - "futures", - "jwst-core", - "jwst-core-rpc", - "jwst-core-storage", - "jwst-logger", - "nanoid", - "regex", - "reqwest", - "serde", - "serde_json", - "swift-bridge", - "swift-bridge-build", - "tokio", -] - -[[package]] -name = "jwst-core-swift-integrate" -version = "0.1.0" -dependencies = [ - "swift-bridge-build", -] - [[package]] name = "jwst-jni" version = "0.1.0" @@ -2973,12 +2946,10 @@ version = "0.1.0" dependencies = [ "chrono", "futures", - "jwst", "jwst-core", + "jwst-core-rpc", + "jwst-core-storage", "jwst-logger", - "jwst-rpc", - "jwst-storage", - "lib0", "nanoid", "regex", "reqwest", @@ -2987,7 +2958,6 @@ dependencies = [ "swift-bridge", "swift-bridge-build", "tokio", - "yrs", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9d8b86b9b..5b51626cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,8 +13,6 @@ members = [ # "libs/jwst-binding/jwst-py", "libs/jwst-binding/jwst-swift", "libs/jwst-binding/jwst-swift/jwst-swift-integrate", - "libs/jwst-binding/jwst-core-swift", - "libs/jwst-binding/jwst-core-swift/jwst-swift-integrate", # "libs/jwst-binding/jwst-wasm", "libs/jwst-codec", "libs/jwst-codec-util", diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist index b3d1aee06..067a2b752 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist @@ -8,7 +8,7 @@ HeadersPath Headers LibraryIdentifier - ios-arm64-simulator + macos-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -16,15 +16,13 @@ arm64 SupportedPlatform - ios - SupportedPlatformVariant - simulator + macos HeadersPath Headers LibraryIdentifier - macos-arm64 + ios-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -32,13 +30,13 @@ arm64 SupportedPlatform - macos + ios HeadersPath Headers LibraryIdentifier - ios-arm64 + ios-arm64-simulator LibraryPath liboctobase.a SupportedArchitectures @@ -47,6 +45,8 @@ SupportedPlatform ios + SupportedPlatformVariant + simulator CFBundlePackageType diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/jwst-core-swift.h b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/jwst-swift.h similarity index 100% rename from apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/jwst-core-swift.h rename to apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/jwst-swift.h diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/module.modulemap b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/module.modulemap index 49cb3768c..f09070708 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/module.modulemap +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/module.modulemap @@ -1,5 +1,5 @@ module RustXcframework { header "SwiftBridgeCore.h" - header "jwst-core-swift.h" + header "jwst-swift.h" export * } diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/jwst-core-swift.h b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/jwst-swift.h similarity index 100% rename from apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/jwst-core-swift.h rename to apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/jwst-swift.h diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/module.modulemap b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/module.modulemap index 49cb3768c..f09070708 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/module.modulemap +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/module.modulemap @@ -1,5 +1,5 @@ module RustXcframework { header "SwiftBridgeCore.h" - header "jwst-core-swift.h" + header "jwst-swift.h" export * } diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/jwst-core-swift.h b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/jwst-swift.h similarity index 100% rename from apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/jwst-core-swift.h rename to apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/jwst-swift.h diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/module.modulemap b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/module.modulemap index 49cb3768c..f09070708 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/module.modulemap +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/module.modulemap @@ -1,5 +1,5 @@ module RustXcframework { header "SwiftBridgeCore.h" - header "jwst-core-swift.h" + header "jwst-swift.h" export * } diff --git a/libs/jwst-binding/jwst-core-swift/.gitignore b/libs/jwst-binding/jwst-core-swift/.gitignore deleted file mode 100644 index 7a2cab5f8..000000000 --- a/libs/jwst-binding/jwst-core-swift/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -generated -octobase \ No newline at end of file diff --git a/libs/jwst-binding/jwst-core-swift/Cargo.toml b/libs/jwst-binding/jwst-core-swift/Cargo.toml deleted file mode 100644 index 3b83939d4..000000000 --- a/libs/jwst-binding/jwst-core-swift/Cargo.toml +++ /dev/null @@ -1,35 +0,0 @@ -[package] -name = "jwst-core-swift" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -chrono = "0.4.26" -futures = "0.3.28" -swift-bridge = "0.1.51" -tokio = "1.27.0" -nanoid = "0.4.0" -serde = { version = "1.0.183", features = ["derive"] } -serde_json = "1.0.104" - -# ======= workspace dependencies ======= -jwst-core = { workspace = true } -jwst-core-rpc = { workspace = true } -jwst-core-storage = { workspace = true, features = ["sqlite"] } -jwst-logger = { workspace = true } - -[lib] -name = "octobase" -crate-type = ["staticlib"] - -[build-dependencies] -swift-bridge-build = "0.1.51" - -[dev-dependencies] -reqwest = { version = "0.11.14", default-features = false, features = [ - "json", - "rustls-tls", -] } -regex = "1.7.1" diff --git a/libs/jwst-binding/jwst-core-swift/binding.h b/libs/jwst-binding/jwst-core-swift/binding.h deleted file mode 100644 index 707d2f418..000000000 --- a/libs/jwst-binding/jwst-core-swift/binding.h +++ /dev/null @@ -1,105 +0,0 @@ - -#ifndef JWST_FFI_H -#define JWST_FFI_H -typedef struct JWSTWorkspace {} JWSTWorkspace; -typedef struct JWSTBlock {} JWSTBlock; -typedef struct YTransaction {} YTransaction; - - -#include -#include -#include -#include - -#define BLOCK_TAG_NUM 1 - -#define BLOCK_TAG_INT 2 - -#define BLOCK_TAG_BOOL 3 - -#define BLOCK_TAG_STR 4 - -typedef struct BlockChildren { - uintptr_t len; - char **data; -} BlockChildren; - -typedef union BlockValue { - double num; - int64_t int; - bool bool; - char *str; -} BlockValue; - -typedef struct BlockContent { - int8_t tag; - union BlockValue value; -} BlockContent; - -JWSTBlock *block_new(const JWSTWorkspace *workspace, - const char *block_id, - const char *flavor, - uint64_t operator_); - -void block_destroy(JWSTBlock *block); - -uint64_t block_get_created(const JWSTBlock *block); - -uint64_t block_get_updated(const JWSTBlock *block); - -char *block_get_flavor(const JWSTBlock *block); - -struct BlockChildren *block_get_children(const JWSTBlock *block); - -void block_push_children(const JWSTBlock *block, YTransaction *trx, const JWSTBlock *child); - -void block_insert_children_at(const JWSTBlock *block, - YTransaction *trx, - const JWSTBlock *child, - uint32_t pos); - -void block_insert_children_before(const JWSTBlock *block, - YTransaction *trx, - const JWSTBlock *child, - const char *reference); - -void block_insert_children_after(const JWSTBlock *block, - YTransaction *trx, - const JWSTBlock *child, - const char *reference); - -void block_children_destroy(struct BlockChildren *children); - -struct BlockContent *block_get_content(const JWSTBlock *block, const char *key); - -void block_set_content(JWSTBlock *block, - const char *key, - YTransaction *trx, - struct BlockContent content); - -void block_content_destroy(struct BlockContent *content); - -JWSTWorkspace *workspace_new(const char *id); - -void workspace_destroy(JWSTWorkspace *workspace); - -JWSTBlock *workspace_get_block(const JWSTWorkspace *workspace, const char *block_id); - -JWSTBlock *workspace_create_block(const JWSTWorkspace *workspace, - const char *block_id, - const char *flavor); - -bool workspace_remove_block(const JWSTWorkspace *workspace, const char *block_id); - -bool workspace_exists_block(const JWSTWorkspace *workspace, const char *block_id); - -void trx_commit(YTransaction *trx); - -Subscription *workspace_observe(JWSTWorkspace *workspace, - void *env, - void (*func)(void*, const YTransaction*, const UpdateEvent*)); - -void workspace_unobserve(Subscription *subscription); - - -#endif diff --git a/libs/jwst-binding/jwst-core-swift/build.rs b/libs/jwst-binding/jwst-core-swift/build.rs deleted file mode 100644 index cc99dbdef..000000000 --- a/libs/jwst-binding/jwst-core-swift/build.rs +++ /dev/null @@ -1,13 +0,0 @@ -const XCODE_CONFIGURATION_ENV: &str = "CONFIGURATION"; - -fn main() { - let out_dir = "./generated"; - - let bridges = vec!["src/lib.rs"]; - for path in &bridges { - println!("cargo:rerun-if-changed={path}"); - } - println!("cargo:rerun-if-env-changed={XCODE_CONFIGURATION_ENV}"); - - swift_bridge_build::parse_bridges(bridges).write_all_concatenated(out_dir, env!("CARGO_PKG_NAME")); -} diff --git a/libs/jwst-binding/jwst-core-swift/generated/SwiftBridgeCore.h b/libs/jwst-binding/jwst-core-swift/generated/SwiftBridgeCore.h deleted file mode 100644 index 3fad0536d..000000000 --- a/libs/jwst-binding/jwst-core-swift/generated/SwiftBridgeCore.h +++ /dev/null @@ -1,148 +0,0 @@ -#include -#include -typedef struct RustStr { uint8_t* const start; uintptr_t len; } RustStr; -typedef struct __private__FfiSlice { void* const start; uintptr_t len; } __private__FfiSlice; -void* __swift_bridge__null_pointer(void); - -typedef struct __private__OptionU8 { uint8_t val; bool is_some; } __private__OptionU8; -typedef struct __private__OptionI8 { int8_t val; bool is_some; } __private__OptionI8; -typedef struct __private__OptionU16 { uint16_t val; bool is_some; } __private__OptionU16; -typedef struct __private__OptionI16 { int16_t val; bool is_some; } __private__OptionI16; -typedef struct __private__OptionU32 { uint32_t val; bool is_some; } __private__OptionU32; -typedef struct __private__OptionI32 { int32_t val; bool is_some; } __private__OptionI32; -typedef struct __private__OptionU64 { uint64_t val; bool is_some; } __private__OptionU64; -typedef struct __private__OptionI64 { int64_t val; bool is_some; } __private__OptionI64; -typedef struct __private__OptionUsize { uintptr_t val; bool is_some; } __private__OptionUsize; -typedef struct __private__OptionIsize { intptr_t val; bool is_some; } __private__OptionIsize; -typedef struct __private__OptionF32 { float val; bool is_some; } __private__OptionF32; -typedef struct __private__OptionF64 { double val; bool is_some; } __private__OptionDouble; -typedef struct __private__OptionBool { bool val; bool is_some; } __private__OptionBool; - -void* __swift_bridge__$Vec_u8$new(); -void __swift_bridge__$Vec_u8$_free(void* const vec); -uintptr_t __swift_bridge__$Vec_u8$len(void* const vec); -void __swift_bridge__$Vec_u8$push(void* const vec, uint8_t val); -__private__OptionU8 __swift_bridge__$Vec_u8$pop(void* const vec); -__private__OptionU8 __swift_bridge__$Vec_u8$get(void* const vec, uintptr_t index); -__private__OptionU8 __swift_bridge__$Vec_u8$get_mut(void* const vec, uintptr_t index); -uint8_t const * __swift_bridge__$Vec_u8$as_ptr(void* const vec); - -void* __swift_bridge__$Vec_u16$new(); -void __swift_bridge__$Vec_u16$_free(void* const vec); -uintptr_t __swift_bridge__$Vec_u16$len(void* const vec); -void __swift_bridge__$Vec_u16$push(void* const vec, uint16_t val); -__private__OptionU16 __swift_bridge__$Vec_u16$pop(void* const vec); -__private__OptionU16 __swift_bridge__$Vec_u16$get(void* const vec, uintptr_t index); -__private__OptionU16 __swift_bridge__$Vec_u16$get_mut(void* const vec, uintptr_t index); -uint16_t const * __swift_bridge__$Vec_u16$as_ptr(void* const vec); - -void* __swift_bridge__$Vec_u32$new(); -void __swift_bridge__$Vec_u32$_free(void* const vec); -uintptr_t __swift_bridge__$Vec_u32$len(void* const vec); -void __swift_bridge__$Vec_u32$push(void* const vec, uint32_t val); -__private__OptionU32 __swift_bridge__$Vec_u32$pop(void* const vec); -__private__OptionU32 __swift_bridge__$Vec_u32$get(void* const vec, uintptr_t index); -__private__OptionU32 __swift_bridge__$Vec_u32$get_mut(void* const vec, uintptr_t index); -uint32_t const * __swift_bridge__$Vec_u32$as_ptr(void* const vec); - -void* __swift_bridge__$Vec_u64$new(); -void __swift_bridge__$Vec_u64$_free(void* const vec); -uintptr_t __swift_bridge__$Vec_u64$len(void* const vec); -void __swift_bridge__$Vec_u64$push(void* const vec, uint64_t val); -__private__OptionU64 __swift_bridge__$Vec_u64$pop(void* const vec); -__private__OptionU64 __swift_bridge__$Vec_u64$get(void* const vec, uintptr_t index); -__private__OptionU64 __swift_bridge__$Vec_u64$get_mut(void* const vec, uintptr_t index); -uint64_t const * __swift_bridge__$Vec_u64$as_ptr(void* const vec); - -void* __swift_bridge__$Vec_usize$new(); -void __swift_bridge__$Vec_usize$_free(void* const vec); -uintptr_t __swift_bridge__$Vec_usize$len(void* const vec); -void __swift_bridge__$Vec_usize$push(void* const vec, uintptr_t val); -__private__OptionUsize __swift_bridge__$Vec_usize$pop(void* const vec); -__private__OptionUsize __swift_bridge__$Vec_usize$get(void* const vec, uintptr_t index); -__private__OptionUsize __swift_bridge__$Vec_usize$get_mut(void* const vec, uintptr_t index); -uintptr_t const * __swift_bridge__$Vec_usize$as_ptr(void* const vec); - -void* __swift_bridge__$Vec_i8$new(); -void __swift_bridge__$Vec_i8$_free(void* const vec); -uintptr_t __swift_bridge__$Vec_i8$len(void* const vec); -void __swift_bridge__$Vec_i8$push(void* const vec, int8_t val); -__private__OptionI8 __swift_bridge__$Vec_i8$pop(void* const vec); -__private__OptionI8 __swift_bridge__$Vec_i8$get(void* const vec, uintptr_t index); -__private__OptionI8 __swift_bridge__$Vec_i8$get_mut(void* const vec, uintptr_t index); -int8_t const * __swift_bridge__$Vec_i8$as_ptr(void* const vec); - -void* __swift_bridge__$Vec_i16$new(); -void __swift_bridge__$Vec_i16$_free(void* const vec); -uintptr_t __swift_bridge__$Vec_i16$len(void* const vec); -void __swift_bridge__$Vec_i16$push(void* const vec, int16_t val); -__private__OptionI16 __swift_bridge__$Vec_i16$pop(void* const vec); -__private__OptionI16 __swift_bridge__$Vec_i16$get(void* const vec, uintptr_t index); -__private__OptionI16 __swift_bridge__$Vec_i16$get_mut(void* const vec, uintptr_t index); -int16_t const * __swift_bridge__$Vec_i16$as_ptr(void* const vec); - -void* __swift_bridge__$Vec_i32$new(); -void __swift_bridge__$Vec_i32$_free(void* const vec); -uintptr_t __swift_bridge__$Vec_i32$len(void* const vec); -void __swift_bridge__$Vec_i32$push(void* const vec, int32_t val); -__private__OptionI32 __swift_bridge__$Vec_i32$pop(void* const vec); -__private__OptionI32 __swift_bridge__$Vec_i32$get(void* const vec, uintptr_t index); -__private__OptionI32 __swift_bridge__$Vec_i32$get_mut(void* const vec, uintptr_t index); -int32_t const * __swift_bridge__$Vec_i32$as_ptr(void* const vec); - -void* __swift_bridge__$Vec_i64$new(); -void __swift_bridge__$Vec_i64$_free(void* const vec); -uintptr_t __swift_bridge__$Vec_i64$len(void* const vec); -void __swift_bridge__$Vec_i64$push(void* const vec, int64_t val); -__private__OptionI64 __swift_bridge__$Vec_i64$pop(void* const vec); -__private__OptionI64 __swift_bridge__$Vec_i64$get(void* const vec, uintptr_t index); -__private__OptionI64 __swift_bridge__$Vec_i64$get_mut(void* const vec, uintptr_t index); -int64_t const * __swift_bridge__$Vec_i64$as_ptr(void* const vec); - -void* __swift_bridge__$Vec_isize$new(); -void __swift_bridge__$Vec_isize$_free(void* const vec); -uintptr_t __swift_bridge__$Vec_isize$len(void* const vec); -void __swift_bridge__$Vec_isize$push(void* const vec, intptr_t val); -__private__OptionIsize __swift_bridge__$Vec_isize$pop(void* const vec); -__private__OptionIsize __swift_bridge__$Vec_isize$get(void* const vec, uintptr_t index); -__private__OptionIsize __swift_bridge__$Vec_isize$get_mut(void* const vec, uintptr_t index); -intptr_t const * __swift_bridge__$Vec_isize$as_ptr(void* const vec); - -void* __swift_bridge__$Vec_bool$new(); -void __swift_bridge__$Vec_bool$_free(void* const vec); -uintptr_t __swift_bridge__$Vec_bool$len(void* const vec); -void __swift_bridge__$Vec_bool$push(void* const vec, bool val); -__private__OptionBool __swift_bridge__$Vec_bool$pop(void* const vec); -__private__OptionBool __swift_bridge__$Vec_bool$get(void* const vec, uintptr_t index); -__private__OptionBool __swift_bridge__$Vec_bool$get_mut(void* const vec, uintptr_t index); -bool const * __swift_bridge__$Vec_bool$as_ptr(void* const vec); - -#include -typedef struct RustString RustString; -void __swift_bridge__$RustString$_free(void* self); - -void* __swift_bridge__$Vec_RustString$new(void); -void __swift_bridge__$Vec_RustString$drop(void* vec_ptr); -void __swift_bridge__$Vec_RustString$push(void* vec_ptr, void* item_ptr); -void* __swift_bridge__$Vec_RustString$pop(void* vec_ptr); -void* __swift_bridge__$Vec_RustString$get(void* vec_ptr, uintptr_t index); -void* __swift_bridge__$Vec_RustString$get_mut(void* vec_ptr, uintptr_t index); -uintptr_t __swift_bridge__$Vec_RustString$len(void* vec_ptr); -void* __swift_bridge__$Vec_RustString$as_ptr(void* vec_ptr); - -void* __swift_bridge__$RustString$new(void); -void* __swift_bridge__$RustString$new_with_str(struct RustStr str); -uintptr_t __swift_bridge__$RustString$len(void* self); -struct RustStr __swift_bridge__$RustString$as_str(void* self); -struct RustStr __swift_bridge__$RustString$trim(void* self); -bool __swift_bridge__$RustStr$partial_eq(struct RustStr lhs, struct RustStr rhs); - - -void __swift_bridge__$call_boxed_fn_once_no_args_no_return(void* boxed_fnonce); -void __swift_bridge__$free_boxed_fn_once_no_args_no_return(void* boxed_fnonce); - - -struct __private__ResultPtrAndPtr { bool is_ok; void* ok_or_err; }; - - -struct __private__ResultVoidAndPtr { bool is_ok; void* err; }; diff --git a/libs/jwst-binding/jwst-core-swift/generated/SwiftBridgeCore.swift b/libs/jwst-binding/jwst-core-swift/generated/SwiftBridgeCore.swift deleted file mode 100644 index 2c0e973f0..000000000 --- a/libs/jwst-binding/jwst-core-swift/generated/SwiftBridgeCore.swift +++ /dev/null @@ -1,881 +0,0 @@ -import Foundation - -extension RustString { - public func toString() -> String { - let str = self.as_str() - let string = str.toString() - - return string - } -} - -extension RustStr { - func toBufferPointer() -> UnsafeBufferPointer { - let bytes = UnsafeBufferPointer(start: self.start, count: Int(self.len)) - return bytes - } - - public func toString() -> String { - let bytes = self.toBufferPointer() - return String(bytes: bytes, encoding: .utf8)! - } -} -extension RustStr: Identifiable { - public var id: String { - self.toString() - } -} -extension RustStr: Equatable { - public static func == (lhs: RustStr, rhs: RustStr) -> Bool { - return __swift_bridge__$RustStr$partial_eq(lhs, rhs); - } -} - -public protocol IntoRustString { - func intoRustString() -> RustString; -} - -public protocol ToRustStr { - func toRustStr (_ withUnsafeRustStr: (RustStr) -> T) -> T; -} - -extension String: IntoRustString { - public func intoRustString() -> RustString { - // TODO: When passing an owned Swift std String to Rust we've being wasteful here in that - // we're creating a RustString (which involves Boxing a Rust std::string::String) - // only to unbox it back into a String once it gets to the Rust side. - // - // A better approach would be to pass a RustStr to the Rust side and then have Rust - // call `.to_string()` on the RustStr. - RustString(self) - } -} - -extension RustString: IntoRustString { - public func intoRustString() -> RustString { - self - } -} - -/// If the String is Some: -/// Safely get a scoped pointer to the String and then call the callback with a RustStr -/// that uses that pointer. -/// -/// If the String is None: -/// Call the callback with a RustStr that has a null pointer. -/// The Rust side will know to treat this as `None`. -func optionalStringIntoRustString(_ string: Optional) -> RustString? { - if let val = string { - return val.intoRustString() - } else { - return nil - } -} - -extension String: ToRustStr { - /// Safely get a scoped pointer to the String and then call the callback with a RustStr - /// that uses that pointer. - public func toRustStr (_ withUnsafeRustStr: (RustStr) -> T) -> T { - return self.utf8CString.withUnsafeBufferPointer({ bufferPtr in - let rustStr = RustStr( - start: UnsafeMutableRawPointer(mutating: bufferPtr.baseAddress!).assumingMemoryBound(to: UInt8.self), - // Subtract 1 because of the null termination character at the end - len: UInt(bufferPtr.count - 1) - ) - return withUnsafeRustStr(rustStr) - }) - } -} - -extension RustStr: ToRustStr { - public func toRustStr (_ withUnsafeRustStr: (RustStr) -> T) -> T { - return withUnsafeRustStr(self) - } -} - -func optionalRustStrToRustStr(_ str: Optional, _ withUnsafeRustStr: (RustStr) -> T) -> T { - if let val = str { - return val.toRustStr(withUnsafeRustStr) - } else { - return withUnsafeRustStr(RustStr(start: nil, len: 0)) - } -} -// TODO: -// Implement iterator https://developer.apple.com/documentation/swift/iteratorprotocol - -public class RustVec { - var ptr: UnsafeMutableRawPointer - var isOwned: Bool = true - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } - - public init() { - ptr = T.vecOfSelfNew() - isOwned = true - } - - public func push (value: T) { - T.vecOfSelfPush(vecPtr: ptr, value: value) - } - - public func pop () -> Optional { - T.vecOfSelfPop(vecPtr: ptr) - } - - public func get(index: UInt) -> Optional { - T.vecOfSelfGet(vecPtr: ptr, index: index) - } - - /// Rust returns a UInt, but we cast to an Int because many Swift APIs such as - /// `ForEach(0..rustVec.len())` expect Int. - public func len() -> Int { - Int(T.vecOfSelfLen(vecPtr: ptr)) - } - - deinit { - if isOwned { - T.vecOfSelfFree(vecPtr: ptr) - } - } -} - -extension RustVec: Sequence { - public func makeIterator() -> RustVecIterator { - return RustVecIterator(self) - } -} - -public struct RustVecIterator: IteratorProtocol { - var rustVec: RustVec - var index: UInt = 0 - - init (_ rustVec: RustVec) { - self.rustVec = rustVec - } - - public mutating func next() -> T.SelfRef? { - let val = rustVec.get(index: index) - index += 1 - return val - } -} - -extension RustVec: Collection { - public typealias Index = Int - - public func index(after i: Int) -> Int { - i + 1 - } - - public subscript(position: Int) -> T.SelfRef { - self.get(index: UInt(position))! - } - - public var startIndex: Int { - 0 - } - - public var endIndex: Int { - self.len() - } -} - -extension RustVec: RandomAccessCollection { -} - -extension UnsafeBufferPointer { - func toFfiSlice () -> __private__FfiSlice { - __private__FfiSlice(start: UnsafeMutablePointer(mutating: self.baseAddress), len: UInt(self.count)) - } -} - -extension Array { - /// Get an UnsafeBufferPointer to the array's content's first byte with the array's length. - /// - /// ``` - /// // BAD! Swift will immediately free the arrays memory and so your pointer is invalid. - /// let pointer = useMyPointer([1, 2, 3].toUnsafeBufferPointer()) - /// - /// // GOOD! The array will outlive the buffer pointer. - /// let array = [1, 2, 3] - /// useMyPointer(array.toUnsafeBufferPointer()) - /// ``` - func toUnsafeBufferPointer() -> UnsafeBufferPointer { - UnsafeBufferPointer(start: UnsafePointer(self), count: self.count) - } -} - -public protocol Vectorizable { - associatedtype SelfRef - associatedtype SelfRefMut - - static func vecOfSelfNew() -> UnsafeMutableRawPointer; - - static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) - - static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) - - static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional - - static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional - - static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional - - static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt -} - -extension UInt8: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_u8$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_u8$_free(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { - __swift_bridge__$Vec_u8$push(vecPtr, value) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let val = __swift_bridge__$Vec_u8$pop(vecPtr) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_u8$get(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_u8$get_mut(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_u8$len(vecPtr) - } -} - -extension UInt16: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_u16$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_u16$_free(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { - __swift_bridge__$Vec_u16$push(vecPtr, value) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let val = __swift_bridge__$Vec_u16$pop(vecPtr) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_u16$get(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_u16$get_mut(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_u16$len(vecPtr) - } -} - -extension UInt32: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_u32$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_u32$_free(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { - __swift_bridge__$Vec_u32$push(vecPtr, value) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let val = __swift_bridge__$Vec_u32$pop(vecPtr) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_u32$get(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_u32$get_mut(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_u32$len(vecPtr) - } -} - -extension UInt64: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_u64$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_u64$_free(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { - __swift_bridge__$Vec_u64$push(vecPtr, value) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let val = __swift_bridge__$Vec_u64$pop(vecPtr) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_u64$get(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_u64$get_mut(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_u64$len(vecPtr) - } -} - -extension UInt: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_usize$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_usize$_free(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { - __swift_bridge__$Vec_usize$push(vecPtr, value) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let val = __swift_bridge__$Vec_usize$pop(vecPtr) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_usize$get(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_usize$get_mut(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_usize$len(vecPtr) - } -} - -extension Int8: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_i8$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_i8$_free(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { - __swift_bridge__$Vec_i8$push(vecPtr, value) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let val = __swift_bridge__$Vec_i8$pop(vecPtr) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_i8$get(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_i8$get_mut(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_i8$len(vecPtr) - } -} - -extension Int16: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_i16$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_i16$_free(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { - __swift_bridge__$Vec_i16$push(vecPtr, value) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let val = __swift_bridge__$Vec_i16$pop(vecPtr) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_i16$get(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_i16$get_mut(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_i16$len(vecPtr) - } -} - -extension Int32: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_i32$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_i32$_free(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { - __swift_bridge__$Vec_i32$push(vecPtr, value) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let val = __swift_bridge__$Vec_i32$pop(vecPtr) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_i32$get(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_i32$get_mut(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_i32$len(vecPtr) - } -} - -extension Int64: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_i64$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_i64$_free(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { - __swift_bridge__$Vec_i64$push(vecPtr, value) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let val = __swift_bridge__$Vec_i64$pop(vecPtr) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_i64$get(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_i64$get_mut(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_i64$len(vecPtr) - } -} - -extension Int: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_isize$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_isize$_free(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { - __swift_bridge__$Vec_isize$push(vecPtr, value) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let val = __swift_bridge__$Vec_isize$pop(vecPtr) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_isize$get(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_isize$get_mut(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_isize$len(vecPtr) - } -} - -extension Bool: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_bool$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_bool$_free(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { - __swift_bridge__$Vec_bool$push(vecPtr, value) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let val = __swift_bridge__$Vec_bool$pop(vecPtr) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_bool$get(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let val = __swift_bridge__$Vec_bool$get_mut(vecPtr, index) - if val.is_some { - return val.val - } else { - return nil - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_bool$len(vecPtr) - } -} - -protocol SwiftBridgeGenericFreer { - func rust_free(); -} - -protocol SwiftBridgeGenericCopyTypeFfiRepr {} - -public class RustString: RustStringRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$RustString$_free(ptr) - } - } -} -extension RustString { - public convenience init() { - self.init(ptr: __swift_bridge__$RustString$new()) - } - - public convenience init(_ str: GenericToRustStr) { - self.init(ptr: str.toRustStr({ strAsRustStr in - __swift_bridge__$RustString$new_with_str(strAsRustStr) - })) - } -} -public class RustStringRefMut: RustStringRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -public class RustStringRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension RustStringRef { - public func len() -> UInt { - __swift_bridge__$RustString$len(ptr) - } - - public func as_str() -> RustStr { - __swift_bridge__$RustString$as_str(ptr) - } - - public func trim() -> RustStr { - __swift_bridge__$RustString$trim(ptr) - } -} -extension RustString: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_RustString$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_RustString$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: RustString) { - __swift_bridge__$Vec_RustString$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_RustString$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (RustString(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_RustString$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return RustStringRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_RustString$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return RustStringRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_RustString$len(vecPtr) - } -} - -public class __private__RustFnOnceCallbackNoArgsNoRet { - var ptr: UnsafeMutableRawPointer - var called = false - - init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } - - deinit { - if !called { - __swift_bridge__$free_boxed_fn_once_no_args_no_return(ptr) - } - } - - func call() { - if called { - fatalError("Cannot call a Rust FnOnce function twice") - } - called = true - return __swift_bridge__$call_boxed_fn_once_no_args_no_return(ptr) - } -} - - -public enum RustResult { - case Ok(T) - case Err(E) -} - -extension RustResult { - func ok() -> T? { - switch self { - case .Ok(let ok): - return ok - case .Err(_): - return nil - } - } - - func err() -> E? { - switch self { - case .Ok(_): - return nil - case .Err(let err): - return err - } - } - - func toResult() -> Result - where E: Error { - switch self { - case .Ok(let ok): - return .success(ok) - case .Err(let err): - return .failure(err) - } - } -} diff --git a/libs/jwst-binding/jwst-core-swift/generated/jwst-core-swift/jwst-core-swift.h b/libs/jwst-binding/jwst-core-swift/generated/jwst-core-swift/jwst-core-swift.h deleted file mode 100644 index 7fa5b3b6f..000000000 --- a/libs/jwst-binding/jwst-core-swift/generated/jwst-core-swift/jwst-core-swift.h +++ /dev/null @@ -1,128 +0,0 @@ -// File automatically generated by swift-bridge. -#include -#include -typedef struct Block Block; -void __swift_bridge__$Block$_free(void* self); - -void* __swift_bridge__$Vec_Block$new(void); -void __swift_bridge__$Vec_Block$drop(void* vec_ptr); -void __swift_bridge__$Vec_Block$push(void* vec_ptr, void* item_ptr); -void* __swift_bridge__$Vec_Block$pop(void* vec_ptr); -void* __swift_bridge__$Vec_Block$get(void* vec_ptr, uintptr_t index); -void* __swift_bridge__$Vec_Block$get_mut(void* vec_ptr, uintptr_t index); -uintptr_t __swift_bridge__$Vec_Block$len(void* vec_ptr); -void* __swift_bridge__$Vec_Block$as_ptr(void* vec_ptr); - -typedef struct DynamicValueMap DynamicValueMap; -void __swift_bridge__$DynamicValueMap$_free(void* self); - -void* __swift_bridge__$Vec_DynamicValueMap$new(void); -void __swift_bridge__$Vec_DynamicValueMap$drop(void* vec_ptr); -void __swift_bridge__$Vec_DynamicValueMap$push(void* vec_ptr, void* item_ptr); -void* __swift_bridge__$Vec_DynamicValueMap$pop(void* vec_ptr); -void* __swift_bridge__$Vec_DynamicValueMap$get(void* vec_ptr, uintptr_t index); -void* __swift_bridge__$Vec_DynamicValueMap$get_mut(void* vec_ptr, uintptr_t index); -uintptr_t __swift_bridge__$Vec_DynamicValueMap$len(void* vec_ptr); -void* __swift_bridge__$Vec_DynamicValueMap$as_ptr(void* vec_ptr); - -typedef struct DynamicValue DynamicValue; -void __swift_bridge__$DynamicValue$_free(void* self); - -void* __swift_bridge__$Vec_DynamicValue$new(void); -void __swift_bridge__$Vec_DynamicValue$drop(void* vec_ptr); -void __swift_bridge__$Vec_DynamicValue$push(void* vec_ptr, void* item_ptr); -void* __swift_bridge__$Vec_DynamicValue$pop(void* vec_ptr); -void* __swift_bridge__$Vec_DynamicValue$get(void* vec_ptr, uintptr_t index); -void* __swift_bridge__$Vec_DynamicValue$get_mut(void* vec_ptr, uintptr_t index); -uintptr_t __swift_bridge__$Vec_DynamicValue$len(void* vec_ptr); -void* __swift_bridge__$Vec_DynamicValue$as_ptr(void* vec_ptr); - -typedef struct Workspace Workspace; -void __swift_bridge__$Workspace$_free(void* self); - -void* __swift_bridge__$Vec_Workspace$new(void); -void __swift_bridge__$Vec_Workspace$drop(void* vec_ptr); -void __swift_bridge__$Vec_Workspace$push(void* vec_ptr, void* item_ptr); -void* __swift_bridge__$Vec_Workspace$pop(void* vec_ptr); -void* __swift_bridge__$Vec_Workspace$get(void* vec_ptr, uintptr_t index); -void* __swift_bridge__$Vec_Workspace$get_mut(void* vec_ptr, uintptr_t index); -uintptr_t __swift_bridge__$Vec_Workspace$len(void* vec_ptr); -void* __swift_bridge__$Vec_Workspace$as_ptr(void* vec_ptr); - -typedef struct JwstWorkSpaceResult JwstWorkSpaceResult; -void __swift_bridge__$JwstWorkSpaceResult$_free(void* self); - -void* __swift_bridge__$Vec_JwstWorkSpaceResult$new(void); -void __swift_bridge__$Vec_JwstWorkSpaceResult$drop(void* vec_ptr); -void __swift_bridge__$Vec_JwstWorkSpaceResult$push(void* vec_ptr, void* item_ptr); -void* __swift_bridge__$Vec_JwstWorkSpaceResult$pop(void* vec_ptr); -void* __swift_bridge__$Vec_JwstWorkSpaceResult$get(void* vec_ptr, uintptr_t index); -void* __swift_bridge__$Vec_JwstWorkSpaceResult$get_mut(void* vec_ptr, uintptr_t index); -uintptr_t __swift_bridge__$Vec_JwstWorkSpaceResult$len(void* vec_ptr); -void* __swift_bridge__$Vec_JwstWorkSpaceResult$as_ptr(void* vec_ptr); - -typedef struct Storage Storage; -void __swift_bridge__$Storage$_free(void* self); - -void* __swift_bridge__$Vec_Storage$new(void); -void __swift_bridge__$Vec_Storage$drop(void* vec_ptr); -void __swift_bridge__$Vec_Storage$push(void* vec_ptr, void* item_ptr); -void* __swift_bridge__$Vec_Storage$pop(void* vec_ptr); -void* __swift_bridge__$Vec_Storage$get(void* vec_ptr, uintptr_t index); -void* __swift_bridge__$Vec_Storage$get_mut(void* vec_ptr, uintptr_t index); -uintptr_t __swift_bridge__$Vec_Storage$len(void* vec_ptr); -void* __swift_bridge__$Vec_Storage$as_ptr(void* vec_ptr); - -void* __swift_bridge__$Block$get(void* self, void* block_id); -void* __swift_bridge__$Block$children(void* self); -void __swift_bridge__$Block$push_children(void* self, void* block); -void __swift_bridge__$Block$insert_children_at(void* self, void* block, uint32_t pos); -void __swift_bridge__$Block$insert_children_before(void* self, void* block, struct RustStr reference); -void __swift_bridge__$Block$insert_children_after(void* self, void* block, struct RustStr reference); -void __swift_bridge__$Block$remove_children(void* self, void* block); -int32_t __swift_bridge__$Block$exists_children(void* self, struct RustStr block_id); -void* __swift_bridge__$Block$parent(void* self); -uint64_t __swift_bridge__$Block$updated(void* self); -void* __swift_bridge__$Block$id(void* self); -void* __swift_bridge__$Block$flavour(void* self); -uint64_t __swift_bridge__$Block$created(void* self); -void __swift_bridge__$Block$set_bool(void* self, void* key, bool value); -void __swift_bridge__$Block$set_string(void* self, void* key, void* value); -void __swift_bridge__$Block$set_float(void* self, void* key, double value); -void __swift_bridge__$Block$set_integer(void* self, void* key, int64_t value); -void __swift_bridge__$Block$set_null(void* self, void* key); -bool __swift_bridge__$Block$is_bool(void* self, void* key); -bool __swift_bridge__$Block$is_string(void* self, void* key); -bool __swift_bridge__$Block$is_float(void* self, void* key); -bool __swift_bridge__$Block$is_integer(void* self, void* key); -struct __private__OptionBool __swift_bridge__$Block$get_bool(void* self, void* key); -void* __swift_bridge__$Block$get_string(void* self, void* key); -struct __private__OptionF64 __swift_bridge__$Block$get_float(void* self, void* key); -struct __private__OptionI64 __swift_bridge__$Block$get_integer(void* self, void* key); -struct __private__OptionBool __swift_bridge__$DynamicValue$as_bool(void* self); -struct __private__OptionF64 __swift_bridge__$DynamicValue$as_number(void* self); -struct __private__OptionI64 __swift_bridge__$DynamicValue$as_int(void* self); -void* __swift_bridge__$DynamicValue$as_string(void* self); -void* __swift_bridge__$DynamicValue$as_map(void* self); -void* __swift_bridge__$DynamicValue$as_array(void* self); -void* __swift_bridge__$DynamicValue$as_buffer(void* self); -void* __swift_bridge__$Workspace$id(void* self); -uint64_t __swift_bridge__$Workspace$client_id(void* self); -void* __swift_bridge__$Workspace$get(void* self, void* block_id); -void* __swift_bridge__$Workspace$create(void* self, void* block_id, void* flavour); -void* __swift_bridge__$Workspace$search(void* self, void* query); -void* __swift_bridge__$Workspace$get_blocks_by_flavour(void* self, struct RustStr flavour); -void* __swift_bridge__$Workspace$get_search_index(void* self); -bool __swift_bridge__$Workspace$set_search_index(void* self, void* fields); -void* __swift_bridge__$Storage$new(void* path); -void* __swift_bridge__$Storage$new_with_log_level(void* path, void* level); -void* __swift_bridge__$Storage$error(void* self); -bool __swift_bridge__$Storage$is_offline(void* self); -bool __swift_bridge__$Storage$is_connected(void* self); -bool __swift_bridge__$Storage$is_finished(void* self); -bool __swift_bridge__$Storage$is_error(void* self); -void* __swift_bridge__$Storage$get_sync_state(void* self); -void* __swift_bridge__$Storage$connect(void* self, void* workspace_id, void* remote); -void* __swift_bridge__$Storage$get_last_synced(void* self); - - diff --git a/libs/jwst-binding/jwst-core-swift/generated/jwst-core-swift/jwst-core-swift.swift b/libs/jwst-binding/jwst-core-swift/generated/jwst-core-swift/jwst-core-swift.swift deleted file mode 100644 index 770e135dc..000000000 --- a/libs/jwst-binding/jwst-core-swift/generated/jwst-core-swift/jwst-core-swift.swift +++ /dev/null @@ -1,646 +0,0 @@ - -public class Block: BlockRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$Block$_free(ptr) - } - } -} -public class BlockRefMut: BlockRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -public class BlockRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension BlockRef { - public func get(_ block_id: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get(ptr, { let rustString = block_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return DynamicValue(ptr: val!) } else { return nil } }() - } - - public func children() -> RustVec { - RustVec(ptr: __swift_bridge__$Block$children(ptr)) - } - - public func push_children(_ block: BlockRef) { - __swift_bridge__$Block$push_children(ptr, block.ptr) - } - - public func insert_children_at(_ block: BlockRef, _ pos: UInt32) { - __swift_bridge__$Block$insert_children_at(ptr, block.ptr, pos) - } - - public func insert_children_before(_ block: BlockRef, _ reference: GenericToRustStr) { - reference.toRustStr({ referenceAsRustStr in - __swift_bridge__$Block$insert_children_before(ptr, block.ptr, referenceAsRustStr) - }) - } - - public func insert_children_after(_ block: BlockRef, _ reference: GenericToRustStr) { - reference.toRustStr({ referenceAsRustStr in - __swift_bridge__$Block$insert_children_after(ptr, block.ptr, referenceAsRustStr) - }) - } - - public func remove_children(_ block: BlockRef) { - __swift_bridge__$Block$remove_children(ptr, block.ptr) - } - - public func exists_children(_ block_id: GenericToRustStr) -> Int32 { - return block_id.toRustStr({ block_idAsRustStr in - __swift_bridge__$Block$exists_children(ptr, block_idAsRustStr) - }) - } - - public func parent() -> RustString { - RustString(ptr: __swift_bridge__$Block$parent(ptr)) - } - - public func updated() -> UInt64 { - __swift_bridge__$Block$updated(ptr) - } - - public func id() -> RustString { - RustString(ptr: __swift_bridge__$Block$id(ptr)) - } - - public func flavour() -> RustString { - RustString(ptr: __swift_bridge__$Block$flavour(ptr)) - } - - public func created() -> UInt64 { - __swift_bridge__$Block$created(ptr) - } - - public func set_bool(_ key: GenericIntoRustString, _ value: Bool) { - __swift_bridge__$Block$set_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), value) - } - - public func set_string(_ key: GenericIntoRustString, _ value: GenericIntoRustString) { - __swift_bridge__$Block$set_string(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = value.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func set_float(_ key: GenericIntoRustString, _ value: Double) { - __swift_bridge__$Block$set_float(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), value) - } - - public func set_integer(_ key: GenericIntoRustString, _ value: Int64) { - __swift_bridge__$Block$set_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), value) - } - - public func set_null(_ key: GenericIntoRustString) { - __swift_bridge__$Block$set_null(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func is_bool(_ key: GenericIntoRustString) -> Bool { - __swift_bridge__$Block$is_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func is_string(_ key: GenericIntoRustString) -> Bool { - __swift_bridge__$Block$is_string(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func is_float(_ key: GenericIntoRustString) -> Bool { - __swift_bridge__$Block$is_float(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func is_integer(_ key: GenericIntoRustString) -> Bool { - __swift_bridge__$Block$is_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func get_bool(_ key: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() - } - - public func get_string(_ key: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get_string(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return RustString(ptr: val!) } else { return nil } }() - } - - public func get_float(_ key: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get_float(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() - } - - public func get_integer(_ key: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() - } -} -extension Block: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_Block$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_Block$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Block) { - __swift_bridge__$Vec_Block$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_Block$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (Block(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Block$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return BlockRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Block$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return BlockRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_Block$len(vecPtr) - } -} - - -public class DynamicValueMap: DynamicValueMapRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$DynamicValueMap$_free(ptr) - } - } -} -public class DynamicValueMapRefMut: DynamicValueMapRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -public class DynamicValueMapRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension DynamicValueMap: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_DynamicValueMap$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_DynamicValueMap$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: DynamicValueMap) { - __swift_bridge__$Vec_DynamicValueMap$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValueMap$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (DynamicValueMap(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValueMap$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return DynamicValueMapRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValueMap$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return DynamicValueMapRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_DynamicValueMap$len(vecPtr) - } -} - - -public class DynamicValue: DynamicValueRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$DynamicValue$_free(ptr) - } - } -} -public class DynamicValueRefMut: DynamicValueRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -public class DynamicValueRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension DynamicValueRef { - public func as_bool() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_bool(ptr); if val.is_some { return val.val } else { return nil } }() - } - - public func as_number() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_number(ptr); if val.is_some { return val.val } else { return nil } }() - } - - public func as_int() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_int(ptr); if val.is_some { return val.val } else { return nil } }() - } - - public func as_string() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_string(ptr); if val != nil { return RustString(ptr: val!) } else { return nil } }() - } - - public func as_map() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_map(ptr); if val != nil { return DynamicValueMap(ptr: val!) } else { return nil } }() - } - - public func as_array() -> Optional> { - { let val = __swift_bridge__$DynamicValue$as_array(ptr); if val != nil { return RustVec(ptr: val!) } else { return nil } }() - } - - public func as_buffer() -> Optional> { - { let val = __swift_bridge__$DynamicValue$as_buffer(ptr); if val != nil { return RustVec(ptr: val!) } else { return nil } }() - } -} -extension DynamicValue: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_DynamicValue$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_DynamicValue$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: DynamicValue) { - __swift_bridge__$Vec_DynamicValue$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValue$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (DynamicValue(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValue$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return DynamicValueRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValue$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return DynamicValueRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_DynamicValue$len(vecPtr) - } -} - - -public class Workspace: WorkspaceRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$Workspace$_free(ptr) - } - } -} -public class WorkspaceRefMut: WorkspaceRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -public class WorkspaceRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension WorkspaceRef { - public func id() -> RustString { - RustString(ptr: __swift_bridge__$Workspace$id(ptr)) - } - - public func client_id() -> UInt64 { - __swift_bridge__$Workspace$client_id(ptr) - } - - public func get(_ block_id: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Workspace$get(ptr, { let rustString = block_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return Block(ptr: val!) } else { return nil } }() - } - - public func create(_ block_id: GenericIntoRustString, _ flavour: GenericIntoRustString) -> Block { - Block(ptr: __swift_bridge__$Workspace$create(ptr, { let rustString = block_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = flavour.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) - } - - public func search(_ query: GenericIntoRustString) -> RustString { - RustString(ptr: __swift_bridge__$Workspace$search(ptr, { let rustString = query.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) - } - - public func get_blocks_by_flavour(_ flavour: GenericToRustStr) -> RustVec { - return flavour.toRustStr({ flavourAsRustStr in - RustVec(ptr: __swift_bridge__$Workspace$get_blocks_by_flavour(ptr, flavourAsRustStr)) - }) - } - - public func get_search_index() -> RustVec { - RustVec(ptr: __swift_bridge__$Workspace$get_search_index(ptr)) - } - - public func set_search_index(_ fields: RustVec) -> Bool { - __swift_bridge__$Workspace$set_search_index(ptr, { let val = fields; val.isOwned = false; return val.ptr }()) - } -} -extension Workspace: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_Workspace$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_Workspace$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Workspace) { - __swift_bridge__$Vec_Workspace$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_Workspace$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (Workspace(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Workspace$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return WorkspaceRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Workspace$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return WorkspaceRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_Workspace$len(vecPtr) - } -} - - -public class JwstWorkSpaceResult: JwstWorkSpaceResultRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$JwstWorkSpaceResult$_free(ptr) - } - } -} -public class JwstWorkSpaceResultRefMut: JwstWorkSpaceResultRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -public class JwstWorkSpaceResultRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension JwstWorkSpaceResult: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_JwstWorkSpaceResult$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_JwstWorkSpaceResult$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: JwstWorkSpaceResult) { - __swift_bridge__$Vec_JwstWorkSpaceResult$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_JwstWorkSpaceResult$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (JwstWorkSpaceResult(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_JwstWorkSpaceResult$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return JwstWorkSpaceResultRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_JwstWorkSpaceResult$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return JwstWorkSpaceResultRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_JwstWorkSpaceResult$len(vecPtr) - } -} - - -public class Storage: StorageRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$Storage$_free(ptr) - } - } -} -extension Storage { - public convenience init(_ path: GenericIntoRustString) { - self.init(ptr: __swift_bridge__$Storage$new({ let rustString = path.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) - } - - public convenience init(_ path: GenericIntoRustString, _ level: GenericIntoRustString) { - self.init(ptr: __swift_bridge__$Storage$new_with_log_level({ let rustString = path.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = level.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) - } -} -public class StorageRefMut: StorageRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -extension StorageRefMut { - public func connect(_ workspace_id: GenericIntoRustString, _ remote: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Storage$connect(ptr, { let rustString = workspace_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = remote.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return Workspace(ptr: val!) } else { return nil } }() - } -} -public class StorageRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension StorageRef { - public func error() -> Optional { - { let val = __swift_bridge__$Storage$error(ptr); if val != nil { return RustString(ptr: val!) } else { return nil } }() - } - - public func is_offline() -> Bool { - __swift_bridge__$Storage$is_offline(ptr) - } - - public func is_connected() -> Bool { - __swift_bridge__$Storage$is_connected(ptr) - } - - public func is_finished() -> Bool { - __swift_bridge__$Storage$is_finished(ptr) - } - - public func is_error() -> Bool { - __swift_bridge__$Storage$is_error(ptr) - } - - public func get_sync_state() -> RustString { - RustString(ptr: __swift_bridge__$Storage$get_sync_state(ptr)) - } - - public func get_last_synced() -> RustVec { - RustVec(ptr: __swift_bridge__$Storage$get_last_synced(ptr)) - } -} -extension Storage: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_Storage$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_Storage$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Storage) { - __swift_bridge__$Vec_Storage$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_Storage$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (Storage(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Storage$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return StorageRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Storage$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return StorageRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_Storage$len(vecPtr) - } -} - - - diff --git a/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/Cargo.toml b/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/Cargo.toml deleted file mode 100644 index a17051532..000000000 --- a/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "jwst-core-swift-integrate" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -swift-bridge-build = "0.1.48" diff --git a/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs b/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs deleted file mode 100644 index 37e70c411..000000000 --- a/libs/jwst-binding/jwst-core-swift/jwst-swift-integrate/src/main.rs +++ /dev/null @@ -1,56 +0,0 @@ -use std::{collections::HashMap, path::PathBuf, process::Command}; - -use swift_bridge_build::{create_package, ApplePlatform as Platform, CreatePackageConfig}; - -fn main() { - let common_commands = [ - "-p", - "jwst-core-swift", - "--target", - "aarch64-apple-ios", - "--target", - "aarch64-apple-ios-sim", - "--target", - "aarch64-apple-darwin", - ]; - Command::new("rustup") - .args(["target", "add", "aarch64-apple-ios"]) - .status() - .expect("Failed to add target aarch64-apple-ios"); - Command::new("rustup") - .args(["target", "add", "aarch64-apple-ios-sim"]) - .status() - .expect("Failed to add target aarch64-apple-ios-sim"); - Command::new("rustup") - .args(["target", "add", "aarch64-apple-darwin"]) - .status() - .expect("Failed to add target aarch64-apple-darwin"); - Command::new("cargo") - .args(if cfg!(debug_assertions) { - ["build"].iter().chain(common_commands.iter()) - } else { - ["build", "--release"].iter().chain(common_commands.iter()) - }) - .status() - .expect("Failed to build jwst-core-swift"); - let dir = if cfg!(debug_assertions) { "debug" } else { "release" }; - create_package(CreatePackageConfig { - bridge_dir: PathBuf::from("libs/jwst-binding/jwst-core-swift/generated"), - paths: HashMap::from([ - ( - Platform::IOS, - PathBuf::from(format!("target/aarch64-apple-ios/{dir}/liboctobase.a")), - ), - ( - Platform::Simulator, - PathBuf::from(format!("target/aarch64-apple-ios-sim/{dir}/liboctobase.a",)), - ), - ( - Platform::MacOS, - PathBuf::from(format!("target/aarch64-apple-darwin/{dir}/liboctobase.a")), - ), - ]), - out_dir: PathBuf::from("apps/swift/OctoBaseSwift"), - package_name: "OctoBase".to_string(), - }); -} diff --git a/libs/jwst-binding/jwst-core-swift/src/block.rs b/libs/jwst-binding/jwst-core-swift/src/block.rs deleted file mode 100644 index d55e59d1e..000000000 --- a/libs/jwst-binding/jwst-core-swift/src/block.rs +++ /dev/null @@ -1,182 +0,0 @@ -use jwst_core::{Any, Block as JwstBlock}; - -use super::*; - -pub struct Block { - pub(crate) block: JwstBlock, -} - -impl Block { - pub fn new(block: JwstBlock) -> Self { - Self { block } - } - - pub fn get(&self, key: String) -> Option { - self.block.get(&key).map(DynamicValue::new) - } - - pub fn children(&self) -> Vec { - self.block.children() - } - - pub fn push_children(&self, target_block: &Block) { - let mut block = self.block.clone(); - let mut target_block = target_block.block.clone(); - - block.push_children(&mut target_block).expect("failed to push children"); - } - - pub fn insert_children_at(&self, target_block: &Block, pos: u32) { - let mut block = self.block.clone(); - let mut target_block = target_block.block.clone(); - - block - .insert_children_at(&mut target_block, pos as u64) - .expect("failed to insert children at position"); - } - - pub fn insert_children_before(&self, target_block: &Block, reference: &str) { - let mut block = self.block.clone(); - let mut target_block = target_block.block.clone(); - - block - .insert_children_before(&mut target_block, &reference) - .expect("failed to insert children before"); - } - - pub fn insert_children_after(&self, target_block: &Block, reference: &str) { - let mut block = self.block.clone(); - let mut target_block = target_block.block.clone(); - - block - .insert_children_after(&mut target_block, &reference) - .expect("failed to insert children after"); - } - - pub fn remove_children(&self, target_block: &Block) { - let mut block = self.block.clone(); - let mut target_block = target_block.block.clone(); - - block - .remove_children(&mut target_block) - .expect("failed to remove jwst block"); - } - - pub fn exists_children(&self, block_id: &str) -> i32 { - self.block.exists_children(block_id).map(|i| i as i32).unwrap_or(-1) - } - - pub fn parent(&self) -> String { - self.block.parent().unwrap() - } - - pub fn updated(&self) -> u64 { - self.block.updated() - } - - pub fn id(&self) -> String { - self.block.block_id() - } - - pub fn flavour(&self) -> String { - self.block.flavour() - } - - pub fn created(&self) -> u64 { - self.block.created() - } - - pub fn set_bool(&self, key: String, value: bool) { - let mut block = self.block.clone(); - block - .set(&key, value) - .expect(&format!("failed to set bool: {} {}", key, value)) - } - - pub fn set_string(&self, key: String, value: String) { - let mut block = self.block.clone(); - block - .set(&key, value.clone()) - .expect(&format!("failed to set string: {} {}", key, value)) - } - - pub fn set_float(&self, key: String, value: f64) { - let mut block = self.block.clone(); - block - .set(&key, value) - .expect(&format!("failed to set float: {} {}", key, value)); - } - - pub fn set_integer(&self, key: String, value: i64) { - let mut block = self.block.clone(); - block - .set(&key, value) - .expect(&format!("failed to set integer: {} {}", key, value)); - } - - pub fn set_null(&self, key: String) { - let mut block = self.block.clone(); - block - .set(&key, Any::Null) - .expect(&format!("failed to set null: {}", key)); - } - - pub fn is_bool(&self, key: String) -> bool { - self.block - .get(&key) - .map(|a| matches!(a, Any::True | Any::False)) - .unwrap_or(false) - } - - pub fn is_string(&self, key: String) -> bool { - self.block - .get(&key) - .map(|a| matches!(a, Any::String(_))) - .unwrap_or(false) - } - - pub fn is_float(&self, key: String) -> bool { - self.block - .get(&key) - .map(|a| matches!(a, Any::Float32(_) | Any::Float64(_))) - .unwrap_or(false) - } - - pub fn is_integer(&self, key: String) -> bool { - self.block - .get(&key) - .map(|a| matches!(a, Any::Integer(_) | Any::BigInt64(_))) - .unwrap_or(false) - } - - pub fn get_bool(&self, key: String) -> Option { - self.block.get(&key).and_then(|a| match a { - Any::True => Some(true), - Any::False => Some(false), - _ => None, - }) - } - - pub fn get_string(&self, key: String) -> Option { - self.block.get(&key).and_then(|a| match a { - Any::String(s) => Some(s), - _ => None, - }) - } - - pub fn get_float(&self, key: String) -> Option { - self.block.get(&key).and_then(|a| match a { - Any::Float32(f) => Some(f.0 as f64), - Any::Float64(f) => Some(f.0), - _ => None, - }) - } - - pub fn get_integer(&self, key: String) -> Option { - self.block.get(&key).and_then(|a| match a { - Any::Integer(i) => Some(i as i64), - Any::BigInt64(i) => Some(i), - _ => None, - }) - } -} diff --git a/libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs b/libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs deleted file mode 100644 index 5e5f7f5be..000000000 --- a/libs/jwst-binding/jwst-core-swift/src/dynamic_value.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::collections::HashMap; - -use jwst_core::Any; - -pub type DynamicValueMap = HashMap; - -pub struct DynamicValue { - any: Any, -} - -impl DynamicValue { - pub fn new(any: Any) -> Self { - Self { any } - } - - pub fn as_bool(&self) -> Option { - match self.any { - Any::True => Some(true), - Any::False => Some(false), - _ => None, - } - } - - pub fn as_number(&self) -> Option { - match self.any { - Any::Float32(value) => Some(value.0 as f64), - Any::Float64(value) => Some(value.0), - _ => None, - } - } - - pub fn as_int(&self) -> Option { - match self.any { - Any::Integer(value) => Some(value as i64), - Any::BigInt64(value) => Some(value), - _ => None, - } - } - - pub fn as_string(&self) -> Option { - match &self.any { - Any::String(value) => Some(value.to_string()), - _ => None, - } - } - - pub fn as_buffer(&self) -> Option> { - match &self.any { - Any::Binary(value) => Some(value.clone()), - _ => None, - } - } - - pub fn as_array(&self) -> Option> { - match &self.any { - Any::Array(value) => Some(value.iter().map(|a| DynamicValue::new(a.clone())).collect()), - _ => None, - } - } - - pub fn as_map(&self) -> Option> { - match &self.any { - Any::Object(value) => Some( - value - .iter() - .map(|(key, value)| (key.clone(), DynamicValue::new(value.clone()))) - .collect(), - ), - _ => None, - } - } -} diff --git a/libs/jwst-binding/jwst-core-swift/src/lib.rs b/libs/jwst-binding/jwst-core-swift/src/lib.rs deleted file mode 100644 index 8bd239c8c..000000000 --- a/libs/jwst-binding/jwst-core-swift/src/lib.rs +++ /dev/null @@ -1,142 +0,0 @@ -mod block; -mod dynamic_value; -mod storage; -mod workspace; - -pub use block::Block; -pub use dynamic_value::{DynamicValue, DynamicValueMap}; -use jwst_core::{error, warn, JwstError}; -use jwst_logger::init_logger_with; -pub use storage::Storage; -pub use workspace::Workspace; - -type JwstWorkSpaceResult = Result; - -#[swift_bridge::bridge] -mod ffi { - extern "Rust" { - type Block; - - fn get(self: &Block, block_id: String) -> Option; - - pub fn children(self: &Block) -> Vec; - - pub fn push_children(self: &Block, block: &Block); - - pub fn insert_children_at(&self, block: &Block, pos: u32); - - pub fn insert_children_before(self: &Block, block: &Block, reference: &str); - - pub fn insert_children_after(self: &Block, block: &Block, reference: &str); - - pub fn remove_children(self: &Block, block: &Block); - - pub fn exists_children(self: &Block, block_id: &str) -> i32; - - pub fn parent(self: &Block) -> String; - - pub fn updated(self: &Block) -> u64; - - pub fn id(self: &Block) -> String; - - pub fn flavour(self: &Block) -> String; - - pub fn created(self: &Block) -> u64; - - pub fn set_bool(self: &Block, key: String, value: bool); - - pub fn set_string(self: &Block, key: String, value: String); - - pub fn set_float(self: &Block, key: String, value: f64); - - pub fn set_integer(self: &Block, key: String, value: i64); - - pub fn set_null(self: &Block, key: String); - - pub fn is_bool(self: &Block, key: String) -> bool; - - pub fn is_string(self: &Block, key: String) -> bool; - - pub fn is_float(&self, key: String) -> bool; - - pub fn is_integer(&self, key: String) -> bool; - - pub fn get_bool(&self, key: String) -> Option; - - pub fn get_string(&self, key: String) -> Option; - - pub fn get_float(&self, key: String) -> Option; - - pub fn get_integer(&self, key: String) -> Option; - } - - extern "Rust" { - type DynamicValue; - type DynamicValueMap; - - fn as_bool(self: &DynamicValue) -> Option; - - fn as_number(self: &DynamicValue) -> Option; - - fn as_int(self: &DynamicValue) -> Option; - - fn as_string(self: &DynamicValue) -> Option; - - fn as_map(self: &DynamicValue) -> Option; - - fn as_array(self: &DynamicValue) -> Option>; - - fn as_buffer(self: &DynamicValue) -> Option>; - } - - extern "Rust" { - type Workspace; - - fn id(self: &Workspace) -> String; - - fn client_id(self: &Workspace) -> u64; - - fn get(self: &Workspace, block_id: String) -> Option; - - fn create(self: &Workspace, block_id: String, flavour: String) -> Block; - - fn search(self: &Workspace, query: String) -> String; - - fn get_blocks_by_flavour(self: &Workspace, flavour: &str) -> Vec; - - fn get_search_index(self: &Workspace) -> Vec; - - fn set_search_index(self: &Workspace, fields: Vec) -> bool; - } - - extern "Rust" { - type JwstWorkSpaceResult; - } - - extern "Rust" { - type Storage; - - #[swift_bridge(init)] - fn new(path: String) -> Storage; - - #[swift_bridge(init)] - fn new_with_log_level(path: String, level: String) -> Storage; - - fn error(self: &Storage) -> Option; - - fn is_offline(self: &Storage) -> bool; - - fn is_connected(self: &Storage) -> bool; - - fn is_finished(self: &Storage) -> bool; - - fn is_error(self: &Storage) -> bool; - - fn get_sync_state(self: &Storage) -> String; - - fn connect(self: &mut Storage, workspace_id: String, remote: String) -> Option; - - fn get_last_synced(self: &Storage) -> Vec; - - } -} diff --git a/libs/jwst-binding/jwst-core-swift/src/storage.rs b/libs/jwst-binding/jwst-core-swift/src/storage.rs deleted file mode 100644 index d0b725bcd..000000000 --- a/libs/jwst-binding/jwst-core-swift/src/storage.rs +++ /dev/null @@ -1,207 +0,0 @@ -use std::sync::{Arc, RwLock}; - -use futures::TryFutureExt; -use jwst_core_rpc::{start_websocket_client_sync, BroadcastChannels, CachedLastSynced, RpcContextImpl, SyncState}; -use jwst_core_storage::{BlobStorageType, JwstStorage as AutoStorage, JwstStorageResult}; -use nanoid::nanoid; -use tokio::{ - runtime::{Builder, Runtime}, - sync::mpsc::channel, -}; - -use super::*; - -#[derive(Clone)] -pub struct Storage { - storage: Arc, - channel: Arc, - error: Option, - sync_state: Arc>, - last_sync: CachedLastSynced, -} - -impl Storage { - pub fn new(path: String) -> Self { - Self::new_with_log_level(path, "info".to_string()) - } - - pub fn new_with_log_level(path: String, level: String) -> Self { - init_logger_with( - &format!( - "{level},mio=off,hyper=off,rustls=off,tantivy=off,sqlx::query=off,tokio_tungstenite=off,\ - tungstenite=off" - ), - false, - ); - - let rt = Runtime::new().unwrap(); - - let storage = rt - .block_on( - AutoStorage::new_with_migration(&format!("sqlite:{path}?mode=rwc"), BlobStorageType::DB).or_else(|e| { - warn!("Failed to open storage, falling back to memory storage: {}", e); - AutoStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) - }), - ) - .unwrap(); - - Self { - storage: Arc::new(storage), - channel: Arc::default(), - error: None, - sync_state: Arc::new(RwLock::new(SyncState::Offline)), - last_sync: CachedLastSynced::default(), - } - } - - pub fn error(&self) -> Option { - self.error.clone() - } - - pub fn is_offline(&self) -> bool { - let sync_state = self.sync_state.read().unwrap(); - matches!(*sync_state, SyncState::Offline) - } - - pub fn is_connected(&self) -> bool { - let sync_state = self.sync_state.read().unwrap(); - matches!(*sync_state, SyncState::Connected) - } - - pub fn is_finished(&self) -> bool { - let sync_state = self.sync_state.read().unwrap(); - matches!(*sync_state, SyncState::Finished) - } - - pub fn is_error(&self) -> bool { - let sync_state = self.sync_state.read().unwrap(); - matches!(*sync_state, SyncState::Error(_)) - } - - pub fn get_sync_state(&self) -> String { - let sync_state = self.sync_state.read().unwrap(); - match sync_state.clone() { - SyncState::Offline => "offline".to_string(), - SyncState::Connected => "connected".to_string(), - SyncState::Finished => "finished".to_string(), - SyncState::Error(e) => format!("Error: {e}"), - } - } - - pub fn connect(&mut self, workspace_id: String, remote: String) -> Option { - match self.sync(workspace_id, remote) { - Ok(workspace) => Some(workspace), - Err(e) => { - error!("Failed to connect to workspace: {:?}", e); - self.error = Some(e.to_string()); - None - } - } - } - - fn sync(&mut self, workspace_id: String, remote: String) -> JwstStorageResult { - let rt = Arc::new( - Builder::new_multi_thread() - .worker_threads(1) - .enable_all() - .thread_name("jwst-core-swift") - .build() - .map_err(JwstError::Io)?, - ); - let is_offline = remote.is_empty(); - - let workspace = rt.block_on(async { self.get_workspace(&workspace_id).await }); - - match workspace { - Ok(mut workspace) => { - // disable remote temporarily - if is_offline || true { - let identifier = nanoid!(); - let (last_synced_tx, last_synced_rx) = channel::(128); - self.last_sync.add_receiver(rt.clone(), last_synced_rx); - - rt.block_on(async { - self.join_broadcast(&mut workspace, identifier.clone(), last_synced_tx) - .await; - }); - } else { - self.last_sync = start_websocket_client_sync( - rt.clone(), - Arc::new(self.clone()), - self.sync_state.clone(), - remote, - workspace_id.clone(), - ); - } - - Ok(Workspace { workspace, rt }) - } - Err(e) => Err(e), - } - } - - pub fn get_last_synced(&self) -> Vec { - self.last_sync.pop() - } -} - -impl RpcContextImpl<'_> for Storage { - fn get_storage(&self) -> &AutoStorage { - &self.storage - } - - fn get_channel(&self) -> &BroadcastChannels { - &self.channel - } -} - -#[cfg(test)] -mod tests { - use tokio::runtime::Runtime; - - use crate::{Storage, Workspace}; - - #[test] - #[ignore = "need manually start collaboration server"] - fn collaboration_test() { - let (workspace_id, block_id) = ("1", "1"); - let workspace = get_workspace(workspace_id, None); - let block = workspace.create(block_id.to_string(), "list".to_string()); - block.set_bool("bool_prop".to_string(), true); - block.set_float("float_prop".to_string(), 1.0); - block.push_children(&workspace.create("2".to_string(), "list".to_string())); - - let resp = get_block_from_server(workspace_id.to_string(), block.id().to_string()); - assert!(!resp.is_empty()); - let prop_extractor = r#"("prop:bool_prop":true)|("prop:float_prop":1\.0)|("sys:children":\["2"\])"#; - let re = regex::Regex::new(prop_extractor).unwrap(); - assert_eq!(re.find_iter(resp.as_str()).count(), 3); - } - - fn get_workspace(workspace_id: &str, offline: Option<()>) -> Workspace { - let mut storage = Storage::new("memory".to_string()); - storage - .connect( - workspace_id.to_string(), - if offline.is_some() { - "".to_string() - } else { - format!("ws://localhost:3000/collaboration/{workspace_id}").to_string() - }, - ) - .unwrap() - } - - fn get_block_from_server(workspace_id: String, block_id: String) -> String { - let rt = Runtime::new().unwrap(); - rt.block_on(async { - let client = reqwest::Client::builder().no_proxy().build().unwrap(); - let resp = client - .get(format!("http://localhost:3000/api/block/{}/{}", workspace_id, block_id)) - .send() - .await - .unwrap(); - resp.text().await.unwrap() - }) - } -} diff --git a/libs/jwst-binding/jwst-core-swift/src/workspace.rs b/libs/jwst-binding/jwst-core-swift/src/workspace.rs deleted file mode 100644 index 147450401..000000000 --- a/libs/jwst-binding/jwst-core-swift/src/workspace.rs +++ /dev/null @@ -1,75 +0,0 @@ -use std::sync::Arc; - -use jwst_core::Workspace as JwstWorkspace; -use tokio::runtime::Runtime; - -use super::*; - -pub struct Workspace { - pub(crate) workspace: JwstWorkspace, - pub(crate) rt: Arc, -} - -impl Workspace { - pub fn new(id: String) -> Self { - Self { - workspace: JwstWorkspace::new(&id).unwrap(), - rt: Arc::new(Runtime::new().unwrap()), - } - } - - pub fn id(&self) -> String { - self.workspace.id() - } - - pub fn client_id(&self) -> u64 { - self.workspace.client_id() - } - - pub fn get(&self, block_id: String) -> Option { - let mut workspace = self.workspace.clone(); - - workspace - .get_blocks() - .ok() - .and_then(|s| s.get(&block_id)) - .map(Block::new) - } - - pub fn create(&self, block_id: String, flavour: String) -> Block { - let mut workspace = self.workspace.clone(); - - Block::new( - workspace - .get_blocks() - .and_then(|mut b| b.create(block_id, flavour)) - .expect("failed to create jwst block"), - ) - } - - pub fn search(self: &Workspace, query: String) -> String { - // self.workspace.search_result(query) - "".to_owned() - } - - pub fn get_blocks_by_flavour(&self, flavour: &str) -> Vec { - let mut workspace = self.workspace.clone(); - - workspace - .get_blocks() - .map(|s| s.get_blocks_by_flavour(flavour).into_iter().map(Block::new).collect()) - .unwrap_or_default() - } - - pub fn get_search_index(self: &Workspace) -> Vec { - // self.workspace.metadata().search_index - vec![] - } - - pub fn set_search_index(self: &Workspace, fields: Vec) -> bool { - // self.workspace - // .set_search_index(fields) - // .expect("failed to set search index") - false - } -} diff --git a/libs/jwst-binding/jwst-swift/Cargo.toml b/libs/jwst-binding/jwst-swift/Cargo.toml index 1b670cb42..2b449eb36 100644 --- a/libs/jwst-binding/jwst-swift/Cargo.toml +++ b/libs/jwst-binding/jwst-swift/Cargo.toml @@ -8,19 +8,16 @@ edition = "2021" [dependencies] chrono = "0.4.26" futures = "0.3.28" -lib0 = "0.16.5" swift-bridge = "0.1.51" tokio = "1.27.0" nanoid = "0.4.0" serde = { version = "1.0.183", features = ["derive"] } serde_json = "1.0.104" -yrs = "0.16.5" # ======= workspace dependencies ======= -jwst = { workspace = true } jwst-core = { workspace = true } -jwst-rpc = { workspace = true } -jwst-storage = { workspace = true, features = ["sqlite"] } +jwst-core-rpc = { workspace = true } +jwst-core-storage = { workspace = true, features = ["sqlite"] } jwst-logger = { workspace = true } [lib] diff --git a/libs/jwst-binding/jwst-swift/src/block.rs b/libs/jwst-binding/jwst-swift/src/block.rs index c1f87e093..d55e59d1e 100644 --- a/libs/jwst-binding/jwst-swift/src/block.rs +++ b/libs/jwst-binding/jwst-swift/src/block.rs @@ -1,385 +1,77 @@ -use std::sync::Arc; - -use jwst::{Block as JwstBlock, Workspace}; -use jwst_rpc::workspace_compare; -use lib0::any::Any; -use tokio::{runtime::Runtime, sync::mpsc::Sender}; +use jwst_core::{Any, Block as JwstBlock}; use super::*; pub struct Block { - pub workspace: Workspace, - pub block: JwstBlock, - runtime: Arc, - - // just for data verify - pub(crate) jwst_workspace: Option, - pub(crate) jwst_block: Option, - pub(crate) sender: Sender, + pub(crate) block: JwstBlock, } impl Block { - pub fn new( - workspace: Workspace, - block: JwstBlock, - runtime: Arc, - jwst_workspace: Option, - sender: Sender, - ) -> Self { - Self { - workspace, - block: block.clone(), - runtime, - - // just for data verify - jwst_workspace: jwst_workspace.clone(), - jwst_block: jwst_workspace - .and_then(|mut w| w.get_blocks().ok()) - .and_then(|s| s.get(block.block_id())), - sender, - } + pub fn new(block: JwstBlock) -> Self { + Self { block } } pub fn get(&self, key: String) -> Option { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { workspace.with_trx(|trx| block.get(&trx.trx, &key).map(DynamicValue::new)) }) - .await - .unwrap() - }) + self.block.get(&key).map(DynamicValue::new) } pub fn children(&self) -> Vec { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { workspace.with_trx(|trx| block.children(&trx.trx)) }) - .await - .unwrap() - }) + self.block.children() } - pub fn push_children(&self, block: &Block) { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let curr_block = self.block.clone(); - let target_block = block.block.clone(); - - // just for data verify - let mut jwst_workspace = self.jwst_workspace.clone(); - let jwst_block = self.jwst_block.clone(); - let sender = self.sender.clone(); - let target_jwst_block = block.jwst_block.clone(); - - self.runtime - .spawn(async move { - // just for data verify - if let Some(mut jwst_block) = jwst_block { - if let Some(mut block) = target_jwst_block { - jwst_block.push_children(&mut block).expect("failed to push children"); - } else if let Err(e) = sender - .send(Log::new( - workspace.id(), - format!("target jwst block not exists: {}", target_block.block_id()), - )) - .await - { - warn!("failed to send log: {}", e); - } - } - - if let Some(content) = workspace - .with_trx(|mut trx| { - curr_block.push_children(&mut trx.trx, &target_block).map(|_| { - if let Some(jwst_workspace) = jwst_workspace.as_mut() { - let content = - workspace_compare(trx.trx, jwst_workspace, Some(&curr_block.block_id())); - Some(content) - } else { - None - } - }) - }) - .expect("failed to push children") - { - if let Err(e) = sender.send(Log::new(workspace.id(), content)).await { - warn!("failed to send log: {}", e); - } - } - }) - .await - .unwrap() - }) + pub fn push_children(&self, target_block: &Block) { + let mut block = self.block.clone(); + let mut target_block = target_block.block.clone(); + + block.push_children(&mut target_block).expect("failed to push children"); } - pub fn insert_children_at(&self, block: &Block, pos: u32) { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let curr_block = self.block.clone(); - let target_block = block.block.clone(); - - // just for data verify - let mut jwst_workspace = self.jwst_workspace.clone(); - let jwst_block = self.jwst_block.clone(); - let sender = self.sender.clone(); - let target_jwst_block = block.jwst_block.clone(); - - self.runtime - .spawn(async move { - // just for data verify - if let Some(mut jwst_block) = jwst_block { - if let Some(mut block) = target_jwst_block { - jwst_block - .insert_children_at(&mut block, pos as u64) - .expect("failed to insert children at position"); - } else if let Err(e) = sender - .send(Log::new( - workspace.id(), - format!("target jwst block not exists: {}", target_block.block_id()), - )) - .await - { - warn!("failed to send log: {}", e); - } - } - - if let Some(content) = workspace - .with_trx(|mut trx| { - curr_block - .insert_children_at(&mut trx.trx, &target_block, pos) - .map(|_| { - if let Some(jwst_workspace) = jwst_workspace.as_mut() { - let content = - workspace_compare(trx.trx, jwst_workspace, Some(&curr_block.block_id())); - Some(content) - } else { - None - } - }) - }) - .expect("failed to insert children at position") - { - if let Err(e) = sender.send(Log::new(workspace.id(), content)).await { - warn!("failed to send log: {}", e); - } - } - }) - .await - .unwrap() - }) + pub fn insert_children_at(&self, target_block: &Block, pos: u32) { + let mut block = self.block.clone(); + let mut target_block = target_block.block.clone(); + + block + .insert_children_at(&mut target_block, pos as u64) + .expect("failed to insert children at position"); } - pub fn insert_children_before(&self, block: &Block, reference: &str) { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let curr_block = self.block.clone(); - let target_block = block.block.clone(); - let reference = reference.to_string(); - - // just for data verify - let mut jwst_workspace = self.jwst_workspace.clone(); - let jwst_block = self.jwst_block.clone(); - let sender = self.sender.clone(); - let target_jwst_block = block.jwst_block.clone(); - - self.runtime - .spawn(async move { - // just for data verify - if let Some(mut jwst_block) = jwst_block { - if let Some(mut block) = target_jwst_block { - jwst_block - .insert_children_before(&mut block, &reference) - .expect("failed to insert children before"); - } else if let Err(e) = sender - .send(Log::new( - workspace.id(), - format!("target jwst block not exists: {}", target_block.block_id()), - )) - .await - { - warn!("failed to send log: {}", e); - } - } - - if let Some(content) = workspace - .with_trx(|mut trx| { - curr_block - .insert_children_before(&mut trx.trx, &target_block, &reference) - .map(|_| { - if let Some(jwst_workspace) = jwst_workspace.as_mut() { - let content = - workspace_compare(trx.trx, jwst_workspace, Some(&curr_block.block_id())); - Some(content) - } else { - None - } - }) - }) - .expect("failed to insert children before") - { - if let Err(e) = sender.send(Log::new(workspace.id(), content)).await { - warn!("failed to send log: {}", e); - } - } - }) - .await - .unwrap() - }) + pub fn insert_children_before(&self, target_block: &Block, reference: &str) { + let mut block = self.block.clone(); + let mut target_block = target_block.block.clone(); + + block + .insert_children_before(&mut target_block, &reference) + .expect("failed to insert children before"); } - pub fn insert_children_after(&self, block: &Block, reference: &str) { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let curr_block = self.block.clone(); - let target_block = block.block.clone(); - let reference = reference.to_string(); - - // just for data verify - let mut jwst_workspace = self.jwst_workspace.clone(); - let jwst_block = self.jwst_block.clone(); - let sender = self.sender.clone(); - let target_jwst_block = block.jwst_block.clone(); - - self.runtime - .spawn(async move { - // just for data verify - if let Some(mut jwst_block) = jwst_block { - if let Some(mut block) = target_jwst_block { - jwst_block - .insert_children_after(&mut block, &reference) - .expect("failed to insert children after"); - } else if let Err(e) = sender - .send(Log::new( - workspace.id(), - format!("target jwst block not exists: {}", target_block.block_id()), - )) - .await - { - warn!("failed to send log: {}", e); - } - } - - if let Some(content) = workspace - .with_trx(|mut trx| { - curr_block - .insert_children_after(&mut trx.trx, &target_block, &reference) - .map(|_| { - if let Some(jwst_workspace) = jwst_workspace.as_mut() { - let content = - workspace_compare(trx.trx, jwst_workspace, Some(&curr_block.block_id())); - Some(content) - } else { - None - } - }) - }) - .expect("failed to insert children after") - { - if let Err(e) = sender.send(Log::new(workspace.id(), content)).await { - warn!("failed to send log: {}", e); - } - } - }) - .await - .unwrap() - }) + pub fn insert_children_after(&self, target_block: &Block, reference: &str) { + let mut block = self.block.clone(); + let mut target_block = target_block.block.clone(); + + block + .insert_children_after(&mut target_block, &reference) + .expect("failed to insert children after"); } - pub fn remove_children(&self, block: &Block) { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let curr_block = self.block.clone(); - let target_block = block.block.clone(); - - // just for data verify - let mut jwst_workspace = self.jwst_workspace.clone(); - let jwst_block = self.jwst_block.clone(); - let sender = self.sender.clone(); - let target_jwst_block = block.jwst_block.clone(); - - self.runtime - .spawn(async move { - // just for data verify - if let Some(mut jwst_block) = jwst_block { - if let Some(mut block) = target_jwst_block { - jwst_block - .remove_children(&mut block) - .expect("failed to remove jwst block"); - } else if let Err(e) = sender - .send(Log::new( - workspace.id(), - format!("target jwst block not exists: {}", target_block.block_id()), - )) - .await - { - warn!("failed to send log: {}", e); - } - } - - if let Some(content) = workspace - .with_trx(|mut trx| { - curr_block.remove_children(&mut trx.trx, &target_block).map(|_| { - if let Some(jwst_workspace) = jwst_workspace.as_mut() { - let content = - workspace_compare(trx.trx, jwst_workspace, Some(&curr_block.block_id())); - Some(content) - } else { - None - } - }) - }) - .expect("failed to remove children") - { - if let Err(e) = sender.send(Log::new(workspace.id(), content)).await { - warn!("failed to send log: {}", e); - } - } - }) - .await - .unwrap() - }) + pub fn remove_children(&self, target_block: &Block) { + let mut block = self.block.clone(); + let mut target_block = target_block.block.clone(); + + block + .remove_children(&mut target_block) + .expect("failed to remove jwst block"); } pub fn exists_children(&self, block_id: &str) -> i32 { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let curr_block = self.block.clone(); - let block_id = block_id.to_string(); - self.runtime - .spawn(async move { - workspace - .with_trx(|trx| curr_block.exists_children(&trx.trx, &block_id)) - .map(|i| i as i32) - .unwrap_or(-1) - }) - .await - .unwrap() - }) + self.block.exists_children(block_id).map(|i| i as i32).unwrap_or(-1) } pub fn parent(&self) -> String { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let curr_block = self.block.clone(); - self.runtime - .spawn(async move { workspace.with_trx(|trx| curr_block.parent(&trx.trx).unwrap()) }) - .await - .unwrap() - }) + self.block.parent().unwrap() } pub fn updated(&self) -> u64 { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { workspace.with_trx(|trx| block.updated(&trx.trx)) }) - .await - .unwrap() - }) + self.block.updated() } pub fn id(&self) -> String { @@ -387,370 +79,104 @@ impl Block { } pub fn flavour(&self) -> String { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { workspace.with_trx(|trx| block.flavour(&trx.trx)) }) - .await - .unwrap() - }) + self.block.flavour() } pub fn created(&self) -> u64 { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { workspace.with_trx(|trx| block.created(&trx.trx)) }) - .await - .unwrap() - }) + self.block.created() } pub fn set_bool(&self, key: String, value: bool) { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - - // just for data verify - let mut jwst_workspace = self.jwst_workspace.clone(); - let jwst_block = self.jwst_block.clone(); - let sender = self.sender.clone(); - - self.runtime - .spawn(async move { - // just for data verify - if let Some(mut jwst_block) = jwst_block { - jwst_block.set(&key, value).expect("failed to set bool"); - } - - if let Some(content) = workspace - .with_trx(|mut trx| { - block.set(&mut trx.trx, &key, value).map(|_| { - if let Some(jwst_workspace) = jwst_workspace.as_mut() { - let content = workspace_compare(trx.trx, jwst_workspace, Some(&block.block_id())); - Some(content) - } else { - None - } - }) - }) - .expect("failed to set bool") - { - if let Err(e) = sender.send(Log::new(workspace.id(), content)).await { - warn!("failed to send log: {}", e); - } - } - }) - .await - .unwrap() - }) + let mut block = self.block.clone(); + block + .set(&key, value) + .expect(&format!("failed to set bool: {} {}", key, value)) } pub fn set_string(&self, key: String, value: String) { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - - // just for data verify - let mut jwst_workspace = self.jwst_workspace.clone(); - let jwst_block = self.jwst_block.clone(); - let sender = self.sender.clone(); - - self.runtime - .spawn(async move { - // just for data verify - if let Some(mut jwst_block) = jwst_block { - jwst_block.set(&key, value.clone()).expect("failed to set string"); - } - - if let Some(content) = workspace - .with_trx(|mut trx| { - block.set(&mut trx.trx, &key, value).map(|_| { - if let Some(jwst_workspace) = jwst_workspace.as_mut() { - let content = workspace_compare(trx.trx, jwst_workspace, Some(&block.block_id())); - Some(content) - } else { - None - } - }) - }) - .expect("failed to set string") - { - if let Err(e) = sender.send(Log::new(workspace.id(), content)).await { - warn!("failed to send log: {}", e); - } - } - }) - .await - .unwrap() - }) + let mut block = self.block.clone(); + block + .set(&key, value.clone()) + .expect(&format!("failed to set string: {} {}", key, value)) } pub fn set_float(&self, key: String, value: f64) { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - - // just for data verify - let mut jwst_workspace = self.jwst_workspace.clone(); - let jwst_block = self.jwst_block.clone(); - let sender = self.sender.clone(); - - self.runtime - .spawn(async move { - // just for data verify - if let Some(mut jwst_block) = jwst_block { - jwst_block.set(&key, value).expect("failed to set float"); - } - - if let Some(content) = workspace - .with_trx(|mut trx| { - block.set(&mut trx.trx, &key, value).map(|_| { - if let Some(jwst_workspace) = jwst_workspace.as_mut() { - let content = workspace_compare(trx.trx, jwst_workspace, Some(&block.block_id())); - Some(content) - } else { - None - } - }) - }) - .expect("failed to set float") - { - if let Err(e) = sender.send(Log::new(workspace.id(), content)).await { - warn!("failed to send log: {}", e); - } - } - }) - .await - .unwrap() - }) + let mut block = self.block.clone(); + block + .set(&key, value) + .expect(&format!("failed to set float: {} {}", key, value)); } pub fn set_integer(&self, key: String, value: i64) { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - - // just for data verify - let mut jwst_workspace = self.jwst_workspace.clone(); - let jwst_block = self.jwst_block.clone(); - let sender = self.sender.clone(); - - self.runtime.block_on(async { - self.runtime - .spawn(async move { - // just for data verify - if let Some(mut jwst_block) = jwst_block { - jwst_block - .set(&key, jwst_core::Any::BigInt64(value)) - .expect("failed to set integer"); - } - - if let Some(content) = workspace - .with_trx(|mut trx| { - block.set(&mut trx.trx, &key, value).map(|_| { - if let Some(jwst_workspace) = jwst_workspace.as_mut() { - let content = workspace_compare(trx.trx, jwst_workspace, Some(&block.block_id())); - Some(content) - } else { - None - } - }) - }) - .expect("failed to set integer") - { - if let Err(e) = sender.send(Log::new(workspace.id(), content)).await { - warn!("failed to send log: {}", e); - } - } - }) - .await - .unwrap() - }) + let mut block = self.block.clone(); + block + .set(&key, value) + .expect(&format!("failed to set integer: {} {}", key, value)); } pub fn set_null(&self, key: String) { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - - // just for data verify - let mut jwst_workspace = self.jwst_workspace.clone(); - let jwst_block = self.jwst_block.clone(); - let sender = self.sender.clone(); - - self.runtime - .spawn(async move { - // just for data verify - if let Some(mut jwst_block) = jwst_block { - jwst_block.set(&key, jwst_core::Any::Null).expect("failed to set null"); - } - - if let Some(content) = workspace - .with_trx(|mut trx| { - block.set(&mut trx.trx, &key, Any::Null).map(|_| { - if let Some(jwst_workspace) = jwst_workspace.as_mut() { - let content = workspace_compare(trx.trx, jwst_workspace, Some(&block.block_id())); - Some(content) - } else { - None - } - }) - }) - .expect("failed to set null") - { - if let Err(e) = sender.send(Log::new(workspace.id(), content)).await { - warn!("failed to send log: {}", e); - } - } - }) - .await - .unwrap() - }) + let mut block = self.block.clone(); + block + .set(&key, Any::Null) + .expect(&format!("failed to set null: {}", key)); } pub fn is_bool(&self, key: String) -> bool { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { - workspace.with_trx(|trx| { - block - .get(&trx.trx, &key) - .map(|a| matches!(a, Any::Bool(_))) - .unwrap_or(false) - }) - }) - .await - .unwrap() - }) + self.block + .get(&key) + .map(|a| matches!(a, Any::True | Any::False)) + .unwrap_or(false) } pub fn is_string(&self, key: String) -> bool { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { - workspace.with_trx(|trx| { - block - .get(&trx.trx, &key) - .map(|a| matches!(a, Any::String(_))) - .unwrap_or(false) - }) - }) - .await - .unwrap() - }) + self.block + .get(&key) + .map(|a| matches!(a, Any::String(_))) + .unwrap_or(false) } pub fn is_float(&self, key: String) -> bool { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { - workspace.with_trx(|trx| { - block - .get(&trx.trx, &key) - .map(|a| matches!(a, Any::Number(_))) - .unwrap_or(false) - }) - }) - .await - .unwrap() - }) + self.block + .get(&key) + .map(|a| matches!(a, Any::Float32(_) | Any::Float64(_))) + .unwrap_or(false) } pub fn is_integer(&self, key: String) -> bool { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { - workspace.with_trx(|trx| { - block - .get(&trx.trx, &key) - .map(|a| matches!(a, Any::BigInt(_))) - .unwrap_or(false) - }) - }) - .await - .unwrap() - }) + self.block + .get(&key) + .map(|a| matches!(a, Any::Integer(_) | Any::BigInt64(_))) + .unwrap_or(false) } - pub fn get_bool(&self, key: String) -> Option { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { - workspace.with_trx(|trx| { - block.get(&trx.trx, &key).and_then(|a| match a { - Any::Bool(i) => Some(i.into()), - _ => None, - }) - }) - }) - .await - .unwrap() + pub fn get_bool(&self, key: String) -> Option { + self.block.get(&key).and_then(|a| match a { + Any::True => Some(true), + Any::False => Some(false), + _ => None, }) } pub fn get_string(&self, key: String) -> Option { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { - workspace.with_trx(|trx| { - block.get(&trx.trx, &key).and_then(|a| match a { - Any::String(i) => Some(i.into()), - _ => None, - }) - }) - }) - .await - .unwrap() + self.block.get(&key).and_then(|a| match a { + Any::String(s) => Some(s), + _ => None, }) } pub fn get_float(&self, key: String) -> Option { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { - workspace.with_trx(|trx| { - block.get(&trx.trx, &key).and_then(|a| match a { - Any::Number(i) => Some(i), - _ => None, - }) - }) - }) - .await - .unwrap() + self.block.get(&key).and_then(|a| match a { + Any::Float32(f) => Some(f.0 as f64), + Any::Float64(f) => Some(f.0), + _ => None, }) } pub fn get_integer(&self, key: String) -> Option { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let block = self.block.clone(); - self.runtime - .spawn(async move { - workspace.with_trx(|trx| { - block.get(&trx.trx, &key).and_then(|a| match a { - Any::BigInt(i) => Some(i), - _ => None, - }) - }) - }) - .await - .unwrap() + self.block.get(&key).and_then(|a| match a { + Any::Integer(i) => Some(i as i64), + Any::BigInt64(i) => Some(i), + _ => None, }) } } diff --git a/libs/jwst-binding/jwst-swift/src/difflog.rs b/libs/jwst-binding/jwst-swift/src/difflog.rs deleted file mode 100644 index b33fe6e57..000000000 --- a/libs/jwst-binding/jwst-swift/src/difflog.rs +++ /dev/null @@ -1,64 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use chrono::{DateTime, Utc}; -use jwst_storage::JwstStorage; -use serde::Serialize; -use tokio::{ - runtime::Runtime, - sync::{mpsc::Receiver, Mutex}, - time::sleep, -}; - -use super::*; - -#[derive(Clone, Debug, Serialize)] -pub struct Log { - content: String, - timestamp: DateTime, - workspace: String, -} - -impl Log { - pub fn new(workspace: String, content: String) -> Self { - Self { - content, - timestamp: chrono::Utc::now(), - workspace, - } - } -} - -#[derive(Clone, Default)] -pub struct CachedDiffLog { - synced: Arc>>, -} - -impl CachedDiffLog { - pub fn add_receiver(&self, mut receiver: Receiver, rt: Arc, storage: Arc) { - let synced = self.synced.clone(); - - rt.spawn(async move { - loop { - tokio::select! { - Some(last_synced) = receiver.recv() => { - let mut synced = synced.lock().await; - synced.push(last_synced); - } - _ = sleep(Duration::from_secs(5)) => { - let mut synced = synced.lock().await; - for log in synced.iter() { - if let Err(e) = storage - .difflog() - .insert(log.workspace.clone(), log.timestamp, log.content.clone()) - .await - { - error!("failed to insert diff log: {:?}", e); - } - } - synced.clear(); - } - } - } - }); - } -} diff --git a/libs/jwst-binding/jwst-swift/src/dynamic_value.rs b/libs/jwst-binding/jwst-swift/src/dynamic_value.rs index 754ae9141..5e5f7f5be 100644 --- a/libs/jwst-binding/jwst-swift/src/dynamic_value.rs +++ b/libs/jwst-binding/jwst-swift/src/dynamic_value.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use lib0::any::Any; +use jwst_core::Any; pub type DynamicValueMap = HashMap; @@ -15,21 +15,24 @@ impl DynamicValue { pub fn as_bool(&self) -> Option { match self.any { - Any::Bool(value) => Some(value), + Any::True => Some(true), + Any::False => Some(false), _ => None, } } pub fn as_number(&self) -> Option { match self.any { - Any::Number(value) => Some(value), + Any::Float32(value) => Some(value.0 as f64), + Any::Float64(value) => Some(value.0), _ => None, } } pub fn as_int(&self) -> Option { match self.any { - Any::BigInt(value) => Some(value), + Any::Integer(value) => Some(value as i64), + Any::BigInt64(value) => Some(value), _ => None, } } @@ -43,7 +46,7 @@ impl DynamicValue { pub fn as_buffer(&self) -> Option> { match &self.any { - Any::Buffer(value) => Some(value.to_vec()), + Any::Binary(value) => Some(value.clone()), _ => None, } } @@ -57,7 +60,7 @@ impl DynamicValue { pub fn as_map(&self) -> Option> { match &self.any { - Any::Map(value) => Some( + Any::Object(value) => Some( value .iter() .map(|(key, value)| (key.clone(), DynamicValue::new(value.clone()))) diff --git a/libs/jwst-binding/jwst-swift/src/lib.rs b/libs/jwst-binding/jwst-swift/src/lib.rs index 3011b8dca..8bd239c8c 100644 --- a/libs/jwst-binding/jwst-swift/src/lib.rs +++ b/libs/jwst-binding/jwst-swift/src/lib.rs @@ -1,13 +1,11 @@ mod block; -mod difflog; mod dynamic_value; mod storage; mod workspace; pub use block::Block; -use difflog::{CachedDiffLog, Log}; pub use dynamic_value::{DynamicValue, DynamicValueMap}; -use jwst::{error, info, warn, JwstError}; +use jwst_core::{error, warn, JwstError}; use jwst_logger::init_logger_with; pub use storage::Storage; pub use workspace::Workspace; @@ -63,7 +61,7 @@ mod ffi { pub fn is_integer(&self, key: String) -> bool; - pub fn get_bool(&self, key: String) -> Option; + pub fn get_bool(&self, key: String) -> Option; pub fn get_string(&self, key: String) -> Option; @@ -109,8 +107,6 @@ mod ffi { fn get_search_index(self: &Workspace) -> Vec; fn set_search_index(self: &Workspace, fields: Vec) -> bool; - - fn compare(self: &mut Workspace) -> Option; } extern "Rust" { diff --git a/libs/jwst-binding/jwst-swift/src/storage.rs b/libs/jwst-binding/jwst-swift/src/storage.rs index f078b2ac7..f587bbd38 100644 --- a/libs/jwst-binding/jwst-swift/src/storage.rs +++ b/libs/jwst-binding/jwst-swift/src/storage.rs @@ -1,14 +1,13 @@ use std::sync::{Arc, RwLock}; use futures::TryFutureExt; -use jwst_rpc::{start_websocket_client_sync, BroadcastChannels, CachedLastSynced, RpcContextImpl, SyncState}; -use jwst_storage::{BlobStorageType, JwstStorage as AutoStorage, JwstStorageResult}; +use jwst_core_rpc::{start_websocket_client_sync, BroadcastChannels, CachedLastSynced, RpcContextImpl, SyncState}; +use jwst_core_storage::{BlobStorageType, JwstStorage as AutoStorage, JwstStorageResult}; use nanoid::nanoid; use tokio::{ runtime::{Builder, Runtime}, sync::mpsc::channel, }; -use yrs::{ReadTxn, StateVector, Transact}; use super::*; @@ -19,7 +18,6 @@ pub struct Storage { error: Option, sync_state: Arc>, last_sync: CachedLastSynced, - difflog: CachedDiffLog, } impl Storage { @@ -29,7 +27,10 @@ impl Storage { pub fn new_with_log_level(path: String, level: String) -> Self { init_logger_with( - &format!("{level},mio=off,hyper=off,rustls=off,tantivy=off,sqlx::query=off"), + &format!( + "{level},mio=off,hyper=off,rustls=off,tantivy=off,sqlx::query=off,tokio_tungstenite=off,\ + tungstenite=off" + ), false, ); @@ -50,7 +51,6 @@ impl Storage { error: None, sync_state: Arc::new(RwLock::new(SyncState::Offline)), last_sync: CachedLastSynced::default(), - difflog: CachedDiffLog::default(), } } @@ -114,7 +114,8 @@ impl Storage { match workspace { Ok(mut workspace) => { - if is_offline { + // disable remote temporarily + if is_offline || true { let identifier = nanoid!(); let (last_synced_tx, last_synced_rx) = channel::(128); self.last_sync.add_receiver(rt.clone(), last_synced_rx); @@ -133,51 +134,7 @@ impl Storage { ); } - let update = workspace - .doc() - .transact() - .encode_state_as_update_v1(&StateVector::default()) - .unwrap(); - - let jwst_workspace = match jwst_core::Workspace::from_binary(update, &workspace_id) { - Ok(mut ws) => { - info!( - "Successfully applied to jwst workspace, jwst blocks: {}, yrs blocks: {}", - ws.get_blocks().map(|s| s.block_count()).unwrap_or_default(), - workspace - .retry_with_trx(|mut t| t.get_blocks(), 50) - .map(|s| s.block_count()) - .unwrap_or_default() - ); - - Some(ws) - } - Err(e) => { - error!("Failed to apply to jwst workspace: {:?}", e); - None - } - }; - - let (sender, receiver) = channel::(10240); - self.difflog.add_receiver(receiver, rt.clone(), self.storage.clone()); - - let mut ws = Workspace { - workspace, - jwst_workspace, - runtime: rt, - sender, - }; - - if let Some(ret) = Workspace::compare(&mut ws) { - info!("Run first compare at workspace init: {}, {}", workspace_id, ret); - } else { - warn!( - "Failed to run first compare, jwst workspace not initialed: {}", - workspace_id - ); - } - - Ok(ws) + Ok(Workspace { workspace, rt }) } Err(e) => Err(e), } diff --git a/libs/jwst-binding/jwst-swift/src/workspace.rs b/libs/jwst-binding/jwst-swift/src/workspace.rs index ff2f75906..147450401 100644 --- a/libs/jwst-binding/jwst-swift/src/workspace.rs +++ b/libs/jwst-binding/jwst-swift/src/workspace.rs @@ -1,31 +1,20 @@ use std::sync::Arc; -use jwst::Workspace as JwstWorkspace; -use jwst_rpc::workspace_compare; -use tokio::{ - runtime::Runtime, - sync::mpsc::{channel, Sender}, -}; +use jwst_core::Workspace as JwstWorkspace; +use tokio::runtime::Runtime; use super::*; pub struct Workspace { pub(crate) workspace: JwstWorkspace, - pub(crate) jwst_workspace: Option, - pub(crate) runtime: Arc, - - pub(crate) sender: Sender, + pub(crate) rt: Arc, } impl Workspace { - pub fn new(id: String, runtime: Arc) -> Self { - let (sender, _receiver) = channel(10240); - + pub fn new(id: String) -> Self { Self { - workspace: JwstWorkspace::new(&id), - jwst_workspace: jwst_core::Workspace::new(id).ok(), - runtime, - sender, + workspace: JwstWorkspace::new(&id).unwrap(), + rt: Arc::new(Runtime::new().unwrap()), } } @@ -38,143 +27,49 @@ impl Workspace { } pub fn get(&self, block_id: String) -> Option { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let runtime = self.runtime.clone(); + let mut workspace = self.workspace.clone(); - let jwst_workspace = self.jwst_workspace.clone(); - let sender = self.sender.clone(); - - self.runtime - .spawn(async move { - workspace.with_trx(|mut trx| { - let block = trx - .get_blocks() - .get(&trx.trx, &block_id) - .map(|b| Block::new(workspace.clone(), b, runtime, jwst_workspace, sender)); - drop(trx); - block - }) - }) - .await - .unwrap() - }) + workspace + .get_blocks() + .ok() + .and_then(|s| s.get(&block_id)) + .map(Block::new) } pub fn create(&self, block_id: String, flavour: String) -> Block { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - - let jwst_workspace = self.jwst_workspace.clone(); - let sender = self.sender.clone(); - - let runtime = self.runtime.clone(); - self.runtime - .spawn(async move { - workspace.with_trx(|mut trx| { - let block = trx - .get_blocks() - .create(&mut trx.trx, block_id.clone(), flavour.clone()) - .expect("failed to create block"); - - let created = block.created(&trx.trx); - - let block = Block::new( - workspace.clone(), - block, - runtime, - { - // just for data verify - if let Some(mut jwst_workspace) = jwst_workspace.clone() { - jwst_workspace - .get_blocks() - .and_then(|mut b| b.create_ffi(block_id, flavour, created)) - .expect("failed to create jwst block"); - - // let ret = workspace_compare(&workspace, - // &jwst_workspace); - // sender.send(Log::new(workspace.id(), - // ret)).unwrap(); - } - - jwst_workspace - }, - sender.clone(), - ); - - drop(trx); - - block - }) - }) - .await - .unwrap() - }) + let mut workspace = self.workspace.clone(); + + Block::new( + workspace + .get_blocks() + .and_then(|mut b| b.create(block_id, flavour)) + .expect("failed to create jwst block"), + ) } pub fn search(self: &Workspace, query: String) -> String { - self.workspace.search_result(query) + // self.workspace.search_result(query) + "".to_owned() } pub fn get_blocks_by_flavour(&self, flavour: &str) -> Vec { - self.runtime.block_on(async { - let workspace = self.workspace.clone(); - let runtime = self.runtime.clone(); - let flavour = flavour.to_string(); - - let jwst_workspace = self.jwst_workspace.clone(); - let sender = self.sender.clone(); + let mut workspace = self.workspace.clone(); - self.runtime - .spawn(async move { - workspace - .with_trx(|mut trx| trx.get_blocks().get_blocks_by_flavour(&trx.trx, &flavour)) - .iter() - .map(|block| { - Block::new( - workspace.clone(), - block.clone(), - runtime.clone(), - jwst_workspace.clone(), - sender.clone(), - ) - }) - .collect::>() - }) - .await - .unwrap() - }) + workspace + .get_blocks() + .map(|s| s.get_blocks_by_flavour(flavour).into_iter().map(Block::new).collect()) + .unwrap_or_default() } pub fn get_search_index(self: &Workspace) -> Vec { - self.workspace.metadata().search_index + // self.workspace.metadata().search_index + vec![] } pub fn set_search_index(self: &Workspace, fields: Vec) -> bool { - self.workspace - .set_search_index(fields) - .expect("failed to set search index") - } - - pub fn compare(self: &mut Workspace) -> Option { - if let Some(jwst_workspace) = self.jwst_workspace.as_mut() { - match self - .workspace - .retry_with_trx(|trx| workspace_compare(trx.trx, jwst_workspace, None), 50) - { - Ok(ret) => { - self.runtime.block_on(async { - if let Err(e) = self.sender.send(Log::new(self.workspace.id(), ret.clone())).await { - warn!("failed to send log: {}", e); - } - }); - return Some(ret); - } - Err(e) => { - warn!("failed to compare: {}", e); - } - } - } - None + // self.workspace + // .set_search_index(fields) + // .expect("failed to set search index") + false } } From c315b5fe6fc8db39f0dd15a78459cb44bbf2810b Mon Sep 17 00:00:00 2001 From: DarkSky Date: Tue, 22 Aug 2023 00:19:32 +0800 Subject: [PATCH 10/49] feat: clearly update encode --- Cargo.lock | 2 -- .../RustXcframework.xcframework/Info.plist | 8 ++--- libs/jwst-codec/src/doc/document.rs | 5 ++- libs/jwst-core-rpc/Cargo.toml | 2 -- libs/jwst-core-rpc/src/broadcast.rs | 16 ++-------- libs/jwst-core-rpc/src/types.rs | 2 -- libs/jwst-core/src/workspace/sync.rs | 31 ++++++------------- 7 files changed, 19 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9f8d88f8e..26f0762d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2779,9 +2779,7 @@ dependencies = [ "jwst-core", "jwst-core-storage", "jwst-logger", - "lru_time_cache", "nanoid", - "nom", "rand 0.8.5", "reqwest", "serde", diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist index 067a2b752..625948fb2 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist @@ -22,7 +22,7 @@ HeadersPath Headers LibraryIdentifier - ios-arm64 + ios-arm64-simulator LibraryPath liboctobase.a SupportedArchitectures @@ -31,12 +31,14 @@ SupportedPlatform ios + SupportedPlatformVariant + simulator HeadersPath Headers LibraryIdentifier - ios-arm64-simulator + ios-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -45,8 +47,6 @@ SupportedPlatform ios - SupportedPlatformVariant - simulator CFBundlePackageType diff --git a/libs/jwst-codec/src/doc/document.rs b/libs/jwst-codec/src/doc/document.rs index e16c1a30c..19b2989dc 100644 --- a/libs/jwst-codec/src/doc/document.rs +++ b/libs/jwst-codec/src/doc/document.rs @@ -87,11 +87,10 @@ impl Doc { Ok(doc) } - pub fn apply_update_from_binary(&mut self, update: Vec) -> JwstCodecResult { + pub fn apply_update_from_binary(&mut self, update: Vec) -> JwstCodecResult { let mut decoder = RawDecoder::new(update); let update = Update::read(&mut decoder)?; - self.apply_update(update)?; - Ok(()) + self.apply_update(update) } pub fn apply_update(&mut self, mut update: Update) -> JwstCodecResult { diff --git a/libs/jwst-core-rpc/Cargo.toml b/libs/jwst-core-rpc/Cargo.toml index ce2a2be49..8799d70a5 100644 --- a/libs/jwst-core-rpc/Cargo.toml +++ b/libs/jwst-core-rpc/Cargo.toml @@ -17,9 +17,7 @@ async-trait = "0.1.68" byteorder = "1.4.3" chrono = "0.4.26" futures = "0.3.28" -lru_time_cache = "0.11.11" nanoid = "0.4.0" -nom = "7.1.3" rand = "0.8.5" serde = "1.0.183" serde_json = "1.0.104" diff --git a/libs/jwst-core-rpc/src/broadcast.rs b/libs/jwst-core-rpc/src/broadcast.rs index 1767410e3..4592a939b 100644 --- a/libs/jwst-core-rpc/src/broadcast.rs +++ b/libs/jwst-core-rpc/src/broadcast.rs @@ -1,8 +1,7 @@ -use std::{collections::HashMap, sync::Mutex}; +use std::collections::HashMap; use jwst_codec::{encode_update_as_message, encode_update_with_guid, write_sync_message, SyncMessage}; use jwst_core::Workspace; -use lru_time_cache::LruCache; use tokio::sync::{broadcast::Sender, RwLock}; use super::*; @@ -24,11 +23,6 @@ pub async fn subscribe(workspace: &mut Workspace, identifier: String, sender: Br let sender = sender.clone(); let workspace_id = workspace.id(); - let dedup_cache = Arc::new(Mutex::new(LruCache::with_expiry_duration_and_capacity( - Duration::from_micros(100), - 128, - ))); - workspace .on_awareness_update(move |awareness, e| { let mut buffer = Vec::new(); @@ -40,12 +34,8 @@ pub async fn subscribe(workspace: &mut Workspace, identifier: String, sender: Br return; } - let mut dedup_cache = dedup_cache.lock().unwrap_or_else(|e| e.into_inner()); - if !dedup_cache.contains_key(&buffer) { - if sender.send(BroadcastType::BroadcastAwareness(buffer.clone())).is_err() { - debug!("broadcast channel {workspace_id} has been closed",) - } - dedup_cache.insert(buffer, ()); + if sender.send(BroadcastType::BroadcastAwareness(buffer.clone())).is_err() { + debug!("broadcast channel {workspace_id} has been closed",) } }) .await; diff --git a/libs/jwst-core-rpc/src/types.rs b/libs/jwst-core-rpc/src/types.rs index 177f1f7bb..3182f90c0 100644 --- a/libs/jwst-core-rpc/src/types.rs +++ b/libs/jwst-core-rpc/src/types.rs @@ -12,8 +12,6 @@ pub enum JwstRpcError { #[allow(dead_code)] #[error("failed to encode sync message")] ProtocolEncode(std::io::Error), - #[error("failed to decode sync message")] - ProtocolDecode(#[from] nom::Err>), #[cfg(feature = "websocket")] #[error("failed to parse url")] UrlParse(#[from] url::ParseError), diff --git a/libs/jwst-core/src/workspace/sync.rs b/libs/jwst-core/src/workspace/sync.rs index 48745e689..f170aa521 100644 --- a/libs/jwst-core/src/workspace/sync.rs +++ b/libs/jwst-core/src/workspace/sync.rs @@ -97,27 +97,16 @@ impl Workspace { } None } - DocMessage::Update(update) => { - if let Ok(update) = Update::read(&mut RawDecoder::new(update)) { - match doc.apply_update(update) { - Ok(update) => { - let mut encoder = RawEncoder::default(); - if let Err(e) = update.write(&mut encoder) { - warn!("failed to encode update: {:?}", e); - None - } else { - Some(SyncMessage::Doc(DocMessage::Update(encoder.into_inner()))) - } - } - Err(e) => { - warn!("failed to apply update: {:?}", e); - None - } - } - } else { - None - } - } + DocMessage::Update(update) => doc + .apply_update_from_binary(update) + .and_then(|update| { + let mut encoder = RawEncoder::default(); + update.write(&mut encoder)?; + Ok(encoder.into_inner()) + }) + .map(|u| SyncMessage::Doc(DocMessage::Update(u))) + .map_err(|e| warn!("failed to apply update: {:?}", e)) + .ok(), }, _ => None, } From 9969ff833fbf430b623daaa3014cd70328e7f3c2 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Tue, 22 Aug 2023 00:44:08 +0800 Subject: [PATCH 11/49] feat: make image minify feature optional --- libs/jwst-core-storage/Cargo.toml | 5 +- .../src/storage/blobs/auto_blob_storage.rs | 423 +++++++++++++++ .../blobs/{local_db.rs => blob_storage.rs} | 0 .../{bucket_local_db.rs => bucket_storage.rs} | 10 +- .../src/storage/blobs/mod.rs | 512 +++--------------- .../src/storage/blobs/utils.rs | 22 +- libs/jwst-core-storage/src/storage/mod.rs | 18 +- 7 files changed, 525 insertions(+), 465 deletions(-) create mode 100644 libs/jwst-core-storage/src/storage/blobs/auto_blob_storage.rs rename libs/jwst-core-storage/src/storage/blobs/{local_db.rs => blob_storage.rs} (100%) rename libs/jwst-core-storage/src/storage/blobs/{bucket_local_db.rs => bucket_storage.rs} (97%) diff --git a/libs/jwst-core-storage/Cargo.toml b/libs/jwst-core-storage/Cargo.toml index 1119ece02..e3a82ba57 100644 --- a/libs/jwst-core-storage/Cargo.toml +++ b/libs/jwst-core-storage/Cargo.toml @@ -7,6 +7,7 @@ license = "AGPL-3.0-only" [features] default = ["sqlite"] +image = ["dep:image"] mysql = ["sea-orm/sqlx-mysql"] postgres = ["sea-orm/sqlx-postgres"] sqlite = ["sea-orm/sqlx-sqlite"] @@ -18,7 +19,6 @@ bytes = "1.4.0" chrono = { version = "0.4.24", features = ["serde"] } futures = "0.3.28" governor = "0.5.1" -image = { version = "0.24.6", features = ["webp-encoder"] } path-ext = "0.1.0" sha2 = "0.10.6" sea-orm = { version = "0.12.2", features = ["runtime-tokio-rustls", "macros"] } @@ -33,6 +33,9 @@ opendal = { version = "0.38.0", default-features = false, features = [ ] } dotenvy = "0.15.7" +# ======= image dependencies ====== +image = { version = "0.24.6", features = ["webp-encoder"], optional = true } + # ======= workspace dependencies ======= jwst-core = { workspace = true } jwst-codec = { workspace = true } diff --git a/libs/jwst-core-storage/src/storage/blobs/auto_blob_storage.rs b/libs/jwst-core-storage/src/storage/blobs/auto_blob_storage.rs new file mode 100644 index 000000000..d0240f309 --- /dev/null +++ b/libs/jwst-core-storage/src/storage/blobs/auto_blob_storage.rs @@ -0,0 +1,423 @@ +#[cfg(test)] +pub use blob_storage::blobs_storage_test; +use blob_storage::BlobDBStorage; +use bytes::Bytes; +use image::ImageError; +use jwst_core::{BlobMetadata, BlobStorage}; +use thiserror::Error; +use tokio::task::JoinError; +pub use utils::BucketStorageBuilder; +use utils::{ImageParams, InternalBlobMetadata}; + +pub(super) type OptimizedBlobModel = ::Model; +type OptimizedBlobActiveModel = super::entities::optimized_blobs::ActiveModel; +type OptimizedBlobColumn = ::Column; + +#[derive(Clone)] +pub struct BlobAutoStorage { + pub(super) db: Arc, + pool: DatabaseConnection, +} + +impl BlobAutoStorage { + pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { + let db = Arc::new(BlobDBStorage::init_with_pool(pool, bucket).await?); + let pool = db.pool.clone(); + Ok(Self { db, pool }) + } + + pub async fn init_pool(database: &str) -> JwstStorageResult { + let db = Arc::new(BlobDBStorage::init_pool(database).await?); + let pool = db.pool.clone(); + Ok(Self { db, pool }) + } + + async fn exists(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { + Ok(OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) + .count(&self.pool) + .await + .map(|c| c > 0)?) + } + + async fn insert(&self, table: &str, hash: &str, params: &str, blob: &[u8]) -> JwstBlobResult<()> { + if !self.exists(table, hash, params).await? { + OptimizedBlobs::insert(OptimizedBlobActiveModel { + workspace_id: Set(table.into()), + hash: Set(hash.into()), + blob: Set(blob.into()), + length: Set(blob.len().try_into().unwrap()), + params: Set(params.into()), + created_at: Set(Utc::now().into()), + }) + .exec(&self.pool) + .await?; + } + + Ok(()) + } + + async fn get(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { + OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) + .one(&self.pool) + .await + .map_err(|e| e.into()) + .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) + } + + async fn metadata(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { + OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) + .select_only() + .column_as(OptimizedBlobColumn::Length, "size") + .column_as(OptimizedBlobColumn::CreatedAt, "created_at") + .into_model::() + .one(&self.pool) + .await + .map_err(|e| e.into()) + .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) + } + + async fn get_metadata_auto( + &self, + workspace: Option, + id: String, + params: Option>, + ) -> JwstBlobResult { + let workspace_id = workspace.as_deref().unwrap_or("__default__"); + if let Some(params) = params { + if let Ok(params) = ImageParams::try_from(¶ms) { + let params_token = params.to_string(); + if self.exists(workspace_id, &id, ¶ms_token).await? { + let metadata = self.metadata(workspace_id, &id, ¶ms_token).await?; + Ok(BlobMetadata { + content_type: format!("image/{}", params.format()), + ..metadata.into() + }) + } else { + self.db.metadata(workspace_id, &id).await.map(Into::into) + } + } else { + Err(JwstBlobError::ImageParams(params)) + } + } else { + self.db.metadata(workspace_id, &id).await.map(Into::into) + } + } + + async fn get_auto( + &self, + workspace: Option, + id: String, + params: Option>, + ) -> JwstBlobResult> { + let workspace_id = workspace.as_deref().unwrap_or("__default__"); + if let Some(params) = params { + if let Ok(params) = ImageParams::try_from(¶ms) { + let params_token = params.to_string(); + if let Ok(blob) = self.get(workspace_id, &id, ¶ms_token).await { + info!( + "exists optimized image: {} {} {}, {}bytes", + workspace_id, + id, + params_token, + blob.blob.len() + ); + Ok(blob.blob) + } else { + // TODO: need ddos mitigation + let blob = self.db.get(workspace_id, &id).await?; + let blob_len = blob.blob.len(); + let image = tokio::task::spawn_blocking(move || params.optimize_image(&blob.blob)).await??; + self.insert(workspace_id, &id, ¶ms_token, &image).await?; + info!( + "optimized image: {} {} {}, {}bytes -> {}bytes", + workspace_id, + id, + params_token, + blob_len, + image.len() + ); + Ok(image) + } + } else { + Err(JwstBlobError::ImageParams(params)) + } + } else { + self.db.get(workspace_id, &id).await.map(|m| m.blob) + } + } + + async fn delete(&self, table: &str, hash: &str) -> JwstBlobResult { + Ok(OptimizedBlobs::delete_many() + .filter( + OptimizedBlobColumn::WorkspaceId + .eq(table) + .and(OptimizedBlobColumn::Hash.eq(hash)), + ) + .exec(&self.pool) + .await + .map(|r| r.rows_affected)?) + } + + async fn drop(&self, table: &str) -> Result<(), DbErr> { + OptimizedBlobs::delete_many() + .filter(OptimizedBlobColumn::WorkspaceId.eq(table)) + .exec(&self.pool) + .await?; + + Ok(()) + } +} + +#[async_trait] +impl BlobStorage for BlobAutoStorage { + async fn list_blobs(&self, workspace: Option) -> JwstStorageResult> { + self.db.list_blobs(workspace).await + } + + async fn check_blob(&self, workspace: Option, id: String) -> JwstStorageResult { + self.db.check_blob(workspace, id).await + } + + async fn get_blob( + &self, + workspace: Option, + id: String, + params: Option>, + ) -> JwstStorageResult> { + let blob = self.get_auto(workspace, id, params).await?; + Ok(blob) + } + + async fn get_metadata( + &self, + workspace: Option, + id: String, + params: Option>, + ) -> JwstStorageResult { + let metadata = self.get_metadata_auto(workspace, id, params).await?; + Ok(metadata) + } + + async fn put_blob_stream( + &self, + workspace: Option, + stream: impl Stream + Send, + ) -> JwstStorageResult { + self.db.put_blob_stream(workspace, stream).await + } + + async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstStorageResult { + self.db.put_blob(workspace, blob).await + } + + async fn delete_blob(&self, workspace_id: Option, id: String) -> JwstStorageResult { + // delete origin blobs + let success = self.db.delete_blob(workspace_id.clone(), id.clone()).await?; + if success { + // delete optimized blobs + let workspace_id = workspace_id.unwrap_or("__default__".into()); + self.delete(&workspace_id, &id).await?; + } + Ok(success) + } + + async fn delete_workspace(&self, workspace_id: String) -> JwstStorageResult<()> { + // delete origin blobs + self.db.delete_workspace(workspace_id.clone()).await?; + + // delete optimized blobs + self.drop(&workspace_id).await?; + + Ok(()) + } + + async fn get_blobs_size(&self, workspace_id: String) -> JwstStorageResult { + let size = self.db.get_blobs_size(&workspace_id).await?; + + return Ok(size.unwrap_or(0)); + } +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use futures::FutureExt; + use image::{DynamicImage, ImageOutputFormat}; + + use super::*; + + #[tokio::test] + async fn test_blob_auto_storage() { + let storage = BlobAutoStorage::init_pool("sqlite::memory:").await.unwrap(); + Migrator::up(&storage.pool, None).await.unwrap(); + + let blob = Vec::from_iter((0..100).map(|_| rand::random())); + + let stream = async { Bytes::from(blob.clone()) }.into_stream(); + let hash1 = storage.put_blob_stream(Some("blob".into()), stream).await.unwrap(); + + // check origin blob result + assert_eq!( + storage + .get_blob(Some("blob".into()), hash1.clone(), None) + .await + .unwrap(), + blob + ); + assert_eq!( + storage + .get_metadata(Some("blob".into()), hash1.clone(), None) + .await + .unwrap() + .size as usize, + blob.len() + ); + + // optimize must failed if blob not supported + assert!(storage + .get_blob( + Some("blob".into()), + hash1.clone(), + Some(HashMap::from([("format".into(), "jpeg".into())])) + ) + .await + .is_err()); + + // generate image + let image = { + let mut image = Cursor::new(vec![]); + DynamicImage::new_rgba8(32, 32) + .write_to(&mut image, ImageOutputFormat::Png) + .unwrap(); + image.into_inner() + }; + let stream = async { Bytes::from(image.clone()) }.into_stream(); + let hash2 = storage.put_blob_stream(Some("blob".into()), stream).await.unwrap(); + + // check origin blob result + assert_eq!( + storage + .get_blob(Some("blob".into()), hash2.clone(), None) + .await + .unwrap(), + image + ); + assert_eq!( + storage + .get_metadata(Some("blob".into()), hash2.clone(), None) + .await + .unwrap() + .size as usize, + image.len() + ); + + // check optimized jpeg result + let jpeg_params = HashMap::from([("format".into(), "jpeg".into())]); + let jpeg = storage + .get_blob(Some("blob".into()), hash2.clone(), Some(jpeg_params.clone())) + .await + .unwrap(); + + assert!(jpeg.starts_with(&[0xff, 0xd8, 0xff])); + assert_eq!( + storage + .get_metadata(Some("blob".into()), hash2.clone(), Some(jpeg_params)) + .await + .unwrap() + .size as usize, + jpeg.len() + ); + + // check optimized webp result + let webp_params = HashMap::from([("format".into(), "webp".into())]); + let webp = storage + .get_blob(Some("blob".into()), hash2.clone(), Some(webp_params.clone())) + .await + .unwrap(); + + assert!(webp.starts_with(b"RIFF")); + assert_eq!( + storage + .get_metadata(Some("blob".into()), hash2.clone(), Some(webp_params.clone())) + .await + .unwrap() + .size as usize, + webp.len() + ); + + // optimize must failed if image params error + assert!(storage + .get_blob( + Some("blob".into()), + hash2.clone(), + Some(HashMap::from([("format".into(), "error_value".into()),])) + ) + .await + .is_err()); + assert!(storage + .get_blob( + Some("blob".into()), + hash2.clone(), + Some(HashMap::from([ + ("format".into(), "webp".into()), + ("size".into(), "error_value".into()) + ])) + ) + .await + .is_err()); + assert!(storage + .get_blob( + Some("blob".into()), + hash2.clone(), + Some(HashMap::from([ + ("format".into(), "webp".into()), + ("width".into(), "111".into()) + ])) + ) + .await + .is_err()); + assert!(storage + .get_blob( + Some("blob".into()), + hash2.clone(), + Some(HashMap::from([ + ("format".into(), "webp".into()), + ("height".into(), "111".into()) + ])) + ) + .await + .is_err()); + + assert_eq!( + storage.get_blobs_size("blob".into()).await.unwrap() as usize, + 100 + image.len() + ); + + assert!(storage.delete_blob(Some("blob".into()), hash2.clone()).await.unwrap()); + assert_eq!( + storage.check_blob(Some("blob".into()), hash2.clone()).await.unwrap(), + false + ); + assert!(storage + .get_blob(Some("blob".into()), hash2.clone(), None) + .await + .is_err()); + assert!(storage + .get_metadata(Some("blob".into()), hash2.clone(), None) + .await + .is_err()); + assert!(storage + .get_metadata(Some("blob".into()), hash2.clone(), Some(webp_params)) + .await + .is_err()); + + assert_eq!(storage.get_blobs_size("blob".into()).await.unwrap() as usize, 100); + + assert_eq!(storage.list_blobs(Some("blob".into())).await.unwrap(), vec![hash1]); + assert_eq!( + storage.list_blobs(Some("not_exists_workspace".into())).await.unwrap(), + Vec::::new() + ); + } +} diff --git a/libs/jwst-core-storage/src/storage/blobs/local_db.rs b/libs/jwst-core-storage/src/storage/blobs/blob_storage.rs similarity index 100% rename from libs/jwst-core-storage/src/storage/blobs/local_db.rs rename to libs/jwst-core-storage/src/storage/blobs/blob_storage.rs diff --git a/libs/jwst-core-storage/src/storage/blobs/bucket_local_db.rs b/libs/jwst-core-storage/src/storage/blobs/bucket_storage.rs similarity index 97% rename from libs/jwst-core-storage/src/storage/blobs/bucket_local_db.rs rename to libs/jwst-core-storage/src/storage/blobs/bucket_storage.rs index 1b19910d9..8830c3ed2 100644 --- a/libs/jwst-core-storage/src/storage/blobs/bucket_local_db.rs +++ b/libs/jwst-core-storage/src/storage/blobs/bucket_storage.rs @@ -19,19 +19,19 @@ type BucketBlobActiveModel = entities::bucket_blobs::ActiveModel; type BucketBlobColumn = ::Column; #[derive(Clone)] -pub struct BlobBucketDBStorage { +pub struct BlobBucketStorage { bucket: Arc, pub(super) pool: DatabaseConnection, pub(super) bucket_storage: BucketStorage, } -impl AsRef for BlobBucketDBStorage { +impl AsRef for BlobBucketStorage { fn as_ref(&self) -> &DatabaseConnection { &self.pool } } -impl BlobBucketDBStorage { +impl BlobBucketStorage { pub async fn init_with_pool( pool: DatabaseConnection, bucket: Arc, @@ -209,7 +209,7 @@ impl BucketBlobStorage for BucketStorage { } #[async_trait] -impl BlobStorage for BlobBucketDBStorage { +impl BlobStorage for BlobBucketStorage { async fn list_blobs(&self, workspace: Option) -> JwstResult, JwstStorageError> { let _lock = self.bucket.read().await; let workspace = workspace.unwrap_or("__default__".into()); @@ -336,7 +336,7 @@ mod tests { .build() .unwrap(); - BlobBucketDBStorage::init_pool("sqlite::memory:", Some(bucket_storage)) + BlobBucketStorage::init_pool("sqlite::memory:", Some(bucket_storage)) .await .unwrap(); } diff --git a/libs/jwst-core-storage/src/storage/blobs/mod.rs b/libs/jwst-core-storage/src/storage/blobs/mod.rs index fb52037ed..d5b34d3bf 100644 --- a/libs/jwst-core-storage/src/storage/blobs/mod.rs +++ b/libs/jwst-core-storage/src/storage/blobs/mod.rs @@ -1,18 +1,26 @@ -mod bucket_local_db; -mod local_db; +#[cfg(feature = "image")] +mod auto_blob_storage; +#[cfg(feature = "image")] +pub use auto_blob_storage::BlobAutoStorage; +#[cfg(feature = "image")] +use image::ImageError; +#[cfg(feature = "image")] +use utils::ImageParams; + +mod blob_storage; +mod bucket_storage; mod utils; -pub use bucket_local_db::BlobBucketDBStorage; +#[cfg(test)] +pub use blob_storage::blobs_storage_test; +pub use blob_storage::BlobDBStorage; +pub use bucket_storage::BlobBucketStorage; use bytes::Bytes; -use image::ImageError; use jwst_core::{BlobMetadata, BlobStorage}; -#[cfg(test)] -pub use local_db::blobs_storage_test; -use local_db::BlobDBStorage; use thiserror::Error; use tokio::task::JoinError; pub use utils::BucketStorageBuilder; -use utils::{ImageParams, InternalBlobMetadata}; +use utils::InternalBlobMetadata; use super::{entities::prelude::*, *}; @@ -22,6 +30,7 @@ pub enum JwstBlobError { BlobNotFound(String), #[error("database error")] Database(#[from] DbErr), + #[cfg(feature = "image")] #[error("failed to optimize image")] Image(#[from] ImageError), #[error("failed to optimize image")] @@ -31,19 +40,11 @@ pub enum JwstBlobError { } pub type JwstBlobResult = Result; -pub(super) type OptimizedBlobModel = ::Model; -type OptimizedBlobActiveModel = super::entities::optimized_blobs::ActiveModel; -type OptimizedBlobColumn = ::Column; - -#[derive(Clone)] -pub struct BlobAutoStorage { - pub(super) db: Arc, - pool: DatabaseConnection, -} - pub enum JwstBlobStorage { - DB(BlobAutoStorage), - MixedBucketDB(BlobBucketDBStorage), + RawStorage(Arc), + #[cfg(feature = "image")] + AutoStorage(BlobAutoStorage), + BucketStorage(BlobBucketStorage), } pub enum BlobStorageType { @@ -87,238 +88,23 @@ impl MixedBucketDBParam { } } -impl BlobAutoStorage { - pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { - let db = Arc::new(BlobDBStorage::init_with_pool(pool, bucket).await?); - let pool = db.pool.clone(); - Ok(Self { db, pool }) - } - - pub async fn init_pool(database: &str) -> JwstStorageResult { - let db = Arc::new(BlobDBStorage::init_pool(database).await?); - let pool = db.pool.clone(); - Ok(Self { db, pool }) - } - - async fn exists(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { - Ok(OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) - .count(&self.pool) - .await - .map(|c| c > 0)?) - } - - async fn insert(&self, table: &str, hash: &str, params: &str, blob: &[u8]) -> JwstBlobResult<()> { - if !self.exists(table, hash, params).await? { - OptimizedBlobs::insert(OptimizedBlobActiveModel { - workspace_id: Set(table.into()), - hash: Set(hash.into()), - blob: Set(blob.into()), - length: Set(blob.len().try_into().unwrap()), - params: Set(params.into()), - created_at: Set(Utc::now().into()), - }) - .exec(&self.pool) - .await?; - } - - Ok(()) - } - - async fn get(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { - OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) - .one(&self.pool) - .await - .map_err(|e| e.into()) - .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) - } - - async fn metadata(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { - OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) - .select_only() - .column_as(OptimizedBlobColumn::Length, "size") - .column_as(OptimizedBlobColumn::CreatedAt, "created_at") - .into_model::() - .one(&self.pool) - .await - .map_err(|e| e.into()) - .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) - } - - async fn get_metadata_auto( - &self, - workspace: Option, - id: String, - params: Option>, - ) -> JwstBlobResult { - let workspace_id = workspace.as_deref().unwrap_or("__default__"); - if let Some(params) = params { - if let Ok(params) = ImageParams::try_from(¶ms) { - let params_token = params.to_string(); - if self.exists(workspace_id, &id, ¶ms_token).await? { - let metadata = self.metadata(workspace_id, &id, ¶ms_token).await?; - Ok(BlobMetadata { - content_type: format!("image/{}", params.format()), - ..metadata.into() - }) - } else { - self.db.metadata(workspace_id, &id).await.map(Into::into) - } - } else { - Err(JwstBlobError::ImageParams(params)) - } - } else { - self.db.metadata(workspace_id, &id).await.map(Into::into) - } - } - - async fn get_auto( - &self, - workspace: Option, - id: String, - params: Option>, - ) -> JwstBlobResult> { - let workspace_id = workspace.as_deref().unwrap_or("__default__"); - if let Some(params) = params { - if let Ok(params) = ImageParams::try_from(¶ms) { - let params_token = params.to_string(); - if let Ok(blob) = self.get(workspace_id, &id, ¶ms_token).await { - info!( - "exists optimized image: {} {} {}, {}bytes", - workspace_id, - id, - params_token, - blob.blob.len() - ); - Ok(blob.blob) - } else { - // TODO: need ddos mitigation - let blob = self.db.get(workspace_id, &id).await?; - let blob_len = blob.blob.len(); - let image = tokio::task::spawn_blocking(move || params.optimize_image(&blob.blob)).await??; - self.insert(workspace_id, &id, ¶ms_token, &image).await?; - info!( - "optimized image: {} {} {}, {}bytes -> {}bytes", - workspace_id, - id, - params_token, - blob_len, - image.len() - ); - Ok(image) - } - } else { - Err(JwstBlobError::ImageParams(params)) - } - } else { - self.db.get(workspace_id, &id).await.map(|m| m.blob) - } - } - - async fn delete(&self, table: &str, hash: &str) -> JwstBlobResult { - Ok(OptimizedBlobs::delete_many() - .filter( - OptimizedBlobColumn::WorkspaceId - .eq(table) - .and(OptimizedBlobColumn::Hash.eq(hash)), - ) - .exec(&self.pool) - .await - .map(|r| r.rows_affected)?) - } - - async fn drop(&self, table: &str) -> Result<(), DbErr> { - OptimizedBlobs::delete_many() - .filter(OptimizedBlobColumn::WorkspaceId.eq(table)) - .exec(&self.pool) - .await?; - - Ok(()) - } -} - -#[async_trait] -impl BlobStorage for BlobAutoStorage { - async fn list_blobs(&self, workspace: Option) -> JwstStorageResult> { - self.db.list_blobs(workspace).await - } - - async fn check_blob(&self, workspace: Option, id: String) -> JwstStorageResult { - self.db.check_blob(workspace, id).await - } - - async fn get_blob( - &self, - workspace: Option, - id: String, - params: Option>, - ) -> JwstStorageResult> { - let blob = self.get_auto(workspace, id, params).await?; - Ok(blob) - } - - async fn get_metadata( - &self, - workspace: Option, - id: String, - params: Option>, - ) -> JwstStorageResult { - let metadata = self.get_metadata_auto(workspace, id, params).await?; - Ok(metadata) - } - - async fn put_blob_stream( - &self, - workspace: Option, - stream: impl Stream + Send, - ) -> JwstStorageResult { - self.db.put_blob_stream(workspace, stream).await - } - - async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstStorageResult { - self.db.put_blob(workspace, blob).await - } - - async fn delete_blob(&self, workspace_id: Option, id: String) -> JwstStorageResult { - // delete origin blobs - let success = self.db.delete_blob(workspace_id.clone(), id.clone()).await?; - if success { - // delete optimized blobs - let workspace_id = workspace_id.unwrap_or("__default__".into()); - self.delete(&workspace_id, &id).await?; - } - Ok(success) - } - - async fn delete_workspace(&self, workspace_id: String) -> JwstStorageResult<()> { - // delete origin blobs - self.db.delete_workspace(workspace_id.clone()).await?; - - // delete optimized blobs - self.drop(&workspace_id).await?; - - Ok(()) - } - - async fn get_blobs_size(&self, workspace_id: String) -> JwstStorageResult { - let size = self.db.get_blobs_size(&workspace_id).await?; - - return Ok(size.unwrap_or(0)); - } -} - #[async_trait] impl BlobStorage for JwstBlobStorage { async fn list_blobs(&self, workspace: Option) -> JwstResult, JwstStorageError> { match self { - JwstBlobStorage::DB(db) => db.list_blobs(workspace).await, - JwstBlobStorage::MixedBucketDB(db) => db.list_blobs(workspace).await, + JwstBlobStorage::RawStorage(db) => db.list_blobs(workspace).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.list_blobs(workspace).await, + JwstBlobStorage::BucketStorage(db) => db.list_blobs(workspace).await, } } async fn check_blob(&self, workspace: Option, id: String) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.check_blob(workspace, id).await, - JwstBlobStorage::MixedBucketDB(db) => db.check_blob(workspace, id).await, + JwstBlobStorage::RawStorage(db) => db.check_blob(workspace, id).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.check_blob(workspace, id).await, + JwstBlobStorage::BucketStorage(db) => db.check_blob(workspace, id).await, } } @@ -329,8 +115,10 @@ impl BlobStorage for JwstBlobStorage { params: Option>, ) -> JwstResult, JwstStorageError> { match self { - JwstBlobStorage::DB(db) => db.get_blob(workspace, id, params).await, - JwstBlobStorage::MixedBucketDB(db) => db.get_blob(workspace, id, params).await, + JwstBlobStorage::RawStorage(db) => db.get_blob(workspace, id, params).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.get_blob(workspace, id, params).await, + JwstBlobStorage::BucketStorage(db) => db.get_blob(workspace, id, params).await, } } @@ -341,8 +129,10 @@ impl BlobStorage for JwstBlobStorage { params: Option>, ) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.get_metadata(workspace, id, params).await, - JwstBlobStorage::MixedBucketDB(db) => db.get_metadata(workspace, id, params).await, + JwstBlobStorage::RawStorage(db) => db.get_metadata(workspace, id, params).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.get_metadata(workspace, id, params).await, + JwstBlobStorage::BucketStorage(db) => db.get_metadata(workspace, id, params).await, } } @@ -352,36 +142,46 @@ impl BlobStorage for JwstBlobStorage { stream: impl Stream + Send, ) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.put_blob_stream(workspace, stream).await, - JwstBlobStorage::MixedBucketDB(db) => db.put_blob_stream(workspace, stream).await, + JwstBlobStorage::RawStorage(db) => db.put_blob_stream(workspace, stream).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.put_blob_stream(workspace, stream).await, + JwstBlobStorage::BucketStorage(db) => db.put_blob_stream(workspace, stream).await, } } async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.put_blob(workspace, blob).await, - JwstBlobStorage::MixedBucketDB(db) => db.put_blob(workspace, blob).await, + JwstBlobStorage::RawStorage(db) => db.put_blob(workspace, blob).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.put_blob(workspace, blob).await, + JwstBlobStorage::BucketStorage(db) => db.put_blob(workspace, blob).await, } } async fn delete_blob(&self, workspace: Option, id: String) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.delete_blob(workspace, id).await, - JwstBlobStorage::MixedBucketDB(db) => db.delete_blob(workspace, id).await, + JwstBlobStorage::RawStorage(db) => db.delete_blob(workspace, id).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.delete_blob(workspace, id).await, + JwstBlobStorage::BucketStorage(db) => db.delete_blob(workspace, id).await, } } async fn delete_workspace(&self, workspace_id: String) -> JwstResult<(), JwstStorageError> { match self { - JwstBlobStorage::DB(db) => db.delete_workspace(workspace_id).await, - JwstBlobStorage::MixedBucketDB(db) => db.delete_workspace(workspace_id).await, + JwstBlobStorage::RawStorage(db) => db.delete_workspace(workspace_id).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.delete_workspace(workspace_id).await, + JwstBlobStorage::BucketStorage(db) => db.delete_workspace(workspace_id).await, } } async fn get_blobs_size(&self, workspace_id: String) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.get_blobs_size(workspace_id).await, - JwstBlobStorage::MixedBucketDB(db) => db.get_blobs_size(workspace_id).await, + JwstBlobStorage::RawStorage(db) => Ok(db.get_blobs_size(&workspace_id).await?.unwrap_or(0)), + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.get_blobs_size(workspace_id).await, + JwstBlobStorage::BucketStorage(db) => db.get_blobs_size(workspace_id).await, } } } @@ -389,199 +189,19 @@ impl BlobStorage for JwstBlobStorage { impl JwstBlobStorage { pub fn get_blob_db(&self) -> Option> { match self { - JwstBlobStorage::DB(db) => Some(db.db.clone()), - JwstBlobStorage::MixedBucketDB(_) => None, + JwstBlobStorage::RawStorage(db) => Some(db.clone()), + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => Some(db.db.clone()), + JwstBlobStorage::BucketStorage(_) => None, } } - pub fn get_mixed_bucket_db(&self) -> Option { + pub fn get_mixed_bucket_db(&self) -> Option { match self { - JwstBlobStorage::DB(_) => None, - JwstBlobStorage::MixedBucketDB(db) => Some(db.clone()), + JwstBlobStorage::RawStorage(_) => None, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(_) => None, + JwstBlobStorage::BucketStorage(db) => Some(db.clone()), } } } - -#[cfg(test)] -mod tests { - use std::io::Cursor; - - use futures::FutureExt; - use image::{DynamicImage, ImageOutputFormat}; - - use super::*; - - #[tokio::test] - async fn test_blob_auto_storage() { - let storage = BlobAutoStorage::init_pool("sqlite::memory:").await.unwrap(); - Migrator::up(&storage.pool, None).await.unwrap(); - - let blob = Vec::from_iter((0..100).map(|_| rand::random())); - - let stream = async { Bytes::from(blob.clone()) }.into_stream(); - let hash1 = storage.put_blob_stream(Some("blob".into()), stream).await.unwrap(); - - // check origin blob result - assert_eq!( - storage - .get_blob(Some("blob".into()), hash1.clone(), None) - .await - .unwrap(), - blob - ); - assert_eq!( - storage - .get_metadata(Some("blob".into()), hash1.clone(), None) - .await - .unwrap() - .size as usize, - blob.len() - ); - - // optimize must failed if blob not supported - assert!(storage - .get_blob( - Some("blob".into()), - hash1.clone(), - Some(HashMap::from([("format".into(), "jpeg".into())])) - ) - .await - .is_err()); - - // generate image - let image = { - let mut image = Cursor::new(vec![]); - DynamicImage::new_rgba8(32, 32) - .write_to(&mut image, ImageOutputFormat::Png) - .unwrap(); - image.into_inner() - }; - let stream = async { Bytes::from(image.clone()) }.into_stream(); - let hash2 = storage.put_blob_stream(Some("blob".into()), stream).await.unwrap(); - - // check origin blob result - assert_eq!( - storage - .get_blob(Some("blob".into()), hash2.clone(), None) - .await - .unwrap(), - image - ); - assert_eq!( - storage - .get_metadata(Some("blob".into()), hash2.clone(), None) - .await - .unwrap() - .size as usize, - image.len() - ); - - // check optimized jpeg result - let jpeg_params = HashMap::from([("format".into(), "jpeg".into())]); - let jpeg = storage - .get_blob(Some("blob".into()), hash2.clone(), Some(jpeg_params.clone())) - .await - .unwrap(); - - assert!(jpeg.starts_with(&[0xff, 0xd8, 0xff])); - assert_eq!( - storage - .get_metadata(Some("blob".into()), hash2.clone(), Some(jpeg_params)) - .await - .unwrap() - .size as usize, - jpeg.len() - ); - - // check optimized webp result - let webp_params = HashMap::from([("format".into(), "webp".into())]); - let webp = storage - .get_blob(Some("blob".into()), hash2.clone(), Some(webp_params.clone())) - .await - .unwrap(); - - assert!(webp.starts_with(b"RIFF")); - assert_eq!( - storage - .get_metadata(Some("blob".into()), hash2.clone(), Some(webp_params.clone())) - .await - .unwrap() - .size as usize, - webp.len() - ); - - // optimize must failed if image params error - assert!(storage - .get_blob( - Some("blob".into()), - hash2.clone(), - Some(HashMap::from([("format".into(), "error_value".into()),])) - ) - .await - .is_err()); - assert!(storage - .get_blob( - Some("blob".into()), - hash2.clone(), - Some(HashMap::from([ - ("format".into(), "webp".into()), - ("size".into(), "error_value".into()) - ])) - ) - .await - .is_err()); - assert!(storage - .get_blob( - Some("blob".into()), - hash2.clone(), - Some(HashMap::from([ - ("format".into(), "webp".into()), - ("width".into(), "111".into()) - ])) - ) - .await - .is_err()); - assert!(storage - .get_blob( - Some("blob".into()), - hash2.clone(), - Some(HashMap::from([ - ("format".into(), "webp".into()), - ("height".into(), "111".into()) - ])) - ) - .await - .is_err()); - - assert_eq!( - storage.get_blobs_size("blob".into()).await.unwrap() as usize, - 100 + image.len() - ); - - assert!(storage.delete_blob(Some("blob".into()), hash2.clone()).await.unwrap()); - assert_eq!( - storage.check_blob(Some("blob".into()), hash2.clone()).await.unwrap(), - false - ); - assert!(storage - .get_blob(Some("blob".into()), hash2.clone(), None) - .await - .is_err()); - assert!(storage - .get_metadata(Some("blob".into()), hash2.clone(), None) - .await - .is_err()); - assert!(storage - .get_metadata(Some("blob".into()), hash2.clone(), Some(webp_params)) - .await - .is_err()); - - assert_eq!(storage.get_blobs_size("blob".into()).await.unwrap() as usize, 100); - - assert_eq!(storage.list_blobs(Some("blob".into())).await.unwrap(), vec![hash1]); - assert_eq!( - storage.list_blobs(Some("not_exists_workspace".into())).await.unwrap(), - Vec::::new() - ); - } -} diff --git a/libs/jwst-core-storage/src/storage/blobs/utils.rs b/libs/jwst-core-storage/src/storage/blobs/utils.rs index a967df3b9..2ef7dd4f9 100644 --- a/libs/jwst-core-storage/src/storage/blobs/utils.rs +++ b/libs/jwst-core-storage/src/storage/blobs/utils.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, io::Cursor}; +use std::collections::HashMap; use bytes::Bytes; use chrono::{DateTime, Utc}; @@ -6,28 +6,30 @@ use futures::{ stream::{iter, StreamExt}, Stream, }; -use image::{load_from_memory, ImageOutputFormat, ImageResult}; use jwst_core::{Base64Engine, BlobMetadata, URL_SAFE_ENGINE}; use opendal::{services::S3, Operator}; use sea_orm::FromQueryResult; use sha2::{Digest, Sha256}; use crate::{ - storage::blobs::{bucket_local_db::BucketStorage, MixedBucketDBParam}, + storage::blobs::{bucket_storage::BucketStorage, MixedBucketDBParam}, JwstStorageError, JwstStorageResult, }; +#[cfg(feature = "image")] enum ImageFormat { Jpeg, WebP, } +#[cfg(feature = "image")] pub struct ImageParams { format: ImageFormat, width: Option, height: Option, } +#[cfg(feature = "image")] impl ImageParams { #[inline] fn check_size(w: Option, h: Option) -> bool { @@ -51,21 +53,22 @@ impl ImageParams { } } - fn output_format(&self) -> ImageOutputFormat { + fn output_format(&self) -> image::ImageOutputFormat { match self.format { - ImageFormat::Jpeg => ImageOutputFormat::Jpeg(80), - ImageFormat::WebP => ImageOutputFormat::WebP, + image::ImageFormat::Jpeg => image::ImageOutputFormat::Jpeg(80), + image::ImageFormat::WebP => image::ImageOutputFormat::WebP, } } - pub fn optimize_image(&self, data: &[u8]) -> ImageResult> { - let mut buffer = Cursor::new(vec![]); - let image = load_from_memory(data)?; + pub fn optimize_image(&self, data: &[u8]) -> image::ImageResult> { + let mut buffer = std::io::Cursor::new(vec![]); + let image = image::load_from_memory(data)?; image.write_to(&mut buffer, self.output_format())?; Ok(buffer.into_inner()) } } +#[cfg(feature = "image")] impl TryFrom<&HashMap> for ImageParams { type Error = (); @@ -97,6 +100,7 @@ impl TryFrom<&HashMap> for ImageParams { } } +#[cfg(feature = "image")] impl ToString for ImageParams { fn to_string(&self) -> String { let mut params = String::new(); diff --git a/libs/jwst-core-storage/src/storage/mod.rs b/libs/jwst-core-storage/src/storage/mod.rs index 8bc6fdb0f..730fab4c9 100644 --- a/libs/jwst-core-storage/src/storage/mod.rs +++ b/libs/jwst-core-storage/src/storage/mod.rs @@ -5,7 +5,9 @@ mod test; use std::{collections::HashMap, time::Instant}; +#[cfg(feature = "image")] use blobs::BlobAutoStorage; +use blobs::BlobDBStorage; use docs::SharedDocDBStorage; use jwst_storage_migration::{Migrator, MigratorTrait}; use tokio::sync::Mutex; @@ -13,7 +15,7 @@ use tokio::sync::Mutex; use self::difflog::DiffLogRecord; use super::*; use crate::{ - storage::blobs::{BlobBucketDBStorage, BlobStorageType, JwstBlobStorage}, + storage::blobs::{BlobBucketStorage, BlobStorageType, JwstBlobStorage}, types::JwstStorageError, }; @@ -37,10 +39,18 @@ impl JwstStorage { let blobs = match blob_storage_type { BlobStorageType::DB => { - JwstBlobStorage::DB(BlobAutoStorage::init_with_pool(pool.clone(), bucket.clone()).await?) + #[cfg(feature = "image")] + { + JwstBlobStorage::AutoStorage(BlobAutoStorage::init_with_pool(pool.clone(), bucket.clone()).await?) + } + #[cfg(not(feature = "image"))] + { + let db = BlobDBStorage::init_with_pool(pool.clone(), bucket.clone()).await?; + JwstBlobStorage::RawStorage(Arc::new(db)) + } } - BlobStorageType::MixedBucketDB(param) => JwstBlobStorage::MixedBucketDB( - BlobBucketDBStorage::init_with_pool(pool.clone(), bucket.clone(), Some(param.try_into()?)).await?, + BlobStorageType::MixedBucketDB(param) => JwstBlobStorage::BucketStorage( + BlobBucketStorage::init_with_pool(pool.clone(), bucket.clone(), Some(param.try_into()?)).await?, ), }; let docs = SharedDocDBStorage::init_with_pool(pool.clone(), bucket.clone()).await?; From d6c70de8f6134f5a386e7dcec18311741ee98fd9 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Tue, 22 Aug 2023 00:47:44 +0800 Subject: [PATCH 12/49] chore: bump deps --- Cargo.lock | 1151 ++++++++++++++++------------- libs/jwst-core-storage/Cargo.toml | 18 +- 2 files changed, 646 insertions(+), 523 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26f0762d9..b8f227b15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,6 +12,15 @@ dependencies = [ "regex", ] +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -78,9 +87,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ "cfg-if 1.0.0", "cipher 0.4.4", @@ -94,7 +103,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "209b47e8954a928e1d72e86eca7000ebb6655fe1436d33eefc2201cad027e237" dependencies = [ "aead 0.5.2", - "aes 0.8.2", + "aes 0.8.3", "cipher 0.4.4", "ctr 0.9.2", "ghash", @@ -198,9 +207,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" dependencies = [ "memchr", ] @@ -213,9 +222,9 @@ checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" [[package]] name = "allocator-api2" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" [[package]] name = "android-tzdata" @@ -225,15 +234,15 @@ checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" [[package]] name = "android_log-sys" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f0fc03f560e1aebde41c2398b691cb98b5ea5996a6184a7a67bbbb77448969" +checksum = "5ecc8056bf6ab9892dcd53216c83d1597487d7dacac16c8df6b877d127df9937" [[package]] name = "android_logger" -version = "0.13.1" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fa490e751f3878eb9accb9f18988eca52c2337ce000a8bf31ef50d4c723ca9e" +checksum = "c494134f746c14dc653a35a4ea5aca24ac368529da5370ecf41fe0341c35772f" dependencies = [ "android_log-sys", "env_logger", @@ -273,9 +282,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" [[package]] name = "anstyle-parse" @@ -297,9 +306,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -307,9 +316,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "arbitrary" @@ -328,9 +337,9 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "arrayvec" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8868f09ff8cea88b079da74ae569d9b8c62a23c68c746240b704ee6f7525c89c" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "asn1-rs" @@ -345,7 +354,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.22", + "time 0.3.25", ] [[package]] @@ -361,7 +370,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.22", + "time 0.3.25", ] [[package]] @@ -371,7 +380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db8b7511298d5b7784b40b092d9e9dcd3a627a5707e4b5e507931ab0d44eeebf" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", "synstructure", ] @@ -383,7 +392,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", "synstructure", ] @@ -395,7 +404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -440,19 +449,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -472,9 +481,9 @@ checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" [[package]] name = "atomic_refcell" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79d6dc922a2792b006573f60b2648076355daeae5ce9cb59507e5908c9625d31" +checksum = "112ef6b3f6cb3cb6fc5b6b494ef7a848492cff1ab0ef4de10b0f7d572861c905" [[package]] name = "autocfg" @@ -484,9 +493,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", @@ -512,7 +521,7 @@ dependencies = [ "sha1", "sync_wrapper", "tokio", - "tokio-tungstenite 0.18.0", + "tokio-tungstenite", "tower", "tower-layer", "tower-service", @@ -559,12 +568,27 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" dependencies = [ - "fastrand", + "fastrand 1.9.0", "futures-core", "pin-project", "tokio", ] +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base16ct" version = "0.1.1" @@ -577,12 +601,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "base64" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" - [[package]] name = "base64" version = "0.21.2" @@ -727,7 +745,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -738,15 +756,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] [[package]] name = "bstr" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a246e68bb43f6cd9db24bea052a53e40405417c5fb372e3d1a8a7f770a564ef5" +checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" dependencies = [ "memchr", "serde", @@ -776,7 +794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -817,11 +835,12 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -935,9 +954,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.19" +version = "4.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d" +checksum = "03aef18ddf7d879c15ce20f04826ef8418101c7e528014c3eeea13321047dca3" dependencies = [ "clap_builder", "clap_derive", @@ -946,9 +965,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.19" +version = "4.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1" +checksum = "f8ce6fffb678c9b80a70b6b6de0aad31df727623a70fd9a842c30cd573e2fa98" dependencies = [ "anstream", "anstyle", @@ -964,8 +983,8 @@ checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" dependencies = [ "heck", "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -1020,7 +1039,7 @@ dependencies = [ "url-escape", "utoipa", "utoipa-swagger-ui", - "x509-parser 0.15.0", + "x509-parser 0.15.1", ] [[package]] @@ -1059,9 +1078,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.2" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" +checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" [[package]] name = "const-random" @@ -1112,9 +1131,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -1294,18 +1313,31 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.0.0-rc.2" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d928d978dbec61a1167414f5ec534f24bea0d7a0d24dd9b6233d3d8223e585" +checksum = "f711ade317dd348950a9910f81c5947e3d8907ebd2b83f76203ff1807e6a2bc2" dependencies = [ "cfg-if 1.0.0", + "cpufeatures", + "curve25519-dalek-derive", "fiat-crypto", - "packed_simd_2", "platforms", + "rustc_version", "subtle", "zeroize", ] +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", +] + [[package]] name = "darling" version = "0.14.4" @@ -1325,7 +1357,7 @@ dependencies = [ "fnv", "ident_case", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "strsim", "syn 1.0.109", ] @@ -1337,7 +1369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ "darling_core", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -1373,9 +1405,9 @@ dependencies = [ [[package]] name = "der" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56acb310e15652100da43d130af8d97b509e95af61aab1c5a7939ef24337ee17" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" dependencies = [ "const-oid", "pem-rfc7468 0.7.0", @@ -1410,6 +1442,15 @@ dependencies = [ "rusticata-macros", ] +[[package]] +name = "deranged" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +dependencies = [ + "serde", +] + [[package]] name = "derivative" version = "2.2.0" @@ -1417,7 +1458,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -1428,7 +1469,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -1439,8 +1480,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -1460,7 +1501,7 @@ checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ "darling", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -1522,8 +1563,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -1558,9 +1599,9 @@ checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" [[package]] name = "dyn-clone" -version = "1.0.11" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" +checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" [[package]] name = "ecdsa" @@ -1576,9 +1617,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" dependencies = [ "serde", ] @@ -1652,8 +1693,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -1674,9 +1715,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", @@ -1712,9 +1753,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "exr" -version = "1.6.4" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279d3efcc55e19917fff7ab3ddd6c14afb6a90881a0078465196fe2f99d08c56" +checksum = "d1e481eb11a482815d3e9d618db8c42a93207134662873809335a92327440c18" dependencies = [ "bit_field", "flume", @@ -1766,6 +1807,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + [[package]] name = "fdeflate" version = "0.3.0" @@ -1814,7 +1861,7 @@ dependencies = [ "log", "petgraph", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "rustc-hash", "smallvec", "smol_str 0.1.24", @@ -1825,9 +1872,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" dependencies = [ "crc32fast", "miniz_oxide", @@ -1943,8 +1990,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -1994,9 +2041,9 @@ dependencies = [ [[package]] name = "generator" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3e123d9ae7c02966b4d892e550bdc32164f05853cd40ab570650ad600596a8a" +checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" dependencies = [ "cc", "libc", @@ -2049,7 +2096,7 @@ checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" dependencies = [ "proc-macro-error", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -2073,6 +2120,12 @@ dependencies = [ "weezl", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "git2" version = "0.16.1" @@ -2094,11 +2147,11 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "globset" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc" +checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" dependencies = [ - "aho-corasick 0.7.20", + "aho-corasick 1.0.4", "bstr", "fnv", "log", @@ -2118,7 +2171,25 @@ dependencies = [ "no-std-compat", "nonzero_ext", "parking_lot", - "quanta", + "quanta 0.9.3", + "rand 0.8.5", + "smallvec", +] + +[[package]] +name = "governor" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "821239e5672ff23e2a7060901fa622950bbd80b649cdaadd78d1c1767ed14eb4" +dependencies = [ + "cfg-if 1.0.0", + "dashmap", + "futures", + "futures-timer", + "no-std-compat", + "nonzero_ext", + "parking_lot", + "quanta 0.11.1", "rand 0.8.5", "smallvec", ] @@ -2136,9 +2207,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ "bytes", "fnv", @@ -2255,18 +2326,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hex" @@ -2331,9 +2393,9 @@ dependencies = [ [[package]] name = "http-range-header" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" [[package]] name = "httparse" @@ -2343,15 +2405,15 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -2364,7 +2426,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -2373,13 +2435,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" dependencies = [ + "futures-util", "http", "hyper", - "rustls 0.21.2", + "rustls 0.21.6", "tokio", "tokio-rustls", ] @@ -2435,9 +2498,9 @@ dependencies = [ [[package]] name = "image" -version = "0.24.6" +version = "0.24.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527909aa81e20ac3a44803521443a765550f09b5130c2c2fa1ea59c2f8f50a3a" +checksum = "6f3dfdbdd72063086ff443e297b61695500514b1e41095b6fb9a5ab48a70a711" dependencies = [ "bytemuck", "byteorder", @@ -2455,9 +2518,9 @@ dependencies = [ [[package]] name = "include-flate" -version = "0.1.4" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfdcb449c721557c1cf89bbd3412bf33fa963289e26e9badbd824a960912e148" +checksum = "c2e11569346406931d20276cc460215ee2826e7cad43aa986999cb244dd7adb0" dependencies = [ "include-flate-codegen-exports", "lazy_static", @@ -2473,7 +2536,7 @@ dependencies = [ "libflate", "proc-macro-hack", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -2495,7 +2558,6 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", - "serde", ] [[package]] @@ -2506,13 +2568,14 @@ checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", "hashbrown 0.14.0", + "serde", ] [[package]] name = "indicatif" -version = "0.17.5" +version = "0.17.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ff8cc23a7393a397ed1d7f56e6365cba772aba9f9912ab968b03043c395d057" +checksum = "0b297dc40733f23a0e52728a58fa9489a5b7638a324932de16b41adc3ef80730" dependencies = [ "console", "instant", @@ -2523,13 +2586,13 @@ dependencies = [ [[package]] name = "inherent" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3822df42143878e35e37b1d1511f911db64a1c5b07a7130569db8dc59a8805" +checksum = "ce243b1bfa62ffc028f1cc3b6034ec63d649f3031bc8a4fbbb004e1ac17d1f68" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -2572,31 +2635,19 @@ dependencies = [ "webrtc-util", ] -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "ipnet" -version = "2.7.2" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "is-terminal" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", + "hermit-abi", "rustix", "windows-sys 0.48.0", ] @@ -2612,19 +2663,19 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jieba-rs" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37228e06c75842d1097432d94d02f37fe3ebfca9791c2e8fef6e9db17ed128c1" +checksum = "93f0c1347cd3ac8d7c6e3a2dc33ac496d365cf09fc0831aa61111e1a6738983e" dependencies = [ "cedarwood", "fxhash", - "hashbrown 0.12.3", + "hashbrown 0.14.0", "lazy_static", "phf", "phf_codegen", @@ -2717,7 +2768,7 @@ dependencies = [ "criterion", "jwst-logger", "lib0", - "loom 0.6.0", + "loom 0.6.1", "nanoid", "nom", "ordered-float", @@ -2787,7 +2838,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", - "tokio-tungstenite 0.20.0", + "tokio-tungstenite", "url", "webrtc", ] @@ -2802,13 +2853,13 @@ dependencies = [ "chrono", "dotenvy", "futures", - "governor", + "governor 0.6.0", "image", "jwst-codec", "jwst-core", "jwst-core-storage-migration", "jwst-logger", - "opendal", + "opendal 0.39.0", "path-ext", "rand 0.8.5", "sea-orm", @@ -2894,7 +2945,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", - "tokio-tungstenite 0.20.0", + "tokio-tungstenite", "url", "webrtc", "yrs", @@ -2910,14 +2961,14 @@ dependencies = [ "chrono", "dotenvy", "futures", - "governor", + "governor 0.5.1", "image", "jwst", "jwst-codec", "jwst-logger", "jwst-storage-migration", "lib0", - "opendal", + "opendal 0.38.1", "path-ext", "rand 0.8.5", "sea-orm", @@ -2991,7 +3042,7 @@ dependencies = [ "serde_json", "sqlx", "thiserror", - "time 0.3.22", + "time 0.3.25", "tokio", "tower", "tower-http", @@ -3025,7 +3076,7 @@ dependencies = [ "base64 0.21.2", "email-encoding", "email_address", - "fastrand", + "fastrand 1.9.0", "futures-io", "futures-util", "httpdate", @@ -3034,9 +3085,9 @@ dependencies = [ "nom", "once_cell", "quoted_printable", - "rustls 0.21.2", + "rustls 0.21.6", "rustls-pemfile", - "socket2", + "socket2 0.4.9", "tokio", "tokio-rustls", "webpki-roots 0.23.1", @@ -3060,9 +3111,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libflate" @@ -3096,12 +3147,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "libm" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" - [[package]] name = "libm" version = "0.2.7" @@ -3131,18 +3176,19 @@ dependencies = [ [[package]] name = "libwebp-sys" -version = "0.4.2" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439fd1885aa28937e7edcd68d2e793cb4a22f8733460d2519fbafd2b215672bf" +checksum = "a5df1e76f0acef0058aa2164ccf74e610e716e7f9eeb3ee2283de7d43659d823" dependencies = [ "cc", + "glob", ] [[package]] name = "libz-sys" -version = "1.1.9" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" dependencies = [ "cc", "libc", @@ -3152,9 +3198,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "lock_api" @@ -3168,9 +3214,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "log-panics" @@ -3197,9 +3243,9 @@ dependencies = [ [[package]] name = "loom" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27537a24ea22f4818f17260442099a6ed6e21c445ece117aa1873688c399cb79" +checksum = "ce9394216e2be01e607cf9e9e2b64c387506df1e768b14cbd2854a3650c3c03e" dependencies = [ "cfg-if 1.0.0", "generator", @@ -3240,20 +3286,29 @@ dependencies = [ "libc", ] +[[package]] +name = "mach2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +dependencies = [ + "libc", +] + [[package]] name = "matchers" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] name = "matchit" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" [[package]] name = "md-5" @@ -3443,13 +3498,13 @@ dependencies = [ [[package]] name = "num-bigint-dig" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2399c9463abc5f909349d8aa9ba080e0b88b3ce2885389b60b993f39b1a56905" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" dependencies = [ "byteorder", "lazy_static", - "libm 0.2.7", + "libm", "num-integer", "num-iter", "num-traits", @@ -3492,21 +3547,21 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", - "libm 0.2.7", + "libm", ] [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] @@ -3516,6 +3571,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "oid-registry" version = "0.4.0" @@ -3563,9 +3627,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "opendal" -version = "0.38.0" +version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "717f47be1760a6a651f81eeba8239444a077d0d229409a016298d2b2483c408c" +checksum = "4df645b6012162c04c8949e9b96ae2ef002e79189cfb154e507e51ac5be76a09" dependencies = [ "anyhow", "async-compat", @@ -3585,7 +3649,39 @@ dependencies = [ "percent-encoding", "pin-project", "quick-xml 0.27.1", - "reqsign", + "reqsign 0.13.0", + "reqwest", + "serde", + "serde_json", + "tokio", + "uuid", +] + +[[package]] +name = "opendal" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad95e460e5976ab1b74f398ab856c59f8417b3dd32202329e3491dcbe3a6b84" +dependencies = [ + "anyhow", + "async-compat", + "async-trait", + "backon", + "base64 0.21.2", + "bytes", + "chrono", + "flagset", + "futures", + "http", + "hyper", + "log", + "md-5", + "once_cell", + "parking_lot", + "percent-encoding", + "pin-project", + "quick-xml 0.29.0", + "reqsign 0.14.1", "reqwest", "serde", "serde_json", @@ -3601,9 +3697,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "ordered-float" -version = "3.7.0" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc2dbde8f8a79f2102cc474ceb0ad68e3b80b85289ea62389b60e66777e4213" +checksum = "126d3e6f3926bfb0fb24495b4f4da50626f547e54956594748e3d8882a0320b4" dependencies = [ "arbitrary", "num-traits", @@ -3640,8 +3736,8 @@ dependencies = [ "heck", "proc-macro-error", "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -3681,16 +3777,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "packed_simd_2" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1914cd452d8fccd6f9db48147b29fd4ae05bea9dc5d9ad578509f72415de282" -dependencies = [ - "cfg-if 1.0.0", - "libm 0.1.4", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -3711,14 +3797,14 @@ dependencies = [ "libc", "redox_syscall 0.3.5", "smallvec", - "windows-targets 0.48.0", + "windows-targets 0.48.5", ] [[package]] name = "paste" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "path-ext" @@ -3764,9 +3850,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.6.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e68e84bfb01f0507134eac1e9b410a12ba379d064eab48c50ba4ce329a527b70" +checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" dependencies = [ "thiserror", "ucd-trie", @@ -3774,9 +3860,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.6.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b79d4c71c865a25a4322296122e3924d30bc8ee0834c8bfc8b95f7f054afbfb" +checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" dependencies = [ "pest", "pest_generator", @@ -3784,22 +3870,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.6.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" +checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" dependencies = [ "pest", "pest_meta", "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] name = "pest_meta" -version = "2.6.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745a452f8eb71e39ffd8ee32b3c5f51d03845f99786fa9b68db6ff509c505411" +checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" dependencies = [ "once_cell", "pest", @@ -3808,19 +3894,19 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 1.9.3", + "indexmap 2.0.0", ] [[package]] name = "phf" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_macros", "phf_shared", @@ -3828,9 +3914,9 @@ dependencies = [ [[package]] name = "phf_codegen" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" dependencies = [ "phf_generator", "phf_shared", @@ -3838,9 +3924,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", "rand 0.8.5", @@ -3848,51 +3934,51 @@ dependencies = [ [[package]] name = "phf_macros" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92aacdc5f16768709a569e913f7451034034178b05bdc8acda226659a3dccc66" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ "phf_generator", "phf_shared", "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 1.0.109", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", ] [[package]] name = "pin-project" -version = "1.1.0" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.0" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" [[package]] name = "pin-utils" @@ -3906,7 +3992,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.7.6", + "der 0.7.8", "pkcs8 0.10.2", "spki 0.7.2", ] @@ -3927,7 +4013,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.6", + "der 0.7.8", "spki 0.7.2", ] @@ -3945,9 +4031,9 @@ checksum = "e3d7ddaed09e0eb771a79ab0fd64609ba0afb0a8366421957936ad14cbd13630" [[package]] name = "plotters" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" dependencies = [ "num-traits", "plotters-backend", @@ -3958,24 +4044,24 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" [[package]] name = "plotters-svg" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" dependencies = [ "plotters-backend", ] [[package]] name = "png" -version = "0.17.9" +version = "0.17.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59871cc5b6cce7eaccca5a802b4173377a1c2ba90654246789a8fa2334426d11" +checksum = "dd75bf2d8dd3702b9707cdbc56a5b9ef42cec752eb8b3bafc01234558442aa64" dependencies = [ "bitflags 1.3.2", "crc32fast", @@ -3986,9 +4072,9 @@ dependencies = [ [[package]] name = "polyval" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef234e08c11dfcb2e56f79fd70f6f2eb7f025c0ce2333e82f4f0518ecad30c6" +checksum = "d52cff9d1d4dee5fe6d03729099f4a310a41179e0a10dbf542039873f2e826fb" dependencies = [ "cfg-if 1.0.0", "cpufeatures", @@ -3998,9 +4084,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.3.3" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "767eb9f07d4a5ebcb39bbf2d452058a93c011373abf6832e24194a1c3f004794" +checksum = "f32154ba0af3a075eefa1eda8bb414ee928f62303a54ea85b8d6638ff1a6ee9e" [[package]] name = "ppv-lite86" @@ -4025,7 +4111,7 @@ checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", "version_check", ] @@ -4037,7 +4123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "version_check", ] @@ -4112,7 +4198,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -4141,6 +4227,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "quanta" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +dependencies = [ + "crossbeam-utils", + "libc", + "mach2", + "once_cell", + "raw-cpuid", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -4167,6 +4269,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "0.6.13" @@ -4178,9 +4290,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2 1.0.66", ] @@ -4316,7 +4428,7 @@ checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", "ring", - "time 0.3.22", + "time 0.3.25", "x509-parser 0.14.0", "yasna", ] @@ -4352,13 +4464,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.4" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ - "aho-corasick 1.0.2", + "aho-corasick 1.0.4", "memchr", - "regex-syntax 0.7.2", + "regex-automata 0.3.6", + "regex-syntax 0.7.4", ] [[package]] @@ -4370,6 +4483,17 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +dependencies = [ + "aho-corasick 1.0.4", + "memchr", + "regex-syntax 0.7.4", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -4378,9 +4502,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "rend" @@ -4420,6 +4544,35 @@ dependencies = [ "sha2", ] +[[package]] +name = "reqsign" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3228e570df74d69d3d3236a71371f1edd748a3e4eb728ea1f29d403bc10fc727" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.21.2", + "chrono", + "form_urlencoded", + "hex", + "hmac", + "home", + "http", + "log", + "once_cell", + "percent-encoding", + "quick-xml 0.29.0", + "rand 0.8.5", + "reqwest", + "rsa", + "rust-ini", + "serde", + "serde_json", + "sha1", + "sha2", +] + [[package]] name = "reqwest" version = "0.11.18" @@ -4444,7 +4597,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.2", + "rustls 0.21.6", "rustls-native-certs", "rustls-pemfile", "serde", @@ -4492,7 +4645,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3df05e0adb96c94ddb287013558ba7ff67f097219f6afa3c789506472e71272" dependencies = [ - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -4535,7 +4688,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -4593,9 +4746,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "6.7.0" +version = "6.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73e721f488c353141288f223b599b4ae9303ecf3e62923f40a492f0634a4dc3" +checksum = "a36224c3276f8c4ebc8c20f158eca7ca4359c8db89991c4925132aaaf6702661" dependencies = [ "include-flate", "rust-embed-impl", @@ -4605,23 +4758,23 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "6.6.0" +version = "6.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22ce362f5561923889196595504317a4372b84210e6e335da529a65ea5452b5" +checksum = "49b94b81e5b2c284684141a2fb9e2a31be90638caf040bf9afbc5a0416afe1ac" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "rust-embed-utils", "shellexpand", - "syn 2.0.28", + "syn 2.0.29", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "7.5.0" +version = "7.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512b0ab6853f7e14e3c8754acb43d6f748bb9ced66aa5915a6553ac8213f7731" +checksum = "9d38ff6bf570dc3bb7100fce9f7b60c33fa71d80e88da3f2580df4ff2bdded74" dependencies = [ "globset", "mime_guess", @@ -4651,13 +4804,12 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.30.0" +version = "1.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0446843641c69436765a35a5a77088e28c2e6a12da93e84aa3ab1cd4aa5a042" +checksum = "4a2ab0025103a60ecaaf3abf24db1db240a4e1c15837090d2c32f625ac98abea" dependencies = [ "arrayvec", "borsh", - "bytecheck", "byteorder", "bytes", "num-traits", @@ -4667,6 +4819,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -4693,13 +4851,12 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.20" +version = "0.38.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" +checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "errno", - "io-lifetimes", "libc", "linux-raw-sys", "windows-sys 0.48.0", @@ -4720,13 +4877,13 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.2" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" +checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" dependencies = [ "log", "ring", - "rustls-webpki 0.100.1", + "rustls-webpki 0.101.3", "sct 0.7.0", ] @@ -4744,9 +4901,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ "base64 0.21.2", ] @@ -4773,9 +4930,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "rusty-fork" @@ -4791,9 +4948,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -4806,11 +4963,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.42.0", + "windows-sys 0.48.0", ] [[package]] @@ -4832,7 +4989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "serde_derive_internals", "syn 1.0.109", ] @@ -4845,9 +5002,9 @@ checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -4890,8 +5047,8 @@ dependencies = [ "heck", "proc-macro-error", "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -4916,7 +5073,7 @@ dependencies = [ "sqlx", "strum 0.25.0", "thiserror", - "time 0.3.22", + "time 0.3.25", "tracing", "url", "uuid", @@ -4947,9 +5104,9 @@ checksum = "cd90e73d5f5b184bad525767da29fbfec132b4e62ebd6f60d2f2737ec6468f62" dependencies = [ "heck", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "sea-bae", - "syn 2.0.28", + "syn 2.0.29", "unicode-ident", ] @@ -4984,7 +5141,7 @@ dependencies = [ "rust_decimal", "sea-query-derive", "serde_json", - "time 0.3.22", + "time 0.3.25", "uuid", ] @@ -5000,7 +5157,7 @@ dependencies = [ "sea-query", "serde_json", "sqlx", - "time 0.3.22", + "time 0.3.25", "uuid", ] @@ -5012,7 +5169,7 @@ checksum = "bd78f2e0ee8e537e9195d1049b752e0433e2cac125426bccb7b5c3e508096117" dependencies = [ "heck", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", "thiserror", ] @@ -5036,7 +5193,7 @@ checksum = "c6f686050f76bffc4f635cda8aea6df5548666b830b52387e8bc7de11056d11e" dependencies = [ "heck", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -5062,9 +5219,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -5075,9 +5232,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" dependencies = [ "core-foundation-sys", "libc", @@ -5085,28 +5242,28 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.183" +version = "1.0.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "dc59dfdcbad1437773485e0367fea4b090a2e0a16d9ffc46af47764536a298ec" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -5116,7 +5273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] @@ -5133,22 +5290,23 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" +checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" dependencies = [ + "itoa", "serde", ] [[package]] name = "serde_repr" -version = "0.1.12" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" +checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -5176,9 +5334,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if 1.0.0", "cpufeatures", @@ -5234,9 +5392,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238abfbb77c1915110ad968465608b68e869e0772622c9656714e73e5a1a522f" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "simdutf8" @@ -5253,7 +5411,7 @@ dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.22", + "time 0.3.25", ] [[package]] @@ -5282,9 +5440,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "smol_str" @@ -5314,6 +5472,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "spin" version = "0.5.2" @@ -5346,7 +5514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", - "der 0.7.6", + "der 0.7.8", ] [[package]] @@ -5404,7 +5572,7 @@ dependencies = [ "paste", "percent-encoding", "rust_decimal", - "rustls 0.21.2", + "rustls 0.21.6", "rustls-pemfile", "serde", "serde_json", @@ -5412,7 +5580,7 @@ dependencies = [ "smallvec", "sqlformat", "thiserror", - "time 0.3.22", + "time 0.3.25", "tokio", "tokio-stream", "tracing", @@ -5428,7 +5596,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "sqlx-core", "sqlx-macros-core", "syn 1.0.109", @@ -5446,7 +5614,7 @@ dependencies = [ "hex", "once_cell", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "serde", "serde_json", "sha2", @@ -5501,7 +5669,7 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", - "time 0.3.22", + "time 0.3.25", "tracing", "uuid", "whoami", @@ -5546,7 +5714,7 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", - "time 0.3.22", + "time 0.3.25", "tracing", "uuid", "whoami", @@ -5571,7 +5739,7 @@ dependencies = [ "percent-encoding", "serde", "sqlx-core", - "time 0.3.22", + "time 0.3.25", "tracing", "url", "uuid", @@ -5591,9 +5759,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "stringprep" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -5628,7 +5796,7 @@ checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "rustversion", "syn 1.0.109", ] @@ -5663,15 +5831,15 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "swift-bridge" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa07c7cd2b2d7ca48d96f5abd159e3fd3eee3457e7bd03adc1994bfbdabd2f" +checksum = "f72aa49fc376893673e6b7ace589802877d2dad554da1fde5aa1445498e1929c" dependencies = [ "swift-bridge-build", "swift-bridge-macro", @@ -5679,9 +5847,9 @@ dependencies = [ [[package]] name = "swift-bridge-build" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "286f727dc922736a1ed74c06bebf43d08b8295a7ba38c77326c74e2b9dfd43df" +checksum = "ba4282dc94afa1b7d8f2445532a7c0b7508d470cee261132e26ec001b3185c3b" dependencies = [ "proc-macro2 1.0.66", "swift-bridge-ir", @@ -5691,23 +5859,23 @@ dependencies = [ [[package]] name = "swift-bridge-ir" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4de97e9abde20abc1c01f6d4faa8072d723c73aba288264481a83a1e2787dc" +checksum = "d2096a82fb42280699d9b5dde6674cb4f802931eca8623f1edc4587c50dc58da" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", ] [[package]] name = "swift-bridge-macro" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64fabad38a0fc643ceeafefed79e08408c30eeec325b629e426a15b13d055d0a" +checksum = "6c2f4d5b41976c351ecb735eb9336bdd3cbaaebc9990300023ea2b6874a2d68c" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "swift-bridge-ir", "syn 1.0.109", ] @@ -5730,18 +5898,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.28" +version = "2.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "unicode-ident", ] @@ -5758,7 +5926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.33", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -5810,7 +5978,7 @@ dependencies = [ "tantivy-query-grammar", "tempfile", "thiserror", - "time 0.3.22", + "time 0.3.25", "uuid", "winapi", ] @@ -5861,13 +6029,12 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.6.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ - "autocfg", "cfg-if 1.0.0", - "fastrand", + "fastrand 2.0.0", "redox_syscall 0.3.5", "rustix", "windows-sys 0.48.0", @@ -5875,22 +6042,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -5905,9 +6072,9 @@ dependencies = [ [[package]] name = "tiff" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7449334f9ff2baf290d55d73983a7d6fa15e01198faef72af07e2a8db851e471" +checksum = "6d172b0f4d3fba17ba89811858b9d3d97f928aece846475bbda076ca46736211" dependencies = [ "flate2", "jpeg-decoder", @@ -5927,10 +6094,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.22" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" +checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" dependencies = [ + "deranged", "itoa", "serde", "time-core", @@ -5945,9 +6113,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" dependencies = [ "time-core", ] @@ -5988,11 +6156,11 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.2" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", "mio", @@ -6000,7 +6168,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.3", "tokio-macros", "windows-sys 0.48.0", ] @@ -6012,8 +6180,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -6022,7 +6190,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.2", + "rustls 0.21.6", "tokio", ] @@ -6037,18 +6205,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.18.0", -] - [[package]] name = "tokio-tungstenite" version = "0.20.0" @@ -6057,10 +6213,10 @@ checksum = "2b2dbec703c26b00d74844519606ef15d09a7d6857860f84ad223dec002ddea2" dependencies = [ "futures-util", "log", - "rustls 0.21.2", + "rustls 0.21.6", "tokio", "tokio-rustls", - "tungstenite 0.20.0", + "tungstenite", "webpki-roots 0.23.1", ] @@ -6105,12 +6261,12 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.4.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" +checksum = "55ae70283aba8d2a8b411c695c437fe25b8b5e44e23e780662002fc72fb47a82" dependencies = [ - "base64 0.20.0", - "bitflags 1.3.2", + "base64 0.21.2", + "bitflags 2.4.0", "bytes", "futures-core", "futures-util", @@ -6152,13 +6308,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -6202,7 +6358,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "time 0.3.22", + "time 0.3.25", "tracing-core", "tracing-subscriber", ] @@ -6234,25 +6390,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" -[[package]] -name = "tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" -dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes", - "http", - "httparse", - "log", - "rand 0.8.5", - "sha1", - "thiserror", - "url", - "utf-8", -] - [[package]] name = "tungstenite" version = "0.20.0" @@ -6266,7 +6403,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls 0.21.2", + "rustls 0.21.6", "sha1", "thiserror", "url", @@ -6309,9 +6446,9 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "ucd-trie" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] name = "unarray" @@ -6336,9 +6473,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -6435,11 +6572,11 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utoipa" -version = "3.3.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ae74ef183fae36d650f063ae7bde1cacbe1cd7e72b617cbe1e985551878b98" +checksum = "d82b1bc5417102a73e8464c686eef947bdfb99fcdfc0a4f228e81afa9526470a" dependencies = [ - "indexmap 1.9.3", + "indexmap 2.0.0", "serde", "serde_json", "utoipa-gen", @@ -6447,21 +6584,22 @@ dependencies = [ [[package]] name = "utoipa-gen" -version = "3.3.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ea8ac818da7e746a63285594cce8a96f5e00ee31994e655bd827569cb8b137b" +checksum = "05d96dcd6fc96f3df9b3280ef480770af1b7c5d14bc55192baa9b067976d920c" dependencies = [ "proc-macro-error", "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "regex", + "syn 2.0.29", ] [[package]] name = "utoipa-swagger-ui" -version = "3.1.3" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062bba5a3568e126ac72049a63254f4cb1da2eb713db0c1ab2a4c76be191db8c" +checksum = "84614caa239fb25b2bb373a52859ffd94605ceb256eeb1d63436325cf81e3653" dependencies = [ "axum", "mime_guess", @@ -6475,9 +6613,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.3.4" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa2982af2eec27de306107c027578ff7f423d65f7250e40ce0fea8f45248b81" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ "getrandom 0.2.10", "serde", @@ -6509,7 +6647,7 @@ dependencies = [ "rustc_version", "rustversion", "thiserror", - "time 0.3.22", + "time 0.3.25", ] [[package]] @@ -6593,8 +6731,8 @@ dependencies = [ "log", "once_cell", "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", "wasm-bindgen-shared", ] @@ -6616,7 +6754,7 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ - "quote 1.0.28", + "quote 1.0.33", "wasm-bindgen-macro-support", ] @@ -6627,8 +6765,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6664,9 +6802,9 @@ dependencies = [ [[package]] name = "webp" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf022f821f166079a407d000ab57e84de020e66ffbbf4edde999bc7d6e371cae" +checksum = "12ff0ebb440d1db63b778cb609db8a8abfda825a7841664a76a70b628502c7e1" dependencies = [ "libwebp-sys", ] @@ -6746,7 +6884,7 @@ dependencies = [ "smol_str 0.2.0", "stun", "thiserror", - "time 0.3.22", + "time 0.3.25", "tokio", "turn", "url", @@ -6847,7 +6985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f08dfd7a6e3987e255c4dbe710dde5d94d0f0574f8a21afa95d171376c143106" dependencies = [ "log", - "socket2", + "socket2 0.4.9", "thiserror", "tokio", "webrtc-util", @@ -6946,9 +7084,9 @@ dependencies = [ [[package]] name = "whoami" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c70234412ca409cc04e864e89523cb0fc37f5e1344ebed5a3ebf4192b6b9f68" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" [[package]] name = "winapi" @@ -6987,22 +7125,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets 0.48.5", ] [[package]] @@ -7020,7 +7143,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.5", ] [[package]] @@ -7040,17 +7163,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -7061,9 +7184,9 @@ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" @@ -7073,9 +7196,9 @@ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" @@ -7085,9 +7208,9 @@ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" @@ -7097,9 +7220,9 @@ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" @@ -7109,9 +7232,9 @@ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" @@ -7121,9 +7244,9 @@ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" @@ -7133,9 +7256,9 @@ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winreg" @@ -7157,11 +7280,11 @@ dependencies = [ [[package]] name = "x25519-dalek" -version = "2.0.0-rc.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabd6e16dd08033932fc3265ad4510cc2eab24656058a6dcb107ffe274abcc95" +checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96" dependencies = [ - "curve25519-dalek 4.0.0-rc.2", + "curve25519-dalek 4.0.0", "rand_core 0.6.4", "serde", "zeroize", @@ -7182,7 +7305,7 @@ dependencies = [ "oid-registry 0.4.0", "rusticata-macros", "thiserror", - "time 0.3.22", + "time 0.3.25", ] [[package]] @@ -7201,14 +7324,14 @@ dependencies = [ "ring", "rusticata-macros", "thiserror", - "time 0.3.22", + "time 0.3.25", ] [[package]] name = "x509-parser" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab0c2f54ae1d92f4fcb99c0b7ccf0b1e3451cbd395e5f115ccbdbcb18d4f634" +checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" dependencies = [ "asn1-rs 0.5.2", "data-encoding", @@ -7218,7 +7341,7 @@ dependencies = [ "oid-registry 0.6.1", "rusticata-macros", "thiserror", - "time 0.3.22", + "time 0.3.25", ] [[package]] @@ -7237,7 +7360,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.22", + "time 0.3.25", ] [[package]] @@ -7269,8 +7392,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] diff --git a/libs/jwst-core-storage/Cargo.toml b/libs/jwst-core-storage/Cargo.toml index e3a82ba57..6a6429cfb 100644 --- a/libs/jwst-core-storage/Cargo.toml +++ b/libs/jwst-core-storage/Cargo.toml @@ -13,21 +13,21 @@ postgres = ["sea-orm/sqlx-postgres"] sqlite = ["sea-orm/sqlx-sqlite"] [dependencies] -anyhow = "1.0.70" -async-trait = "0.1.68" +anyhow = "1.0.75" +async-trait = "0.1.73" bytes = "1.4.0" chrono = { version = "0.4.24", features = ["serde"] } futures = "0.3.28" -governor = "0.5.1" +governor = "0.6.0" path-ext = "0.1.0" -sha2 = "0.10.6" +sha2 = "0.10.7" sea-orm = { version = "0.12.2", features = ["runtime-tokio-rustls", "macros"] } sea-orm-migration = "0.12.2" -thiserror = "1.0.40" -tokio = { version = "1.27.0", features = ["fs", "macros", "sync"] } -tokio-util = { version = "0.7.7", features = ["io"] } -url = "2.3.1" -opendal = { version = "0.38.0", default-features = false, features = [ +thiserror = "1.0.47" +tokio = { version = "1.32.0", features = ["fs", "macros", "sync"] } +tokio-util = { version = "0.7.8", features = ["io"] } +url = "2.4.0" +opendal = { version = "0.39.0", default-features = false, features = [ "rustls", "services-s3", ] } From f39620d90842a61cea0c343fd7e0397b68d4cec9 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Tue, 22 Aug 2023 00:57:53 +0800 Subject: [PATCH 13/49] chore: update headers --- .../Headers/SwiftBridgeCore.h | 24 +- .../ios-arm64/Headers/SwiftBridgeCore.h | 24 +- .../macos-arm64/Headers/SwiftBridgeCore.h | 24 +- .../Sources/OctoBase/SwiftBridgeCore.swift | 474 +++++++++++++++++- .../Sources/OctoBase/jwst-swift.swift | 36 +- 5 files changed, 543 insertions(+), 39 deletions(-) diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/SwiftBridgeCore.h b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/SwiftBridgeCore.h index 3fad0536d..b1b1f60ea 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/SwiftBridgeCore.h +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64-simulator/Headers/SwiftBridgeCore.h @@ -4,6 +4,7 @@ typedef struct RustStr { uint8_t* const start; uintptr_t len; } RustStr; typedef struct __private__FfiSlice { void* const start; uintptr_t len; } __private__FfiSlice; void* __swift_bridge__null_pointer(void); + typedef struct __private__OptionU8 { uint8_t val; bool is_some; } __private__OptionU8; typedef struct __private__OptionI8 { int8_t val; bool is_some; } __private__OptionI8; typedef struct __private__OptionU16 { uint16_t val; bool is_some; } __private__OptionU16; @@ -15,7 +16,7 @@ typedef struct __private__OptionI64 { int64_t val; bool is_some; } __private__Op typedef struct __private__OptionUsize { uintptr_t val; bool is_some; } __private__OptionUsize; typedef struct __private__OptionIsize { intptr_t val; bool is_some; } __private__OptionIsize; typedef struct __private__OptionF32 { float val; bool is_some; } __private__OptionF32; -typedef struct __private__OptionF64 { double val; bool is_some; } __private__OptionDouble; +typedef struct __private__OptionF64 { double val; bool is_some; } __private__OptionF64; typedef struct __private__OptionBool { bool val; bool is_some; } __private__OptionBool; void* __swift_bridge__$Vec_u8$new(); @@ -117,6 +118,24 @@ __private__OptionBool __swift_bridge__$Vec_bool$get(void* const vec, uintptr_t i __private__OptionBool __swift_bridge__$Vec_bool$get_mut(void* const vec, uintptr_t index); bool const * __swift_bridge__$Vec_bool$as_ptr(void* const vec); +void* __swift_bridge__$Vec_f32$new(); +void __swift_bridge__$Vec_f32$_free(void* const vec); +uintptr_t __swift_bridge__$Vec_f32$len(void* const vec); +void __swift_bridge__$Vec_f32$push(void* const vec, float val); +__private__OptionF32 __swift_bridge__$Vec_f32$pop(void* const vec); +__private__OptionF32 __swift_bridge__$Vec_f32$get(void* const vec, uintptr_t index); +__private__OptionF32 __swift_bridge__$Vec_f32$get_mut(void* const vec, uintptr_t index); +float const * __swift_bridge__$Vec_f32$as_ptr(void* const vec); + +void* __swift_bridge__$Vec_f64$new(); +void __swift_bridge__$Vec_f64$_free(void* const vec); +uintptr_t __swift_bridge__$Vec_f64$len(void* const vec); +void __swift_bridge__$Vec_f64$push(void* const vec, double val); +__private__OptionF64 __swift_bridge__$Vec_f64$pop(void* const vec); +__private__OptionF64 __swift_bridge__$Vec_f64$get(void* const vec, uintptr_t index); +__private__OptionF64 __swift_bridge__$Vec_f64$get_mut(void* const vec, uintptr_t index); +double const * __swift_bridge__$Vec_f64$as_ptr(void* const vec); + #include typedef struct RustString RustString; void __swift_bridge__$RustString$_free(void* self); @@ -143,6 +162,3 @@ void __swift_bridge__$free_boxed_fn_once_no_args_no_return(void* boxed_fnonce); struct __private__ResultPtrAndPtr { bool is_ok; void* ok_or_err; }; - - -struct __private__ResultVoidAndPtr { bool is_ok; void* err; }; diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/SwiftBridgeCore.h b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/SwiftBridgeCore.h index 3fad0536d..b1b1f60ea 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/SwiftBridgeCore.h +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/ios-arm64/Headers/SwiftBridgeCore.h @@ -4,6 +4,7 @@ typedef struct RustStr { uint8_t* const start; uintptr_t len; } RustStr; typedef struct __private__FfiSlice { void* const start; uintptr_t len; } __private__FfiSlice; void* __swift_bridge__null_pointer(void); + typedef struct __private__OptionU8 { uint8_t val; bool is_some; } __private__OptionU8; typedef struct __private__OptionI8 { int8_t val; bool is_some; } __private__OptionI8; typedef struct __private__OptionU16 { uint16_t val; bool is_some; } __private__OptionU16; @@ -15,7 +16,7 @@ typedef struct __private__OptionI64 { int64_t val; bool is_some; } __private__Op typedef struct __private__OptionUsize { uintptr_t val; bool is_some; } __private__OptionUsize; typedef struct __private__OptionIsize { intptr_t val; bool is_some; } __private__OptionIsize; typedef struct __private__OptionF32 { float val; bool is_some; } __private__OptionF32; -typedef struct __private__OptionF64 { double val; bool is_some; } __private__OptionDouble; +typedef struct __private__OptionF64 { double val; bool is_some; } __private__OptionF64; typedef struct __private__OptionBool { bool val; bool is_some; } __private__OptionBool; void* __swift_bridge__$Vec_u8$new(); @@ -117,6 +118,24 @@ __private__OptionBool __swift_bridge__$Vec_bool$get(void* const vec, uintptr_t i __private__OptionBool __swift_bridge__$Vec_bool$get_mut(void* const vec, uintptr_t index); bool const * __swift_bridge__$Vec_bool$as_ptr(void* const vec); +void* __swift_bridge__$Vec_f32$new(); +void __swift_bridge__$Vec_f32$_free(void* const vec); +uintptr_t __swift_bridge__$Vec_f32$len(void* const vec); +void __swift_bridge__$Vec_f32$push(void* const vec, float val); +__private__OptionF32 __swift_bridge__$Vec_f32$pop(void* const vec); +__private__OptionF32 __swift_bridge__$Vec_f32$get(void* const vec, uintptr_t index); +__private__OptionF32 __swift_bridge__$Vec_f32$get_mut(void* const vec, uintptr_t index); +float const * __swift_bridge__$Vec_f32$as_ptr(void* const vec); + +void* __swift_bridge__$Vec_f64$new(); +void __swift_bridge__$Vec_f64$_free(void* const vec); +uintptr_t __swift_bridge__$Vec_f64$len(void* const vec); +void __swift_bridge__$Vec_f64$push(void* const vec, double val); +__private__OptionF64 __swift_bridge__$Vec_f64$pop(void* const vec); +__private__OptionF64 __swift_bridge__$Vec_f64$get(void* const vec, uintptr_t index); +__private__OptionF64 __swift_bridge__$Vec_f64$get_mut(void* const vec, uintptr_t index); +double const * __swift_bridge__$Vec_f64$as_ptr(void* const vec); + #include typedef struct RustString RustString; void __swift_bridge__$RustString$_free(void* self); @@ -143,6 +162,3 @@ void __swift_bridge__$free_boxed_fn_once_no_args_no_return(void* boxed_fnonce); struct __private__ResultPtrAndPtr { bool is_ok; void* ok_or_err; }; - - -struct __private__ResultVoidAndPtr { bool is_ok; void* err; }; diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/SwiftBridgeCore.h b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/SwiftBridgeCore.h index 3fad0536d..b1b1f60ea 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/SwiftBridgeCore.h +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/macos-arm64/Headers/SwiftBridgeCore.h @@ -4,6 +4,7 @@ typedef struct RustStr { uint8_t* const start; uintptr_t len; } RustStr; typedef struct __private__FfiSlice { void* const start; uintptr_t len; } __private__FfiSlice; void* __swift_bridge__null_pointer(void); + typedef struct __private__OptionU8 { uint8_t val; bool is_some; } __private__OptionU8; typedef struct __private__OptionI8 { int8_t val; bool is_some; } __private__OptionI8; typedef struct __private__OptionU16 { uint16_t val; bool is_some; } __private__OptionU16; @@ -15,7 +16,7 @@ typedef struct __private__OptionI64 { int64_t val; bool is_some; } __private__Op typedef struct __private__OptionUsize { uintptr_t val; bool is_some; } __private__OptionUsize; typedef struct __private__OptionIsize { intptr_t val; bool is_some; } __private__OptionIsize; typedef struct __private__OptionF32 { float val; bool is_some; } __private__OptionF32; -typedef struct __private__OptionF64 { double val; bool is_some; } __private__OptionDouble; +typedef struct __private__OptionF64 { double val; bool is_some; } __private__OptionF64; typedef struct __private__OptionBool { bool val; bool is_some; } __private__OptionBool; void* __swift_bridge__$Vec_u8$new(); @@ -117,6 +118,24 @@ __private__OptionBool __swift_bridge__$Vec_bool$get(void* const vec, uintptr_t i __private__OptionBool __swift_bridge__$Vec_bool$get_mut(void* const vec, uintptr_t index); bool const * __swift_bridge__$Vec_bool$as_ptr(void* const vec); +void* __swift_bridge__$Vec_f32$new(); +void __swift_bridge__$Vec_f32$_free(void* const vec); +uintptr_t __swift_bridge__$Vec_f32$len(void* const vec); +void __swift_bridge__$Vec_f32$push(void* const vec, float val); +__private__OptionF32 __swift_bridge__$Vec_f32$pop(void* const vec); +__private__OptionF32 __swift_bridge__$Vec_f32$get(void* const vec, uintptr_t index); +__private__OptionF32 __swift_bridge__$Vec_f32$get_mut(void* const vec, uintptr_t index); +float const * __swift_bridge__$Vec_f32$as_ptr(void* const vec); + +void* __swift_bridge__$Vec_f64$new(); +void __swift_bridge__$Vec_f64$_free(void* const vec); +uintptr_t __swift_bridge__$Vec_f64$len(void* const vec); +void __swift_bridge__$Vec_f64$push(void* const vec, double val); +__private__OptionF64 __swift_bridge__$Vec_f64$pop(void* const vec); +__private__OptionF64 __swift_bridge__$Vec_f64$get(void* const vec, uintptr_t index); +__private__OptionF64 __swift_bridge__$Vec_f64$get_mut(void* const vec, uintptr_t index); +double const * __swift_bridge__$Vec_f64$as_ptr(void* const vec); + #include typedef struct RustString RustString; void __swift_bridge__$RustString$_free(void* self); @@ -143,6 +162,3 @@ void __swift_bridge__$free_boxed_fn_once_no_args_no_return(void* boxed_fnonce); struct __private__ResultPtrAndPtr { bool is_ok; void* ok_or_err; }; - - -struct __private__ResultVoidAndPtr { bool is_ok; void* err; }; diff --git a/apps/swift/OctoBaseSwift/Sources/OctoBase/SwiftBridgeCore.swift b/apps/swift/OctoBaseSwift/Sources/OctoBase/SwiftBridgeCore.swift index 1d23df774..e0ab62f65 100644 --- a/apps/swift/OctoBaseSwift/Sources/OctoBase/SwiftBridgeCore.swift +++ b/apps/swift/OctoBaseSwift/Sources/OctoBase/SwiftBridgeCore.swift @@ -101,9 +101,6 @@ func optionalRustStrToRustStr(_ str: Optional, _ withUnsafeR return withUnsafeRustStr(RustStr(start: nil, len: 0)) } } -// TODO: -// Implement iterator https://developer.apple.com/documentation/swift/iteratorprotocol - public class RustVec { var ptr: UnsafeMutableRawPointer var isOwned: Bool = true @@ -129,6 +126,10 @@ public class RustVec { T.vecOfSelfGet(vecPtr: ptr, index: index) } + public func as_ptr() -> UnsafePointer { + UnsafePointer(OpaquePointer(T.vecOfSelfAsPtr(vecPtr: ptr))) + } + /// Rust returns a UInt, but we cast to an Int because many Swift APIs such as /// `ForEach(0..rustVec.len())` expect Int. public func len() -> Int { @@ -183,8 +184,7 @@ extension RustVec: Collection { } } -extension RustVec: RandomAccessCollection { -} +extension RustVec: RandomAccessCollection {} extension UnsafeBufferPointer { func toFfiSlice () -> __private__FfiSlice { @@ -192,22 +192,6 @@ extension UnsafeBufferPointer { } } -extension Array { - /// Get an UnsafeBufferPointer to the array's content's first byte with the array's length. - /// - /// ``` - /// // BAD! Swift will immediately free the arrays memory and so your pointer is invalid. - /// let pointer = useMyPointer([1, 2, 3].toUnsafeBufferPointer()) - /// - /// // GOOD! The array will outlive the buffer pointer. - /// let array = [1, 2, 3] - /// useMyPointer(array.toUnsafeBufferPointer()) - /// ``` - func toUnsafeBufferPointer() -> UnsafeBufferPointer { - UnsafeBufferPointer(start: UnsafePointer(self), count: self.count) - } -} - public protocol Vectorizable { associatedtype SelfRef associatedtype SelfRefMut @@ -224,6 +208,8 @@ public protocol Vectorizable { static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional + static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer + static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt } @@ -267,6 +253,10 @@ extension UInt8: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_u8$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_u8$len(vecPtr) } @@ -312,6 +302,10 @@ extension UInt16: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_u16$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_u16$len(vecPtr) } @@ -357,6 +351,10 @@ extension UInt32: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_u32$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_u32$len(vecPtr) } @@ -402,6 +400,10 @@ extension UInt64: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_u64$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_u64$len(vecPtr) } @@ -447,6 +449,10 @@ extension UInt: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_usize$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_usize$len(vecPtr) } @@ -492,6 +498,10 @@ extension Int8: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_i8$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_i8$len(vecPtr) } @@ -537,6 +547,10 @@ extension Int16: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_i16$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_i16$len(vecPtr) } @@ -582,6 +596,10 @@ extension Int32: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_i32$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_i32$len(vecPtr) } @@ -627,6 +645,10 @@ extension Int64: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_i64$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_i64$len(vecPtr) } @@ -672,6 +694,10 @@ extension Int: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_isize$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_isize$len(vecPtr) } @@ -717,11 +743,113 @@ extension Bool: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_bool$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_bool$len(vecPtr) } } +extension Float: Vectorizable { + public static func vecOfSelfNew() -> UnsafeMutableRawPointer { + __swift_bridge__$Vec_f32$new() + } + + public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { + __swift_bridge__$Vec_f32$_free(vecPtr) + } + + public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { + __swift_bridge__$Vec_f32$push(vecPtr, value) + } + + public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { + let val = __swift_bridge__$Vec_f32$pop(vecPtr) + if val.is_some { + return val.val + } else { + return nil + } + } + + public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let val = __swift_bridge__$Vec_f32$get(vecPtr, index) + if val.is_some { + return val.val + } else { + return nil + } + } + + public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let val = __swift_bridge__$Vec_f32$get_mut(vecPtr, index) + if val.is_some { + return val.val + } else { + return nil + } + } + + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_f32$as_ptr(vecPtr))) + } + + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { + __swift_bridge__$Vec_f32$len(vecPtr) + } +} + +extension Double: Vectorizable { + public static func vecOfSelfNew() -> UnsafeMutableRawPointer { + __swift_bridge__$Vec_f64$new() + } + + public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { + __swift_bridge__$Vec_f64$_free(vecPtr) + } + + public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Self) { + __swift_bridge__$Vec_f64$push(vecPtr, value) + } + + public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { + let val = __swift_bridge__$Vec_f64$pop(vecPtr) + if val.is_some { + return val.val + } else { + return nil + } + } + + public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let val = __swift_bridge__$Vec_f64$get(vecPtr, index) + if val.is_some { + return val.val + } else { + return nil + } + } + + public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { + let val = __swift_bridge__$Vec_f64$get_mut(vecPtr, index) + if val.is_some { + return val.val + } else { + return nil + } + } + + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_f64$as_ptr(vecPtr))) + } + + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { + __swift_bridge__$Vec_f64$len(vecPtr) + } +} + protocol SwiftBridgeGenericFreer { func rust_free(); } @@ -817,6 +945,10 @@ extension RustString: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_RustString$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_RustString$len(vecPtr) } @@ -880,3 +1012,303 @@ extension RustResult { } } } + + +extension __private__OptionU8 { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123, is_some: false) + } + } +} +extension Optional where Wrapped == UInt8 { + func intoFfiRepr() -> __private__OptionU8 { + __private__OptionU8(self) + } +} + +extension __private__OptionI8 { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123, is_some: false) + } + } +} +extension Optional where Wrapped == Int8 { + func intoFfiRepr() -> __private__OptionI8 { + __private__OptionI8(self) + } +} + +extension __private__OptionU16 { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123, is_some: false) + } + } +} +extension Optional where Wrapped == UInt16 { + func intoFfiRepr() -> __private__OptionU16 { + __private__OptionU16(self) + } +} + +extension __private__OptionI16 { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123, is_some: false) + } + } +} +extension Optional where Wrapped == Int16 { + func intoFfiRepr() -> __private__OptionI16 { + __private__OptionI16(self) + } +} + +extension __private__OptionU32 { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123, is_some: false) + } + } +} +extension Optional where Wrapped == UInt32 { + func intoFfiRepr() -> __private__OptionU32 { + __private__OptionU32(self) + } +} + +extension __private__OptionI32 { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123, is_some: false) + } + } +} +extension Optional where Wrapped == Int32 { + func intoFfiRepr() -> __private__OptionI32 { + __private__OptionI32(self) + } +} + +extension __private__OptionU64 { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123, is_some: false) + } + } +} +extension Optional where Wrapped == UInt64 { + func intoFfiRepr() -> __private__OptionU64 { + __private__OptionU64(self) + } +} + +extension __private__OptionI64 { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123, is_some: false) + } + } +} +extension Optional where Wrapped == Int64 { + func intoFfiRepr() -> __private__OptionI64 { + __private__OptionI64(self) + } +} + +extension __private__OptionUsize { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123, is_some: false) + } + } +} +extension Optional where Wrapped == UInt { + func intoFfiRepr() -> __private__OptionUsize { + __private__OptionUsize(self) + } +} + +extension __private__OptionIsize { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123, is_some: false) + } + } +} +extension Optional where Wrapped == Int { + func intoFfiRepr() -> __private__OptionIsize { + __private__OptionIsize(self) + } +} + +extension __private__OptionF32 { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123.4, is_some: false) + } + } +} +extension Optional where Wrapped == Float { + func intoFfiRepr() -> __private__OptionF32 { + __private__OptionF32(self) + } +} + +extension __private__OptionF64 { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: 123.4, is_some: false) + } + } +} +extension Optional where Wrapped == Double { + func intoFfiRepr() -> __private__OptionF64 { + __private__OptionF64(self) + } +} + +extension __private__OptionBool { + func intoSwiftRepr() -> Optional { + if self.is_some { + return self.val + } else { + return nil + } + } + + init(_ val: Optional) { + if let val = val { + self = Self(val: val, is_some: true) + } else { + self = Self(val: false, is_some: false) + } + } +} +extension Optional where Wrapped == Bool { + func intoFfiRepr() -> __private__OptionBool { + __private__OptionBool(self) + } +} diff --git a/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-swift.swift b/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-swift.swift index 0556654ee..19d3b7832 100644 --- a/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-swift.swift +++ b/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-swift.swift @@ -121,7 +121,7 @@ extension BlockRef { } public func get_bool(_ key: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() + __swift_bridge__$Block$get_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()).intoSwiftRepr() } public func get_string(_ key: GenericIntoRustString) -> Optional { @@ -129,11 +129,11 @@ extension BlockRef { } public func get_float(_ key: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get_float(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() + __swift_bridge__$Block$get_float(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()).intoSwiftRepr() } public func get_integer(_ key: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() + __swift_bridge__$Block$get_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()).intoSwiftRepr() } } extension Block: Vectorizable { @@ -176,6 +176,10 @@ extension Block: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_Block$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_Block$len(vecPtr) } @@ -247,6 +251,10 @@ extension DynamicValueMap: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_DynamicValueMap$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_DynamicValueMap$len(vecPtr) } @@ -280,15 +288,15 @@ public class DynamicValueRef { } extension DynamicValueRef { public func as_bool() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_bool(ptr); if val.is_some { return val.val } else { return nil } }() + __swift_bridge__$DynamicValue$as_bool(ptr).intoSwiftRepr() } public func as_number() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_number(ptr); if val.is_some { return val.val } else { return nil } }() + __swift_bridge__$DynamicValue$as_number(ptr).intoSwiftRepr() } public func as_int() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_int(ptr); if val.is_some { return val.val } else { return nil } }() + __swift_bridge__$DynamicValue$as_int(ptr).intoSwiftRepr() } public func as_string() -> Optional { @@ -347,6 +355,10 @@ extension DynamicValue: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_DynamicValue$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_DynamicValue$len(vecPtr) } @@ -453,6 +465,10 @@ extension Workspace: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_Workspace$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_Workspace$len(vecPtr) } @@ -524,6 +540,10 @@ extension JwstWorkSpaceResult: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_JwstWorkSpaceResult$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_JwstWorkSpaceResult$len(vecPtr) } @@ -638,6 +658,10 @@ extension Storage: Vectorizable { } } + public static func vecOfSelfAsPtr(vecPtr: UnsafeMutableRawPointer) -> UnsafePointer { + UnsafePointer(OpaquePointer(__swift_bridge__$Vec_Storage$as_ptr(vecPtr))) + } + public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { __swift_bridge__$Vec_Storage$len(vecPtr) } From c98cc52e28e14f3ac87e4c37ae641c2d95adbd96 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Tue, 22 Aug 2023 02:02:49 +0800 Subject: [PATCH 14/49] feat: reorganize the structure of storage --- Cargo.lock | 91 +------- .../RustXcframework.xcframework/Info.plist | 12 +- libs/cloud-database/Cargo.toml | 2 +- libs/cloud-database/migration/Cargo.toml | 1 + libs/jwst-core-storage/Cargo.toml | 9 +- libs/jwst-core-storage/src/lib.rs | 9 +- .../auto_storage.rs} | 11 +- .../src/storage/blobs/auto_storage/mod.rs | 8 + .../src/storage/blobs/auto_storage/utils.rs | 96 ++++++++ .../src/storage/blobs/blob_storage.rs | 4 +- .../{bucket_storage.rs => bucket/bucket.rs} | 37 ++- .../src/storage/blobs/bucket/mod.rs | 8 + .../src/storage/blobs/bucket/utils.rs | 116 ++++++++++ .../src/storage/blobs/mod.rs | 67 ++---- .../src/storage/blobs/utils.rs | 214 ------------------ libs/jwst-core-storage/src/storage/difflog.rs | 43 ---- libs/jwst-core-storage/src/storage/mod.rs | 20 +- libs/jwst-core-storage/src/types.rs | 2 + libs/jwst-storage/Cargo.toml | 4 +- libs/jwst-storage/src/migration/Cargo.toml | 1 + 20 files changed, 319 insertions(+), 436 deletions(-) rename libs/jwst-core-storage/src/storage/blobs/{auto_blob_storage.rs => auto_storage/auto_storage.rs} (98%) create mode 100644 libs/jwst-core-storage/src/storage/blobs/auto_storage/mod.rs create mode 100644 libs/jwst-core-storage/src/storage/blobs/auto_storage/utils.rs rename libs/jwst-core-storage/src/storage/blobs/{bucket_storage.rs => bucket/bucket.rs} (91%) create mode 100644 libs/jwst-core-storage/src/storage/blobs/bucket/mod.rs create mode 100644 libs/jwst-core-storage/src/storage/blobs/bucket/utils.rs delete mode 100644 libs/jwst-core-storage/src/storage/difflog.rs diff --git a/Cargo.lock b/Cargo.lock index b8f227b15..0905cb264 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2859,7 +2859,7 @@ dependencies = [ "jwst-core", "jwst-core-storage-migration", "jwst-logger", - "opendal 0.39.0", + "opendal", "path-ext", "rand 0.8.5", "sea-orm", @@ -2968,7 +2968,7 @@ dependencies = [ "jwst-logger", "jwst-storage-migration", "lib0", - "opendal 0.38.1", + "opendal", "path-ext", "rand 0.8.5", "sea-orm", @@ -3625,38 +3625,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" -[[package]] -name = "opendal" -version = "0.38.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4df645b6012162c04c8949e9b96ae2ef002e79189cfb154e507e51ac5be76a09" -dependencies = [ - "anyhow", - "async-compat", - "async-trait", - "backon", - "base64 0.21.2", - "bytes", - "chrono", - "flagset", - "futures", - "http", - "hyper", - "log", - "md-5", - "once_cell", - "parking_lot", - "percent-encoding", - "pin-project", - "quick-xml 0.27.1", - "reqsign 0.13.0", - "reqwest", - "serde", - "serde_json", - "tokio", - "uuid", -] - [[package]] name = "opendal" version = "0.39.0" @@ -3680,8 +3648,8 @@ dependencies = [ "parking_lot", "percent-encoding", "pin-project", - "quick-xml 0.29.0", - "reqsign 0.14.1", + "quick-xml", + "reqsign", "reqwest", "serde", "serde_json", @@ -4249,26 +4217,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-xml" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc053f057dd768a56f62cd7e434c42c831d296968997e9ac1f76ea7c2d14c41" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quick-xml" -version = "0.28.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.29.0" @@ -4515,35 +4463,6 @@ dependencies = [ "bytecheck", ] -[[package]] -name = "reqsign" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6cb65eb3405f9c2de5c18bfc37338d6bbdb2c35eb8eb0e946208cbb564e4833" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.21.2", - "chrono", - "form_urlencoded", - "hex", - "hmac", - "home", - "http", - "log", - "once_cell", - "percent-encoding", - "quick-xml 0.28.2", - "rand 0.8.5", - "reqwest", - "rsa", - "rust-ini", - "serde", - "serde_json", - "sha1", - "sha2", -] - [[package]] name = "reqsign" version = "0.14.1" @@ -4562,7 +4481,7 @@ dependencies = [ "log", "once_cell", "percent-encoding", - "quick-xml 0.29.0", + "quick-xml", "rand 0.8.5", "reqwest", "rsa", diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist index 625948fb2..b3d1aee06 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist @@ -8,7 +8,7 @@ HeadersPath Headers LibraryIdentifier - macos-arm64 + ios-arm64-simulator LibraryPath liboctobase.a SupportedArchitectures @@ -16,13 +16,15 @@ arm64 SupportedPlatform - macos + ios + SupportedPlatformVariant + simulator HeadersPath Headers LibraryIdentifier - ios-arm64-simulator + macos-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -30,9 +32,7 @@ arm64 SupportedPlatform - ios - SupportedPlatformVariant - simulator + macos HeadersPath diff --git a/libs/cloud-database/Cargo.toml b/libs/cloud-database/Cargo.toml index 646b3efa4..5c16ec3d6 100644 --- a/libs/cloud-database/Cargo.toml +++ b/libs/cloud-database/Cargo.toml @@ -23,7 +23,7 @@ sqlx = { version = "0.7.1", features = [ "runtime-tokio-rustls", ] } sea-orm = { version = "0.12.2", features = ["runtime-tokio-rustls", "macros"] } -sea-orm-migration = "0.12.2" +sea-orm-migration = { version = "0.12.2", default-features = false } tokio = { version = "1.27.0", features = ["fs", "macros", "sync"] } yrs = "0.16.5" diff --git a/libs/cloud-database/migration/Cargo.toml b/libs/cloud-database/migration/Cargo.toml index 4ca1fbd55..8f2b47042 100644 --- a/libs/cloud-database/migration/Cargo.toml +++ b/libs/cloud-database/migration/Cargo.toml @@ -13,4 +13,5 @@ tokio = { version = "1.27.0", features = ["macros"] } [dependencies.sea-orm-migration] version = "0.12.2" +default-features = false features = ["runtime-tokio-rustls", "sqlx-postgres"] diff --git a/libs/jwst-core-storage/Cargo.toml b/libs/jwst-core-storage/Cargo.toml index 6a6429cfb..f481ed1ae 100644 --- a/libs/jwst-core-storage/Cargo.toml +++ b/libs/jwst-core-storage/Cargo.toml @@ -7,6 +7,7 @@ license = "AGPL-3.0-only" [features] default = ["sqlite"] +bucket = ["dotenvy", "opendal"] image = ["dep:image"] mysql = ["sea-orm/sqlx-mysql"] postgres = ["sea-orm/sqlx-postgres"] @@ -22,16 +23,18 @@ governor = "0.6.0" path-ext = "0.1.0" sha2 = "0.10.7" sea-orm = { version = "0.12.2", features = ["runtime-tokio-rustls", "macros"] } -sea-orm-migration = "0.12.2" +sea-orm-migration = { version = "0.12.2", default-features = false } thiserror = "1.0.47" tokio = { version = "1.32.0", features = ["fs", "macros", "sync"] } tokio-util = { version = "0.7.8", features = ["io"] } url = "2.4.0" + +# ======== bucket dependencies ======= opendal = { version = "0.39.0", default-features = false, features = [ "rustls", "services-s3", -] } -dotenvy = "0.15.7" +], optional = true } +dotenvy = { version = "0.15.7", optional = true } # ======= image dependencies ====== image = { version = "0.24.6", features = ["webp-encoder"], optional = true } diff --git a/libs/jwst-core-storage/src/lib.rs b/libs/jwst-core-storage/src/lib.rs index 2612da8bf..438447d49 100644 --- a/libs/jwst-core-storage/src/lib.rs +++ b/libs/jwst-core-storage/src/lib.rs @@ -7,17 +7,16 @@ mod types; use std::{path::PathBuf, sync::Arc, time::Duration}; use async_trait::async_trait; -use chrono::{DateTime, Utc}; +use chrono::Utc; use futures::{Future, Stream}; use jwst_core::{DocStorage, JwstResult, Workspace}; use jwst_logger::{debug, error, info, trace, warn}; use path_ext::PathExt; use rate_limiter::{get_bucket, is_sqlite, Bucket}; use sea_orm::{prelude::*, ConnectOptions, Database, DbErr, QuerySelect, Set}; -pub use storage::{ - blobs::{BlobStorageType, MixedBucketDBParam}, - JwstStorage, -}; +#[cfg(feature = "bucket")] +pub use storage::blobs::MixedBucketDBParam; +pub use storage::{blobs::BlobStorageType, JwstStorage}; pub use types::{JwstStorageError, JwstStorageResult}; #[inline] diff --git a/libs/jwst-core-storage/src/storage/blobs/auto_blob_storage.rs b/libs/jwst-core-storage/src/storage/blobs/auto_storage/auto_storage.rs similarity index 98% rename from libs/jwst-core-storage/src/storage/blobs/auto_blob_storage.rs rename to libs/jwst-core-storage/src/storage/blobs/auto_storage/auto_storage.rs index d0240f309..176079118 100644 --- a/libs/jwst-core-storage/src/storage/blobs/auto_blob_storage.rs +++ b/libs/jwst-core-storage/src/storage/blobs/auto_storage/auto_storage.rs @@ -1,13 +1,8 @@ -#[cfg(test)] -pub use blob_storage::blobs_storage_test; use blob_storage::BlobDBStorage; use bytes::Bytes; -use image::ImageError; use jwst_core::{BlobMetadata, BlobStorage}; -use thiserror::Error; -use tokio::task::JoinError; -pub use utils::BucketStorageBuilder; -use utils::{ImageParams, InternalBlobMetadata}; + +use super::*; pub(super) type OptimizedBlobModel = ::Model; type OptimizedBlobActiveModel = super::entities::optimized_blobs::ActiveModel; @@ -15,7 +10,7 @@ type OptimizedBlobColumn = ::Column; #[derive(Clone)] pub struct BlobAutoStorage { - pub(super) db: Arc, + pub(crate) db: Arc, pool: DatabaseConnection, } diff --git a/libs/jwst-core-storage/src/storage/blobs/auto_storage/mod.rs b/libs/jwst-core-storage/src/storage/blobs/auto_storage/mod.rs new file mode 100644 index 000000000..21b5e0006 --- /dev/null +++ b/libs/jwst-core-storage/src/storage/blobs/auto_storage/mod.rs @@ -0,0 +1,8 @@ +mod auto_storage; +mod utils; + +pub use auto_storage::BlobAutoStorage; +pub use image::ImageError; +pub use utils::ImageParams; + +use super::*; diff --git a/libs/jwst-core-storage/src/storage/blobs/auto_storage/utils.rs b/libs/jwst-core-storage/src/storage/blobs/auto_storage/utils.rs new file mode 100644 index 000000000..c7568ca0d --- /dev/null +++ b/libs/jwst-core-storage/src/storage/blobs/auto_storage/utils.rs @@ -0,0 +1,96 @@ +use std::collections::HashMap; + +enum ImageFormat { + Jpeg, + WebP, +} + +pub struct ImageParams { + format: ImageFormat, + width: Option, + height: Option, +} + +impl ImageParams { + #[inline] + fn check_size(w: Option, h: Option) -> bool { + if let Some(w) = w { + if w % 320 != 0 || w > 1920 { + return false; + } + } + if let Some(h) = h { + if h % 180 != 0 || h > 1080 { + return false; + } + } + true + } + + pub(super) fn format(&self) -> String { + match self.format { + ImageFormat::Jpeg => "jpeg".to_string(), + ImageFormat::WebP => "webp".to_string(), + } + } + + fn output_format(&self) -> image::ImageOutputFormat { + match self.format { + ImageFormat::Jpeg => image::ImageOutputFormat::Jpeg(80), + ImageFormat::WebP => image::ImageOutputFormat::WebP, + } + } + + pub fn optimize_image(&self, data: &[u8]) -> image::ImageResult> { + let mut buffer = std::io::Cursor::new(vec![]); + let image = image::load_from_memory(data)?; + image.write_to(&mut buffer, self.output_format())?; + Ok(buffer.into_inner()) + } +} + +impl TryFrom<&HashMap> for ImageParams { + type Error = (); + + fn try_from(value: &HashMap) -> Result { + let mut format = None; + let mut width = None; + let mut height = None; + for (key, value) in value { + match key.as_str() { + "format" => { + format = match value.as_str() { + "jpeg" => Some(ImageFormat::Jpeg), + "webp" => Some(ImageFormat::WebP), + _ => return Err(()), + } + } + "width" => width = value.parse().ok(), + "height" => height = value.parse().ok(), + _ => return Err(()), + } + } + + if let Some(format) = format { + if Self::check_size(width, height) { + return Ok(Self { format, width, height }); + } + } + Err(()) + } +} + +impl ToString for ImageParams { + fn to_string(&self) -> String { + let mut params = String::new(); + + params.push_str(&format!("format={}", self.format())); + if let Some(width) = &self.width { + params.push_str(&format!("width={}", width)); + } + if let Some(height) = &self.height { + params.push_str(&format!("height={}", height)); + } + params + } +} diff --git a/libs/jwst-core-storage/src/storage/blobs/blob_storage.rs b/libs/jwst-core-storage/src/storage/blobs/blob_storage.rs index fd5b82308..8c2229c04 100644 --- a/libs/jwst-core-storage/src/storage/blobs/blob_storage.rs +++ b/libs/jwst-core-storage/src/storage/blobs/blob_storage.rs @@ -1,8 +1,8 @@ use jwst_core::{Base64Engine, URL_SAFE_ENGINE}; use sha2::{Digest, Sha256}; -use super::{utils::get_hash, *}; -use crate::types::JwstStorageResult; +use super::*; + pub(super) type BlobModel = ::Model; type BlobActiveModel = super::entities::blobs::ActiveModel; type BlobColumn = ::Column; diff --git a/libs/jwst-core-storage/src/storage/blobs/bucket_storage.rs b/libs/jwst-core-storage/src/storage/blobs/bucket/bucket.rs similarity index 91% rename from libs/jwst-core-storage/src/storage/blobs/bucket_storage.rs rename to libs/jwst-core-storage/src/storage/blobs/bucket/bucket.rs index 8830c3ed2..fa28a1e6d 100644 --- a/libs/jwst-core-storage/src/storage/blobs/bucket_storage.rs +++ b/libs/jwst-core-storage/src/storage/blobs/bucket/bucket.rs @@ -8,10 +8,7 @@ use opendal::{services::S3, Operator}; use sea_orm::{DatabaseConnection, EntityTrait}; use sea_orm_migration::MigratorTrait; -use super::{ - utils::{calculate_hash, get_hash}, - *, -}; +use super::*; use crate::{rate_limiter::Bucket, JwstStorageError}; pub(super) type BucketBlobModel = ::Model; @@ -208,6 +205,37 @@ impl BucketBlobStorage for BucketStorage { } } +impl TryFrom> for BucketStorage { + type Error = JwstStorageError; + + fn try_from(map: HashMap) -> Result { + let mut builder = BucketStorageBuilder::new(); + let access_token = map.get("BUCKET_ACCESS_TOKEN"); + let secret_access_key = map.get("BUCKET_SECRET_TOKEN"); + let endpoint = map.get("BUCKET_ENDPOINT"); + let bucket = map.get("BUCKET_NAME"); + let root = map.get("BUCKET_ROOT"); + + if let Some(access_token) = access_token { + builder = builder.access_key(access_token); + } + if let Some(secret_access_key) = secret_access_key { + builder = builder.secret_access_key(secret_access_key); + } + if let Some(endpoint) = endpoint { + builder = builder.endpoint(endpoint); + } + if let Some(bucket) = bucket { + builder = builder.bucket(bucket); + } + if let Some(root) = root { + builder = builder.root(root); + } + + builder.build() + } +} + #[async_trait] impl BlobStorage for BlobBucketStorage { async fn list_blobs(&self, workspace: Option) -> JwstResult, JwstStorageError> { @@ -322,7 +350,6 @@ impl BlobStorage for BlobBucketStorage { #[cfg(test)] mod tests { use super::*; - use crate::storage::blobs::utils::BucketStorageBuilder; #[tokio::test] #[ignore = "need to config bucket auth"] diff --git a/libs/jwst-core-storage/src/storage/blobs/bucket/mod.rs b/libs/jwst-core-storage/src/storage/blobs/bucket/mod.rs new file mode 100644 index 000000000..5ffb26493 --- /dev/null +++ b/libs/jwst-core-storage/src/storage/blobs/bucket/mod.rs @@ -0,0 +1,8 @@ +mod bucket; +mod utils; + +pub use bucket::BlobBucketStorage; +use bucket::BucketStorage; +pub use utils::{calculate_hash, BucketStorageBuilder, MixedBucketDBParam}; + +use super::*; diff --git a/libs/jwst-core-storage/src/storage/blobs/bucket/utils.rs b/libs/jwst-core-storage/src/storage/blobs/bucket/utils.rs new file mode 100644 index 000000000..e01f3ba1e --- /dev/null +++ b/libs/jwst-core-storage/src/storage/blobs/bucket/utils.rs @@ -0,0 +1,116 @@ +use jwst_core::{Base64Engine, JwstResult, URL_SAFE_ENGINE}; +use opendal::{services::S3, Operator}; +use sha2::{Digest, Sha256}; + +use super::*; + +pub struct MixedBucketDBParam { + pub(crate) access_key: String, + pub(crate) secret_access_key: String, + pub(crate) endpoint: String, + pub(crate) bucket: Option, + pub(crate) root: Option, +} + +impl MixedBucketDBParam { + pub fn new_from_env() -> JwstResult { + Ok(MixedBucketDBParam { + access_key: dotenvy::var("BUCKET_ACCESS_TOKEN")?, + secret_access_key: dotenvy::var("BUCKET_SECRET_TOKEN")?, + endpoint: dotenvy::var("BUCKET_ENDPOINT")?, + bucket: dotenvy::var("BUCKET_NAME").ok(), + root: dotenvy::var("BUCKET_ROOT").ok(), + }) + } + + pub fn new( + access_key: String, + secret_access_key: String, + endpoint: String, + bucket: Option, + root: Option, + ) -> Self { + MixedBucketDBParam { + access_key, + secret_access_key, + endpoint, + bucket, + root, + } + } +} + +impl TryFrom for BucketStorage { + type Error = JwstStorageError; + + fn try_from(value: MixedBucketDBParam) -> Result { + let mut builder = BucketStorageBuilder::new(); + builder = builder.access_key(&value.access_key); + builder = builder.secret_access_key(&value.secret_access_key); + builder = builder.endpoint(&value.endpoint); + builder = builder.bucket(&value.bucket.unwrap_or("__default_bucket__".to_string())); + builder = builder.root(&value.root.unwrap_or("__default_root__".to_string())); + builder.build() + } +} + +#[derive(Default)] +pub struct BucketStorageBuilder { + access_key: String, + secret_access_key: String, + endpoint: String, + bucket: String, + root: String, +} + +impl BucketStorageBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn access_key(mut self, access_key: &str) -> Self { + self.access_key = access_key.to_string(); + self + } + + pub fn secret_access_key(mut self, secret_access_key: &str) -> Self { + self.secret_access_key = secret_access_key.to_string(); + self + } + + pub fn endpoint(mut self, endpoint: &str) -> Self { + self.endpoint = endpoint.to_string(); + self + } + + pub fn bucket(mut self, bucket: &str) -> Self { + self.bucket = bucket.to_string(); + self + } + + pub fn root(mut self, root: &str) -> Self { + self.root = root.to_string(); + self + } + + pub fn build(self) -> JwstStorageResult { + let mut builder = S3::default(); + + builder.bucket(self.bucket.as_str()); + builder.root(self.root.as_str()); + builder.endpoint(self.endpoint.as_str()); + builder.access_key_id(self.access_key.as_str()); + builder.secret_access_key(self.secret_access_key.as_str()); + + Ok(BucketStorage { + op: Operator::new(builder)?.finish(), + }) + } +} + +/// Calculate sha256 hash for given blob +pub fn calculate_hash(blob: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(blob); + URL_SAFE_ENGINE.encode(hasher.finalize()) +} diff --git a/libs/jwst-core-storage/src/storage/blobs/mod.rs b/libs/jwst-core-storage/src/storage/blobs/mod.rs index d5b34d3bf..db57f9767 100644 --- a/libs/jwst-core-storage/src/storage/blobs/mod.rs +++ b/libs/jwst-core-storage/src/storage/blobs/mod.rs @@ -1,26 +1,24 @@ +#[cfg(feature = "bucket")] +mod bucket; +#[cfg(feature = "bucket")] +pub use bucket::{BlobBucketStorage, MixedBucketDBParam}; + #[cfg(feature = "image")] -mod auto_blob_storage; -#[cfg(feature = "image")] -pub use auto_blob_storage::BlobAutoStorage; -#[cfg(feature = "image")] -use image::ImageError; +mod auto_storage; #[cfg(feature = "image")] -use utils::ImageParams; +pub use auto_storage::{BlobAutoStorage, ImageError, ImageParams}; mod blob_storage; -mod bucket_storage; mod utils; #[cfg(test)] pub use blob_storage::blobs_storage_test; pub use blob_storage::BlobDBStorage; -pub use bucket_storage::BlobBucketStorage; use bytes::Bytes; use jwst_core::{BlobMetadata, BlobStorage}; use thiserror::Error; use tokio::task::JoinError; -pub use utils::BucketStorageBuilder; -use utils::InternalBlobMetadata; +use utils::{get_hash, InternalBlobMetadata}; use super::{entities::prelude::*, *}; @@ -44,50 +42,16 @@ pub enum JwstBlobStorage { RawStorage(Arc), #[cfg(feature = "image")] AutoStorage(BlobAutoStorage), + #[cfg(feature = "bucket")] BucketStorage(BlobBucketStorage), } pub enum BlobStorageType { DB, + #[cfg(feature = "bucket")] MixedBucketDB(MixedBucketDBParam), } -pub struct MixedBucketDBParam { - pub(crate) access_key: String, - pub(crate) secret_access_key: String, - pub(crate) endpoint: String, - pub(crate) bucket: Option, - pub(crate) root: Option, -} - -impl MixedBucketDBParam { - pub fn new_from_env() -> JwstResult { - Ok(MixedBucketDBParam { - access_key: dotenvy::var("BUCKET_ACCESS_TOKEN")?, - secret_access_key: dotenvy::var("BUCKET_SECRET_TOKEN")?, - endpoint: dotenvy::var("BUCKET_ENDPOINT")?, - bucket: dotenvy::var("BUCKET_NAME").ok(), - root: dotenvy::var("BUCKET_ROOT").ok(), - }) - } - - pub fn new( - access_key: String, - secret_access_key: String, - endpoint: String, - bucket: Option, - root: Option, - ) -> Self { - MixedBucketDBParam { - access_key, - secret_access_key, - endpoint, - bucket, - root, - } - } -} - #[async_trait] impl BlobStorage for JwstBlobStorage { async fn list_blobs(&self, workspace: Option) -> JwstResult, JwstStorageError> { @@ -95,6 +59,7 @@ impl BlobStorage for JwstBlobStorage { JwstBlobStorage::RawStorage(db) => db.list_blobs(workspace).await, #[cfg(feature = "image")] JwstBlobStorage::AutoStorage(db) => db.list_blobs(workspace).await, + #[cfg(feature = "bucket")] JwstBlobStorage::BucketStorage(db) => db.list_blobs(workspace).await, } } @@ -104,6 +69,7 @@ impl BlobStorage for JwstBlobStorage { JwstBlobStorage::RawStorage(db) => db.check_blob(workspace, id).await, #[cfg(feature = "image")] JwstBlobStorage::AutoStorage(db) => db.check_blob(workspace, id).await, + #[cfg(feature = "bucket")] JwstBlobStorage::BucketStorage(db) => db.check_blob(workspace, id).await, } } @@ -118,6 +84,7 @@ impl BlobStorage for JwstBlobStorage { JwstBlobStorage::RawStorage(db) => db.get_blob(workspace, id, params).await, #[cfg(feature = "image")] JwstBlobStorage::AutoStorage(db) => db.get_blob(workspace, id, params).await, + #[cfg(feature = "bucket")] JwstBlobStorage::BucketStorage(db) => db.get_blob(workspace, id, params).await, } } @@ -132,6 +99,7 @@ impl BlobStorage for JwstBlobStorage { JwstBlobStorage::RawStorage(db) => db.get_metadata(workspace, id, params).await, #[cfg(feature = "image")] JwstBlobStorage::AutoStorage(db) => db.get_metadata(workspace, id, params).await, + #[cfg(feature = "bucket")] JwstBlobStorage::BucketStorage(db) => db.get_metadata(workspace, id, params).await, } } @@ -145,6 +113,7 @@ impl BlobStorage for JwstBlobStorage { JwstBlobStorage::RawStorage(db) => db.put_blob_stream(workspace, stream).await, #[cfg(feature = "image")] JwstBlobStorage::AutoStorage(db) => db.put_blob_stream(workspace, stream).await, + #[cfg(feature = "bucket")] JwstBlobStorage::BucketStorage(db) => db.put_blob_stream(workspace, stream).await, } } @@ -154,6 +123,7 @@ impl BlobStorage for JwstBlobStorage { JwstBlobStorage::RawStorage(db) => db.put_blob(workspace, blob).await, #[cfg(feature = "image")] JwstBlobStorage::AutoStorage(db) => db.put_blob(workspace, blob).await, + #[cfg(feature = "bucket")] JwstBlobStorage::BucketStorage(db) => db.put_blob(workspace, blob).await, } } @@ -163,6 +133,7 @@ impl BlobStorage for JwstBlobStorage { JwstBlobStorage::RawStorage(db) => db.delete_blob(workspace, id).await, #[cfg(feature = "image")] JwstBlobStorage::AutoStorage(db) => db.delete_blob(workspace, id).await, + #[cfg(feature = "bucket")] JwstBlobStorage::BucketStorage(db) => db.delete_blob(workspace, id).await, } } @@ -172,6 +143,7 @@ impl BlobStorage for JwstBlobStorage { JwstBlobStorage::RawStorage(db) => db.delete_workspace(workspace_id).await, #[cfg(feature = "image")] JwstBlobStorage::AutoStorage(db) => db.delete_workspace(workspace_id).await, + #[cfg(feature = "bucket")] JwstBlobStorage::BucketStorage(db) => db.delete_workspace(workspace_id).await, } } @@ -181,6 +153,7 @@ impl BlobStorage for JwstBlobStorage { JwstBlobStorage::RawStorage(db) => Ok(db.get_blobs_size(&workspace_id).await?.unwrap_or(0)), #[cfg(feature = "image")] JwstBlobStorage::AutoStorage(db) => db.get_blobs_size(workspace_id).await, + #[cfg(feature = "bucket")] JwstBlobStorage::BucketStorage(db) => db.get_blobs_size(workspace_id).await, } } @@ -192,10 +165,12 @@ impl JwstBlobStorage { JwstBlobStorage::RawStorage(db) => Some(db.clone()), #[cfg(feature = "image")] JwstBlobStorage::AutoStorage(db) => Some(db.db.clone()), + #[cfg(feature = "bucket")] JwstBlobStorage::BucketStorage(_) => None, } } + #[cfg(feature = "bucket")] pub fn get_mixed_bucket_db(&self) -> Option { match self { JwstBlobStorage::RawStorage(_) => None, diff --git a/libs/jwst-core-storage/src/storage/blobs/utils.rs b/libs/jwst-core-storage/src/storage/blobs/utils.rs index 2ef7dd4f9..b129fb3af 100644 --- a/libs/jwst-core-storage/src/storage/blobs/utils.rs +++ b/libs/jwst-core-storage/src/storage/blobs/utils.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use bytes::Bytes; use chrono::{DateTime, Utc}; use futures::{ @@ -7,115 +5,9 @@ use futures::{ Stream, }; use jwst_core::{Base64Engine, BlobMetadata, URL_SAFE_ENGINE}; -use opendal::{services::S3, Operator}; use sea_orm::FromQueryResult; use sha2::{Digest, Sha256}; -use crate::{ - storage::blobs::{bucket_storage::BucketStorage, MixedBucketDBParam}, - JwstStorageError, JwstStorageResult, -}; - -#[cfg(feature = "image")] -enum ImageFormat { - Jpeg, - WebP, -} - -#[cfg(feature = "image")] -pub struct ImageParams { - format: ImageFormat, - width: Option, - height: Option, -} - -#[cfg(feature = "image")] -impl ImageParams { - #[inline] - fn check_size(w: Option, h: Option) -> bool { - if let Some(w) = w { - if w % 320 != 0 || w > 1920 { - return false; - } - } - if let Some(h) = h { - if h % 180 != 0 || h > 1080 { - return false; - } - } - true - } - - pub(super) fn format(&self) -> String { - match self.format { - ImageFormat::Jpeg => "jpeg".to_string(), - ImageFormat::WebP => "webp".to_string(), - } - } - - fn output_format(&self) -> image::ImageOutputFormat { - match self.format { - image::ImageFormat::Jpeg => image::ImageOutputFormat::Jpeg(80), - image::ImageFormat::WebP => image::ImageOutputFormat::WebP, - } - } - - pub fn optimize_image(&self, data: &[u8]) -> image::ImageResult> { - let mut buffer = std::io::Cursor::new(vec![]); - let image = image::load_from_memory(data)?; - image.write_to(&mut buffer, self.output_format())?; - Ok(buffer.into_inner()) - } -} - -#[cfg(feature = "image")] -impl TryFrom<&HashMap> for ImageParams { - type Error = (); - - fn try_from(value: &HashMap) -> Result { - let mut format = None; - let mut width = None; - let mut height = None; - for (key, value) in value { - match key.as_str() { - "format" => { - format = match value.as_str() { - "jpeg" => Some(ImageFormat::Jpeg), - "webp" => Some(ImageFormat::WebP), - _ => return Err(()), - } - } - "width" => width = value.parse().ok(), - "height" => height = value.parse().ok(), - _ => return Err(()), - } - } - - if let Some(format) = format { - if Self::check_size(width, height) { - return Ok(Self { format, width, height }); - } - } - Err(()) - } -} - -#[cfg(feature = "image")] -impl ToString for ImageParams { - fn to_string(&self) -> String { - let mut params = String::new(); - - params.push_str(&format!("format={}", self.format())); - if let Some(width) = &self.width { - params.push_str(&format!("width={}", width)); - } - if let Some(height) = &self.height { - params.push_str(&format!("height={}", height)); - } - params - } -} - pub async fn get_hash(stream: impl Stream + Send) -> (String, Vec) { let mut hasher = Sha256::new(); @@ -131,13 +23,6 @@ pub async fn get_hash(stream: impl Stream + Send) -> (String, Vec< (hash, buffer) } -/// Calculate sha256 hash for given blob -pub fn calculate_hash(blob: &[u8]) -> String { - let mut hasher = Sha256::new(); - hasher.update(blob); - URL_SAFE_ENGINE.encode(hasher.finalize()) -} - #[derive(FromQueryResult)] pub(super) struct InternalBlobMetadata { pub(super) size: i64, @@ -153,102 +38,3 @@ impl From for BlobMetadata { } } } - -impl TryFrom> for BucketStorage { - type Error = JwstStorageError; - - fn try_from(map: HashMap) -> Result { - let mut builder = BucketStorageBuilder::new(); - let access_token = map.get("BUCKET_ACCESS_TOKEN"); - let secret_access_key = map.get("BUCKET_SECRET_TOKEN"); - let endpoint = map.get("BUCKET_ENDPOINT"); - let bucket = map.get("BUCKET_NAME"); - let root = map.get("BUCKET_ROOT"); - - if let Some(access_token) = access_token { - builder = builder.access_key(access_token); - } - if let Some(secret_access_key) = secret_access_key { - builder = builder.secret_access_key(secret_access_key); - } - if let Some(endpoint) = endpoint { - builder = builder.endpoint(endpoint); - } - if let Some(bucket) = bucket { - builder = builder.bucket(bucket); - } - if let Some(root) = root { - builder = builder.root(root); - } - - builder.build() - } -} - -impl TryFrom for BucketStorage { - type Error = JwstStorageError; - - fn try_from(value: MixedBucketDBParam) -> Result { - let mut builder = BucketStorageBuilder::new(); - builder = builder.access_key(&value.access_key); - builder = builder.secret_access_key(&value.secret_access_key); - builder = builder.endpoint(&value.endpoint); - builder = builder.bucket(&value.bucket.unwrap_or("__default_bucket__".to_string())); - builder = builder.root(&value.root.unwrap_or("__default_root__".to_string())); - builder.build() - } -} - -#[derive(Default)] -pub struct BucketStorageBuilder { - access_key: String, - secret_access_key: String, - endpoint: String, - bucket: String, - root: String, -} - -impl BucketStorageBuilder { - pub fn new() -> Self { - Self::default() - } - - pub fn access_key(mut self, access_key: &str) -> Self { - self.access_key = access_key.to_string(); - self - } - - pub fn secret_access_key(mut self, secret_access_key: &str) -> Self { - self.secret_access_key = secret_access_key.to_string(); - self - } - - pub fn endpoint(mut self, endpoint: &str) -> Self { - self.endpoint = endpoint.to_string(); - self - } - - pub fn bucket(mut self, bucket: &str) -> Self { - self.bucket = bucket.to_string(); - self - } - - pub fn root(mut self, root: &str) -> Self { - self.root = root.to_string(); - self - } - - pub fn build(self) -> JwstStorageResult { - let mut builder = S3::default(); - - builder.bucket(self.bucket.as_str()); - builder.root(self.root.as_str()); - builder.endpoint(self.endpoint.as_str()); - builder.access_key_id(self.access_key.as_str()); - builder.secret_access_key(self.secret_access_key.as_str()); - - Ok(BucketStorage { - op: Operator::new(builder)?.finish(), - }) - } -} diff --git a/libs/jwst-core-storage/src/storage/difflog.rs b/libs/jwst-core-storage/src/storage/difflog.rs deleted file mode 100644 index 4c7ec4bfd..000000000 --- a/libs/jwst-core-storage/src/storage/difflog.rs +++ /dev/null @@ -1,43 +0,0 @@ -use super::{entities::prelude::*, *}; -use crate::types::JwstStorageResult; - -// type DiffLogModel = ::Model; -type DiffLogActiveModel = super::entities::diff_log::ActiveModel; -// type DiffLogColumn = ::Column; - -pub struct DiffLogRecord { - bucket: Arc, - pool: DatabaseConnection, -} - -impl DiffLogRecord { - pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { - Ok(Self { bucket, pool }) - } - - pub async fn init_pool(database: &str) -> JwstStorageResult { - let is_sqlite = is_sqlite(database); - let pool = create_connection(database, is_sqlite).await?; - - Self::init_with_pool(pool, get_bucket(is_sqlite)).await - } - - pub async fn insert(&self, workspace: String, ts: DateTime, log: String) -> JwstStorageResult<()> { - let _lock = self.bucket.write().await; - DiffLog::insert(DiffLogActiveModel { - workspace: Set(workspace), - timestamp: Set(ts.into()), - log: Set(log), - ..Default::default() - }) - .exec(&self.pool) - .await?; - Ok(()) - } - - pub async fn count(&self) -> JwstStorageResult { - let _lock = self.bucket.read().await; - let count = DiffLog::find().count(&self.pool).await?; - Ok(count) - } -} diff --git a/libs/jwst-core-storage/src/storage/mod.rs b/libs/jwst-core-storage/src/storage/mod.rs index 730fab4c9..99a1c09f1 100644 --- a/libs/jwst-core-storage/src/storage/mod.rs +++ b/libs/jwst-core-storage/src/storage/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod blobs; -mod difflog; mod docs; mod test; @@ -7,23 +6,19 @@ use std::{collections::HashMap, time::Instant}; #[cfg(feature = "image")] use blobs::BlobAutoStorage; -use blobs::BlobDBStorage; +#[cfg(feature = "bucket")] +use blobs::BlobBucketStorage; +use blobs::{BlobDBStorage, BlobStorageType, JwstBlobStorage}; use docs::SharedDocDBStorage; use jwst_storage_migration::{Migrator, MigratorTrait}; use tokio::sync::Mutex; -use self::difflog::DiffLogRecord; use super::*; -use crate::{ - storage::blobs::{BlobBucketStorage, BlobStorageType, JwstBlobStorage}, - types::JwstStorageError, -}; pub struct JwstStorage { pool: DatabaseConnection, blobs: JwstBlobStorage, docs: SharedDocDBStorage, - difflog: DiffLogRecord, last_migrate: Mutex>, } @@ -49,18 +44,17 @@ impl JwstStorage { JwstBlobStorage::RawStorage(Arc::new(db)) } } + #[cfg(feature = "bucket")] BlobStorageType::MixedBucketDB(param) => JwstBlobStorage::BucketStorage( BlobBucketStorage::init_with_pool(pool.clone(), bucket.clone(), Some(param.try_into()?)).await?, ), }; let docs = SharedDocDBStorage::init_with_pool(pool.clone(), bucket.clone()).await?; - let difflog = DiffLogRecord::init_with_pool(pool.clone(), bucket).await?; Ok(Self { pool, blobs, docs, - difflog, last_migrate: Mutex::new(HashMap::new()), }) } @@ -108,10 +102,6 @@ impl JwstStorage { &self.docs } - pub fn difflog(&self) -> &DiffLogRecord { - &self.difflog - } - pub async fn with_pool(&self, func: F) -> JwstStorageResult where F: Fn(DatabaseConnection) -> Fut, @@ -199,7 +189,6 @@ impl JwstStorage { #[cfg(test)] mod tests { use super::*; - use crate::storage::blobs::MixedBucketDBParam; #[tokio::test] async fn test_sqlite_storage() { @@ -210,6 +199,7 @@ mod tests { } #[tokio::test] + #[cfg(feature = "bucket")] #[ignore = "need to config bucket auth"] async fn test_bucket_storage() { let bucket_params = MixedBucketDBParam { diff --git a/libs/jwst-core-storage/src/types.rs b/libs/jwst-core-storage/src/types.rs index 81d4be76e..2b2259e9a 100644 --- a/libs/jwst-core-storage/src/types.rs +++ b/libs/jwst-core-storage/src/types.rs @@ -19,8 +19,10 @@ pub enum JwstStorageError { Jwst(#[from] jwst_core::JwstError), #[error("failed to process blob")] JwstBlob(#[from] crate::storage::blobs::JwstBlobError), + #[cfg(feature = "bucket")] #[error("bucket error")] JwstBucket(#[from] opendal::Error), + #[cfg(feature = "bucket")] #[error("env variables read error")] DotEnvy(#[from] dotenvy::Error), } diff --git a/libs/jwst-storage/Cargo.toml b/libs/jwst-storage/Cargo.toml index c27febd9a..286df9281 100644 --- a/libs/jwst-storage/Cargo.toml +++ b/libs/jwst-storage/Cargo.toml @@ -23,13 +23,13 @@ lib0 = "0.16.5" path-ext = "0.1.0" sha2 = "0.10.6" sea-orm = { version = "0.12.2", features = ["runtime-tokio-rustls", "macros"] } -sea-orm-migration = "0.12.2" +sea-orm-migration = { version = "0.12.2", default-features = false } thiserror = "1.0.40" tokio = { version = "1.27.0", features = ["fs", "macros", "sync"] } tokio-util = { version = "0.7.7", features = ["io"] } url = "2.3.1" yrs = "0.16.5" -opendal = { version = "0.38.0", default-features = false, features = [ +opendal = { version = "0.39.0", default-features = false, features = [ "rustls", "services-s3", ] } diff --git a/libs/jwst-storage/src/migration/Cargo.toml b/libs/jwst-storage/src/migration/Cargo.toml index 6c740b997..5b8bba191 100644 --- a/libs/jwst-storage/src/migration/Cargo.toml +++ b/libs/jwst-storage/src/migration/Cargo.toml @@ -13,4 +13,5 @@ tokio = { version = "^1", features = ["macros", "rt-multi-thread"] } [dependencies.sea-orm-migration] version = "0.12.2" +default-features = false features = ["runtime-tokio-rustls", "sqlx-postgres"] From c97c623f45f396547360c804a31f0a141eb0aca6 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Tue, 22 Aug 2023 02:03:54 +0800 Subject: [PATCH 15/49] chore: cleanup codes --- .../Sources/OctoBase/jwst-core-swift.swift | 647 ------------------ 1 file changed, 647 deletions(-) delete mode 100644 apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-core-swift.swift diff --git a/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-core-swift.swift b/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-core-swift.swift deleted file mode 100644 index 0556654ee..000000000 --- a/apps/swift/OctoBaseSwift/Sources/OctoBase/jwst-core-swift.swift +++ /dev/null @@ -1,647 +0,0 @@ -import RustXcframework - -public class Block: BlockRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$Block$_free(ptr) - } - } -} -public class BlockRefMut: BlockRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -public class BlockRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension BlockRef { - public func get(_ block_id: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get(ptr, { let rustString = block_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return DynamicValue(ptr: val!) } else { return nil } }() - } - - public func children() -> RustVec { - RustVec(ptr: __swift_bridge__$Block$children(ptr)) - } - - public func push_children(_ block: BlockRef) { - __swift_bridge__$Block$push_children(ptr, block.ptr) - } - - public func insert_children_at(_ block: BlockRef, _ pos: UInt32) { - __swift_bridge__$Block$insert_children_at(ptr, block.ptr, pos) - } - - public func insert_children_before(_ block: BlockRef, _ reference: GenericToRustStr) { - reference.toRustStr({ referenceAsRustStr in - __swift_bridge__$Block$insert_children_before(ptr, block.ptr, referenceAsRustStr) - }) - } - - public func insert_children_after(_ block: BlockRef, _ reference: GenericToRustStr) { - reference.toRustStr({ referenceAsRustStr in - __swift_bridge__$Block$insert_children_after(ptr, block.ptr, referenceAsRustStr) - }) - } - - public func remove_children(_ block: BlockRef) { - __swift_bridge__$Block$remove_children(ptr, block.ptr) - } - - public func exists_children(_ block_id: GenericToRustStr) -> Int32 { - return block_id.toRustStr({ block_idAsRustStr in - __swift_bridge__$Block$exists_children(ptr, block_idAsRustStr) - }) - } - - public func parent() -> RustString { - RustString(ptr: __swift_bridge__$Block$parent(ptr)) - } - - public func updated() -> UInt64 { - __swift_bridge__$Block$updated(ptr) - } - - public func id() -> RustString { - RustString(ptr: __swift_bridge__$Block$id(ptr)) - } - - public func flavour() -> RustString { - RustString(ptr: __swift_bridge__$Block$flavour(ptr)) - } - - public func created() -> UInt64 { - __swift_bridge__$Block$created(ptr) - } - - public func set_bool(_ key: GenericIntoRustString, _ value: Bool) { - __swift_bridge__$Block$set_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), value) - } - - public func set_string(_ key: GenericIntoRustString, _ value: GenericIntoRustString) { - __swift_bridge__$Block$set_string(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = value.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func set_float(_ key: GenericIntoRustString, _ value: Double) { - __swift_bridge__$Block$set_float(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), value) - } - - public func set_integer(_ key: GenericIntoRustString, _ value: Int64) { - __swift_bridge__$Block$set_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), value) - } - - public func set_null(_ key: GenericIntoRustString) { - __swift_bridge__$Block$set_null(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func is_bool(_ key: GenericIntoRustString) -> Bool { - __swift_bridge__$Block$is_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func is_string(_ key: GenericIntoRustString) -> Bool { - __swift_bridge__$Block$is_string(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func is_float(_ key: GenericIntoRustString) -> Bool { - __swift_bridge__$Block$is_float(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func is_integer(_ key: GenericIntoRustString) -> Bool { - __swift_bridge__$Block$is_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()) - } - - public func get_bool(_ key: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get_bool(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() - } - - public func get_string(_ key: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get_string(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return RustString(ptr: val!) } else { return nil } }() - } - - public func get_float(_ key: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get_float(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() - } - - public func get_integer(_ key: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Block$get_integer(ptr, { let rustString = key.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val.is_some { return val.val } else { return nil } }() - } -} -extension Block: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_Block$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_Block$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Block) { - __swift_bridge__$Vec_Block$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_Block$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (Block(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Block$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return BlockRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Block$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return BlockRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_Block$len(vecPtr) - } -} - - -public class DynamicValueMap: DynamicValueMapRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$DynamicValueMap$_free(ptr) - } - } -} -public class DynamicValueMapRefMut: DynamicValueMapRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -public class DynamicValueMapRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension DynamicValueMap: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_DynamicValueMap$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_DynamicValueMap$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: DynamicValueMap) { - __swift_bridge__$Vec_DynamicValueMap$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValueMap$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (DynamicValueMap(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValueMap$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return DynamicValueMapRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValueMap$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return DynamicValueMapRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_DynamicValueMap$len(vecPtr) - } -} - - -public class DynamicValue: DynamicValueRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$DynamicValue$_free(ptr) - } - } -} -public class DynamicValueRefMut: DynamicValueRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -public class DynamicValueRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension DynamicValueRef { - public func as_bool() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_bool(ptr); if val.is_some { return val.val } else { return nil } }() - } - - public func as_number() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_number(ptr); if val.is_some { return val.val } else { return nil } }() - } - - public func as_int() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_int(ptr); if val.is_some { return val.val } else { return nil } }() - } - - public func as_string() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_string(ptr); if val != nil { return RustString(ptr: val!) } else { return nil } }() - } - - public func as_map() -> Optional { - { let val = __swift_bridge__$DynamicValue$as_map(ptr); if val != nil { return DynamicValueMap(ptr: val!) } else { return nil } }() - } - - public func as_array() -> Optional> { - { let val = __swift_bridge__$DynamicValue$as_array(ptr); if val != nil { return RustVec(ptr: val!) } else { return nil } }() - } - - public func as_buffer() -> Optional> { - { let val = __swift_bridge__$DynamicValue$as_buffer(ptr); if val != nil { return RustVec(ptr: val!) } else { return nil } }() - } -} -extension DynamicValue: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_DynamicValue$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_DynamicValue$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: DynamicValue) { - __swift_bridge__$Vec_DynamicValue$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValue$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (DynamicValue(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValue$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return DynamicValueRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_DynamicValue$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return DynamicValueRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_DynamicValue$len(vecPtr) - } -} - - -public class Workspace: WorkspaceRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$Workspace$_free(ptr) - } - } -} -public class WorkspaceRefMut: WorkspaceRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -public class WorkspaceRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension WorkspaceRef { - public func id() -> RustString { - RustString(ptr: __swift_bridge__$Workspace$id(ptr)) - } - - public func client_id() -> UInt64 { - __swift_bridge__$Workspace$client_id(ptr) - } - - public func get(_ block_id: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Workspace$get(ptr, { let rustString = block_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return Block(ptr: val!) } else { return nil } }() - } - - public func create(_ block_id: GenericIntoRustString, _ flavour: GenericIntoRustString) -> Block { - Block(ptr: __swift_bridge__$Workspace$create(ptr, { let rustString = block_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = flavour.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) - } - - public func search(_ query: GenericIntoRustString) -> RustString { - RustString(ptr: __swift_bridge__$Workspace$search(ptr, { let rustString = query.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) - } - - public func get_blocks_by_flavour(_ flavour: GenericToRustStr) -> RustVec { - return flavour.toRustStr({ flavourAsRustStr in - RustVec(ptr: __swift_bridge__$Workspace$get_blocks_by_flavour(ptr, flavourAsRustStr)) - }) - } - - public func get_search_index() -> RustVec { - RustVec(ptr: __swift_bridge__$Workspace$get_search_index(ptr)) - } - - public func set_search_index(_ fields: RustVec) -> Bool { - __swift_bridge__$Workspace$set_search_index(ptr, { let val = fields; val.isOwned = false; return val.ptr }()) - } -} -extension Workspace: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_Workspace$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_Workspace$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Workspace) { - __swift_bridge__$Vec_Workspace$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_Workspace$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (Workspace(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Workspace$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return WorkspaceRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Workspace$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return WorkspaceRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_Workspace$len(vecPtr) - } -} - - -public class JwstWorkSpaceResult: JwstWorkSpaceResultRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$JwstWorkSpaceResult$_free(ptr) - } - } -} -public class JwstWorkSpaceResultRefMut: JwstWorkSpaceResultRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -public class JwstWorkSpaceResultRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension JwstWorkSpaceResult: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_JwstWorkSpaceResult$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_JwstWorkSpaceResult$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: JwstWorkSpaceResult) { - __swift_bridge__$Vec_JwstWorkSpaceResult$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_JwstWorkSpaceResult$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (JwstWorkSpaceResult(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_JwstWorkSpaceResult$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return JwstWorkSpaceResultRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_JwstWorkSpaceResult$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return JwstWorkSpaceResultRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_JwstWorkSpaceResult$len(vecPtr) - } -} - - -public class Storage: StorageRefMut { - var isOwned: Bool = true - - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } - - deinit { - if isOwned { - __swift_bridge__$Storage$_free(ptr) - } - } -} -extension Storage { - public convenience init(_ path: GenericIntoRustString) { - self.init(ptr: __swift_bridge__$Storage$new({ let rustString = path.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) - } - - public convenience init(_ path: GenericIntoRustString, _ level: GenericIntoRustString) { - self.init(ptr: __swift_bridge__$Storage$new_with_log_level({ let rustString = path.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = level.intoRustString(); rustString.isOwned = false; return rustString.ptr }())) - } -} -public class StorageRefMut: StorageRef { - public override init(ptr: UnsafeMutableRawPointer) { - super.init(ptr: ptr) - } -} -extension StorageRefMut { - public func connect(_ workspace_id: GenericIntoRustString, _ remote: GenericIntoRustString) -> Optional { - { let val = __swift_bridge__$Storage$connect(ptr, { let rustString = workspace_id.intoRustString(); rustString.isOwned = false; return rustString.ptr }(), { let rustString = remote.intoRustString(); rustString.isOwned = false; return rustString.ptr }()); if val != nil { return Workspace(ptr: val!) } else { return nil } }() - } -} -public class StorageRef { - var ptr: UnsafeMutableRawPointer - - public init(ptr: UnsafeMutableRawPointer) { - self.ptr = ptr - } -} -extension StorageRef { - public func error() -> Optional { - { let val = __swift_bridge__$Storage$error(ptr); if val != nil { return RustString(ptr: val!) } else { return nil } }() - } - - public func is_offline() -> Bool { - __swift_bridge__$Storage$is_offline(ptr) - } - - public func is_connected() -> Bool { - __swift_bridge__$Storage$is_connected(ptr) - } - - public func is_finished() -> Bool { - __swift_bridge__$Storage$is_finished(ptr) - } - - public func is_error() -> Bool { - __swift_bridge__$Storage$is_error(ptr) - } - - public func get_sync_state() -> RustString { - RustString(ptr: __swift_bridge__$Storage$get_sync_state(ptr)) - } - - public func get_last_synced() -> RustVec { - RustVec(ptr: __swift_bridge__$Storage$get_last_synced(ptr)) - } -} -extension Storage: Vectorizable { - public static func vecOfSelfNew() -> UnsafeMutableRawPointer { - __swift_bridge__$Vec_Storage$new() - } - - public static func vecOfSelfFree(vecPtr: UnsafeMutableRawPointer) { - __swift_bridge__$Vec_Storage$drop(vecPtr) - } - - public static func vecOfSelfPush(vecPtr: UnsafeMutableRawPointer, value: Storage) { - __swift_bridge__$Vec_Storage$push(vecPtr, {value.isOwned = false; return value.ptr;}()) - } - - public static func vecOfSelfPop(vecPtr: UnsafeMutableRawPointer) -> Optional { - let pointer = __swift_bridge__$Vec_Storage$pop(vecPtr) - if pointer == nil { - return nil - } else { - return (Storage(ptr: pointer!) as! Self) - } - } - - public static func vecOfSelfGet(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Storage$get(vecPtr, index) - if pointer == nil { - return nil - } else { - return StorageRef(ptr: pointer!) - } - } - - public static func vecOfSelfGetMut(vecPtr: UnsafeMutableRawPointer, index: UInt) -> Optional { - let pointer = __swift_bridge__$Vec_Storage$get_mut(vecPtr, index) - if pointer == nil { - return nil - } else { - return StorageRefMut(ptr: pointer!) - } - } - - public static func vecOfSelfLen(vecPtr: UnsafeMutableRawPointer) -> UInt { - __swift_bridge__$Vec_Storage$len(vecPtr) - } -} - - - From 8d87a2955ca32543b8fc3e63251ef64e8fe08bf3 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Wed, 23 Aug 2023 22:55:53 +0800 Subject: [PATCH 16/49] feat: keck core --- .vscode/launch.json | 6 +- Cargo.lock | 261 +++++----- Cargo.toml | 1 + apps/keck-core/.env.template | 2 + apps/keck-core/.gitignore | 1 + apps/keck-core/Cargo.toml | 65 +++ apps/keck-core/src/main.rs | 16 + apps/keck-core/src/server/api/blobs.rs | 238 +++++++++ apps/keck-core/src/server/api/blocks/block.rs | 468 ++++++++++++++++++ apps/keck-core/src/server/api/blocks/mod.rs | 154 ++++++ .../keck-core/src/server/api/blocks/schema.rs | 51 ++ .../src/server/api/blocks/workspace.rs | 406 +++++++++++++++ apps/keck-core/src/server/api/mod.rs | 117 +++++ apps/keck-core/src/server/mod.rs | 137 +++++ apps/keck-core/src/server/subscribe.rs | 69 +++ apps/keck-core/src/server/sync/blobs.rs | 135 +++++ .../src/server/sync/collaboration.rs | 305 ++++++++++++ apps/keck-core/src/server/sync/mod.rs | 25 + apps/keck-core/src/server/utils.rs | 3 + .../RustXcframework.xcframework/Info.plist | 14 +- libs/jwst-binding/jwst-swift/src/storage.rs | 3 +- libs/jwst-core-rpc/Cargo.toml | 6 +- libs/jwst-core-rpc/src/context.rs | 2 +- libs/jwst-core-rpc/src/handler.rs | 1 + libs/jwst-core-storage/Cargo.toml | 2 +- libs/jwst-core/src/workspace/sync.rs | 18 +- 26 files changed, 2347 insertions(+), 159 deletions(-) create mode 100644 apps/keck-core/.env.template create mode 100644 apps/keck-core/.gitignore create mode 100644 apps/keck-core/Cargo.toml create mode 100644 apps/keck-core/src/main.rs create mode 100644 apps/keck-core/src/server/api/blobs.rs create mode 100644 apps/keck-core/src/server/api/blocks/block.rs create mode 100644 apps/keck-core/src/server/api/blocks/mod.rs create mode 100644 apps/keck-core/src/server/api/blocks/schema.rs create mode 100644 apps/keck-core/src/server/api/blocks/workspace.rs create mode 100644 apps/keck-core/src/server/api/mod.rs create mode 100644 apps/keck-core/src/server/mod.rs create mode 100644 apps/keck-core/src/server/subscribe.rs create mode 100644 apps/keck-core/src/server/sync/blobs.rs create mode 100644 apps/keck-core/src/server/sync/collaboration.rs create mode 100644 apps/keck-core/src/server/sync/mod.rs create mode 100644 apps/keck-core/src/server/utils.rs diff --git a/.vscode/launch.json b/.vscode/launch.json index 26d4b6e5f..37d715ec8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -26,11 +26,11 @@ { "type": "lldb", "request": "launch", - "name": "Debug executable 'keck'", + "name": "Debug executable 'keck-core'", "cargo": { - "args": ["build", "--bin=keck", "--package=keck"], + "args": ["build", "--bin=keck-core", "--package=keck-core"], "filter": { - "name": "keck", + "name": "keck-core", "kind": "bin" } }, diff --git a/Cargo.lock b/Cargo.lock index 0905cb264..c9eb8c054 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,15 +12,6 @@ dependencies = [ "regex", ] -[[package]] -name = "addr2line" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" -dependencies = [ - "gimli", -] - [[package]] name = "adler" version = "1.0.2" @@ -135,7 +126,7 @@ name = "affine-cloud" version = "0.1.0" dependencies = [ "axum", - "axum-test-helper", + "axum-test-helper 0.2.0", "bytes", "chrono", "cloud-database", @@ -354,7 +345,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.25", + "time 0.3.27", ] [[package]] @@ -370,7 +361,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.25", + "time 0.3.27", ] [[package]] @@ -562,6 +553,24 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-test-helper" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298f62fa902c2515c169ab0bfb56c593229f33faa01131215d58e3d4898e3aa9" +dependencies = [ + "axum", + "bytes", + "http", + "http-body", + "hyper", + "reqwest", + "serde", + "tokio", + "tower", + "tower-service", +] + [[package]] name = "backon" version = "0.4.1" @@ -574,21 +583,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "backtrace" -version = "0.3.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" -dependencies = [ - "addr2line", - "cc", - "cfg-if 1.0.0", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - [[package]] name = "base16ct" version = "0.1.1" @@ -1375,9 +1369,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.5.0" +version = "5.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" +checksum = "edd72493923899c6f10c641bdbdeddc7183d6396641d99c1a0d1597f37f92e28" dependencies = [ "cfg-if 1.0.0", "hashbrown 0.14.0", @@ -2120,12 +2114,6 @@ dependencies = [ "weezl", ] -[[package]] -name = "gimli" -version = "0.27.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" - [[package]] name = "git2" version = "0.16.1" @@ -2207,9 +2195,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -2426,7 +2414,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2", "tokio", "tower-service", "tracing", @@ -3022,7 +3010,7 @@ version = "0.1.0" dependencies = [ "anyhow", "axum", - "axum-test-helper", + "axum-test-helper 0.2.0", "cfg-if 1.0.0", "cloud-infra", "dotenvy", @@ -3042,7 +3030,7 @@ dependencies = [ "serde_json", "sqlx", "thiserror", - "time 0.3.25", + "time 0.3.27", "tokio", "tower", "tower-http", @@ -3051,6 +3039,39 @@ dependencies = [ "yrs", ] +[[package]] +name = "keck-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "axum-test-helper 0.3.0", + "cfg-if 1.0.0", + "dotenvy", + "futures", + "jwst-core", + "jwst-core-rpc", + "jwst-core-storage", + "jwst-logger", + "lib0", + "libc", + "log", + "mimalloc", + "nanoid", + "rand 0.8.5", + "reqwest", + "serde", + "serde_json", + "sqlx", + "thiserror", + "time 0.3.27", + "tokio", + "tower", + "tower-http", + "utoipa", + "yrs", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -3087,7 +3108,7 @@ dependencies = [ "quoted_printable", "rustls 0.21.6", "rustls-pemfile", - "socket2 0.4.9", + "socket2", "tokio", "tokio-rustls", "webpki-roots 0.23.1", @@ -3410,6 +3431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", + "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -3487,9 +3509,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg", "num-integer", @@ -3571,15 +3593,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" -[[package]] -name = "object" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" -dependencies = [ - "memchr", -] - [[package]] name = "oid-registry" version = "0.4.0" @@ -3665,9 +3678,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "ordered-float" -version = "3.9.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126d3e6f3926bfb0fb24495b4f4da50626f547e54956594748e3d8882a0320b4" +checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" dependencies = [ "arbitrary", "num-traits", @@ -4376,7 +4389,7 @@ checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", "ring", - "time 0.3.25", + "time 0.3.27", "x509-parser 0.14.0", "yasna", ] @@ -4494,9 +4507,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "20b9b67e2ca7dd9e9f9285b759de30ff538aab981abaaf7bc9bd90b84a0126c3" dependencies = [ "base64 0.21.2", "bytes", @@ -4531,7 +4544,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.22.6", + "webpki-roots 0.25.2", "winreg", ] @@ -4738,12 +4751,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - [[package]] name = "rustc-hash" version = "1.1.0" @@ -4791,7 +4798,7 @@ dependencies = [ "log", "ring", "sct 0.6.1", - "webpki 0.21.4", + "webpki", ] [[package]] @@ -4802,7 +4809,7 @@ checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" dependencies = [ "log", "ring", - "rustls-webpki 0.101.3", + "rustls-webpki 0.101.4", "sct 0.7.0", ] @@ -4829,9 +4836,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.100.1" +version = "0.100.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab" dependencies = [ "ring", "untrusted", @@ -4839,9 +4846,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.3" +version = "0.101.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261e9e0888cba427c3316e6322805653c9425240b6fd96cee7cb671ab70ab8d0" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" dependencies = [ "ring", "untrusted", @@ -4992,7 +4999,7 @@ dependencies = [ "sqlx", "strum 0.25.0", "thiserror", - "time 0.3.25", + "time 0.3.27", "tracing", "url", "uuid", @@ -5060,7 +5067,7 @@ dependencies = [ "rust_decimal", "sea-query-derive", "serde_json", - "time 0.3.25", + "time 0.3.27", "uuid", ] @@ -5076,7 +5083,7 @@ dependencies = [ "sea-query", "serde_json", "sqlx", - "time 0.3.25", + "time 0.3.27", "uuid", ] @@ -5330,20 +5337,20 @@ dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.25", + "time 0.3.27", ] [[package]] name = "siphasher" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] @@ -5391,16 +5398,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "socket2" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" -dependencies = [ - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "spin" version = "0.5.2" @@ -5499,7 +5496,7 @@ dependencies = [ "smallvec", "sqlformat", "thiserror", - "time 0.3.25", + "time 0.3.27", "tokio", "tokio-stream", "tracing", @@ -5588,7 +5585,7 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", - "time 0.3.25", + "time 0.3.27", "tracing", "uuid", "whoami", @@ -5633,7 +5630,7 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", - "time 0.3.25", + "time 0.3.27", "tracing", "uuid", "whoami", @@ -5658,7 +5655,7 @@ dependencies = [ "percent-encoding", "serde", "sqlx-core", - "time 0.3.25", + "time 0.3.27", "tracing", "url", "uuid", @@ -5897,7 +5894,7 @@ dependencies = [ "tantivy-query-grammar", "tempfile", "thiserror", - "time 0.3.25", + "time 0.3.27", "uuid", "winapi", ] @@ -6013,9 +6010,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" +checksum = "0bb39ee79a6d8de55f48f2293a830e040392f1c5f16e336bdd1788cd0aadce07" dependencies = [ "deranged", "itoa", @@ -6032,9 +6029,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.11" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" +checksum = "733d258752e9303d392b94b75230d07b0b9c489350c69b851fc6c065fde3e8f9" dependencies = [ "time-core", ] @@ -6075,11 +6072,11 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" dependencies = [ - "backtrace", + "autocfg", "bytes", "libc", "mio", @@ -6087,7 +6084,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.3", + "socket2", "tokio-macros", "windows-sys 0.48.0", ] @@ -6277,7 +6274,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "time 0.3.25", + "time 0.3.27", "tracing-core", "tracing-subscriber", ] @@ -6377,9 +6374,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] @@ -6566,7 +6563,7 @@ dependencies = [ "rustc_version", "rustversion", "thiserror", - "time 0.3.25", + "time 0.3.27", ] [[package]] @@ -6698,9 +6695,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasm-streams" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" dependencies = [ "futures-util", "js-sys", @@ -6738,32 +6735,13 @@ dependencies = [ "untrusted", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "webpki-roots" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" -dependencies = [ - "webpki 0.22.0", -] - [[package]] name = "webpki-roots" version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" dependencies = [ - "rustls-webpki 0.100.1", + "rustls-webpki 0.100.2", ] [[package]] @@ -6772,9 +6750,15 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" dependencies = [ - "rustls-webpki 0.101.3", + "rustls-webpki 0.101.4", ] +[[package]] +name = "webpki-roots" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" + [[package]] name = "webrtc" version = "0.8.0" @@ -6803,7 +6787,7 @@ dependencies = [ "smol_str 0.2.0", "stun", "thiserror", - "time 0.3.25", + "time 0.3.27", "tokio", "turn", "url", @@ -6867,7 +6851,7 @@ dependencies = [ "subtle", "thiserror", "tokio", - "webpki 0.21.4", + "webpki", "webrtc-util", "x25519-dalek", "x509-parser 0.13.2", @@ -6904,7 +6888,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f08dfd7a6e3987e255c4dbe710dde5d94d0f0574f8a21afa95d171376c143106" dependencies = [ "log", - "socket2 0.4.9", + "socket2", "thiserror", "tokio", "webrtc-util", @@ -7181,11 +7165,12 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if 1.0.0", + "windows-sys 0.48.0", ] [[package]] @@ -7224,7 +7209,7 @@ dependencies = [ "oid-registry 0.4.0", "rusticata-macros", "thiserror", - "time 0.3.25", + "time 0.3.27", ] [[package]] @@ -7243,7 +7228,7 @@ dependencies = [ "ring", "rusticata-macros", "thiserror", - "time 0.3.25", + "time 0.3.27", ] [[package]] @@ -7260,7 +7245,7 @@ dependencies = [ "oid-registry 0.6.1", "rusticata-macros", "thiserror", - "time 0.3.25", + "time 0.3.27", ] [[package]] @@ -7279,7 +7264,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.25", + "time 0.3.27", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5b51626cc..d071dc365 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "apps/cloud", "apps/doc_merger", "apps/keck", + "apps/keck-core", "libs/cloud-database", "libs/cloud-database/migration", "libs/cloud-infra", diff --git a/apps/keck-core/.env.template b/apps/keck-core/.env.template new file mode 100644 index 000000000..e3cca72ae --- /dev/null +++ b/apps/keck-core/.env.template @@ -0,0 +1,2 @@ +# The block observation webhook endpoint +# HOOK_ENDPOINT= diff --git a/apps/keck-core/.gitignore b/apps/keck-core/.gitignore new file mode 100644 index 000000000..adfa211b4 --- /dev/null +++ b/apps/keck-core/.gitignore @@ -0,0 +1 @@ +updates.* \ No newline at end of file diff --git a/apps/keck-core/Cargo.toml b/apps/keck-core/Cargo.toml new file mode 100644 index 000000000..15b160c5c --- /dev/null +++ b/apps/keck-core/Cargo.toml @@ -0,0 +1,65 @@ +[package] +name = "keck-core" +version = "0.1.0" +authors = ["DarkSky "] +edition = "2021" +license = "AGPL-3.0-only" + +[features] +default = ["jwst"] +affine = ["jwst-core-storage/sqlite"] +jwst = [ + # "api", + "jwst-core-storage/sqlite", + "jwst-core-storage/mysql", + "jwst-core-storage/postgres", +] +api = ["utoipa"] + +[dependencies] +anyhow = "1.0.70" +axum = { version = "0.6.20", features = ["headers", "ws"] } +cfg-if = "1.0.0" +futures = "0.3.28" +lib0 = { version = "0.16.5", features = ["lib0-serde"] } +log = { version = "0.4.17", features = [ + "max_level_trace", + "release_max_level_info", +] } +dotenvy = "0.15.7" +mimalloc = "0.1.36" +nanoid = "0.4.0" +serde = { version = "1.0.160", features = ["derive"] } +serde_json = "1.0.96" +sqlx = { version = "0.7.1", features = [ + "chrono", + "macros", + "migrate", + "runtime-tokio-rustls", +] } +tower = "0.4.13" +tower-http = { version = "0.4.0", features = ["cors"] } +thiserror = "1.0.40" +time = "0.3.20" +tokio = { version = "=1.28.0", features = [ + "macros", + "rt-multi-thread", + "signal", +] } +utoipa = { version = "3.5.0", features = ["axum_extras"], optional = true } +yrs = "0.16.5" +libc = "0.2.147" +rand = "0.8.5" +reqwest = { version = "0.11.19", default-features = false, features = [ + "json", + "rustls-tls", +] } + +# ======= workspace dependencies ======= +jwst-core = { workspace = true } +jwst-logger = { workspace = true } +jwst-core-rpc = { workspace = true } +jwst-core-storage = { workspace = true } + +[dev-dependencies] +axum-test-helper = "0.3.0" diff --git a/apps/keck-core/src/main.rs b/apps/keck-core/src/main.rs new file mode 100644 index 000000000..53ea10e9b --- /dev/null +++ b/apps/keck-core/src/main.rs @@ -0,0 +1,16 @@ +mod server; + +use jwst_logger::init_logger; + +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +const PKG_NAME: &str = env!("CARGO_PKG_NAME"); + +#[tokio::main] +async fn main() { + // ignore load error when missing env file + let _ = dotenvy::dotenv(); + init_logger(PKG_NAME); + server::start_server().await; +} diff --git a/apps/keck-core/src/server/api/blobs.rs b/apps/keck-core/src/server/api/blobs.rs new file mode 100644 index 000000000..b4e1e63a3 --- /dev/null +++ b/apps/keck-core/src/server/api/blobs.rs @@ -0,0 +1,238 @@ +use axum::{extract::BodyStream, response::Response, routing::post}; +use futures::{future, StreamExt}; +use jwst_core::BlobStorage; +use utoipa::ToSchema; + +use super::*; + +#[derive(Serialize, ToSchema)] +pub struct BlobStatus { + id: Option, + exists: bool, +} + +/// Check a `Blob` is exists by id +/// - Return 200 if `Blob` is exists. +/// - Return 404 Not Found if `Workspace` or `Blob` not exists. +#[utoipa::path( + head, + tag = "Blobs", + context_path = "/api/blobs", + path = "/{workspace_id}/{hash}", + params( + ("workspace_id", description = "workspace id"), + ("hash", description = "blob hash"), + ), + responses( + (status = 200, description = "Blob exists"), + (status = 404, description = "Workspace or blob content not found"), + (status = 500, description = "Failed to query blobs"), + ) +)] +pub async fn check_blob(Extension(context): Extension>, Path(params): Path<(String, String)>) -> Response { + let (workspace, hash) = params; + info!("check_blob: {}, {}", workspace, hash); + if let Ok(exists) = context.storage.blobs().check_blob(Some(workspace), hash).await { + if exists { StatusCode::OK } else { StatusCode::NOT_FOUND }.into_response() + } else { + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } +} + +/// Get a `Blob` by hash +/// - Return 200 and `Blob` data if `Blob` is exists. +/// - Return 404 Not Found if `Workspace` or `Blob` not exists. +#[utoipa::path( + get, + tag = "Blobs", + context_path = "/api/blobs", + path = "/{workspace_id}/{hash}", + params( + ("workspace_id", description = "workspace id"), + ("hash", description = "blob hash"), + ), + responses( + (status = 200, description = "Get blob", body = Vec), + (status = 404, description = "Workspace or blob content not found"), + ) +)] +pub async fn get_blob(Extension(context): Extension>, Path(params): Path<(String, String)>) -> Response { + let (workspace, hash) = params; + info!("get_blob: {}, {}", workspace, hash); + if let Ok(blob) = context.storage.blobs().get_blob(Some(workspace), hash, None).await { + blob.into_response() + } else { + StatusCode::NOT_FOUND.into_response() + } +} + +/// Save `Blob` if not exists +/// - Return 200 if `Blob` save successful. +/// - Return 404 Not Found if `Workspace` not exists. +#[utoipa::path( + post, + tag = "Blobs", + context_path = "/api/blobs", + path = "/{workspace_id}", + params( + ("workspace_id", description = "workspace id"), + ("hash", description = "blob hash"), + ), + request_body( + content = BodyStream, + content_type="application/octet-stream" + ), + responses( + (status = 200, description = "Blob was saved", body = BlobStatus), + (status = 404, description = "Workspace not found", body = BlobStatus), + ) +)] +pub async fn set_blob( + Extension(context): Extension>, + Path(workspace): Path, + body: BodyStream, +) -> Response { + info!("set_blob: {}", workspace); + + let mut has_error = false; + let body = body + .take_while(|x| { + has_error = x.is_err(); + future::ready(x.is_ok()) + }) + .filter_map(|data| future::ready(data.ok())); + + if let Ok(id) = context + .storage + .blobs() + .put_blob_stream(Some(workspace.clone()), body) + .await + { + if has_error { + let _ = context.storage.blobs().delete_blob(Some(workspace), id).await; + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } else { + Json(BlobStatus { + id: Some(id), + exists: true, + }) + .into_response() + } + } else { + ( + StatusCode::NOT_FOUND, + Json(BlobStatus { + id: None, + exists: false, + }), + ) + .into_response() + } +} + +/// Delete `blob` if exists +/// - Return 204 if `Blob` delete successful. +/// - Return 404 Not Found if `Workspace` or `Blob` not exists. +#[utoipa::path( + delete, + tag = "Blobs", + context_path = "/api/blobs", + path = "/{workspace_id}/{hash}", + params( + ("workspace_id", description = "workspace id"), + ("hash", description = "blob hash"), + ), + responses( + (status = 204, description = "Blob was deleted"), + (status = 404, description = "Workspace or blob not found"), + ) +)] +pub async fn delete_blob( + Extension(context): Extension>, + Path(params): Path<(String, String)>, +) -> Response { + let (workspace, hash) = params; + info!("delete_blob: {}, {}", workspace, hash); + + if let Ok(success) = context.storage.blobs().delete_blob(Some(workspace), hash).await { + if success { + StatusCode::NO_CONTENT + } else { + StatusCode::NOT_FOUND + } + .into_response() + } else { + StatusCode::NOT_FOUND.into_response() + } +} + +pub fn blobs_apis(router: Router) -> Router { + router.route("/blobs/:workspace", post(set_blob)).route( + "/blobs/:workspace/:blob", + head(check_blob).get(get_blob).delete(delete_blob), + ) +} + +#[cfg(test)] +mod tests { + use axum::body::{Body, Bytes}; + use axum_test_helper::TestClient; + use futures::stream; + use jwst_core_storage::BlobStorageType; + + use super::*; + + #[tokio::test] + async fn test_blobs_apis() { + let ctx = Context::new( + JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) + .await + .ok(), + None, + ) + .await; + let client = TestClient::new(blobs_apis(Router::new()).layer(Extension(Arc::new(ctx)))); + + let test_data: Vec = (0..=255).collect(); + let test_data_len = test_data.len(); + let test_data_stream = stream::iter( + test_data + .clone() + .into_iter() + .map(|byte| Ok::<_, std::io::Error>(Bytes::from(vec![byte]))), + ); + + //upload blob in workspace + let resp = client + .post("/blobs/test") + .header("Content-Length", test_data_len.clone().to_string()) + .body(Body::wrap_stream(test_data_stream.clone())) + .send() + .await; + assert_eq!(resp.status(), StatusCode::OK); + let resp_json: serde_json::Value = resp.json().await; + let hash = resp_json["id"].as_str().unwrap().to_string(); + + let resp = client.head(&format!("/blobs/test/{}", hash)).send().await; + assert_eq!(resp.status(), StatusCode::OK); + + let resp = client.get(&format!("/blobs/test/{}", hash)).send().await; + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(test_data, resp.bytes().await.to_vec()); + + let resp = client.delete(&format!("/blobs/test/{}", hash)).send().await; + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let resp = client.delete(&format!("/blobs/test/{}", hash)).send().await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + let resp = client.head(&format!("/blobs/test/{}", hash)).send().await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + let resp = client.get(&format!("/blobs/test/{}", hash)).send().await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + let resp = client.get("/blobs/test/not_exists_id").send().await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + } +} diff --git a/apps/keck-core/src/server/api/blocks/block.rs b/apps/keck-core/src/server/api/blocks/block.rs new file mode 100644 index 000000000..2b8e97a6c --- /dev/null +++ b/apps/keck-core/src/server/api/blocks/block.rs @@ -0,0 +1,468 @@ +use axum::{extract::Query, response::Response}; +use jwst::{constants, DocStorage}; +use lib0::any::Any; +use serde_json::Value as JsonValue; + +use super::*; + +/// Get a `Block` by id +/// - Return 200 and `Block`'s data if `Block` is exists. +/// - Return 404 Not Found if `Workspace` or `Block` not exists. +#[utoipa::path( + get, + tag = "Blocks", + context_path = "/api/block", + path = "/{workspace_id}/{block_id}", + params( + ("workspace_id", description = "workspace id"), + ("block_id", description = "block id"), + ), + responses( + (status = 200, description = "Get block", body = Block), + (status = 404, description = "Workspace or block content not found"), + ) +)] +pub async fn get_block(Extension(context): Extension>, Path(params): Path<(String, String)>) -> Response { + let (ws_id, block) = params; + info!("get_block: {}, {}", ws_id, block); + if let Ok(workspace) = context.get_workspace(ws_id).await { + if let Some(block) = workspace.with_trx(|mut t| t.get_blocks().get(&t.trx, block)) { + Json(block).into_response() + } else { + StatusCode::NOT_FOUND.into_response() + } + } else { + StatusCode::NOT_FOUND.into_response() + } +} + +/// Create or modify `Block` if block exists with specific id. +/// Note that flavour can only be set when creating a block. +/// - Return 200 and `Block`'s data if `Block`'s content set successful. +/// - Return 404 Not Found if `Workspace` not exists. +#[utoipa::path( + post, + tag = "Blocks", + context_path = "/api/block", + path = "/{workspace_id}/{block_id}/?flavour={flavour}", + params( + ("workspace_id", description = "workspace id"), + ("block_id", description = "block id"), + ("flavour", Query, description = "block flavour, default flavour is text. Optional"), + ), + request_body( + content = String, + description = "json", + content_type = "application/json" + ), + responses( + (status = 200, description = "Block created and content was set", body = Block), + (status = 404, description = "Workspace not found"), + ) +)] +pub async fn set_block( + Extension(context): Extension>, + Path(params): Path<(String, String)>, + query_param: Option>>, + Json(payload): Json, +) -> Response { + let (ws_id, block_id) = params; + info!("set_block: {}, {}", ws_id, block_id); + if let Ok(workspace) = context.get_workspace(&ws_id).await { + let mut update = None; + if let Some(block) = workspace.with_trx(|mut t| { + let flavour = if let Some(query_map) = query_param { + query_map + .get("flavour") + .map_or_else(|| String::from("text"), |v| v.clone()) + } else { + String::from("text") + }; + + if let Ok(block) = t + .get_blocks() + .create(&mut t.trx, &block_id, flavour) + .map_err(|e| error!("failed to create block: {:?}", e)) + { + // set block content + if let Some(block_content) = payload.as_object() { + let mut changed = false; + for (key, value) in block_content.iter() { + if key == constants::sys::FLAVOUR { + continue; + } + changed = true; + if let Ok(value) = serde_json::from_value::(value.clone()) { + if let Err(e) = block.set(&mut t.trx, key, value.clone()) { + error!("failed to set block {} content: {}, {}, {:?}", block_id, key, value, e); + } + } + } + + if changed { + update = t.trx.encode_update_v1().ok(); + } + } + + Some(block) + } else { + None + } + }) { + if let Some(update) = update { + if let Err(e) = context + .storage + .docs() + .update_doc(ws_id, workspace.doc_guid().to_string(), &update) + .await + { + error!("db write error: {:?}", e); + } + } + + // response block content + Json(block).into_response() + } else { + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + } else { + StatusCode::NOT_FOUND.into_response() + } +} + +/// Get exists `Blocks` in certain `Workspace` by flavour +/// - Return 200 Ok and `Blocks`'s data if `Blocks` is exists. +/// - Return 404 Not Found if `Workspace` not exists or 500 Internal Server +/// Error when transaction init fails. +#[utoipa::path( + get, + tag = "Blocks", + context_path = "/api/block", + path = "/{workspace_id}/flavour/{flavour}", + params( + ("workspace_id", description = "workspace id"), + ("flavour", description = "block flavour"), + ), + responses( + (status = 200, description = "Get all certain flavour blocks belongs to the given workspace"), + (status = 404, description = "Workspace not found") + ) +)] +pub async fn get_block_by_flavour( + Extension(context): Extension>, + Path(params): Path<(String, String)>, +) -> Response { + let (ws_id, flavour) = params; + info!("get_block_by_flavour: ws_id, {}, flavour, {}", ws_id, flavour); + if let Ok(workspace) = context.get_workspace(&ws_id).await { + match workspace.try_with_trx(|mut trx| trx.get_blocks().get_blocks_by_flavour(&trx.trx, &flavour)) { + Some(blocks) => Json(blocks).into_response(), + None => ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Workspace({ws_id:?}) get transaction error"), + ) + .into_response(), + } + } else { + (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() + } +} + +/// Get `Block` history +/// - Return 200 and `Block`'s history if `Block` exists. +/// - Return 404 Not Found if `Workspace` or `Block` not exists. +#[utoipa::path( + get, + tag = "Blocks", + context_path = "/api/block", + path = "/{workspace_id}/{block_id}/history", + params( + ("workspace_id", description = "workspace id"), + ("block_id", description = "block id"), + ), + responses( + (status = 200, description = "Get block history", body = [BlockHistory]), + (status = 404, description = "Workspace or block not found"), + ) +)] +pub async fn get_block_history( + Extension(context): Extension>, + Path(params): Path<(String, String)>, +) -> Response { + let (ws_id, block) = params; + info!("get_block_history: {}, {}", ws_id, block); + if let Ok(workspace) = context.get_workspace(&ws_id).await { + workspace.with_trx(|mut t| { + if let Some(block) = t.get_blocks().get(&t.trx, block) { + Json(&block.history(&t.trx)).into_response() + } else { + StatusCode::NOT_FOUND.into_response() + } + }) + } else { + StatusCode::NOT_FOUND.into_response() + } +} + +/// Delete block +/// - Return 204 No Content if delete successful. +/// - Return 404 Not Found if `Workspace` or `Block` not exists. +#[utoipa::path( + delete, + tag = "Blocks", + context_path = "/api/block", + path = "/{workspace_id}/{block_id}", + params( + ("workspace_id", description = "workspace id"), + ("block_id", description = "block id"), + ), + responses( + (status = 204, description = "Block successfully deleted"), + (status = 404, description = "Workspace or block not found"), + ) +)] +pub async fn delete_block( + Extension(context): Extension>, + Path(params): Path<(String, String)>, +) -> StatusCode { + let (ws_id, block) = params; + info!("delete_block: {}, {}", ws_id, block); + if let Ok(workspace) = context.get_workspace(&ws_id).await { + if let Some(update) = workspace.with_trx(|mut t| { + if t.get_blocks().remove(&mut t.trx, &block) { + t.trx.encode_update_v1().ok() + } else { + None + } + }) { + if let Err(e) = context + .storage + .docs() + .update_doc(ws_id, workspace.doc_guid().to_string(), &update) + .await + { + error!("db write error: {:?}", e); + } + return StatusCode::NO_CONTENT; + } + } + StatusCode::NOT_FOUND +} + +/// Get children in `Block` +/// - Return 200 and `Block`'s children ID. +/// - Return 404 Not Found if `Workspace` or `Block` not exists. +#[utoipa::path( + get, + tag = "Blocks", + context_path = "/api/block", + path = "/{workspace_id}/{block_id}/children", + params( + ("workspace_id", description = "workspace id"), + ("block_id", description = "block id"), + Pagination + ), + responses( + (status = 200, description = "Get block children", body = PageData<[String]>), + (status = 404, description = "Workspace or block not found"), + ) +)] +pub async fn get_block_children( + Extension(context): Extension>, + Path(params): Path<(String, String)>, + Query(pagination): Query, +) -> Response { + let (ws_id, block) = params; + let Pagination { offset, limit } = pagination; + info!("get_block_children: {}, {}", ws_id, block); + if let Ok(workspace) = context.get_workspace(ws_id).await { + if let Some(block) = workspace.with_trx(|mut t| t.get_blocks().get(&t.trx, &block)) { + let data: Vec = block.children_iter(|children| children.skip(offset).take(limit).collect()); + + let status = if data.is_empty() { + StatusCode::NOT_FOUND + } else { + StatusCode::OK + }; + + ( + status, + Json(PageData { + total: block.children_len() as usize, + data, + }), + ) + .into_response() + } else { + StatusCode::NOT_FOUND.into_response() + } + } else { + StatusCode::NOT_FOUND.into_response() + } +} + +/// Insert a another `Block` into a `Block`'s children +/// - Return 200 and `Block`'s data if insert successful. +/// - Return 404 Not Found if `Workspace` or `Block` not exists. +#[utoipa::path( + post, + tag = "Blocks", + context_path = "/api/block", + path = "/{workspace_id}/{block_id}/children", + params( + ("workspace_id", description = "workspace id"), + ("block_id", description = "block id"), + ), + request_body( + content = InsertChildren, + description = "json", + content_type = "application/json" + ), + responses( + (status = 200, description = "Block inserted", body = Block), + (status = 404, description = "Workspace or block not found"), + (status = 500, description = "Failed to insert block") + ) +)] +pub async fn insert_block_children( + Extension(context): Extension>, + Path(params): Path<(String, String)>, + Json(payload): Json, +) -> Response { + let (ws_id, block) = params; + info!("insert_block: {}, {}", ws_id, block); + if let Ok(workspace) = context.get_workspace(&ws_id).await { + let mut update = None; + + if let Some(block) = workspace.with_trx(|mut t| t.get_blocks().get(&t.trx, block)) { + if let Some(block) = workspace.with_trx(|mut t| { + let space = t.get_blocks(); + let mut changed = false; + match payload { + InsertChildren::Push(block_id) => { + if let Some(child) = space.get(&t.trx, block_id) { + changed = true; + if let Err(e) = block.push_children(&mut t.trx, &child) { + // TODO: handle error correctly + error!("failed to insert block: {:?}", e); + return None; + } + } + } + InsertChildren::InsertBefore { id, before } => { + if let Some(child) = space.get(&t.trx, id) { + changed = true; + if let Err(e) = block.insert_children_before(&mut t.trx, &child, &before) { + // TODO: handle error correctly + error!("failed to insert children before: {:?}", e); + return None; + } + } + } + InsertChildren::InsertAfter { id, after } => { + if let Some(child) = space.get(&t.trx, id) { + changed = true; + if let Err(e) = block.insert_children_after(&mut t.trx, &child, &after) { + // TODO: handle error correctly + error!("failed to insert children after: {:?}", e); + return None; + } + } + } + InsertChildren::InsertAt { id, pos } => { + if let Some(child) = space.get(&t.trx, id) { + changed = true; + if let Err(e) = block.insert_children_at(&mut t.trx, &child, pos) { + // TODO: handle error correctly + error!("failed to insert children at: {:?}", e); + return None; + } + } + } + }; + + if changed { + update = t.trx.encode_update_v1().ok(); + } + + Some(block) + }) { + if let Some(update) = update { + if let Err(e) = context + .storage + .docs() + .update_doc(ws_id, workspace.doc_guid().to_string(), &update) + .await + { + error!("db write error: {:?}", e); + } + } + + // response block content + Json(block).into_response() + } else { + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + } else { + StatusCode::NOT_FOUND.into_response() + } + } else { + StatusCode::NOT_FOUND.into_response() + } +} + +/// Remove children in `Block` +/// - Return 200 and `Block`'s data if remove successful. +/// - Return 404 Not Found if `Workspace` or `Block` not exists. +#[utoipa::path( + delete, + tag = "Blocks", + context_path = "/api/block", + path = "/{workspace_id}/{block_id}/children/{children}", + params( + ("workspace_id", description = "workspace id"), + ("block_id", description = "block id"), + ), + responses( + (status = 200, description = "Block children removed", body = Block), + (status = 404, description = "Workspace or block not found"), + ) +)] +pub async fn remove_block_children( + Extension(context): Extension>, + Path(params): Path<(String, String, String)>, +) -> Response { + let (ws_id, block, child_id) = params; + info!("insert_block: {}, {}", ws_id, block); + if let Ok(workspace) = context.get_workspace(&ws_id).await { + if let Some(update) = workspace.with_trx(|mut t| { + let space = t.get_blocks(); + if let Some(block) = space.get(&t.trx, &block) { + if block.children_exists(&t.trx, &child_id) { + if let Some(child) = space.get(&t.trx, &child_id) { + return block + .remove_children(&mut t.trx, &child) + .and_then(|_| Ok(t.trx.encode_update_v1()?)) + .ok(); + } + } + } + None + }) { + if let Err(e) = context + .storage + .docs() + .update_doc(ws_id, workspace.doc_guid().to_string(), &update) + .await + { + error!("db write error: {:?}", e); + } + // response block content + Json(block).into_response() + } else { + StatusCode::NOT_FOUND.into_response() + } + } else { + StatusCode::NOT_FOUND.into_response() + } +} diff --git a/apps/keck-core/src/server/api/blocks/mod.rs b/apps/keck-core/src/server/api/blocks/mod.rs new file mode 100644 index 000000000..8eb765ded --- /dev/null +++ b/apps/keck-core/src/server/api/blocks/mod.rs @@ -0,0 +1,154 @@ +pub mod block; +pub mod schema; +pub mod workspace; + +pub use block::{delete_block, get_block, get_block_history, insert_block_children, remove_block_children, set_block}; +use schema::InsertChildren; +pub use schema::SubscribeWorkspace; +pub use workspace::{ + delete_workspace, get_workspace, history_workspace, history_workspace_clients, set_workspace, subscribe_workspace, + workspace_client, +}; + +use super::*; + +fn block_apis(router: Router) -> Router { + let block_operation = Router::new() + .route("/history", get(block::get_block_history)) + .route( + "/children", + get(block::get_block_children).post(block::insert_block_children), + ) + .route("/children/:children", delete(block::remove_block_children)); + + router.nest("/block/:workspace/:block/", block_operation).route( + "/block/:workspace/:block", + get(block::get_block).post(block::set_block).delete(block::delete_block), + ) +} + +fn workspace_apis(router: Router) -> Router { + router + .route("/block/:workspace/client", get(workspace::workspace_client)) + .route("/block/:workspace/history", get(workspace::history_workspace_clients)) + .route("/block/:workspace/history/:client", get(workspace::history_workspace)) + .route( + "/block/:workspace", + get(workspace::get_workspace) + .post(workspace::set_workspace) + .delete(workspace::delete_workspace), + ) + .route("/block/:workspace/flavour/:flavour", get(block::get_block_by_flavour)) + .route("/block/:workspace/blocks", get(workspace::get_workspace_block)) + .route("/search/:workspace", get(workspace::workspace_search)) + .route( + "/search/:workspace/index", + get(workspace::get_search_index).post(workspace::set_search_index), + ) + .route("/subscribe", post(subscribe_workspace)) +} + +pub fn blocks_apis(router: Router) -> Router { + workspace_apis(block_apis(router)) +} + +#[cfg(test)] +mod tests { + use axum_test_helper::TestClient; + use serde_json::{from_str, json, to_string, Value}; + + use super::*; + + #[tokio::test] + async fn test_workspace_apis() { + let client = Arc::new(reqwest::Client::builder().no_proxy().build().unwrap()); + let runtime = Arc::new( + runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_time() + .enable_io() + .build() + .expect("Failed to create runtime"), + ); + let workspace_changed_blocks = Arc::new(RwLock::new(HashMap::::new())); + let hook_endpoint = Arc::new(RwLock::new(String::new())); + let cb: WorkspaceRetrievalCallback = { + let workspace_changed_blocks = workspace_changed_blocks.clone(); + let runtime = runtime.clone(); + Some(Arc::new(Box::new(move |workspace: &Workspace| { + workspace.set_callback(generate_ws_callback(&workspace_changed_blocks, &runtime)); + }))) + }; + let ctx = Arc::new( + Context::new( + JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) + .await + .ok(), + cb, + ) + .await, + ); + let client = TestClient::new( + workspace_apis(Router::new()) + .layer(Extension(ctx.clone())) + .layer(Extension(client.clone())) + .layer(Extension(runtime.clone())) + .layer(Extension(workspace_changed_blocks.clone())) + .layer(Extension(hook_endpoint.clone())), + ); + + // basic workspace apis + let resp = client.get("/block/test").send().await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + let resp = client.post("/block/test").send().await; + assert_eq!(resp.status(), StatusCode::OK); + let resp = client.get("/block/test/client").send().await; + assert_eq!( + resp.text().await.parse::().unwrap(), + ctx.storage.get_workspace("test").await.unwrap().client_id() + ); + let resp = client.get("/block/test/history").send().await; + assert_eq!(resp.json::>().await, Vec::::new()); + let resp = client.get("/block/test").send().await; + assert_eq!(resp.status(), StatusCode::OK); + let resp = client.delete("/block/test").send().await; + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + let resp = client.get("/block/test").send().await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + // workspace history apis + let resp = client.post("/block/test").send().await; + assert_eq!(resp.status(), StatusCode::OK); + + let resp = client.get("/search/test/index").send().await; + assert_eq!(resp.status(), StatusCode::OK); + let index = resp.json::>().await; + assert_eq!(index, vec!["title".to_owned(), "text".to_owned()]); + + let body = to_string(&json!(["test"])).unwrap(); + let resp = client + .post("/search/test/index") + .header("content-type", "application/json") + .body(body) + .send() + .await; + assert_eq!(resp.status(), StatusCode::OK); + + let resp = client.get("/search/test/index").send().await; + assert_eq!(resp.status(), StatusCode::OK); + let index = resp.json::>().await; + assert_eq!(index, vec!["test".to_owned()]); + + let body = json!({ + "hookEndpoint": "localhost:3000/api/hook" + }) + .to_string(); + let resp = client + .post("/subscribe") + .header("content-type", "application/json") + .body(body) + .send() + .await; + assert_eq!(resp.status(), StatusCode::OK); + } +} diff --git a/apps/keck-core/src/server/api/blocks/schema.rs b/apps/keck-core/src/server/api/blocks/schema.rs new file mode 100644 index 000000000..c6507c771 --- /dev/null +++ b/apps/keck-core/src/server/api/blocks/schema.rs @@ -0,0 +1,51 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Default, Deserialize, PartialEq, Debug, ToSchema)] +pub struct Workspace { + pub(super) blocks: HashMap, + pub(super) updated: HashMap, +} + +#[derive(Deserialize, PartialEq, Debug, ToSchema)] +#[schema(example = json!({ + "sys_id": "0", + "sys:flavour": "affine:text", + "sys:created": 946684800000_u64, + "sys:children": ["block1", "block2"], + "prop:text": "123", + "prop:color": "#ff0000", +}))] +pub struct Block { + #[serde(rename = "sys:flavour")] + flavour: String, + #[serde(rename = "sys:created")] + created: u64, + #[serde(rename = "sys:children")] + children: Vec, +} + +#[derive(Deserialize, PartialEq, Debug, ToSchema)] +#[schema(example = json!([12345, 946684800000_u64, "add"]))] +pub struct BlockRawHistory(u64, u64, String); + +#[derive(Deserialize, ToSchema)] +#[schema(example = json!({"Push": "jwstRf4rMzua7E"}))] + +pub enum InsertChildren { + Push(String), + InsertBefore { id: String, before: String }, + InsertAfter { id: String, after: String }, + InsertAt { id: String, pos: u32 }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[schema(example=json!({ + "hookEndpoint": "http://localhost:3000/api/hooks" +}))] +pub struct SubscribeWorkspace { + #[serde(rename = "hookEndpoint")] + pub hook_endpoint: String, +} diff --git a/apps/keck-core/src/server/api/blocks/workspace.rs b/apps/keck-core/src/server/api/blocks/workspace.rs new file mode 100644 index 000000000..deb454de8 --- /dev/null +++ b/apps/keck-core/src/server/api/blocks/workspace.rs @@ -0,0 +1,406 @@ +use axum::{ + extract::{Path, Query}, + http::header, + response::Response, +}; +use jwst_core::{parse_history, parse_history_client, DocStorage}; +use utoipa::IntoParams; + +use super::*; +use crate::server::api::blocks::SubscribeWorkspace; + +/// Get a exists `Workspace` by id +/// - Return 200 Ok and `Workspace`'s data if `Workspace` is exists. +/// - Return 404 Not Found if `Workspace` not exists. +#[utoipa::path( + get, + tag = "Workspace", + context_path = "/api/block", + path = "/{workspace}", + params( + ("workspace", description = "workspace id"), + ), + responses( + (status = 200, description = "Get workspace data", body = Workspace), + (status = 404, description = "Workspace not found") + ) +)] +pub async fn get_workspace(Extension(context): Extension>, Path(workspace): Path) -> Response { + info!("get_workspace: {}", workspace); + if let Ok(workspace) = context.get_workspace(&workspace).await { + Json(workspace).into_response() + } else { + (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() + } +} + +/// Create a `Workspace` by id +/// - Return 200 Ok and `Workspace`'s data if init success or `Workspace` is +/// exists. +/// - Return 500 Internal Server Error if init failed. +#[utoipa::path( + post, + tag = "Workspace", + context_path = "/api/block", + path = "/{workspace}", + params( + ("workspace", description = "workspace id"), + ), + responses( + (status = 200, description = "Return workspace data", body = Workspace), + (status = 500, description = "Failed to init a workspace") + ) +)] +pub async fn set_workspace(Extension(context): Extension>, Path(workspace): Path) -> Response { + info!("set_workspace: {}", workspace); + match context.create_workspace(workspace).await { + Ok(workspace) => Json(workspace).into_response(), + Err(e) => { + error!("Failed to init doc: {:?}", e); + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + } +} + +/// Delete a exists `Workspace` by id +/// - Return 204 No Content if delete successful. +/// - Return 404 Not Found if `Workspace` not exists. +/// - Return 500 Internal Server Error if delete failed. +#[utoipa::path( + delete, + tag = "Workspace", + context_path = "/api/block", + path = "/{workspace}", + params( + ("workspace", description = "workspace id"), + ), + responses( + (status = 204, description = "Workspace data deleted"), + (status = 404, description = "Workspace not exists"), + (status = 500, description = "Failed to delete workspace") + ) +)] +pub async fn delete_workspace(Extension(context): Extension>, Path(workspace): Path) -> Response { + info!("delete_workspace: {}", workspace); + if context.storage.docs().delete_workspace(&workspace).await.is_err() { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + + StatusCode::NO_CONTENT.into_response() +} + +/// Get current client id of server +/// +/// When the server initializes or get the `Workspace`, a `Client` will be +/// created. This `Client` will not be destroyed until the server restarts. +/// Therefore, the `Client ID` in the history generated by modifying `Block` +/// through HTTP API will remain unchanged until the server restarts. +/// +/// This interface return the client id that server will used. +#[utoipa::path( + get, + tag = "Workspace", + context_path = "/api/block", + path = "/{workspace}/client", + params( + ("workspace", description = "workspace id"), + ), + responses( + (status = 200, description = "Get workspace client id", body = u64), + (status = 404, description = "Workspace not found") + ) +)] +pub async fn workspace_client(Extension(context): Extension>, Path(workspace): Path) -> Response { + if let Ok(workspace) = context.get_workspace(&workspace).await { + Json(workspace.client_id()).into_response() + } else { + (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() + } +} + +/// Block search query +// See doc for using utoipa search queries example here: https://github.com/juhaku/utoipa/blob/6c7f6a2d/examples/todo-axum/src/main.rs#L124-L130 +#[derive(Deserialize, IntoParams)] +pub struct BlockSearchQuery { + /// Search by title and text. + query: String, +} + +/// Search workspace blocks of server +/// +/// This will return back a list of relevant blocks. +#[utoipa::path( + get, + tag = "Workspace", + context_path = "/api/search", + path = "/{workspace}", + params( + ("workspace", description = "workspace id"), + BlockSearchQuery, + ), + responses( + (status = 200, description = "Search results", body = SearchResults), + ) +)] +pub async fn workspace_search( + Extension(context): Extension>, + Path(workspace): Path, + query: Query, +) -> Response { + let query_text = &query.query; + let ws_id = workspace; + info!("workspace_search: {ws_id:?} query = {query_text:?}"); + if let Ok(workspace) = context.get_workspace(&ws_id).await { + match workspace.search(query_text) { + Ok(list) => { + debug!("workspace_search: {ws_id:?} query = {query_text:?}; {list:#?}"); + Json(list).into_response() + } + Err(err) => { + error!("Internal server error calling workspace_search: {err:?}"); + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + } + } else { + (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() + } +} + +#[utoipa::path( + get, + tag = "Workspace", + context_path = "/api/search", + path = "/{workspace}/index", + params( + ("workspace", description = "workspace id"), + ), + responses( + (status = 200, description = "result", body = Vec), + (status = 404, description = "Workspace not found") + ) +)] +pub async fn get_search_index(Extension(context): Extension>, Path(workspace): Path) -> Response { + info!("get_search_index: {workspace:?}"); + + if let Ok(workspace) = context.get_workspace(&workspace).await { + Json(workspace.metadata().search_index).into_response() + } else { + (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() + } +} + +#[utoipa::path( + post, + tag = "Workspace", + context_path = "/api/search", + path = "/{workspace}/index", + params( + ("workspace", description = "workspace id"), + ), + responses( + (status = 200, description = "success"), + (status = 400, description = "Bad Request"), + (status = 404, description = "Workspace not found") + ) +)] +pub async fn set_search_index( + Extension(context): Extension>, + Path(workspace): Path, + Json(fields): Json>, +) -> Response { + info!("set_search_index: {workspace:?} fields = {fields:?}"); + + if let Ok(workspace) = context.get_workspace(&workspace).await { + if let Ok(true) = workspace.set_search_index(fields) { + StatusCode::OK.into_response() + } else { + StatusCode::BAD_REQUEST.into_response() + } + } else { + (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() + } +} + +/// Get `Block` in `Workspace` +/// - Return 200 and `Block`'s ID. +/// - Return 404 Not Found if `Workspace` or `Block` not exists. +#[utoipa::path( + get, + tag = "Workspace", + context_path = "/api/block", + path = "/{workspace}/blocks", + params( + ("workspace", description = "workspace id"), + Pagination + ), + responses( + (status = 200, description = "Get Blocks", body = PageData<[Block]>), + (status = 404, description = "Workspace or block not found"), + ) +)] +pub async fn get_workspace_block( + Extension(context): Extension>, + Path(workspace): Path, + Query(pagination): Query, +) -> Response { + let Pagination { offset, limit } = pagination; + info!("get_workspace_block: {workspace:?}"); + if let Ok(workspace) = context.get_workspace(&workspace).await { + let (total, data) = workspace.with_trx(|mut t| { + let space = t.get_blocks(); + + let total = space.block_count() as usize; + let data = space.blocks(&t.trx, |blocks| blocks.skip(offset).take(limit).collect::>()); + + (total, data) + }); + + let status = if data.is_empty() { + StatusCode::NOT_FOUND + } else { + StatusCode::OK + }; + + (status, Json(PageData { total, data })).into_response() + } else { + (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() + } +} + +/// Get all client ids of the `Workspace` +/// +/// This interface returns all `Client IDs` that includes history in the +/// `Workspace` +/// +/// Every client write something into a `Workspace` will has a unique id. +/// +/// For example: +/// - A user writes a new `Block` to a `Workspace` through `Client` on the +/// front end, which will generate a series of histories. A `Client ID` +/// contained in these histories will be randomly generated by the `Client` +/// and will remain unchanged until the Client instance is destroyed +/// - When the server initializes or get the `Workspace`, a `Client` will be +/// created. This `Client` will not be destroyed until the server restarts. +/// Therefore, the `Client ID` in the history generated by modifying `Block` +/// through HTTP API will remain unchanged until the server restarts. +#[utoipa::path( + get, + tag = "Workspace", + context_path = "/api/block", + path = "/{workspace}/history", + params( + ("workspace", description = "workspace id"), + ), + responses( + (status = 200, description = "Get workspace history client ids", body = [u64]), + (status = 500, description = "Failed to get workspace history") + ) +)] +pub async fn history_workspace_clients( + Extension(context): Extension>, + Path(workspace): Path, +) -> Response { + if let Ok(workspace) = context.get_workspace(&workspace).await { + if let Some(history) = parse_history_client(&workspace.doc()) { + Json(history).into_response() + } else { + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + } else { + (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() + } +} + +/// Get the history generated by a specific `Client ID` of the `Workspace` +/// +/// If client id set to 0, return all history of the `Workspace`. +#[utoipa::path( + get, + tag = "Workspace", + context_path = "/api/block", + path = "/{workspace}/history/{client}", + params( + ("workspace", description = "workspace id"), + ("client", description = "client id, is give 0 then return all clients histories"), + ), + responses( + (status = 200, description = "Get workspace history", body = [RawHistory]), + (status = 400, description = "Client id invalid"), + (status = 500, description = "Failed to get workspace history") + ) +)] +pub async fn history_workspace( + Extension(context): Extension>, + Path(params): Path<(String, String)>, +) -> Response { + let (ws_id, client) = params; + if let Ok(workspace) = context.get_workspace(&ws_id).await { + if let Ok(client) = client.parse::() { + if let Some(json) = + parse_history(&workspace.doc(), client).and_then(|history| serde_json::to_string(&history).ok()) + { + ([(header::CONTENT_TYPE, "application/json")], json).into_response() + } else { + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + } else { + StatusCode::BAD_REQUEST.into_response() + } + } else { + (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() + } +} + +/// Register a webhook for all block changes from all workspace changes +#[utoipa::path( + post, + tag = "Workspace", + context_path = "/api/subscribe", + path = "", + request_body( + content_type = "application/json", + content = SubscribeWorkspace, + description = "Provide endpoint of webhook server", + ), + responses( + (status = 200, description = "Subscribe workspace succeed"), + (status = 500, description = "Internal Server Error") + ) +)] +pub async fn subscribe_workspace( + Extension(hook_endpoint): Extension>>, + Json(payload): Json, +) -> Response { + info!("subscribe all workspaces, hook endpoint: {}", payload.hook_endpoint); + + let mut write_guard = hook_endpoint.write().await; + *write_guard = payload.hook_endpoint.clone(); + info!("successfully subscribed all workspaces"); + StatusCode::OK.into_response() +} + +#[cfg(all(test, feature = "sqlite"))] +mod test { + use super::*; + + #[tokio::test] + async fn workspace() { + use axum_test_helper::TestClient; + + let pool = DbPool::init_memory_pool().await.unwrap(); + let context = Arc::new(Context::new(Some(pool)).await); + + let app = super::workspace_apis(Router::new()).layer(Extension(context)); + + let client = TestClient::new(app); + + let resp = client.post("/block/test").send().await; + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(resp.json::().await, schema::Workspace::default()); + + let resp = client.get("/block/test").send().await; + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(resp.json::().await, schema::Workspace::default()); + } +} diff --git a/apps/keck-core/src/server/api/mod.rs b/apps/keck-core/src/server/api/mod.rs new file mode 100644 index 000000000..b95bc77ea --- /dev/null +++ b/apps/keck-core/src/server/api/mod.rs @@ -0,0 +1,117 @@ +#[cfg(feature = "api")] +mod blobs; +#[cfg(feature = "api")] +mod blocks; + +use std::collections::HashMap; + +use axum::Router; +#[cfg(feature = "api")] +use axum::{ + extract::{Json, Path}, + http::StatusCode, + response::IntoResponse, + routing::{delete, get, head, post}, +}; +use jwst_core_rpc::{BroadcastChannels, RpcContextImpl}; +use jwst_core_storage::{BlobStorageType, JwstStorage, JwstStorageResult}; +use tokio::sync::RwLock; + +use super::*; + +#[derive(Deserialize)] +#[cfg_attr(feature = "api", derive(utoipa::IntoParams))] +pub struct Pagination { + #[serde(default)] + offset: usize, + #[serde(default = "default_limit")] + limit: usize, +} + +fn default_limit() -> usize { + usize::MAX +} + +#[derive(Serialize)] +pub struct PageData { + total: usize, + data: T, +} + +pub struct Context { + channel: BroadcastChannels, + storage: JwstStorage, + callback: WorkspaceRetrievalCallback, +} + +impl Context { + pub async fn new(storage: Option, cb: WorkspaceRetrievalCallback) -> Self { + let blob_storage_type = BlobStorageType::DB; + + let storage = if let Some(storage) = storage { + info!("use external storage instance: {}", storage.database()); + Ok(storage) + } else if dotenvy::var("USE_MEMORY_SQLITE").is_ok() { + info!("use memory sqlite database"); + JwstStorage::new_with_migration("sqlite::memory:", blob_storage_type).await + } else if let Ok(database_url) = dotenvy::var("DATABASE_URL") { + info!("use external database: {}", database_url); + JwstStorage::new_with_migration(&database_url, blob_storage_type).await + } else { + info!("use sqlite database: jwst.db"); + JwstStorage::new_with_sqlite("jwst", blob_storage_type).await + } + .expect("Cannot create database"); + + Context { + channel: RwLock::new(HashMap::new()), + storage, + callback: cb, + } + } + + pub async fn get_workspace(&self, workspace_id: S) -> JwstStorageResult + where + S: AsRef, + { + let workspace = self.storage.get_workspace(workspace_id).await?; + if let Some(cb) = self.callback.clone() { + cb(&workspace); + } + + Ok(workspace) + } + + pub async fn create_workspace(&self, workspace_id: S) -> JwstStorageResult + where + S: AsRef, + { + let workspace = self.storage.create_workspace(workspace_id).await?; + if let Some(cb) = self.callback.clone() { + cb(&workspace); + } + + Ok(workspace) + } +} + +impl RpcContextImpl<'_> for Context { + fn get_storage(&self) -> &JwstStorage { + &self.storage + } + + fn get_channel(&self) -> &BroadcastChannels { + &self.channel + } +} + +pub fn api_handler(router: Router) -> Router { + #[cfg(feature = "api")] + { + router.nest("/api", blobs::blobs_apis(blocks::blocks_apis(Router::new()))) + } + #[cfg(not(feature = "api"))] + { + router + } +} diff --git a/apps/keck-core/src/server/mod.rs b/apps/keck-core/src/server/mod.rs new file mode 100644 index 000000000..e5bfb6919 --- /dev/null +++ b/apps/keck-core/src/server/mod.rs @@ -0,0 +1,137 @@ +mod api; +mod subscribe; +mod sync; +mod utils; + +use std::{collections::HashMap, net::SocketAddr, sync::Arc, thread::sleep}; + +use api::Context; +use axum::{http::Method, Extension, Router, Server}; +use jwst_core::Workspace; +pub use subscribe::*; +use tokio::{runtime, signal, sync::RwLock}; +use tower_http::cors::{Any, CorsLayer}; +pub use utils::*; + +async fn shutdown_signal() { + let ctrl_c = async { + signal::ctrl_c().await.expect("failed to install Ctrl+C handler"); + }; + + #[cfg(unix)] + let terminate = async { + signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => {}, + _ = terminate => {}, + } + + info!("Shutdown signal received, starting graceful shutdown"); +} + +type WorkspaceRetrievalCallback = Option>>; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WorkspaceChangedBlocks { + #[serde(rename(serialize = "workspaceId"))] + pub workspace_id: String, + #[serde(rename(serialize = "blockIds"))] + pub block_ids: Vec, +} + +impl WorkspaceChangedBlocks { + pub fn new(workspace_id: String) -> WorkspaceChangedBlocks { + WorkspaceChangedBlocks { + workspace_id, + block_ids: Vec::new(), + } + } + + pub fn insert_block_ids(&mut self, mut updated_block_ids: Vec) { + self.block_ids.append(&mut updated_block_ids); + } +} + +pub async fn start_server() { + let origins = [ + "http://localhost:4200".parse().unwrap(), + "http://127.0.0.1:4200".parse().unwrap(), + "http://localhost:3000".parse().unwrap(), + "http://127.0.0.1:3000".parse().unwrap(), + "http://localhost:5173".parse().unwrap(), + "http://127.0.0.1:5173".parse().unwrap(), + ]; + + let cors = CorsLayer::new() + // allow `GET` and `POST` when accessing the resource + .allow_methods(vec![Method::GET, Method::POST, Method::DELETE, Method::OPTIONS]) + // allow requests from any origin + .allow_origin(origins) + .allow_headers(Any); + + let client = Arc::new(reqwest::Client::builder().no_proxy().build().unwrap()); + let runtime = Arc::new( + runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_time() + .enable_io() + .build() + .expect("Failed to create runtime"), + ); + let workspace_changed_blocks = Arc::new(RwLock::new(HashMap::::new())); + let hook_endpoint = Arc::new(RwLock::new(dotenvy::var("HOOK_ENDPOINT").unwrap_or_default())); + let cb: WorkspaceRetrievalCallback = { + let workspace_changed_blocks = workspace_changed_blocks.clone(); + let runtime = runtime.clone(); + Some(Arc::new(Box::new(move |workspace: &Workspace| { + // workspace.set_callback(generate_ws_callback(& + // workspace_changed_blocks, &runtime)); + }))) + }; + let context = Arc::new(Context::new(None, cb).await); + + start_handling_observed_blocks( + runtime.clone(), + workspace_changed_blocks.clone(), + hook_endpoint.clone(), + client.clone(), + ); + + let app = sync::sync_handler(api::api_handler(Router::new())) + .layer(cors) + .layer(Extension(context.clone())) + .layer(Extension(client)) + .layer(Extension(runtime)) + .layer(Extension(workspace_changed_blocks)) + .layer(Extension(hook_endpoint)); + + let addr = SocketAddr::from(( + [0, 0, 0, 0], + dotenvy::var("KECK_PORT") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(3000), + )); + info!("listening on {}", addr); + + if let Err(e) = Server::bind(&addr) + .serve(app.into_make_service()) + .with_graceful_shutdown(shutdown_signal()) + .await + { + error!("Server shutdown due to error: {}", e); + } + + // context.docs.close().await; + // context.blobs.close().await; + + info!("Server shutdown complete"); +} diff --git a/apps/keck-core/src/server/subscribe.rs b/apps/keck-core/src/server/subscribe.rs new file mode 100644 index 000000000..766d48e0c --- /dev/null +++ b/apps/keck-core/src/server/subscribe.rs @@ -0,0 +1,69 @@ +use std::{collections::HashMap, sync::Arc, thread, time::Duration}; + +use reqwest::Client; +use tokio::{runtime::Runtime, sync::RwLock}; + +use super::*; +use crate::server::WorkspaceChangedBlocks; + +pub fn generate_ws_callback( + workspace_changed_blocks: &Arc>>, + runtime: &Arc, +) -> Arc) + Send + Sync>> { + let workspace_changed_blocks = workspace_changed_blocks.clone(); + let runtime = runtime.clone(); + Arc::new(Box::new(move |workspace_id, block_ids| { + let workspace_changed_blocks = workspace_changed_blocks.clone(); + runtime.spawn(async move { + let mut write_guard = workspace_changed_blocks.write().await; + write_guard + .entry(workspace_id.clone()) + .or_insert(WorkspaceChangedBlocks::new(workspace_id.clone())) + .insert_block_ids(block_ids.clone()); + }); + })) +} + +pub fn start_handling_observed_blocks( + runtime: Arc, + workspace_changed_blocks: Arc>>, + hook_endpoint: Arc>, + client: Arc, +) { + thread::spawn(move || loop { + let workspace_changed_blocks = workspace_changed_blocks.clone(); + let hook_endpoint = hook_endpoint.clone(); + let runtime_cloned = runtime.clone(); + let client = client.clone(); + runtime.spawn(async move { + let read_guard = workspace_changed_blocks.read().await; + if !read_guard.is_empty() { + let endpoint = hook_endpoint.read().await; + if !endpoint.is_empty() { + { + let workspace_changed_blocks_clone = read_guard.clone(); + let post_body = workspace_changed_blocks_clone + .into_values() + .collect::>(); + let endpoint = endpoint.clone(); + runtime_cloned.spawn(async move { + let response = client.post(endpoint.to_string()).json(&post_body).send().await; + match response { + Ok(response) => info!( + "notified hook endpoint, endpoint response status: {}", + response.status() + ), + Err(e) => error!("Failed to send notify: {}", e), + } + }); + } + } + drop(read_guard); + let mut write_guard = workspace_changed_blocks.write().await; + info!("workspace_changed_blocks: {:?}", write_guard); + write_guard.clear(); + } + }); + sleep(Duration::from_millis(200)); + }); +} diff --git a/apps/keck-core/src/server/sync/blobs.rs b/apps/keck-core/src/server/sync/blobs.rs new file mode 100644 index 000000000..f893ef4a3 --- /dev/null +++ b/apps/keck-core/src/server/sync/blobs.rs @@ -0,0 +1,135 @@ +use std::sync::Arc; + +use axum::{ + extract::{BodyStream, Path}, + headers::ContentLength, + http::{ + header::{CACHE_CONTROL, CONTENT_LENGTH, CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED}, + HeaderMap, HeaderValue, StatusCode, + }, + response::{IntoResponse, Response}, + Json, TypedHeader, +}; +use futures::{future, StreamExt}; +use jwst_core::BlobStorage; +use jwst_core_rpc::RpcContextImpl; +use time::{format_description::well_known::Rfc2822, OffsetDateTime}; + +use super::*; + +#[derive(Serialize)] +struct BlobStatus { + exists: bool, + id: String, +} + +impl Context { + async fn get_blob(&self, workspace: Option, id: String, method: Method, headers: HeaderMap) -> Response { + if let Some(etag) = headers.get(IF_NONE_MATCH).and_then(|h| h.to_str().ok()) { + if etag == id { + return StatusCode::NOT_MODIFIED.into_response(); + } + } + + let Ok(meta) = self + .get_storage() + .blobs() + .get_metadata(workspace.clone(), id.clone(), None) + .await + else { + return StatusCode::NOT_FOUND.into_response(); + }; + + if let Some(modified_since) = headers + .get(IF_MODIFIED_SINCE) + .and_then(|h| h.to_str().ok()) + .and_then(|s| OffsetDateTime::parse(s, &Rfc2822).ok()) + { + if meta.last_modified.timestamp() <= modified_since.unix_timestamp() { + return StatusCode::NOT_MODIFIED.into_response(); + } + } + + let mut header = HeaderMap::with_capacity(5); + header.insert(ETAG, HeaderValue::from_str(&id).unwrap()); + header.insert(CONTENT_TYPE, HeaderValue::from_static("application/octet-stream")); + header.insert( + LAST_MODIFIED, + HeaderValue::from_str( + &OffsetDateTime::from_unix_timestamp(meta.last_modified.timestamp()) + .unwrap() + .format(&Rfc2822) + .unwrap(), + ) + .unwrap(), + ); + header.insert(CONTENT_LENGTH, HeaderValue::from_str(&meta.size.to_string()).unwrap()); + header.insert( + CACHE_CONTROL, + HeaderValue::from_str("public, immutable, max-age=31536000").unwrap(), + ); + + if method == Method::HEAD { + return header.into_response(); + }; + + let Ok(file) = self.get_storage().blobs().get_blob(workspace, id, None).await else { + return StatusCode::NOT_FOUND.into_response(); + }; + + if meta.size != file.len() as i64 { + header.insert(CONTENT_LENGTH, HeaderValue::from_str(&file.len().to_string()).unwrap()); + } + + (header, file).into_response() + } + + async fn upload_blob(&self, stream: BodyStream, workspace: Option) -> Response { + // TODO: cancel + let mut has_error = false; + let stream = stream + .take_while(|x| { + has_error = x.is_err(); + future::ready(x.is_ok()) + }) + .filter_map(|data| future::ready(data.ok())); + + if let Ok(id) = self + .get_storage() + .blobs() + .put_blob_stream(workspace.clone(), stream) + .await + { + if has_error { + let _ = self.get_storage().blobs().delete_blob(workspace, id).await; + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } else { + Json(BlobStatus { id, exists: true }).into_response() + } + } else { + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + } +} + +pub async fn get_blob_in_workspace( + Extension(ctx): Extension>, + Path((workspace_id, id)): Path<(String, String)>, + method: Method, + headers: HeaderMap, +) -> Response { + ctx.get_blob(Some(workspace_id), id, method, headers).await +} + +pub async fn upload_blob_in_workspace( + Extension(ctx): Extension>, + Path(workspace_id): Path, + TypedHeader(length): TypedHeader, + stream: BodyStream, +) -> Response { + if length.0 > 10 * 1024 * 1024 { + return StatusCode::PAYLOAD_TOO_LARGE.into_response(); + } + + ctx.upload_blob(stream, Some(workspace_id)).await +} diff --git a/apps/keck-core/src/server/sync/collaboration.rs b/apps/keck-core/src/server/sync/collaboration.rs new file mode 100644 index 000000000..cafc1ea92 --- /dev/null +++ b/apps/keck-core/src/server/sync/collaboration.rs @@ -0,0 +1,305 @@ +use std::sync::Arc; + +use axum::{ + extract::{ws::WebSocketUpgrade, Path}, + response::Response, + Json, +}; +use futures::FutureExt; +use jwst_core_rpc::{axum_socket_connector, handle_connector}; +use serde::Serialize; + +use super::*; + +#[derive(Serialize)] +pub struct WebSocketAuthentication { + protocol: String, +} + +pub async fn auth_handler(Path(workspace_id): Path) -> Json { + info!("auth: {}", workspace_id); + Json(WebSocketAuthentication { + protocol: "AFFiNE".to_owned(), + }) +} + +pub async fn upgrade_handler( + Extension(context): Extension>, + Path(workspace): Path, + ws: WebSocketUpgrade, +) -> Response { + let identifier = nanoid!(); + if let Err(e) = context.create_workspace(workspace.clone()).await { + error!("create workspace failed: {:?}", e); + } + ws.protocols(["AFFiNE"]).on_upgrade(move |socket| { + handle_connector(context.clone(), workspace.clone(), identifier, move || { + axum_socket_connector(socket, &workspace) + }) + .map(|_| ()) + }) +} + +#[cfg(test)] +mod test { + use std::{ + ffi::c_int, + io::{BufRead, BufReader}, + process::{Child, Command, Stdio}, + string::String, + sync::Arc, + }; + + use jwst_core::{Block, DocStorage, Workspace}; + use jwst_core_rpc::{start_websocket_client_sync, BroadcastChannels, RpcContextImpl}; + use jwst_core_storage::{BlobStorageType, JwstStorage}; + use jwst_logger::info; + use libc::{kill, SIGTERM}; + use rand::{thread_rng, Rng}; + use tokio::runtime::Runtime; + + struct TestContext { + storage: Arc, + channel: Arc, + } + + impl TestContext { + fn new(storage: Arc) -> Self { + Self { + storage, + channel: Arc::default(), + } + } + } + + impl RpcContextImpl<'_> for TestContext { + fn get_storage(&self) -> &JwstStorage { + &self.storage + } + + fn get_channel(&self) -> &BroadcastChannels { + &self.channel + } + } + + // #[test] + // #[ignore = "not needed in ci"] + // fn client_collaboration_with_server() { + // if dotenvy::var("KECK_DEBUG").is_ok() { + // jwst_logger::init_logger("keck"); + // } + + // let server_port = thread_rng().gen_range(10000..=30000); + // let child = start_collaboration_server(server_port); + + // let rt = Runtime::new().unwrap(); + // let (workspace_id, workspace) = rt.block_on(async move { + // let workspace_id = "1"; + // let context = Arc::new(TestContext::new(Arc::new( + // JwstStorage::new_with_migration("sqlite::memory:", + // BlobStorageType::DB) .await + // .expect("get storage: memory sqlite failed"), + // ))); + // let remote = + // format!("ws://localhost:{server_port}/collaboration/1"); + + // start_websocket_client_sync( + // Arc::new(Runtime::new().unwrap()), + // context.clone(), + // Arc::default(), + // remote, + // workspace_id.to_owned(), + // ); + + // ( + // workspace_id.to_owned(), + // context.get_workspace(workspace_id).await.unwrap(), + // ) + // }); + + // for block_id in 0..3 { + // let block = create_block(&workspace, block_id.to_string(), + // "list".to_string()); info!("from client, create a block: + // {:?}", block); } + + // info!("------------------after sync------------------"); + + // for block_id in 0..3 { + // info!( + // "get block {block_id} from server: {}", + // get_block_from_server(workspace_id.clone(), + // block_id.to_string(), server_port) ); + // assert!(!get_block_from_server(workspace_id.clone(), + // block_id.to_string(), server_port).is_empty()); } + + // workspace.with_trx(|mut trx| { + // let space = trx.get_space("blocks"); + // let blocks = space.get_blocks_by_flavour(&trx.trx, "list"); + // let mut ids: Vec<_> = blocks.iter().map(|block| + // block.block_id()).collect(); assert_eq!(ids.sort(), vec!["7", + // "8", "9"].sort()); info!("blocks from local storage:"); + // for block in blocks { + // info!("block: {:?}", block); + // } + // }); + + // close_collaboration_server(child); + // } + + // #[test] + // #[ignore = "not needed in ci"] + // fn client_collaboration_with_server_with_poor_connection() { + // let server_port = thread_rng().gen_range(30001..=65535); + // let child = start_collaboration_server(server_port); + + // let rt = Runtime::new().unwrap(); + // let workspace_id = String::from("1"); + // let (storage, workspace) = rt.block_on(async { + // let storage: Arc = Arc::new( + // JwstStorage::new_with_migration("sqlite::memory:", + // BlobStorageType::DB) .await + // .expect("get storage: memory sqlite failed"), + // ); + // let workspace = storage + // .docs() + // .get_or_create_workspace(workspace_id.clone()) + // .await + // .expect("get workspace: {workspace_id} failed"); + // (storage, workspace) + // }); + + // // simulate creating a block in offline environment + // let block = create_block(&workspace, "0".to_string(), + // "list".to_string()); info!("from client, create a block: {:?}", + // block); info!( + // "get block 0 from server: {}", + // get_block_from_server(workspace_id.clone(), "0".to_string(), + // server_port) ); + // assert!(get_block_from_server(workspace_id.clone(), "0".to_string(), + // server_port).is_empty()); + + // let rt = Runtime::new().unwrap(); + // let (workspace_id, workspace) = rt.block_on(async move { + // let workspace_id = "1"; + // let context = Arc::new(TestContext::new(storage)); + // let remote = + // format!("ws://localhost:{server_port}/collaboration/1"); + + // start_websocket_client_sync( + // Arc::new(Runtime::new().unwrap()), + // context.clone(), + // Arc::default(), + // remote, + // workspace_id.to_owned(), + // ); + + // ( + // workspace_id.to_owned(), + // context.get_workspace(workspace_id).await.unwrap(), + // ) + // }); + + // info!("----------------start syncing from + // start_sync_thread()----------------"); + + // for block_id in 1..3 { + // let block = create_block(&workspace, block_id.to_string(), + // "list".to_string()); info!("from client, create a block: + // {:?}", block); info!( + // "get block {block_id} from server: {}", + // get_block_from_server(workspace_id.clone(), + // block_id.to_string(), server_port) ); + // assert!(!get_block_from_server(workspace_id.clone(), + // block_id.to_string(), server_port).is_empty()); } + + // workspace.with_trx(|mut trx| { + // let space = trx.get_space("blocks"); + // let blocks = space.get_blocks_by_flavour(&trx.trx, "list"); + // let mut ids: Vec<_> = blocks.iter().map(|block| + // block.block_id()).collect(); assert_eq!(ids.sort(), vec!["0", + // "1", "2"].sort()); info!("blocks from local storage:"); + // for block in blocks { + // info!("block: {:?}", block); + // } + // }); + + // info!("------------------after sync------------------"); + + // for block_id in 0..3 { + // info!( + // "get block {block_id} from server: {}", + // get_block_from_server(workspace_id.clone(), + // block_id.to_string(), server_port) ); + // assert!(!get_block_from_server(workspace_id.clone(), + // block_id.to_string(), server_port).is_empty()); } + + // workspace.with_trx(|mut trx| { + // let space = trx.get_space("blocks"); + // let blocks = space.get_blocks_by_flavour(&trx.trx, "list"); + // let mut ids: Vec<_> = blocks.iter().map(|block| + // block.block_id()).collect(); assert_eq!(ids.sort(), vec!["0", + // "1", "2"].sort()); info!("blocks from local storage:"); + // for block in blocks { + // info!("block: {:?}", block); + // } + // }); + + // close_collaboration_server(child); + // } + + // fn get_block_from_server(workspace_id: String, block_id: String, + // server_port: u16) -> String { let rt = Runtime::new().unwrap(); + // rt.block_on(async { + // let client = reqwest::Client::new(); + // let resp = client + // .get(format!( + // "http://localhost:{server_port}/api/block/{}/{}", + // workspace_id, block_id + // )) + // .send() + // .await + // .unwrap(); + // resp.text().await.unwrap() + // }) + // } + + // fn create_block(workspace: &Workspace, block_id: String, block_flavour: + // String) -> Block { workspace.with_trx(|mut trx| { + // let space = trx.get_space("blocks"); + // space + // .create(&mut trx.trx, block_id, block_flavour) + // .expect("failed to create block") + // }) + // } + + // fn start_collaboration_server(port: u16) -> Child { + // let mut child = Command::new("cargo") + // .args(&["run", "-p", "keck"]) + // .env("KECK_PORT", port.to_string()) + // .env("USE_MEMORY_SQLITE", "true") + // .env("KECK_LOG", "INFO") + // .stdout(Stdio::piped()) + // .spawn() + // .expect("Failed to run command"); + + // if let Some(ref mut stdout) = child.stdout { + // let reader = BufReader::new(stdout); + + // for line in reader.lines() { + // let line = line.expect("Failed to read line"); + // info!("{}", line); + + // if line.contains("listening on 0.0.0.0:") { + // info!("Keck server started"); + // break; + // } + // } + // } + + // child + // } + + // fn close_collaboration_server(child: Child) { + // unsafe { kill(child.id() as c_int, SIGTERM) }; + // } +} diff --git a/apps/keck-core/src/server/sync/mod.rs b/apps/keck-core/src/server/sync/mod.rs new file mode 100644 index 000000000..fc9f94641 --- /dev/null +++ b/apps/keck-core/src/server/sync/mod.rs @@ -0,0 +1,25 @@ +mod blobs; +mod collaboration; + +use axum::routing::{get, post, put}; + +use super::*; + +pub fn sync_handler(router: Router) -> Router { + let router = if cfg!(feature = "api") { + router + } else { + router.nest( + "/api", + Router::new() + .route("/workspace/:id/blob", put(blobs::upload_blob_in_workspace)) + .route("/workspace/:id/blob/:name", get(blobs::get_blob_in_workspace)), + ) + } + .nest_service( + "/collaboration/:workspace", + post(collaboration::auth_handler).get(collaboration::upgrade_handler), + ); + + router +} diff --git a/apps/keck-core/src/server/utils.rs b/apps/keck-core/src/server/utils.rs new file mode 100644 index 000000000..596fa35a0 --- /dev/null +++ b/apps/keck-core/src/server/utils.rs @@ -0,0 +1,3 @@ +pub use jwst_logger::{debug, error, info, warn}; +pub use nanoid::nanoid; +pub use serde::{Deserialize, Serialize}; diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist index b3d1aee06..067a2b752 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist @@ -8,7 +8,7 @@ HeadersPath Headers LibraryIdentifier - ios-arm64-simulator + macos-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -16,15 +16,13 @@ arm64 SupportedPlatform - ios - SupportedPlatformVariant - simulator + macos HeadersPath Headers LibraryIdentifier - macos-arm64 + ios-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -32,13 +30,13 @@ arm64 SupportedPlatform - macos + ios HeadersPath Headers LibraryIdentifier - ios-arm64 + ios-arm64-simulator LibraryPath liboctobase.a SupportedArchitectures @@ -47,6 +45,8 @@ SupportedPlatform ios + SupportedPlatformVariant + simulator CFBundlePackageType diff --git a/libs/jwst-binding/jwst-swift/src/storage.rs b/libs/jwst-binding/jwst-swift/src/storage.rs index f587bbd38..615a355c4 100644 --- a/libs/jwst-binding/jwst-swift/src/storage.rs +++ b/libs/jwst-binding/jwst-swift/src/storage.rs @@ -114,8 +114,7 @@ impl Storage { match workspace { Ok(mut workspace) => { - // disable remote temporarily - if is_offline || true { + if is_offline { let identifier = nanoid!(); let (last_synced_tx, last_synced_rx) = channel::(128); self.last_sync.add_receiver(rt.clone(), last_synced_rx); diff --git a/libs/jwst-core-rpc/Cargo.toml b/libs/jwst-core-rpc/Cargo.toml index 8799d70a5..c2d3a88ab 100644 --- a/libs/jwst-core-rpc/Cargo.toml +++ b/libs/jwst-core-rpc/Cargo.toml @@ -22,11 +22,7 @@ rand = "0.8.5" serde = "1.0.183" serde_json = "1.0.104" thiserror = "1.0.40" -tokio = { version = "1.27.0", features = [ - "macros", - "rt-multi-thread", - "signal", -] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] } # ======== websocket dependencies ======== axum = { version = "0.6.16", features = ["ws"], optional = true } diff --git a/libs/jwst-core-rpc/src/context.rs b/libs/jwst-core-rpc/src/context.rs index dd4a15f89..e442fc3bc 100644 --- a/libs/jwst-core-rpc/src/context.rs +++ b/libs/jwst-core-rpc/src/context.rs @@ -172,7 +172,7 @@ pub trait RpcContextImpl<'a> { // skip empty update continue; } - trace!("apply_change: recv binary: {:?}", binary.len()); + debug!("apply_change: recv binary: {:?}", binary.len()); updates.push(binary); } else { break; diff --git a/libs/jwst-core-rpc/src/handler.rs b/libs/jwst-core-rpc/src/handler.rs index 0df9e1e18..bc2207d64 100644 --- a/libs/jwst-core-rpc/src/handler.rs +++ b/libs/jwst-core-rpc/src/handler.rs @@ -50,6 +50,7 @@ pub async fn handle_connector( // Send initialization message. match ws.sync_init_message().await { Ok(init_data) => { + debug!("send init data:{:?}", init_data); if tx.send(Message::Binary(init_data)).await.is_err() { warn!("failed to send init message: {}", identifier); // client disconnected diff --git a/libs/jwst-core-storage/Cargo.toml b/libs/jwst-core-storage/Cargo.toml index f481ed1ae..a885ca77e 100644 --- a/libs/jwst-core-storage/Cargo.toml +++ b/libs/jwst-core-storage/Cargo.toml @@ -25,7 +25,7 @@ sha2 = "0.10.7" sea-orm = { version = "0.12.2", features = ["runtime-tokio-rustls", "macros"] } sea-orm-migration = { version = "0.12.2", default-features = false } thiserror = "1.0.47" -tokio = { version = "1.32.0", features = ["fs", "macros", "sync"] } +tokio = { version = "1", features = ["fs", "macros", "sync"] } tokio-util = { version = "0.7.8", features = ["io"] } url = "2.4.0" diff --git a/libs/jwst-core/src/workspace/sync.rs b/libs/jwst-core/src/workspace/sync.rs index f170aa521..df84032a4 100644 --- a/libs/jwst-core/src/workspace/sync.rs +++ b/libs/jwst-core/src/workspace/sync.rs @@ -2,6 +2,7 @@ use jwst_codec::{ write_sync_message, CrdtRead, CrdtWrite, DocMessage, RawDecoder, RawEncoder, StateVector, SyncMessage, SyncMessageScanner, Update, }; +use tracing::debug; use super::*; @@ -34,10 +35,17 @@ impl Workspace { let mut content = vec![]; for buffer in buffers { - trace!("sync message: {:?}", buffer); let (awareness_msg, content_msg): (Vec<_>, Vec<_>) = SyncMessageScanner::new(&buffer) .flatten() .partition(|msg| matches!(msg, SyncMessage::Awareness(_) | SyncMessage::AwarenessQuery)); + + debug!( + "sync message: {}, awareness: {}, content: {}", + buffer.len(), + awareness_msg.len(), + content_msg.len() + ); + awareness.extend(awareness_msg); content.extend(content_msg); } @@ -85,11 +93,16 @@ impl Workspace { match msg { SyncMessage::Doc(msg) => match msg { DocMessage::Step1(sv) => StateVector::read(&mut RawDecoder::new(sv)).ok().and_then(|sv| { + debug!("step1 get sv: {sv:?}"); doc.encode_state_as_update_v1(&sv) - .map(|update| SyncMessage::Doc(DocMessage::Step2(update))) + .map(|update| { + debug!("step1 encode update: {}", update.len()); + SyncMessage::Doc(DocMessage::Step2(update)) + }) .ok() }), DocMessage::Step2(update) => { + debug!("step2 get update: {}", update.len()); if let Ok(update) = Update::read(&mut RawDecoder::new(update)) { if let Err(e) = doc.apply_update(update) { warn!("failed to apply update: {:?}", e); @@ -115,6 +128,7 @@ impl Workspace { if let Err(e) = write_sync_message(&mut buffer, &msg) { warn!("failed to encode message: {:?}", e); } else { + debug!("return update: {}", buffer.len()); result.push(buffer); } } From 18489d90591f1f90caec0b7212aab2e638524923 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Thu, 24 Aug 2023 18:38:35 +0800 Subject: [PATCH 17/49] fix: renew value in map --- libs/jwst-codec/src/doc/store.rs | 3 ++- libs/jwst-codec/src/doc/types/map.rs | 10 +++++----- libs/jwst-core/src/block/mod.rs | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/libs/jwst-codec/src/doc/store.rs b/libs/jwst-codec/src/doc/store.rs index 54e792e82..b6ee860ce 100644 --- a/libs/jwst-codec/src/doc/store.rs +++ b/libs/jwst-codec/src/doc/store.rs @@ -360,7 +360,8 @@ impl DocStore { if let Content::Type(ty) = Arc::make_mut(&mut item.content) { ty.store = Arc::downgrade(&store_ref); - // we keep ty owner in dangling_types so the delete of any type will not make it dropped + // we keep ty owner in dangling_types so the delete of any type will not make it + // dropped if ty.inner.is_owned() { let owned_inner = ty.inner.swap_take(); self.dangling_types.insert( diff --git a/libs/jwst-codec/src/doc/types/map.rs b/libs/jwst-codec/src/doc/types/map.rs index e9255285f..26200e403 100644 --- a/libs/jwst-codec/src/doc/types/map.rs +++ b/libs/jwst-codec/src/doc/types/map.rs @@ -13,11 +13,11 @@ impl_type!(Map); pub(crate) trait MapType: AsInner { fn insert(&mut self, key: impl AsRef, value: impl Into) -> JwstCodecResult { if let Some((mut store, mut ty)) = self.as_inner().write() { - let left = ty.map.as_ref().and_then(|map| { - map.get(key.as_ref()) - .and_then(|struct_info| struct_info.left()) - .map(|l| l.as_item()) - }); + let left = ty + .map + .as_ref() + .and_then(|map| map.get(key.as_ref())) + .map(|l| l.as_item()); let item = store.create_item( value.into(), diff --git a/libs/jwst-core/src/block/mod.rs b/libs/jwst-core/src/block/mod.rs index d1933bf17..0a1d65e47 100644 --- a/libs/jwst-core/src/block/mod.rs +++ b/libs/jwst-core/src/block/mod.rs @@ -499,6 +499,22 @@ mod test { ); } + #[test] + fn block_renew_value() { + let mut workspace = Workspace::new("test").unwrap(); + + let mut space = workspace.get_space("space").unwrap(); + let mut block = space.create("test", "affine:text").unwrap(); + + let key = "hello".to_string(); + block.set(&key, "world").unwrap(); + + let mut block = space.get("test").unwrap(); + block.set(&key, "12345678").unwrap(); + + assert_eq!(block.get(&key).unwrap().to_string(), "12345678"); + } + #[test] fn insert_remove_children() { let mut workspace = Workspace::new("text").unwrap(); From 592ab19420cde150dab824e736a65b0a5785300d Mon Sep 17 00:00:00 2001 From: DarkSky Date: Fri, 25 Aug 2023 18:12:01 +0800 Subject: [PATCH 18/49] fix: sync protocol --- .../RustXcframework.xcframework/Info.plist | 14 +++++++------- libs/jwst-codec/src/doc/codec/update.rs | 4 ++++ libs/jwst-core-rpc/src/broadcast.rs | 2 +- libs/jwst-core/src/workspace/sync.rs | 12 ++++++++---- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist index 067a2b752..5a676cdb2 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist @@ -8,7 +8,7 @@ HeadersPath Headers LibraryIdentifier - macos-arm64 + ios-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -16,13 +16,13 @@ arm64 SupportedPlatform - macos + ios HeadersPath Headers LibraryIdentifier - ios-arm64 + ios-arm64-simulator LibraryPath liboctobase.a SupportedArchitectures @@ -31,12 +31,14 @@ SupportedPlatform ios + SupportedPlatformVariant + simulator HeadersPath Headers LibraryIdentifier - ios-arm64-simulator + macos-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -44,9 +46,7 @@ arm64 SupportedPlatform - ios - SupportedPlatformVariant - simulator + macos CFBundlePackageType diff --git a/libs/jwst-codec/src/doc/codec/update.rs b/libs/jwst-codec/src/doc/codec/update.rs index 2848d17aa..0fa21c565 100644 --- a/libs/jwst-codec/src/doc/codec/update.rs +++ b/libs/jwst-codec/src/doc/codec/update.rs @@ -159,6 +159,10 @@ impl Update { } } + pub fn is_content_empty(&self) -> bool { + self.structs.is_empty() + } + pub fn is_empty(&self) -> bool { self.structs.is_empty() && self.delete_set.is_empty() } diff --git a/libs/jwst-core-rpc/src/broadcast.rs b/libs/jwst-core-rpc/src/broadcast.rs index 4592a939b..8e4581613 100644 --- a/libs/jwst-core-rpc/src/broadcast.rs +++ b/libs/jwst-core-rpc/src/broadcast.rs @@ -47,7 +47,7 @@ pub async fn subscribe(workspace: &mut Workspace, identifier: String, sender: Br debug!("workspace {} changed: {}bytes", workspace_id, update.len()); match encode_update_with_guid(update.to_vec(), workspace_id.clone()) - .and_then(|update| encode_update_as_message(update.clone()).map(|u| (update, u))) + .and_then(|update_with_guid| encode_update_as_message(update.to_vec()).map(|u| (update_with_guid, u))) { Ok((broadcast_update, sendable_update)) => { if sender diff --git a/libs/jwst-core/src/workspace/sync.rs b/libs/jwst-core/src/workspace/sync.rs index df84032a4..1edc82fa6 100644 --- a/libs/jwst-core/src/workspace/sync.rs +++ b/libs/jwst-core/src/workspace/sync.rs @@ -102,7 +102,6 @@ impl Workspace { .ok() }), DocMessage::Step2(update) => { - debug!("step2 get update: {}", update.len()); if let Ok(update) = Update::read(&mut RawDecoder::new(update)) { if let Err(e) = doc.apply_update(update) { warn!("failed to apply update: {:?}", e); @@ -113,13 +112,18 @@ impl Workspace { DocMessage::Update(update) => doc .apply_update_from_binary(update) .and_then(|update| { + if update.is_content_empty() { + return Ok(None); + } + let mut encoder = RawEncoder::default(); update.write(&mut encoder)?; - Ok(encoder.into_inner()) + Ok(Some(encoder.into_inner())) }) - .map(|u| SyncMessage::Doc(DocMessage::Update(u))) .map_err(|e| warn!("failed to apply update: {:?}", e)) - .ok(), + .ok() + .flatten() + .map(|u| SyncMessage::Doc(DocMessage::Update(u))), }, _ => None, } From 0ee6a9a0f2cfaf871618ac2a2064886df96c8198 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Sat, 26 Aug 2023 20:20:22 +0800 Subject: [PATCH 19/49] chore: cleanup subscribe --- apps/keck-core/src/server/api/blocks/mod.rs | 9 +- apps/keck-core/src/server/api/mod.rs | 18 +- apps/keck-core/src/server/mod.rs | 45 +--- apps/keck-core/src/server/subscribe.rs | 69 ------ .../src/server/sync/collaboration.rs | 215 +++++++++--------- 5 files changed, 110 insertions(+), 246 deletions(-) delete mode 100644 apps/keck-core/src/server/subscribe.rs diff --git a/apps/keck-core/src/server/api/blocks/mod.rs b/apps/keck-core/src/server/api/blocks/mod.rs index 8eb765ded..48dbcb855 100644 --- a/apps/keck-core/src/server/api/blocks/mod.rs +++ b/apps/keck-core/src/server/api/blocks/mod.rs @@ -72,19 +72,12 @@ mod tests { ); let workspace_changed_blocks = Arc::new(RwLock::new(HashMap::::new())); let hook_endpoint = Arc::new(RwLock::new(String::new())); - let cb: WorkspaceRetrievalCallback = { - let workspace_changed_blocks = workspace_changed_blocks.clone(); - let runtime = runtime.clone(); - Some(Arc::new(Box::new(move |workspace: &Workspace| { - workspace.set_callback(generate_ws_callback(&workspace_changed_blocks, &runtime)); - }))) - }; + let ctx = Arc::new( Context::new( JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) .await .ok(), - cb, ) .await, ); diff --git a/apps/keck-core/src/server/api/mod.rs b/apps/keck-core/src/server/api/mod.rs index b95bc77ea..e4b9c1246 100644 --- a/apps/keck-core/src/server/api/mod.rs +++ b/apps/keck-core/src/server/api/mod.rs @@ -41,11 +41,10 @@ pub struct PageData { pub struct Context { channel: BroadcastChannels, storage: JwstStorage, - callback: WorkspaceRetrievalCallback, } impl Context { - pub async fn new(storage: Option, cb: WorkspaceRetrievalCallback) -> Self { + pub async fn new(storage: Option) -> Self { let blob_storage_type = BlobStorageType::DB; let storage = if let Some(storage) = storage { @@ -66,7 +65,6 @@ impl Context { Context { channel: RwLock::new(HashMap::new()), storage, - callback: cb, } } @@ -74,24 +72,14 @@ impl Context { where S: AsRef, { - let workspace = self.storage.get_workspace(workspace_id).await?; - if let Some(cb) = self.callback.clone() { - cb(&workspace); - } - - Ok(workspace) + self.storage.get_workspace(workspace_id).await } pub async fn create_workspace(&self, workspace_id: S) -> JwstStorageResult where S: AsRef, { - let workspace = self.storage.create_workspace(workspace_id).await?; - if let Some(cb) = self.callback.clone() { - cb(&workspace); - } - - Ok(workspace) + self.storage.create_workspace(workspace_id).await } } diff --git a/apps/keck-core/src/server/mod.rs b/apps/keck-core/src/server/mod.rs index e5bfb6919..f11540ae5 100644 --- a/apps/keck-core/src/server/mod.rs +++ b/apps/keck-core/src/server/mod.rs @@ -1,14 +1,12 @@ mod api; -mod subscribe; mod sync; mod utils; -use std::{collections::HashMap, net::SocketAddr, sync::Arc, thread::sleep}; +use std::{net::SocketAddr, sync::Arc}; use api::Context; use axum::{http::Method, Extension, Router, Server}; use jwst_core::Workspace; -pub use subscribe::*; use tokio::{runtime, signal, sync::RwLock}; use tower_http::cors::{Any, CorsLayer}; pub use utils::*; @@ -37,29 +35,6 @@ async fn shutdown_signal() { info!("Shutdown signal received, starting graceful shutdown"); } -type WorkspaceRetrievalCallback = Option>>; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WorkspaceChangedBlocks { - #[serde(rename(serialize = "workspaceId"))] - pub workspace_id: String, - #[serde(rename(serialize = "blockIds"))] - pub block_ids: Vec, -} - -impl WorkspaceChangedBlocks { - pub fn new(workspace_id: String) -> WorkspaceChangedBlocks { - WorkspaceChangedBlocks { - workspace_id, - block_ids: Vec::new(), - } - } - - pub fn insert_block_ids(&mut self, mut updated_block_ids: Vec) { - self.block_ids.append(&mut updated_block_ids); - } -} - pub async fn start_server() { let origins = [ "http://localhost:4200".parse().unwrap(), @@ -86,31 +61,15 @@ pub async fn start_server() { .build() .expect("Failed to create runtime"), ); - let workspace_changed_blocks = Arc::new(RwLock::new(HashMap::::new())); let hook_endpoint = Arc::new(RwLock::new(dotenvy::var("HOOK_ENDPOINT").unwrap_or_default())); - let cb: WorkspaceRetrievalCallback = { - let workspace_changed_blocks = workspace_changed_blocks.clone(); - let runtime = runtime.clone(); - Some(Arc::new(Box::new(move |workspace: &Workspace| { - // workspace.set_callback(generate_ws_callback(& - // workspace_changed_blocks, &runtime)); - }))) - }; - let context = Arc::new(Context::new(None, cb).await); - start_handling_observed_blocks( - runtime.clone(), - workspace_changed_blocks.clone(), - hook_endpoint.clone(), - client.clone(), - ); + let context = Arc::new(Context::new(None).await); let app = sync::sync_handler(api::api_handler(Router::new())) .layer(cors) .layer(Extension(context.clone())) .layer(Extension(client)) .layer(Extension(runtime)) - .layer(Extension(workspace_changed_blocks)) .layer(Extension(hook_endpoint)); let addr = SocketAddr::from(( diff --git a/apps/keck-core/src/server/subscribe.rs b/apps/keck-core/src/server/subscribe.rs deleted file mode 100644 index 766d48e0c..000000000 --- a/apps/keck-core/src/server/subscribe.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::{collections::HashMap, sync::Arc, thread, time::Duration}; - -use reqwest::Client; -use tokio::{runtime::Runtime, sync::RwLock}; - -use super::*; -use crate::server::WorkspaceChangedBlocks; - -pub fn generate_ws_callback( - workspace_changed_blocks: &Arc>>, - runtime: &Arc, -) -> Arc) + Send + Sync>> { - let workspace_changed_blocks = workspace_changed_blocks.clone(); - let runtime = runtime.clone(); - Arc::new(Box::new(move |workspace_id, block_ids| { - let workspace_changed_blocks = workspace_changed_blocks.clone(); - runtime.spawn(async move { - let mut write_guard = workspace_changed_blocks.write().await; - write_guard - .entry(workspace_id.clone()) - .or_insert(WorkspaceChangedBlocks::new(workspace_id.clone())) - .insert_block_ids(block_ids.clone()); - }); - })) -} - -pub fn start_handling_observed_blocks( - runtime: Arc, - workspace_changed_blocks: Arc>>, - hook_endpoint: Arc>, - client: Arc, -) { - thread::spawn(move || loop { - let workspace_changed_blocks = workspace_changed_blocks.clone(); - let hook_endpoint = hook_endpoint.clone(); - let runtime_cloned = runtime.clone(); - let client = client.clone(); - runtime.spawn(async move { - let read_guard = workspace_changed_blocks.read().await; - if !read_guard.is_empty() { - let endpoint = hook_endpoint.read().await; - if !endpoint.is_empty() { - { - let workspace_changed_blocks_clone = read_guard.clone(); - let post_body = workspace_changed_blocks_clone - .into_values() - .collect::>(); - let endpoint = endpoint.clone(); - runtime_cloned.spawn(async move { - let response = client.post(endpoint.to_string()).json(&post_body).send().await; - match response { - Ok(response) => info!( - "notified hook endpoint, endpoint response status: {}", - response.status() - ), - Err(e) => error!("Failed to send notify: {}", e), - } - }); - } - } - drop(read_guard); - let mut write_guard = workspace_changed_blocks.write().await; - info!("workspace_changed_blocks: {:?}", write_guard); - write_guard.clear(); - } - }); - sleep(Duration::from_millis(200)); - }); -} diff --git a/apps/keck-core/src/server/sync/collaboration.rs b/apps/keck-core/src/server/sync/collaboration.rs index cafc1ea92..4b9dd4e0b 100644 --- a/apps/keck-core/src/server/sync/collaboration.rs +++ b/apps/keck-core/src/server/sync/collaboration.rs @@ -82,69 +82,66 @@ mod test { } } - // #[test] - // #[ignore = "not needed in ci"] - // fn client_collaboration_with_server() { - // if dotenvy::var("KECK_DEBUG").is_ok() { - // jwst_logger::init_logger("keck"); - // } - - // let server_port = thread_rng().gen_range(10000..=30000); - // let child = start_collaboration_server(server_port); - - // let rt = Runtime::new().unwrap(); - // let (workspace_id, workspace) = rt.block_on(async move { - // let workspace_id = "1"; - // let context = Arc::new(TestContext::new(Arc::new( - // JwstStorage::new_with_migration("sqlite::memory:", - // BlobStorageType::DB) .await - // .expect("get storage: memory sqlite failed"), - // ))); - // let remote = - // format!("ws://localhost:{server_port}/collaboration/1"); - - // start_websocket_client_sync( - // Arc::new(Runtime::new().unwrap()), - // context.clone(), - // Arc::default(), - // remote, - // workspace_id.to_owned(), - // ); - - // ( - // workspace_id.to_owned(), - // context.get_workspace(workspace_id).await.unwrap(), - // ) - // }); + #[test] + #[ignore = "not needed in ci"] + fn client_collaboration_with_server() { + if dotenvy::var("KECK_DEBUG").is_ok() { + jwst_logger::init_logger("keck-core"); + } - // for block_id in 0..3 { - // let block = create_block(&workspace, block_id.to_string(), - // "list".to_string()); info!("from client, create a block: - // {:?}", block); } + let server_port = thread_rng().gen_range(10000..=30000); + let child = start_collaboration_server(server_port); + + let rt: Runtime = Runtime::new().unwrap(); + let (workspace_id, mut workspace) = { + let workspace_id = "1"; + let context = rt.block_on(async move { + Arc::new(TestContext::new(Arc::new( + JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) + .await + .expect("get storage: memory sqlite failed"), + ))) + }); + let remote = format!("ws://localhost:{server_port}/collaboration/1"); + + start_websocket_client_sync( + Arc::new(Runtime::new().unwrap()), + context.clone(), + Arc::default(), + remote, + workspace_id.to_owned(), + ); + + ( + workspace_id.to_owned(), + rt.block_on(async move { context.get_workspace(workspace_id).await.unwrap() }), + ) + }; + + for block_id in 0..3 { + let block = create_block(&mut workspace, block_id.to_string(), "list".to_string()); + info!("from client, create a block:{:?}", serde_json::to_string(&block)); + } - // info!("------------------after sync------------------"); + info!("------------------after sync------------------"); - // for block_id in 0..3 { - // info!( - // "get block {block_id} from server: {}", - // get_block_from_server(workspace_id.clone(), - // block_id.to_string(), server_port) ); - // assert!(!get_block_from_server(workspace_id.clone(), - // block_id.to_string(), server_port).is_empty()); } + for block_id in 0..3 { + let ret = get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port); + info!("get block {block_id} from server: {ret}"); + assert!(!ret.is_empty()); + } - // workspace.with_trx(|mut trx| { - // let space = trx.get_space("blocks"); - // let blocks = space.get_blocks_by_flavour(&trx.trx, "list"); - // let mut ids: Vec<_> = blocks.iter().map(|block| - // block.block_id()).collect(); assert_eq!(ids.sort(), vec!["7", - // "8", "9"].sort()); info!("blocks from local storage:"); - // for block in blocks { - // info!("block: {:?}", block); - // } - // }); + let space = workspace.get_space("blocks").unwrap(); + let blocks = space.get_blocks_by_flavour("list"); + let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); + assert_eq!(ids.sort(), vec!["7", "8", "9"].sort()); + info!("blocks from local storage:"); + for block in blocks { + info!("block: {:?}", block); + } - // close_collaboration_server(child); - // } + close_collaboration_server(child); + } // #[test] // #[ignore = "not needed in ci"] @@ -214,7 +211,7 @@ mod test { // workspace.with_trx(|mut trx| { // let space = trx.get_space("blocks"); - // let blocks = space.get_blocks_by_flavour(&trx.trx, "list"); + // let blocks = space.get_blocks_by_flavour( "list"); // let mut ids: Vec<_> = blocks.iter().map(|block| // block.block_id()).collect(); assert_eq!(ids.sort(), vec!["0", // "1", "2"].sort()); info!("blocks from local storage:"); @@ -235,7 +232,7 @@ mod test { // workspace.with_trx(|mut trx| { // let space = trx.get_space("blocks"); - // let blocks = space.get_blocks_by_flavour(&trx.trx, "list"); + // let blocks = space.get_blocks_by_flavour( "list"); // let mut ids: Vec<_> = blocks.iter().map(|block| // block.block_id()).collect(); assert_eq!(ids.sort(), vec!["0", // "1", "2"].sort()); info!("blocks from local storage:"); @@ -247,59 +244,55 @@ mod test { // close_collaboration_server(child); // } - // fn get_block_from_server(workspace_id: String, block_id: String, - // server_port: u16) -> String { let rt = Runtime::new().unwrap(); - // rt.block_on(async { - // let client = reqwest::Client::new(); - // let resp = client - // .get(format!( - // "http://localhost:{server_port}/api/block/{}/{}", - // workspace_id, block_id - // )) - // .send() - // .await - // .unwrap(); - // resp.text().await.unwrap() - // }) - // } + fn get_block_from_server(workspace_id: String, block_id: String, server_port: u16) -> String { + let rt = Runtime::new().unwrap(); + rt.block_on(async { + let client = reqwest::Client::new(); + let resp = client + .get(format!( + "http://localhost:{server_port}/api/block/{}/{}", + workspace_id, block_id + )) + .send() + .await + .unwrap(); + resp.text().await.unwrap() + }) + } - // fn create_block(workspace: &Workspace, block_id: String, block_flavour: - // String) -> Block { workspace.with_trx(|mut trx| { - // let space = trx.get_space("blocks"); - // space - // .create(&mut trx.trx, block_id, block_flavour) - // .expect("failed to create block") - // }) - // } + fn create_block(workspace: &mut Workspace, block_id: String, block_flavour: String) -> Block { + let mut space = workspace.get_space("blocks").unwrap(); + space.create(block_id, block_flavour).expect("failed to create block") + } - // fn start_collaboration_server(port: u16) -> Child { - // let mut child = Command::new("cargo") - // .args(&["run", "-p", "keck"]) - // .env("KECK_PORT", port.to_string()) - // .env("USE_MEMORY_SQLITE", "true") - // .env("KECK_LOG", "INFO") - // .stdout(Stdio::piped()) - // .spawn() - // .expect("Failed to run command"); - - // if let Some(ref mut stdout) = child.stdout { - // let reader = BufReader::new(stdout); - - // for line in reader.lines() { - // let line = line.expect("Failed to read line"); - // info!("{}", line); - - // if line.contains("listening on 0.0.0.0:") { - // info!("Keck server started"); - // break; - // } - // } - // } + fn start_collaboration_server(port: u16) -> Child { + let mut child = Command::new("cargo") + .args(&["run", "-p", "keck-core"]) + .env("KECK_PORT", port.to_string()) + .env("USE_MEMORY_SQLITE", "true") + .env("KECK_LOG", "INFO") + .stdout(Stdio::piped()) + .spawn() + .expect("Failed to run command"); + + if let Some(ref mut stdout) = child.stdout { + let reader = BufReader::new(stdout); + + for line in reader.lines() { + let line = line.expect("Failed to read line"); + info!("{}", line); + + if line.contains("listening on 0.0.0.0:") { + info!("Keck server started"); + break; + } + } + } - // child - // } + child + } - // fn close_collaboration_server(child: Child) { - // unsafe { kill(child.id() as c_int, SIGTERM) }; - // } + fn close_collaboration_server(child: Child) { + unsafe { kill(child.id() as c_int, SIGTERM) }; + } } From 4601ba25f1b847d65a88af1b590f9346d4e147df Mon Sep 17 00:00:00 2001 From: DarkSky Date: Sat, 26 Aug 2023 21:19:44 +0800 Subject: [PATCH 20/49] feat: fix block apis --- apps/keck-core/src/server/api/blobs.rs | 1 - apps/keck-core/src/server/api/blocks/block.rs | 306 +++++++----------- apps/keck-core/src/server/api/blocks/mod.rs | 21 +- .../keck-core/src/server/api/blocks/schema.rs | 2 +- .../src/server/api/blocks/workspace.rs | 288 ++++++++--------- 5 files changed, 255 insertions(+), 363 deletions(-) diff --git a/apps/keck-core/src/server/api/blobs.rs b/apps/keck-core/src/server/api/blobs.rs index b4e1e63a3..6fae433ad 100644 --- a/apps/keck-core/src/server/api/blobs.rs +++ b/apps/keck-core/src/server/api/blobs.rs @@ -188,7 +188,6 @@ mod tests { JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) .await .ok(), - None, ) .await; let client = TestClient::new(blobs_apis(Router::new()).layer(Extension(Arc::new(ctx)))); diff --git a/apps/keck-core/src/server/api/blocks/block.rs b/apps/keck-core/src/server/api/blocks/block.rs index 2b8e97a6c..978c21198 100644 --- a/apps/keck-core/src/server/api/blocks/block.rs +++ b/apps/keck-core/src/server/api/blocks/block.rs @@ -1,6 +1,5 @@ use axum::{extract::Query, response::Response}; -use jwst::{constants, DocStorage}; -use lib0::any::Any; +use jwst_core::{constants, Any, DocStorage}; use serde_json::Value as JsonValue; use super::*; @@ -25,8 +24,12 @@ use super::*; pub async fn get_block(Extension(context): Extension>, Path(params): Path<(String, String)>) -> Response { let (ws_id, block) = params; info!("get_block: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(ws_id).await { - if let Some(block) = workspace.with_trx(|mut t| t.get_blocks().get(&t.trx, block)) { + if let Ok(space) = context + .get_workspace(ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if let Some(block) = space.get(block) { Json(block).into_response() } else { StatusCode::NOT_FOUND.into_response() @@ -68,66 +71,46 @@ pub async fn set_block( ) -> Response { let (ws_id, block_id) = params; info!("set_block: {}, {}", ws_id, block_id); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - let mut update = None; - if let Some(block) = workspace.with_trx(|mut t| { - let flavour = if let Some(query_map) = query_param { - query_map - .get("flavour") - .map_or_else(|| String::from("text"), |v| v.clone()) - } else { - String::from("text") - }; + if let Ok(mut space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + let flavour = if let Some(query_map) = query_param { + query_map + .get("flavour") + .map_or_else(|| String::from("text"), |v| v.clone()) + } else { + String::from("text") + }; - if let Ok(block) = t - .get_blocks() - .create(&mut t.trx, &block_id, flavour) - .map_err(|e| error!("failed to create block: {:?}", e)) - { - // set block content - if let Some(block_content) = payload.as_object() { - let mut changed = false; - for (key, value) in block_content.iter() { - if key == constants::sys::FLAVOUR { - continue; - } - changed = true; - if let Ok(value) = serde_json::from_value::(value.clone()) { - if let Err(e) = block.set(&mut t.trx, key, value.clone()) { - error!("failed to set block {} content: {}, {}, {:?}", block_id, key, value, e); - } - } + if let Ok(mut block) = space + .create(&block_id, flavour) + .map_err(|e| error!("failed to create block: {:?}", e)) + { + // set block content + if let Some(block_content) = payload.as_object() { + for (key, value) in block_content.iter() { + if key == constants::sys::FLAVOUR { + continue; } - if changed { - update = t.trx.encode_update_v1().ok(); + if let Ok(value) = serde_json::from_value::(value.clone()) { + if let Err(e) = block.set(key, value.clone()) { + error!( + "failed to set block {} content: {}, {:?}, {:?}", + block_id, key, value, e + ); + } } } - - Some(block) - } else { - None - } - }) { - if let Some(update) = update { - if let Err(e) = context - .storage - .docs() - .update_doc(ws_id, workspace.doc_guid().to_string(), &update) - .await - { - error!("db write error: {:?}", e); - } } // response block content - Json(block).into_response() - } else { - StatusCode::INTERNAL_SERVER_ERROR.into_response() + return Json(block).into_response(); } - } else { - StatusCode::NOT_FOUND.into_response() } + StatusCode::NOT_FOUND.into_response() } /// Get exists `Blocks` in certain `Workspace` by flavour @@ -154,15 +137,12 @@ pub async fn get_block_by_flavour( ) -> Response { let (ws_id, flavour) = params; info!("get_block_by_flavour: ws_id, {}, flavour, {}", ws_id, flavour); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - match workspace.try_with_trx(|mut trx| trx.get_blocks().get_blocks_by_flavour(&trx.trx, &flavour)) { - Some(blocks) => Json(blocks).into_response(), - None => ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Workspace({ws_id:?}) get transaction error"), - ) - .into_response(), - } + if let Ok(space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + Json(space.get_blocks_by_flavour(&flavour)).into_response() } else { (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() } @@ -191,14 +171,16 @@ pub async fn get_block_history( ) -> Response { let (ws_id, block) = params; info!("get_block_history: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - workspace.with_trx(|mut t| { - if let Some(block) = t.get_blocks().get(&t.trx, block) { - Json(&block.history(&t.trx)).into_response() - } else { - StatusCode::NOT_FOUND.into_response() - } - }) + if let Ok(space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if let Some(block) = space.get(block) { + Json(&block.history()).into_response() + } else { + StatusCode::NOT_FOUND.into_response() + } } else { StatusCode::NOT_FOUND.into_response() } @@ -227,22 +209,12 @@ pub async fn delete_block( ) -> StatusCode { let (ws_id, block) = params; info!("delete_block: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - if let Some(update) = workspace.with_trx(|mut t| { - if t.get_blocks().remove(&mut t.trx, &block) { - t.trx.encode_update_v1().ok() - } else { - None - } - }) { - if let Err(e) = context - .storage - .docs() - .update_doc(ws_id, workspace.doc_guid().to_string(), &update) - .await - { - error!("db write error: {:?}", e); - } + if let Ok(mut space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if space.remove(&block) { return StatusCode::NO_CONTENT; } } @@ -275,8 +247,12 @@ pub async fn get_block_children( let (ws_id, block) = params; let Pagination { offset, limit } = pagination; info!("get_block_children: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(ws_id).await { - if let Some(block) = workspace.with_trx(|mut t| t.get_blocks().get(&t.trx, &block)) { + if let Ok(space) = context + .get_workspace(ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if let Some(block) = space.get(&block) { let data: Vec = block.children_iter(|children| children.skip(offset).take(limit).collect()); let status = if data.is_empty() { @@ -285,20 +261,17 @@ pub async fn get_block_children( StatusCode::OK }; - ( + return ( status, Json(PageData { total: block.children_len() as usize, data, }), ) - .into_response() - } else { - StatusCode::NOT_FOUND.into_response() + .into_response(); } - } else { - StatusCode::NOT_FOUND.into_response() } + StatusCode::NOT_FOUND.into_response() } /// Insert a another `Block` into a `Block`'s children @@ -331,84 +304,54 @@ pub async fn insert_block_children( ) -> Response { let (ws_id, block) = params; info!("insert_block: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - let mut update = None; - - if let Some(block) = workspace.with_trx(|mut t| t.get_blocks().get(&t.trx, block)) { - if let Some(block) = workspace.with_trx(|mut t| { - let space = t.get_blocks(); - let mut changed = false; - match payload { - InsertChildren::Push(block_id) => { - if let Some(child) = space.get(&t.trx, block_id) { - changed = true; - if let Err(e) = block.push_children(&mut t.trx, &child) { - // TODO: handle error correctly - error!("failed to insert block: {:?}", e); - return None; - } + if let Ok(space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if let Some(mut block) = space.get(block) { + match payload { + InsertChildren::Push(block_id) => { + if let Some(mut child) = space.get(block_id) { + if let Err(e) = block.push_children(&mut child) { + error!("failed to insert block: {:?}", e); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } } - InsertChildren::InsertBefore { id, before } => { - if let Some(child) = space.get(&t.trx, id) { - changed = true; - if let Err(e) = block.insert_children_before(&mut t.trx, &child, &before) { - // TODO: handle error correctly - error!("failed to insert children before: {:?}", e); - return None; - } - } - } - InsertChildren::InsertAfter { id, after } => { - if let Some(child) = space.get(&t.trx, id) { - changed = true; - if let Err(e) = block.insert_children_after(&mut t.trx, &child, &after) { - // TODO: handle error correctly - error!("failed to insert children after: {:?}", e); - return None; - } + } + InsertChildren::InsertBefore { id, before } => { + if let Some(mut child) = space.get(id) { + if let Err(e) = block.insert_children_before(&mut child, &before) { + error!("failed to insert children before: {:?}", e); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } } - InsertChildren::InsertAt { id, pos } => { - if let Some(child) = space.get(&t.trx, id) { - changed = true; - if let Err(e) = block.insert_children_at(&mut t.trx, &child, pos) { - // TODO: handle error correctly - error!("failed to insert children at: {:?}", e); - return None; - } + } + InsertChildren::InsertAfter { id, after } => { + if let Some(mut child) = space.get(id) { + if let Err(e) = block.insert_children_after(&mut child, &after) { + // TODO: handle error correctly + error!("failed to insert children after: {:?}", e); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } } - }; - - if changed { - update = t.trx.encode_update_v1().ok(); } - - Some(block) - }) { - if let Some(update) = update { - if let Err(e) = context - .storage - .docs() - .update_doc(ws_id, workspace.doc_guid().to_string(), &update) - .await - { - error!("db write error: {:?}", e); + InsertChildren::InsertAt { id, pos } => { + if let Some(mut child) = space.get(id) { + if let Err(e) = block.insert_children_at(&mut child, pos) { + // TODO: handle error correctly + error!("failed to insert children at: {:?}", e); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } } } + }; - // response block content - Json(block).into_response() - } else { - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } - } else { - StatusCode::NOT_FOUND.into_response() + // response block content + return Json(block).into_response(); } - } else { - StatusCode::NOT_FOUND.into_response() } + StatusCode::NOT_FOUND.into_response() } /// Remove children in `Block` @@ -434,35 +377,24 @@ pub async fn remove_block_children( ) -> Response { let (ws_id, block, child_id) = params; info!("insert_block: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - if let Some(update) = workspace.with_trx(|mut t| { - let space = t.get_blocks(); - if let Some(block) = space.get(&t.trx, &block) { - if block.children_exists(&t.trx, &child_id) { - if let Some(child) = space.get(&t.trx, &child_id) { - return block - .remove_children(&mut t.trx, &child) - .and_then(|_| Ok(t.trx.encode_update_v1()?)) - .ok(); + if let Ok(space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if let Some(mut block) = space.get(&block) { + if block.children_exists(&child_id) { + if let Some(mut child) = space.get(&child_id) { + if let Err(e) = block.remove_children(&mut child) { + error!("failed to remove block: {:?}", e); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } else { + // response block content + return Json(block).into_response(); } } } - None - }) { - if let Err(e) = context - .storage - .docs() - .update_doc(ws_id, workspace.doc_guid().to_string(), &update) - .await - { - error!("db write error: {:?}", e); - } - // response block content - Json(block).into_response() - } else { - StatusCode::NOT_FOUND.into_response() } - } else { - StatusCode::NOT_FOUND.into_response() } + StatusCode::NOT_FOUND.into_response() } diff --git a/apps/keck-core/src/server/api/blocks/mod.rs b/apps/keck-core/src/server/api/blocks/mod.rs index 48dbcb855..70e41d739 100644 --- a/apps/keck-core/src/server/api/blocks/mod.rs +++ b/apps/keck-core/src/server/api/blocks/mod.rs @@ -5,10 +5,7 @@ pub mod workspace; pub use block::{delete_block, get_block, get_block_history, insert_block_children, remove_block_children, set_block}; use schema::InsertChildren; pub use schema::SubscribeWorkspace; -pub use workspace::{ - delete_workspace, get_workspace, history_workspace, history_workspace_clients, set_workspace, subscribe_workspace, - workspace_client, -}; +pub use workspace::{delete_workspace, get_workspace, set_workspace, subscribe_workspace, workspace_client}; use super::*; @@ -30,8 +27,8 @@ fn block_apis(router: Router) -> Router { fn workspace_apis(router: Router) -> Router { router .route("/block/:workspace/client", get(workspace::workspace_client)) - .route("/block/:workspace/history", get(workspace::history_workspace_clients)) - .route("/block/:workspace/history/:client", get(workspace::history_workspace)) + // .route("/block/:workspace/history", get(workspace::history_workspace_clients)) + // .route("/block/:workspace/history/:client", get(workspace::history_workspace)) .route( "/block/:workspace", get(workspace::get_workspace) @@ -40,11 +37,11 @@ fn workspace_apis(router: Router) -> Router { ) .route("/block/:workspace/flavour/:flavour", get(block::get_block_by_flavour)) .route("/block/:workspace/blocks", get(workspace::get_workspace_block)) - .route("/search/:workspace", get(workspace::workspace_search)) - .route( - "/search/:workspace/index", - get(workspace::get_search_index).post(workspace::set_search_index), - ) + // .route("/search/:workspace", get(workspace::workspace_search)) + // .route( + // "/search/:workspace/index", + // get(workspace::get_search_index).post(workspace::set_search_index), + // ) .route("/subscribe", post(subscribe_workspace)) } @@ -70,7 +67,6 @@ mod tests { .build() .expect("Failed to create runtime"), ); - let workspace_changed_blocks = Arc::new(RwLock::new(HashMap::::new())); let hook_endpoint = Arc::new(RwLock::new(String::new())); let ctx = Arc::new( @@ -86,7 +82,6 @@ mod tests { .layer(Extension(ctx.clone())) .layer(Extension(client.clone())) .layer(Extension(runtime.clone())) - .layer(Extension(workspace_changed_blocks.clone())) .layer(Extension(hook_endpoint.clone())), ); diff --git a/apps/keck-core/src/server/api/blocks/schema.rs b/apps/keck-core/src/server/api/blocks/schema.rs index c6507c771..c12ecf1c9 100644 --- a/apps/keck-core/src/server/api/blocks/schema.rs +++ b/apps/keck-core/src/server/api/blocks/schema.rs @@ -38,7 +38,7 @@ pub enum InsertChildren { Push(String), InsertBefore { id: String, before: String }, InsertAfter { id: String, after: String }, - InsertAt { id: String, pos: u32 }, + InsertAt { id: String, pos: u64 }, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] diff --git a/apps/keck-core/src/server/api/blocks/workspace.rs b/apps/keck-core/src/server/api/blocks/workspace.rs index deb454de8..505ccaa4b 100644 --- a/apps/keck-core/src/server/api/blocks/workspace.rs +++ b/apps/keck-core/src/server/api/blocks/workspace.rs @@ -1,9 +1,8 @@ use axum::{ extract::{Path, Query}, - http::header, response::Response, }; -use jwst_core::{parse_history, parse_history_client, DocStorage}; +use jwst_core::DocStorage; use utoipa::IntoParams; use super::*; @@ -129,97 +128,84 @@ pub struct BlockSearchQuery { /// Search workspace blocks of server /// /// This will return back a list of relevant blocks. -#[utoipa::path( - get, - tag = "Workspace", - context_path = "/api/search", - path = "/{workspace}", - params( - ("workspace", description = "workspace id"), - BlockSearchQuery, - ), - responses( - (status = 200, description = "Search results", body = SearchResults), - ) -)] -pub async fn workspace_search( - Extension(context): Extension>, - Path(workspace): Path, - query: Query, -) -> Response { - let query_text = &query.query; - let ws_id = workspace; - info!("workspace_search: {ws_id:?} query = {query_text:?}"); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - match workspace.search(query_text) { - Ok(list) => { - debug!("workspace_search: {ws_id:?} query = {query_text:?}; {list:#?}"); - Json(list).into_response() - } - Err(err) => { - error!("Internal server error calling workspace_search: {err:?}"); - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } - } - } else { - (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() - } -} +// #[utoipa::path( +// get, +// tag = "Workspace", +// context_path = "/api/search", +// path = "/{workspace}", +// params( +// ("workspace", description = "workspace id"), +// BlockSearchQuery, +// ), +// responses( +// (status = 200, description = "Search results", body = SearchResults), +// ) +// )] +// pub async fn workspace_search( +// Extension(context): Extension>, +// Path(workspace): Path, +// query: Query, +// ) -> Response { let query_text = &query.query; let ws_id = workspace; info!("workspace_search: {ws_id:?} query = +// {query_text:?}"); if let Ok(workspace) = context.get_workspace(&ws_id).await { match workspace.search(query_text) { +// Ok(list) => { debug!("workspace_search: {ws_id:?} query = {query_text:?}; {list:#?}"); Json(list).into_response() } +// Err(err) => { error!("Internal server error calling workspace_search: {err:?}"); +// StatusCode::INTERNAL_SERVER_ERROR.into_response() } } } else { (StatusCode::NOT_FOUND, +// format!("Workspace({ws_id:?}) not found")).into_response() } +// } -#[utoipa::path( - get, - tag = "Workspace", - context_path = "/api/search", - path = "/{workspace}/index", - params( - ("workspace", description = "workspace id"), - ), - responses( - (status = 200, description = "result", body = Vec), - (status = 404, description = "Workspace not found") - ) -)] -pub async fn get_search_index(Extension(context): Extension>, Path(workspace): Path) -> Response { - info!("get_search_index: {workspace:?}"); +// #[utoipa::path( +// get, +// tag = "Workspace", +// context_path = "/api/search", +// path = "/{workspace}/index", +// params( +// ("workspace", description = "workspace id"), +// ), +// responses( +// (status = 200, description = "result", body = Vec), +// (status = 404, description = "Workspace not found") +// ) +// )] +// pub async fn get_search_index(Extension(context): Extension>, Path(workspace): Path) -> Response +// { info!("get_search_index: {workspace:?}"); - if let Ok(workspace) = context.get_workspace(&workspace).await { - Json(workspace.metadata().search_index).into_response() - } else { - (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() - } -} +// if let Ok(workspace) = context.get_workspace(&workspace).await { +// Json(workspace.metadata().search_index).into_response() +// } else { +// (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() +// } +// } -#[utoipa::path( - post, - tag = "Workspace", - context_path = "/api/search", - path = "/{workspace}/index", - params( - ("workspace", description = "workspace id"), - ), - responses( - (status = 200, description = "success"), - (status = 400, description = "Bad Request"), - (status = 404, description = "Workspace not found") - ) -)] -pub async fn set_search_index( - Extension(context): Extension>, - Path(workspace): Path, - Json(fields): Json>, -) -> Response { - info!("set_search_index: {workspace:?} fields = {fields:?}"); +// #[utoipa::path( +// post, +// tag = "Workspace", +// context_path = "/api/search", +// path = "/{workspace}/index", +// params( +// ("workspace", description = "workspace id"), +// ), +// responses( +// (status = 200, description = "success"), +// (status = 400, description = "Bad Request"), +// (status = 404, description = "Workspace not found") +// ) +// )] +// pub async fn set_search_index( +// Extension(context): Extension>, +// Path(workspace): Path, +// Json(fields): Json>, +// ) -> Response { info!("set_search_index: {workspace:?} fields = {fields:?}"); - if let Ok(workspace) = context.get_workspace(&workspace).await { - if let Ok(true) = workspace.set_search_index(fields) { - StatusCode::OK.into_response() - } else { - StatusCode::BAD_REQUEST.into_response() - } - } else { - (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() - } -} +// if let Ok(workspace) = context.get_workspace(&workspace).await { +// if let Ok(true) = workspace.set_search_index(fields) { +// StatusCode::OK.into_response() +// } else { +// StatusCode::BAD_REQUEST.into_response() +// } +// } else { +// (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() +// } +// } /// Get `Block` in `Workspace` /// - Return 200 and `Block`'s ID. @@ -245,15 +231,13 @@ pub async fn get_workspace_block( ) -> Response { let Pagination { offset, limit } = pagination; info!("get_workspace_block: {workspace:?}"); - if let Ok(workspace) = context.get_workspace(&workspace).await { - let (total, data) = workspace.with_trx(|mut t| { - let space = t.get_blocks(); - - let total = space.block_count() as usize; - let data = space.blocks(&t.trx, |blocks| blocks.skip(offset).take(limit).collect::>()); - - (total, data) - }); + if let Ok(space) = context + .get_workspace(&workspace) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + let total = space.block_count() as usize; + let data = space.blocks(|blocks| blocks.skip(offset).take(limit).collect::>()); let status = if data.is_empty() { StatusCode::NOT_FOUND @@ -283,73 +267,55 @@ pub async fn get_workspace_block( /// created. This `Client` will not be destroyed until the server restarts. /// Therefore, the `Client ID` in the history generated by modifying `Block` /// through HTTP API will remain unchanged until the server restarts. -#[utoipa::path( - get, - tag = "Workspace", - context_path = "/api/block", - path = "/{workspace}/history", - params( - ("workspace", description = "workspace id"), - ), - responses( - (status = 200, description = "Get workspace history client ids", body = [u64]), - (status = 500, description = "Failed to get workspace history") - ) -)] -pub async fn history_workspace_clients( - Extension(context): Extension>, - Path(workspace): Path, -) -> Response { - if let Ok(workspace) = context.get_workspace(&workspace).await { - if let Some(history) = parse_history_client(&workspace.doc()) { - Json(history).into_response() - } else { - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } - } else { - (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() - } -} +// #[utoipa::path( +// get, +// tag = "Workspace", +// context_path = "/api/block", +// path = "/{workspace}/history", +// params( +// ("workspace", description = "workspace id"), +// ), +// responses( +// (status = 200, description = "Get workspace history client ids", body = [u64]), +// (status = 500, description = "Failed to get workspace history") +// ) +// )] +// pub async fn history_workspace_clients( +// Extension(context): Extension>, +// Path(workspace): Path, +// ) -> Response { if let Ok(workspace) = context.get_workspace(&workspace).await { if let Some(history) = +// parse_history_client(&workspace.doc()) { Json(history).into_response() } else { +// StatusCode::INTERNAL_SERVER_ERROR.into_response() } } else { (StatusCode::NOT_FOUND, +// format!("Workspace({workspace:?}) not found")).into_response() } +// } /// Get the history generated by a specific `Client ID` of the `Workspace` /// /// If client id set to 0, return all history of the `Workspace`. -#[utoipa::path( - get, - tag = "Workspace", - context_path = "/api/block", - path = "/{workspace}/history/{client}", - params( - ("workspace", description = "workspace id"), - ("client", description = "client id, is give 0 then return all clients histories"), - ), - responses( - (status = 200, description = "Get workspace history", body = [RawHistory]), - (status = 400, description = "Client id invalid"), - (status = 500, description = "Failed to get workspace history") - ) -)] -pub async fn history_workspace( - Extension(context): Extension>, - Path(params): Path<(String, String)>, -) -> Response { - let (ws_id, client) = params; - if let Ok(workspace) = context.get_workspace(&ws_id).await { - if let Ok(client) = client.parse::() { - if let Some(json) = - parse_history(&workspace.doc(), client).and_then(|history| serde_json::to_string(&history).ok()) - { - ([(header::CONTENT_TYPE, "application/json")], json).into_response() - } else { - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } - } else { - StatusCode::BAD_REQUEST.into_response() - } - } else { - (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() - } -} +// #[utoipa::path( +// get, +// tag = "Workspace", +// context_path = "/api/block", +// path = "/{workspace}/history/{client}", +// params( +// ("workspace", description = "workspace id"), +// ("client", description = "client id, is give 0 then return all clients histories"), +// ), +// responses( +// (status = 200, description = "Get workspace history", body = [RawHistory]), +// (status = 400, description = "Client id invalid"), +// (status = 500, description = "Failed to get workspace history") +// ) +// )] +// pub async fn history_workspace( +// Extension(context): Extension>, +// Path(params): Path<(String, String)>, +// ) -> Response { let (ws_id, client) = params; if let Ok(workspace) = context.get_workspace(&ws_id).await { if let +// Ok(client) = client.parse::() { if let Some(json) = parse_history(&workspace.doc(), client).and_then(|history| +// serde_json::to_string(&history).ok()) { ([(header::CONTENT_TYPE, "application/json")], json).into_response() } else +// { StatusCode::INTERNAL_SERVER_ERROR.into_response() } } else { StatusCode::BAD_REQUEST.into_response() } } else { +// (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() } +// } /// Register a webhook for all block changes from all workspace changes #[utoipa::path( From 8f00603726a3d41057d0843a5038301be81162f2 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Sat, 26 Aug 2023 21:20:10 +0800 Subject: [PATCH 21/49] fix: collaboration test --- apps/keck-core/Cargo.toml | 2 +- .../src/server/sync/collaboration.rs | 196 +++++++++--------- libs/jwst-core/src/block/mod.rs | 2 +- 3 files changed, 95 insertions(+), 105 deletions(-) diff --git a/apps/keck-core/Cargo.toml b/apps/keck-core/Cargo.toml index 15b160c5c..a48cf8661 100644 --- a/apps/keck-core/Cargo.toml +++ b/apps/keck-core/Cargo.toml @@ -9,7 +9,7 @@ license = "AGPL-3.0-only" default = ["jwst"] affine = ["jwst-core-storage/sqlite"] jwst = [ - # "api", + "api", "jwst-core-storage/sqlite", "jwst-core-storage/mysql", "jwst-core-storage/postgres", diff --git a/apps/keck-core/src/server/sync/collaboration.rs b/apps/keck-core/src/server/sync/collaboration.rs index 4b9dd4e0b..a28e0d4e0 100644 --- a/apps/keck-core/src/server/sync/collaboration.rs +++ b/apps/keck-core/src/server/sync/collaboration.rs @@ -48,6 +48,8 @@ mod test { process::{Child, Command, Stdio}, string::String, sync::Arc, + thread::sleep, + time::Duration, }; use jwst_core::{Block, DocStorage, Workspace}; @@ -92,7 +94,7 @@ mod test { let server_port = thread_rng().gen_range(10000..=30000); let child = start_collaboration_server(server_port); - let rt: Runtime = Runtime::new().unwrap(); + let rt = Arc::new(Runtime::new().unwrap()); let (workspace_id, mut workspace) = { let workspace_id = "1"; let context = rt.block_on(async move { @@ -105,7 +107,7 @@ mod test { let remote = format!("ws://localhost:{server_port}/collaboration/1"); start_websocket_client_sync( - Arc::new(Runtime::new().unwrap()), + rt.clone(), context.clone(), Arc::default(), remote, @@ -123,6 +125,7 @@ mod test { info!("from client, create a block:{:?}", serde_json::to_string(&block)); } + sleep(Duration::from_secs(1)); info!("------------------after sync------------------"); for block_id in 0..3 { @@ -143,106 +146,93 @@ mod test { close_collaboration_server(child); } - // #[test] - // #[ignore = "not needed in ci"] - // fn client_collaboration_with_server_with_poor_connection() { - // let server_port = thread_rng().gen_range(30001..=65535); - // let child = start_collaboration_server(server_port); - - // let rt = Runtime::new().unwrap(); - // let workspace_id = String::from("1"); - // let (storage, workspace) = rt.block_on(async { - // let storage: Arc = Arc::new( - // JwstStorage::new_with_migration("sqlite::memory:", - // BlobStorageType::DB) .await - // .expect("get storage: memory sqlite failed"), - // ); - // let workspace = storage - // .docs() - // .get_or_create_workspace(workspace_id.clone()) - // .await - // .expect("get workspace: {workspace_id} failed"); - // (storage, workspace) - // }); - - // // simulate creating a block in offline environment - // let block = create_block(&workspace, "0".to_string(), - // "list".to_string()); info!("from client, create a block: {:?}", - // block); info!( - // "get block 0 from server: {}", - // get_block_from_server(workspace_id.clone(), "0".to_string(), - // server_port) ); - // assert!(get_block_from_server(workspace_id.clone(), "0".to_string(), - // server_port).is_empty()); - - // let rt = Runtime::new().unwrap(); - // let (workspace_id, workspace) = rt.block_on(async move { - // let workspace_id = "1"; - // let context = Arc::new(TestContext::new(storage)); - // let remote = - // format!("ws://localhost:{server_port}/collaboration/1"); - - // start_websocket_client_sync( - // Arc::new(Runtime::new().unwrap()), - // context.clone(), - // Arc::default(), - // remote, - // workspace_id.to_owned(), - // ); - - // ( - // workspace_id.to_owned(), - // context.get_workspace(workspace_id).await.unwrap(), - // ) - // }); - - // info!("----------------start syncing from - // start_sync_thread()----------------"); - - // for block_id in 1..3 { - // let block = create_block(&workspace, block_id.to_string(), - // "list".to_string()); info!("from client, create a block: - // {:?}", block); info!( - // "get block {block_id} from server: {}", - // get_block_from_server(workspace_id.clone(), - // block_id.to_string(), server_port) ); - // assert!(!get_block_from_server(workspace_id.clone(), - // block_id.to_string(), server_port).is_empty()); } - - // workspace.with_trx(|mut trx| { - // let space = trx.get_space("blocks"); - // let blocks = space.get_blocks_by_flavour( "list"); - // let mut ids: Vec<_> = blocks.iter().map(|block| - // block.block_id()).collect(); assert_eq!(ids.sort(), vec!["0", - // "1", "2"].sort()); info!("blocks from local storage:"); - // for block in blocks { - // info!("block: {:?}", block); - // } - // }); - - // info!("------------------after sync------------------"); - - // for block_id in 0..3 { - // info!( - // "get block {block_id} from server: {}", - // get_block_from_server(workspace_id.clone(), - // block_id.to_string(), server_port) ); - // assert!(!get_block_from_server(workspace_id.clone(), - // block_id.to_string(), server_port).is_empty()); } - - // workspace.with_trx(|mut trx| { - // let space = trx.get_space("blocks"); - // let blocks = space.get_blocks_by_flavour( "list"); - // let mut ids: Vec<_> = blocks.iter().map(|block| - // block.block_id()).collect(); assert_eq!(ids.sort(), vec!["0", - // "1", "2"].sort()); info!("blocks from local storage:"); - // for block in blocks { - // info!("block: {:?}", block); - // } - // }); - - // close_collaboration_server(child); - // } + #[test] + #[ignore = "not needed in ci"] + fn client_collaboration_with_server_with_poor_connection() { + let server_port = thread_rng().gen_range(30001..=65535); + let child = start_collaboration_server(server_port); + + let rt = Runtime::new().unwrap(); + let workspace_id = String::from("1"); + let (storage, mut workspace) = rt.block_on(async { + let storage: Arc = Arc::new( + JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) + .await + .expect("get storage: memory sqlite failed"), + ); + let workspace = storage + .docs() + .get_or_create_workspace(workspace_id.clone()) + .await + .expect("get workspace: {workspace_id} failed"); + (storage, workspace) + }); + + // simulate creating a block in offline environment + let block = create_block(&mut workspace, "0".to_string(), "list".to_string()); + info!("from client, create a block: {:?}", block); + info!( + "get block 0 from server: {}", + get_block_from_server(workspace_id.clone(), "0".to_string(), server_port) + ); + assert!(get_block_from_server(workspace_id.clone(), "0".to_string(), server_port).is_empty()); + + let rt = Arc::new(Runtime::new().unwrap()); + let (workspace_id, mut workspace) = { + let workspace_id = "1"; + let context = Arc::new(TestContext::new(storage)); + let remote = format!("ws://localhost:{server_port}/collaboration/1"); + + start_websocket_client_sync( + rt.clone(), + context.clone(), + Arc::default(), + remote, + workspace_id.to_owned(), + ); + + ( + workspace_id.to_owned(), + rt.block_on(async move { context.get_workspace(workspace_id).await.unwrap() }), + ) + }; + + info!("----------------start syncing from start_sync_thread()----------------"); + + for block_id in 1..3 { + let block = create_block(&mut workspace, block_id.to_string(), "list".to_string()); + info!("from client, create a block: {:?}", serde_json::to_string(&block)); + } + + let space = workspace.get_blocks().unwrap(); + let blocks = space.get_blocks_by_flavour("list"); + let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); + assert_eq!(ids.sort(), vec!["0", "1", "2"].sort()); + info!("blocks from local storage:"); + for block in blocks { + info!("block: {:?}", block); + } + + sleep(Duration::from_secs(1)); + info!("------------------after sync------------------"); + + for block_id in 0..3 { + let ret = get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port); + info!("get block {block_id} from server: {}", ret); + assert!(!ret.is_empty()); + } + + let space = workspace.get_blocks().unwrap(); + let blocks = space.get_blocks_by_flavour("list"); + let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); + assert_eq!(ids.sort(), vec!["0", "1", "2"].sort()); + info!("blocks from local storage:"); + for block in blocks { + info!("block: {:?}", block); + } + + close_collaboration_server(child); + } fn get_block_from_server(workspace_id: String, block_id: String, server_port: u16) -> String { let rt = Runtime::new().unwrap(); @@ -270,7 +260,7 @@ mod test { .args(&["run", "-p", "keck-core"]) .env("KECK_PORT", port.to_string()) .env("USE_MEMORY_SQLITE", "true") - .env("KECK_LOG", "INFO") + .env("KECK_CORE_LOG", "debug") .stdout(Stdio::piped()) .spawn() .expect("Failed to run command"); diff --git a/libs/jwst-core/src/block/mod.rs b/libs/jwst-core/src/block/mod.rs index 0a1d65e47..f975c6234 100644 --- a/libs/jwst-core/src/block/mod.rs +++ b/libs/jwst-core/src/block/mod.rs @@ -308,7 +308,7 @@ impl Block { self.children.len() } - pub fn children_exists(&self, block_id: S) -> bool + pub fn children_exists(&self, block_id: S) -> bool where S: AsRef, { From 58356f4bca97ecfd6a5b7452264c8bef4d107185 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 28 Aug 2023 15:33:54 +0800 Subject: [PATCH 22/49] fix: lint --- libs/jwst-codec/src/doc/types/map.rs | 2 +- libs/jwst-codec/src/doc/types/mod.rs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/libs/jwst-codec/src/doc/types/map.rs b/libs/jwst-codec/src/doc/types/map.rs index de34ee506..941196eaf 100644 --- a/libs/jwst-codec/src/doc/types/map.rs +++ b/libs/jwst-codec/src/doc/types/map.rs @@ -113,7 +113,7 @@ pub(crate) trait MapType: AsInner { }); MapIterator { - nodes: map.unwrap_or(vec![]), + nodes: map.unwrap_or_default(), index: 0, } } diff --git a/libs/jwst-codec/src/doc/types/mod.rs b/libs/jwst-codec/src/doc/types/mod.rs index a4ee90214..e6e2a27e3 100644 --- a/libs/jwst-codec/src/doc/types/mod.rs +++ b/libs/jwst-codec/src/doc/types/mod.rs @@ -105,6 +105,7 @@ impl YTypeRef { self.inner.get().and_then(|ty| ty.write().ok()) } + #[allow(dead_code)] pub fn store<'a>(&self) -> Option> { if let Some(store) = self.store.upgrade() { let ptr = unsafe { &*Arc::as_ptr(&store) }; @@ -125,6 +126,7 @@ impl YTypeRef { } } + #[allow(dead_code)] pub fn read(&self) -> Option<(RwLockReadGuard, RwLockReadGuard)> { self.store().and_then(|store| self.ty().map(|ty| (store, ty))) } From 1313146c510c1b7a91b01dd436d360153fde3b92 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 28 Aug 2023 15:43:42 +0800 Subject: [PATCH 23/49] fix: merge error --- Cargo.lock | 1 + .../RustXcframework.xcframework/Info.plist | 14 +-- libs/jwst-codec-util/Cargo.toml | 17 ++- .../bin/bench_result_render.rs | 112 ++++++++++++++++++ 4 files changed, 133 insertions(+), 11 deletions(-) create mode 100644 libs/jwst-codec-util/bin/bench_result_render.rs diff --git a/Cargo.lock b/Cargo.lock index c9eb8c054..2d938d2f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2779,6 +2779,7 @@ dependencies = [ "arbitrary", "jwst-codec", "phf", + "regex", "yrs", ] diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist index 5a676cdb2..b3d1aee06 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist @@ -8,7 +8,7 @@ HeadersPath Headers LibraryIdentifier - ios-arm64 + ios-arm64-simulator LibraryPath liboctobase.a SupportedArchitectures @@ -17,12 +17,14 @@ SupportedPlatform ios + SupportedPlatformVariant + simulator HeadersPath Headers LibraryIdentifier - ios-arm64-simulator + macos-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -30,15 +32,13 @@ arm64 SupportedPlatform - ios - SupportedPlatformVariant - simulator + macos HeadersPath Headers LibraryIdentifier - macos-arm64 + ios-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -46,7 +46,7 @@ arm64 SupportedPlatform - macos + ios CFBundlePackageType diff --git a/libs/jwst-codec-util/Cargo.toml b/libs/jwst-codec-util/Cargo.toml index 42ea4948b..050b8df01 100644 --- a/libs/jwst-codec-util/Cargo.toml +++ b/libs/jwst-codec-util/Cargo.toml @@ -5,8 +5,17 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +bench = ["regex"] +fuzz = ["arbitrary", "jwst-codec", "phf", "yrs"] + [dependencies] -arbitrary = { version = "1.3.0", features = ["derive"] } -phf = { version = "0.11", features = ["macros"] } -yrs = "0.16.5" -jwst-codec = { workspace = true } +arbitrary = { version = "1.3.0", features = ["derive"], optional = true } +jwst-codec = { workspace = true, optional = true } +phf = { version = "0.11", features = ["macros"], optional = true } +regex = { version = "1.5", optional = true } +yrs = { version = "=0.16.5", optional = true } + +[[bin]] +name = "bench_result_render" +path = "bin/bench_result_render.rs" diff --git a/libs/jwst-codec-util/bin/bench_result_render.rs b/libs/jwst-codec-util/bin/bench_result_render.rs new file mode 100644 index 000000000..dad0195d5 --- /dev/null +++ b/libs/jwst-codec-util/bin/bench_result_render.rs @@ -0,0 +1,112 @@ +#![allow(unused)] + +use std::collections::HashMap; + +fn process_duration(duration: &str) -> (f64, f64) { + let dur_split: Vec = duration.split("±").map(String::from).collect(); + let units = dur_split[1].chars().skip_while(|c| c.is_digit(10)).collect::(); + let dur_secs = convert_dur_to_seconds(dur_split[0].parse::().unwrap(), &units); + let error_secs = convert_dur_to_seconds( + dur_split[1] + .chars() + .take_while(|c| c.is_digit(10)) + .collect::() + .parse::() + .unwrap(), + &units, + ); + (dur_secs, error_secs) +} + +fn convert_dur_to_seconds(dur: f64, units: &str) -> f64 { + let factors = [ + ("s", 1.0), + ("ms", 1.0 / 1000.0), + ("µs", 1.0 / 1_000_000.0), + ("ns", 1.0 / 1_000_000_000.0), + ] + .iter() + .cloned() + .collect::>(); + dur * factors.get(units).unwrap_or(&1.0) +} + +#[allow(dead_code)] +fn is_significant(changes_dur: f64, changes_err: f64, base_dur: f64, base_err: f64) -> bool { + if changes_dur < base_dur { + changes_dur + changes_err < base_dur || base_dur - base_err > changes_dur + } else { + changes_dur - changes_err > base_dur || base_dur + base_err < changes_dur + } +} + +#[cfg(feature = "bench")] +fn convert_to_markdown() -> impl Iterator { + let re = regex::Regex::new(r"\s{2,}").unwrap(); + std::io::stdin() + .lines() + .skip(2) + .map(|l| l.ok()) + .map(move |row| { + if let Some(row) = row { + let columns = re.split(&row).collect::>(); + let name = columns.get(0)?; + let base_duration = columns.get(2)?; + let changes_duration = columns.get(5)?; + Some(( + name.to_string(), + base_duration.to_string(), + changes_duration.to_string(), + )) + } else { + None + } + }) + .flatten() + .map(|(name, base_duration, changes_duration)| { + let mut difference = "N/A".to_string(); + let base_undefined = base_duration == "?"; + let changes_undefined = changes_duration == "?"; + + if !base_undefined && !changes_undefined { + let (base_dur_secs, base_err_secs) = process_duration(&base_duration); + let (changes_dur_secs, changes_err_secs) = process_duration(&changes_duration); + + let diff = -(1.0 - changes_dur_secs / base_dur_secs) * 100.0; + difference = format!("{:+.2}%", diff); + + if is_significant(changes_dur_secs, changes_err_secs, base_dur_secs, base_err_secs) { + difference = format!("**{}**", difference); + } + } + + format!( + "| {} | {} | {} | {} |", + name.replace("|", "\\|"), + if base_undefined { "N/A" } else { &base_duration }, + if changes_undefined { "N/A" } else { &changes_duration }, + difference + ) + }) +} + +fn main() { + let platform = std::env::args().into_iter().skip(1).next().unwrap(); + + #[cfg(feature = "bench")] + for ret in [ + &format!("## Benchmark for {platform}"), + "

", + " Click to view benchmark", + "", + "| Test | Base | PR | % |", + "| --- | --- | --- | --- |", + ] + .into_iter() + .map(|s| s.to_owned()) + .chain(convert_to_markdown()) + .chain(["
".to_owned()]) + { + println!("{}", ret); + } +} From 70caee9f016e8cc44b876846590d93ebc54b8cf3 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 28 Aug 2023 15:48:16 +0800 Subject: [PATCH 24/49] fix: build warning --- libs/jwst-core-storage/src/migration/src/lib.rs | 2 +- .../src/migration/src/m20220101_000001_initial_blob_table.rs | 2 +- .../src/migration/src/m20220101_000002_initial_doc_table.rs | 2 +- .../src/migration/src/m20230321_000001_blob_optimized_table.rs | 2 +- .../migration/src/m20230614_000001_initial_bucket_blob_table.rs | 2 +- .../src/migration/src/m20230626_023319_doc_guid.rs | 2 +- .../src/migration/src/{schema.rs => schemas.rs} | 0 libs/jwst-storage/src/migration/src/lib.rs | 2 +- .../src/migration/src/m20220101_000001_initial_blob_table.rs | 2 +- .../src/migration/src/m20220101_000002_initial_doc_table.rs | 2 +- .../src/migration/src/m20230321_000001_blob_optimized_table.rs | 2 +- .../migration/src/m20230614_000001_initial_bucket_blob_table.rs | 2 +- .../jwst-storage/src/migration/src/m20230626_023319_doc_guid.rs | 2 +- libs/jwst-storage/src/migration/src/{schema.rs => schemas.rs} | 0 14 files changed, 12 insertions(+), 12 deletions(-) rename libs/jwst-core-storage/src/migration/src/{schema.rs => schemas.rs} (100%) rename libs/jwst-storage/src/migration/src/{schema.rs => schemas.rs} (100%) diff --git a/libs/jwst-core-storage/src/migration/src/lib.rs b/libs/jwst-core-storage/src/migration/src/lib.rs index b2b885e82..8861c8ec9 100644 --- a/libs/jwst-core-storage/src/migration/src/lib.rs +++ b/libs/jwst-core-storage/src/migration/src/lib.rs @@ -6,7 +6,7 @@ mod m20230321_000001_blob_optimized_table; mod m20230614_000001_initial_bucket_blob_table; mod m20230626_023319_doc_guid; mod m20230814_061223_initial_diff_log_table; -mod schema; +mod schemas; pub struct Migrator; diff --git a/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs b/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs index f253f6971..f6b4e5185 100644 --- a/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs +++ b/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs @@ -1,6 +1,6 @@ use sea_orm_migration::prelude::*; -use super::schema::Blobs; +use super::schemas::Blobs; pub struct Migration; diff --git a/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs b/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs index 47edd0483..c357701e5 100644 --- a/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs +++ b/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs @@ -1,6 +1,6 @@ use sea_orm_migration::prelude::*; -use super::schema::Docs; +use super::schemas::Docs; pub struct Migration; diff --git a/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs b/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs index d89fab857..8f9676ddb 100644 --- a/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs +++ b/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs @@ -1,6 +1,6 @@ use sea_orm_migration::prelude::*; -use super::schema::OptimizedBlobs; +use super::schemas::OptimizedBlobs; pub struct Migration; diff --git a/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs b/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs index 3f778d8cd..ceb35c575 100644 --- a/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs +++ b/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs @@ -1,6 +1,6 @@ use sea_orm_migration::prelude::*; -use super::schema::BucketBlobs; +use super::schemas::BucketBlobs; pub struct Migration; diff --git a/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs b/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs index bfe91e36b..ccaf1f49a 100644 --- a/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs +++ b/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs @@ -1,6 +1,6 @@ use sea_orm_migration::prelude::*; -use crate::schema::Docs; +use crate::schemas::Docs; #[derive(DeriveMigrationName)] pub struct Migration; diff --git a/libs/jwst-core-storage/src/migration/src/schema.rs b/libs/jwst-core-storage/src/migration/src/schemas.rs similarity index 100% rename from libs/jwst-core-storage/src/migration/src/schema.rs rename to libs/jwst-core-storage/src/migration/src/schemas.rs diff --git a/libs/jwst-storage/src/migration/src/lib.rs b/libs/jwst-storage/src/migration/src/lib.rs index b2b885e82..8861c8ec9 100644 --- a/libs/jwst-storage/src/migration/src/lib.rs +++ b/libs/jwst-storage/src/migration/src/lib.rs @@ -6,7 +6,7 @@ mod m20230321_000001_blob_optimized_table; mod m20230614_000001_initial_bucket_blob_table; mod m20230626_023319_doc_guid; mod m20230814_061223_initial_diff_log_table; -mod schema; +mod schemas; pub struct Migrator; diff --git a/libs/jwst-storage/src/migration/src/m20220101_000001_initial_blob_table.rs b/libs/jwst-storage/src/migration/src/m20220101_000001_initial_blob_table.rs index f253f6971..f6b4e5185 100644 --- a/libs/jwst-storage/src/migration/src/m20220101_000001_initial_blob_table.rs +++ b/libs/jwst-storage/src/migration/src/m20220101_000001_initial_blob_table.rs @@ -1,6 +1,6 @@ use sea_orm_migration::prelude::*; -use super::schema::Blobs; +use super::schemas::Blobs; pub struct Migration; diff --git a/libs/jwst-storage/src/migration/src/m20220101_000002_initial_doc_table.rs b/libs/jwst-storage/src/migration/src/m20220101_000002_initial_doc_table.rs index 47edd0483..c357701e5 100644 --- a/libs/jwst-storage/src/migration/src/m20220101_000002_initial_doc_table.rs +++ b/libs/jwst-storage/src/migration/src/m20220101_000002_initial_doc_table.rs @@ -1,6 +1,6 @@ use sea_orm_migration::prelude::*; -use super::schema::Docs; +use super::schemas::Docs; pub struct Migration; diff --git a/libs/jwst-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs b/libs/jwst-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs index d89fab857..8f9676ddb 100644 --- a/libs/jwst-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs +++ b/libs/jwst-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs @@ -1,6 +1,6 @@ use sea_orm_migration::prelude::*; -use super::schema::OptimizedBlobs; +use super::schemas::OptimizedBlobs; pub struct Migration; diff --git a/libs/jwst-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs b/libs/jwst-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs index 3f778d8cd..ceb35c575 100644 --- a/libs/jwst-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs +++ b/libs/jwst-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs @@ -1,6 +1,6 @@ use sea_orm_migration::prelude::*; -use super::schema::BucketBlobs; +use super::schemas::BucketBlobs; pub struct Migration; diff --git a/libs/jwst-storage/src/migration/src/m20230626_023319_doc_guid.rs b/libs/jwst-storage/src/migration/src/m20230626_023319_doc_guid.rs index bfe91e36b..ccaf1f49a 100644 --- a/libs/jwst-storage/src/migration/src/m20230626_023319_doc_guid.rs +++ b/libs/jwst-storage/src/migration/src/m20230626_023319_doc_guid.rs @@ -1,6 +1,6 @@ use sea_orm_migration::prelude::*; -use crate::schema::Docs; +use crate::schemas::Docs; #[derive(DeriveMigrationName)] pub struct Migration; diff --git a/libs/jwst-storage/src/migration/src/schema.rs b/libs/jwst-storage/src/migration/src/schemas.rs similarity index 100% rename from libs/jwst-storage/src/migration/src/schema.rs rename to libs/jwst-storage/src/migration/src/schemas.rs From 37e914a117264a359efde91b35edf9b128449d08 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 28 Aug 2023 16:06:05 +0800 Subject: [PATCH 25/49] chore: merge keck-core --- Cargo.lock | 95 +---- Cargo.toml | 2 - apps/keck-core/.env.template | 2 - apps/keck-core/.gitignore | 1 - apps/keck-core/Cargo.toml | 65 --- apps/keck-core/src/main.rs | 16 - apps/keck-core/src/server/api/blobs.rs | 237 ----------- apps/keck-core/src/server/api/blocks/block.rs | 400 ------------------ apps/keck-core/src/server/api/blocks/mod.rs | 142 ------- .../keck-core/src/server/api/blocks/schema.rs | 51 --- .../src/server/api/blocks/workspace.rs | 372 ---------------- apps/keck-core/src/server/api/mod.rs | 105 ----- apps/keck-core/src/server/mod.rs | 96 ----- apps/keck-core/src/server/sync/blobs.rs | 135 ------ .../src/server/sync/collaboration.rs | 288 ------------- apps/keck-core/src/server/sync/mod.rs | 25 -- apps/keck-core/src/server/utils.rs | 3 - apps/keck/Cargo.toml | 29 +- apps/keck/build.rs | 17 - apps/keck/src/main.rs | 1 - apps/keck/src/server/api/blobs.rs | 5 +- apps/keck/src/server/api/blocks/block.rs | 306 ++++++-------- apps/keck/src/server/api/blocks/mod.rs | 40 +- apps/keck/src/server/api/blocks/schema.rs | 2 +- apps/keck/src/server/api/blocks/workspace.rs | 288 ++++++------- apps/keck/src/server/api/doc.rs | 56 ++- apps/keck/src/server/api/mod.rs | 37 +- apps/keck/src/server/files.rs | 30 -- apps/keck/src/server/mod.rs | 49 +-- apps/keck/src/server/subscribe.rs | 69 --- apps/keck/src/server/sync/blobs.rs | 4 +- apps/keck/src/server/sync/collaboration.rs | 252 +++-------- apps/keck/src/server/sync/mod.rs | 6 - y-octo-utils/bin/bench_result_render.rs | 112 ----- 34 files changed, 398 insertions(+), 2940 deletions(-) delete mode 100644 apps/keck-core/.env.template delete mode 100644 apps/keck-core/.gitignore delete mode 100644 apps/keck-core/Cargo.toml delete mode 100644 apps/keck-core/src/main.rs delete mode 100644 apps/keck-core/src/server/api/blobs.rs delete mode 100644 apps/keck-core/src/server/api/blocks/block.rs delete mode 100644 apps/keck-core/src/server/api/blocks/mod.rs delete mode 100644 apps/keck-core/src/server/api/blocks/schema.rs delete mode 100644 apps/keck-core/src/server/api/blocks/workspace.rs delete mode 100644 apps/keck-core/src/server/api/mod.rs delete mode 100644 apps/keck-core/src/server/mod.rs delete mode 100644 apps/keck-core/src/server/sync/blobs.rs delete mode 100644 apps/keck-core/src/server/sync/collaboration.rs delete mode 100644 apps/keck-core/src/server/sync/mod.rs delete mode 100644 apps/keck-core/src/server/utils.rs delete mode 100644 apps/keck/build.rs delete mode 100644 apps/keck/src/server/files.rs delete mode 100644 apps/keck/src/server/subscribe.rs delete mode 100644 y-octo-utils/bin/bench_result_render.rs diff --git a/Cargo.lock b/Cargo.lock index 2d938d2f8..5ddfcd65f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,41 +121,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "affine-cloud" -version = "0.1.0" -dependencies = [ - "axum", - "axum-test-helper 0.2.0", - "bytes", - "chrono", - "cloud-database", - "cloud-infra", - "dotenvy", - "form_urlencoded", - "futures", - "futures-util", - "http-body", - "image", - "jsonwebtoken", - "jwst", - "jwst-logger", - "jwst-rpc", - "jwst-storage", - "lettre", - "lib0", - "mimalloc", - "mime", - "nanoid", - "serde", - "serde_json", - "tempfile", - "tokio", - "tower-http", - "utoipa", - "yrs", -] - [[package]] name = "affine-cloud-migration" version = "0.1.0" @@ -535,24 +500,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "axum-test-helper" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d349b3174ceac58442ea1f768233c817e59447c0343be2584fca9f0ed71d3a" -dependencies = [ - "axum", - "bytes", - "http", - "http-body", - "hyper", - "reqwest", - "serde", - "tokio", - "tower", - "tower-service", -] - [[package]] name = "axum-test-helper" version = "0.3.0" @@ -3011,42 +2958,7 @@ version = "0.1.0" dependencies = [ "anyhow", "axum", - "axum-test-helper 0.2.0", - "cfg-if 1.0.0", - "cloud-infra", - "dotenvy", - "futures", - "jwst", - "jwst-logger", - "jwst-rpc", - "jwst-storage", - "lib0", - "libc", - "log", - "mimalloc", - "nanoid", - "rand 0.8.5", - "reqwest", - "serde", - "serde_json", - "sqlx", - "thiserror", - "time 0.3.27", - "tokio", - "tower", - "tower-http", - "utoipa", - "utoipa-swagger-ui", - "yrs", -] - -[[package]] -name = "keck-core" -version = "0.1.0" -dependencies = [ - "anyhow", - "axum", - "axum-test-helper 0.3.0", + "axum-test-helper", "cfg-if 1.0.0", "dotenvy", "futures", @@ -3070,6 +2982,7 @@ dependencies = [ "tower", "tower-http", "utoipa", + "utoipa-swagger-ui", "yrs", ] @@ -6182,7 +6095,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55ae70283aba8d2a8b411c695c437fe25b8b5e44e23e780662002fc72fb47a82" dependencies = [ - "base64 0.21.2", "bitflags 2.4.0", "bytes", "futures-core", @@ -6190,12 +6102,9 @@ dependencies = [ "http", "http-body", "http-range-header", - "mime", "pin-project-lite", "tower-layer", "tower-service", - "tracing", - "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d071dc365..e18a15a48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,8 @@ [workspace] members = [ - "apps/cloud", "apps/doc_merger", "apps/keck", - "apps/keck-core", "libs/cloud-database", "libs/cloud-database/migration", "libs/cloud-infra", diff --git a/apps/keck-core/.env.template b/apps/keck-core/.env.template deleted file mode 100644 index e3cca72ae..000000000 --- a/apps/keck-core/.env.template +++ /dev/null @@ -1,2 +0,0 @@ -# The block observation webhook endpoint -# HOOK_ENDPOINT= diff --git a/apps/keck-core/.gitignore b/apps/keck-core/.gitignore deleted file mode 100644 index adfa211b4..000000000 --- a/apps/keck-core/.gitignore +++ /dev/null @@ -1 +0,0 @@ -updates.* \ No newline at end of file diff --git a/apps/keck-core/Cargo.toml b/apps/keck-core/Cargo.toml deleted file mode 100644 index a48cf8661..000000000 --- a/apps/keck-core/Cargo.toml +++ /dev/null @@ -1,65 +0,0 @@ -[package] -name = "keck-core" -version = "0.1.0" -authors = ["DarkSky "] -edition = "2021" -license = "AGPL-3.0-only" - -[features] -default = ["jwst"] -affine = ["jwst-core-storage/sqlite"] -jwst = [ - "api", - "jwst-core-storage/sqlite", - "jwst-core-storage/mysql", - "jwst-core-storage/postgres", -] -api = ["utoipa"] - -[dependencies] -anyhow = "1.0.70" -axum = { version = "0.6.20", features = ["headers", "ws"] } -cfg-if = "1.0.0" -futures = "0.3.28" -lib0 = { version = "0.16.5", features = ["lib0-serde"] } -log = { version = "0.4.17", features = [ - "max_level_trace", - "release_max_level_info", -] } -dotenvy = "0.15.7" -mimalloc = "0.1.36" -nanoid = "0.4.0" -serde = { version = "1.0.160", features = ["derive"] } -serde_json = "1.0.96" -sqlx = { version = "0.7.1", features = [ - "chrono", - "macros", - "migrate", - "runtime-tokio-rustls", -] } -tower = "0.4.13" -tower-http = { version = "0.4.0", features = ["cors"] } -thiserror = "1.0.40" -time = "0.3.20" -tokio = { version = "=1.28.0", features = [ - "macros", - "rt-multi-thread", - "signal", -] } -utoipa = { version = "3.5.0", features = ["axum_extras"], optional = true } -yrs = "0.16.5" -libc = "0.2.147" -rand = "0.8.5" -reqwest = { version = "0.11.19", default-features = false, features = [ - "json", - "rustls-tls", -] } - -# ======= workspace dependencies ======= -jwst-core = { workspace = true } -jwst-logger = { workspace = true } -jwst-core-rpc = { workspace = true } -jwst-core-storage = { workspace = true } - -[dev-dependencies] -axum-test-helper = "0.3.0" diff --git a/apps/keck-core/src/main.rs b/apps/keck-core/src/main.rs deleted file mode 100644 index 53ea10e9b..000000000 --- a/apps/keck-core/src/main.rs +++ /dev/null @@ -1,16 +0,0 @@ -mod server; - -use jwst_logger::init_logger; - -#[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; - -const PKG_NAME: &str = env!("CARGO_PKG_NAME"); - -#[tokio::main] -async fn main() { - // ignore load error when missing env file - let _ = dotenvy::dotenv(); - init_logger(PKG_NAME); - server::start_server().await; -} diff --git a/apps/keck-core/src/server/api/blobs.rs b/apps/keck-core/src/server/api/blobs.rs deleted file mode 100644 index 6fae433ad..000000000 --- a/apps/keck-core/src/server/api/blobs.rs +++ /dev/null @@ -1,237 +0,0 @@ -use axum::{extract::BodyStream, response::Response, routing::post}; -use futures::{future, StreamExt}; -use jwst_core::BlobStorage; -use utoipa::ToSchema; - -use super::*; - -#[derive(Serialize, ToSchema)] -pub struct BlobStatus { - id: Option, - exists: bool, -} - -/// Check a `Blob` is exists by id -/// - Return 200 if `Blob` is exists. -/// - Return 404 Not Found if `Workspace` or `Blob` not exists. -#[utoipa::path( - head, - tag = "Blobs", - context_path = "/api/blobs", - path = "/{workspace_id}/{hash}", - params( - ("workspace_id", description = "workspace id"), - ("hash", description = "blob hash"), - ), - responses( - (status = 200, description = "Blob exists"), - (status = 404, description = "Workspace or blob content not found"), - (status = 500, description = "Failed to query blobs"), - ) -)] -pub async fn check_blob(Extension(context): Extension>, Path(params): Path<(String, String)>) -> Response { - let (workspace, hash) = params; - info!("check_blob: {}, {}", workspace, hash); - if let Ok(exists) = context.storage.blobs().check_blob(Some(workspace), hash).await { - if exists { StatusCode::OK } else { StatusCode::NOT_FOUND }.into_response() - } else { - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } -} - -/// Get a `Blob` by hash -/// - Return 200 and `Blob` data if `Blob` is exists. -/// - Return 404 Not Found if `Workspace` or `Blob` not exists. -#[utoipa::path( - get, - tag = "Blobs", - context_path = "/api/blobs", - path = "/{workspace_id}/{hash}", - params( - ("workspace_id", description = "workspace id"), - ("hash", description = "blob hash"), - ), - responses( - (status = 200, description = "Get blob", body = Vec), - (status = 404, description = "Workspace or blob content not found"), - ) -)] -pub async fn get_blob(Extension(context): Extension>, Path(params): Path<(String, String)>) -> Response { - let (workspace, hash) = params; - info!("get_blob: {}, {}", workspace, hash); - if let Ok(blob) = context.storage.blobs().get_blob(Some(workspace), hash, None).await { - blob.into_response() - } else { - StatusCode::NOT_FOUND.into_response() - } -} - -/// Save `Blob` if not exists -/// - Return 200 if `Blob` save successful. -/// - Return 404 Not Found if `Workspace` not exists. -#[utoipa::path( - post, - tag = "Blobs", - context_path = "/api/blobs", - path = "/{workspace_id}", - params( - ("workspace_id", description = "workspace id"), - ("hash", description = "blob hash"), - ), - request_body( - content = BodyStream, - content_type="application/octet-stream" - ), - responses( - (status = 200, description = "Blob was saved", body = BlobStatus), - (status = 404, description = "Workspace not found", body = BlobStatus), - ) -)] -pub async fn set_blob( - Extension(context): Extension>, - Path(workspace): Path, - body: BodyStream, -) -> Response { - info!("set_blob: {}", workspace); - - let mut has_error = false; - let body = body - .take_while(|x| { - has_error = x.is_err(); - future::ready(x.is_ok()) - }) - .filter_map(|data| future::ready(data.ok())); - - if let Ok(id) = context - .storage - .blobs() - .put_blob_stream(Some(workspace.clone()), body) - .await - { - if has_error { - let _ = context.storage.blobs().delete_blob(Some(workspace), id).await; - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } else { - Json(BlobStatus { - id: Some(id), - exists: true, - }) - .into_response() - } - } else { - ( - StatusCode::NOT_FOUND, - Json(BlobStatus { - id: None, - exists: false, - }), - ) - .into_response() - } -} - -/// Delete `blob` if exists -/// - Return 204 if `Blob` delete successful. -/// - Return 404 Not Found if `Workspace` or `Blob` not exists. -#[utoipa::path( - delete, - tag = "Blobs", - context_path = "/api/blobs", - path = "/{workspace_id}/{hash}", - params( - ("workspace_id", description = "workspace id"), - ("hash", description = "blob hash"), - ), - responses( - (status = 204, description = "Blob was deleted"), - (status = 404, description = "Workspace or blob not found"), - ) -)] -pub async fn delete_blob( - Extension(context): Extension>, - Path(params): Path<(String, String)>, -) -> Response { - let (workspace, hash) = params; - info!("delete_blob: {}, {}", workspace, hash); - - if let Ok(success) = context.storage.blobs().delete_blob(Some(workspace), hash).await { - if success { - StatusCode::NO_CONTENT - } else { - StatusCode::NOT_FOUND - } - .into_response() - } else { - StatusCode::NOT_FOUND.into_response() - } -} - -pub fn blobs_apis(router: Router) -> Router { - router.route("/blobs/:workspace", post(set_blob)).route( - "/blobs/:workspace/:blob", - head(check_blob).get(get_blob).delete(delete_blob), - ) -} - -#[cfg(test)] -mod tests { - use axum::body::{Body, Bytes}; - use axum_test_helper::TestClient; - use futures::stream; - use jwst_core_storage::BlobStorageType; - - use super::*; - - #[tokio::test] - async fn test_blobs_apis() { - let ctx = Context::new( - JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) - .await - .ok(), - ) - .await; - let client = TestClient::new(blobs_apis(Router::new()).layer(Extension(Arc::new(ctx)))); - - let test_data: Vec = (0..=255).collect(); - let test_data_len = test_data.len(); - let test_data_stream = stream::iter( - test_data - .clone() - .into_iter() - .map(|byte| Ok::<_, std::io::Error>(Bytes::from(vec![byte]))), - ); - - //upload blob in workspace - let resp = client - .post("/blobs/test") - .header("Content-Length", test_data_len.clone().to_string()) - .body(Body::wrap_stream(test_data_stream.clone())) - .send() - .await; - assert_eq!(resp.status(), StatusCode::OK); - let resp_json: serde_json::Value = resp.json().await; - let hash = resp_json["id"].as_str().unwrap().to_string(); - - let resp = client.head(&format!("/blobs/test/{}", hash)).send().await; - assert_eq!(resp.status(), StatusCode::OK); - - let resp = client.get(&format!("/blobs/test/{}", hash)).send().await; - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(test_data, resp.bytes().await.to_vec()); - - let resp = client.delete(&format!("/blobs/test/{}", hash)).send().await; - assert_eq!(resp.status(), StatusCode::NO_CONTENT); - - let resp = client.delete(&format!("/blobs/test/{}", hash)).send().await; - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - let resp = client.head(&format!("/blobs/test/{}", hash)).send().await; - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - let resp = client.get(&format!("/blobs/test/{}", hash)).send().await; - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - let resp = client.get("/blobs/test/not_exists_id").send().await; - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - } -} diff --git a/apps/keck-core/src/server/api/blocks/block.rs b/apps/keck-core/src/server/api/blocks/block.rs deleted file mode 100644 index 978c21198..000000000 --- a/apps/keck-core/src/server/api/blocks/block.rs +++ /dev/null @@ -1,400 +0,0 @@ -use axum::{extract::Query, response::Response}; -use jwst_core::{constants, Any, DocStorage}; -use serde_json::Value as JsonValue; - -use super::*; - -/// Get a `Block` by id -/// - Return 200 and `Block`'s data if `Block` is exists. -/// - Return 404 Not Found if `Workspace` or `Block` not exists. -#[utoipa::path( - get, - tag = "Blocks", - context_path = "/api/block", - path = "/{workspace_id}/{block_id}", - params( - ("workspace_id", description = "workspace id"), - ("block_id", description = "block id"), - ), - responses( - (status = 200, description = "Get block", body = Block), - (status = 404, description = "Workspace or block content not found"), - ) -)] -pub async fn get_block(Extension(context): Extension>, Path(params): Path<(String, String)>) -> Response { - let (ws_id, block) = params; - info!("get_block: {}, {}", ws_id, block); - if let Ok(space) = context - .get_workspace(ws_id) - .await - .and_then(|mut ws| Ok(ws.get_blocks()?)) - { - if let Some(block) = space.get(block) { - Json(block).into_response() - } else { - StatusCode::NOT_FOUND.into_response() - } - } else { - StatusCode::NOT_FOUND.into_response() - } -} - -/// Create or modify `Block` if block exists with specific id. -/// Note that flavour can only be set when creating a block. -/// - Return 200 and `Block`'s data if `Block`'s content set successful. -/// - Return 404 Not Found if `Workspace` not exists. -#[utoipa::path( - post, - tag = "Blocks", - context_path = "/api/block", - path = "/{workspace_id}/{block_id}/?flavour={flavour}", - params( - ("workspace_id", description = "workspace id"), - ("block_id", description = "block id"), - ("flavour", Query, description = "block flavour, default flavour is text. Optional"), - ), - request_body( - content = String, - description = "json", - content_type = "application/json" - ), - responses( - (status = 200, description = "Block created and content was set", body = Block), - (status = 404, description = "Workspace not found"), - ) -)] -pub async fn set_block( - Extension(context): Extension>, - Path(params): Path<(String, String)>, - query_param: Option>>, - Json(payload): Json, -) -> Response { - let (ws_id, block_id) = params; - info!("set_block: {}, {}", ws_id, block_id); - if let Ok(mut space) = context - .get_workspace(&ws_id) - .await - .and_then(|mut ws| Ok(ws.get_blocks()?)) - { - let flavour = if let Some(query_map) = query_param { - query_map - .get("flavour") - .map_or_else(|| String::from("text"), |v| v.clone()) - } else { - String::from("text") - }; - - if let Ok(mut block) = space - .create(&block_id, flavour) - .map_err(|e| error!("failed to create block: {:?}", e)) - { - // set block content - if let Some(block_content) = payload.as_object() { - for (key, value) in block_content.iter() { - if key == constants::sys::FLAVOUR { - continue; - } - - if let Ok(value) = serde_json::from_value::(value.clone()) { - if let Err(e) = block.set(key, value.clone()) { - error!( - "failed to set block {} content: {}, {:?}, {:?}", - block_id, key, value, e - ); - } - } - } - } - - // response block content - return Json(block).into_response(); - } - } - StatusCode::NOT_FOUND.into_response() -} - -/// Get exists `Blocks` in certain `Workspace` by flavour -/// - Return 200 Ok and `Blocks`'s data if `Blocks` is exists. -/// - Return 404 Not Found if `Workspace` not exists or 500 Internal Server -/// Error when transaction init fails. -#[utoipa::path( - get, - tag = "Blocks", - context_path = "/api/block", - path = "/{workspace_id}/flavour/{flavour}", - params( - ("workspace_id", description = "workspace id"), - ("flavour", description = "block flavour"), - ), - responses( - (status = 200, description = "Get all certain flavour blocks belongs to the given workspace"), - (status = 404, description = "Workspace not found") - ) -)] -pub async fn get_block_by_flavour( - Extension(context): Extension>, - Path(params): Path<(String, String)>, -) -> Response { - let (ws_id, flavour) = params; - info!("get_block_by_flavour: ws_id, {}, flavour, {}", ws_id, flavour); - if let Ok(space) = context - .get_workspace(&ws_id) - .await - .and_then(|mut ws| Ok(ws.get_blocks()?)) - { - Json(space.get_blocks_by_flavour(&flavour)).into_response() - } else { - (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() - } -} - -/// Get `Block` history -/// - Return 200 and `Block`'s history if `Block` exists. -/// - Return 404 Not Found if `Workspace` or `Block` not exists. -#[utoipa::path( - get, - tag = "Blocks", - context_path = "/api/block", - path = "/{workspace_id}/{block_id}/history", - params( - ("workspace_id", description = "workspace id"), - ("block_id", description = "block id"), - ), - responses( - (status = 200, description = "Get block history", body = [BlockHistory]), - (status = 404, description = "Workspace or block not found"), - ) -)] -pub async fn get_block_history( - Extension(context): Extension>, - Path(params): Path<(String, String)>, -) -> Response { - let (ws_id, block) = params; - info!("get_block_history: {}, {}", ws_id, block); - if let Ok(space) = context - .get_workspace(&ws_id) - .await - .and_then(|mut ws| Ok(ws.get_blocks()?)) - { - if let Some(block) = space.get(block) { - Json(&block.history()).into_response() - } else { - StatusCode::NOT_FOUND.into_response() - } - } else { - StatusCode::NOT_FOUND.into_response() - } -} - -/// Delete block -/// - Return 204 No Content if delete successful. -/// - Return 404 Not Found if `Workspace` or `Block` not exists. -#[utoipa::path( - delete, - tag = "Blocks", - context_path = "/api/block", - path = "/{workspace_id}/{block_id}", - params( - ("workspace_id", description = "workspace id"), - ("block_id", description = "block id"), - ), - responses( - (status = 204, description = "Block successfully deleted"), - (status = 404, description = "Workspace or block not found"), - ) -)] -pub async fn delete_block( - Extension(context): Extension>, - Path(params): Path<(String, String)>, -) -> StatusCode { - let (ws_id, block) = params; - info!("delete_block: {}, {}", ws_id, block); - if let Ok(mut space) = context - .get_workspace(&ws_id) - .await - .and_then(|mut ws| Ok(ws.get_blocks()?)) - { - if space.remove(&block) { - return StatusCode::NO_CONTENT; - } - } - StatusCode::NOT_FOUND -} - -/// Get children in `Block` -/// - Return 200 and `Block`'s children ID. -/// - Return 404 Not Found if `Workspace` or `Block` not exists. -#[utoipa::path( - get, - tag = "Blocks", - context_path = "/api/block", - path = "/{workspace_id}/{block_id}/children", - params( - ("workspace_id", description = "workspace id"), - ("block_id", description = "block id"), - Pagination - ), - responses( - (status = 200, description = "Get block children", body = PageData<[String]>), - (status = 404, description = "Workspace or block not found"), - ) -)] -pub async fn get_block_children( - Extension(context): Extension>, - Path(params): Path<(String, String)>, - Query(pagination): Query, -) -> Response { - let (ws_id, block) = params; - let Pagination { offset, limit } = pagination; - info!("get_block_children: {}, {}", ws_id, block); - if let Ok(space) = context - .get_workspace(ws_id) - .await - .and_then(|mut ws| Ok(ws.get_blocks()?)) - { - if let Some(block) = space.get(&block) { - let data: Vec = block.children_iter(|children| children.skip(offset).take(limit).collect()); - - let status = if data.is_empty() { - StatusCode::NOT_FOUND - } else { - StatusCode::OK - }; - - return ( - status, - Json(PageData { - total: block.children_len() as usize, - data, - }), - ) - .into_response(); - } - } - StatusCode::NOT_FOUND.into_response() -} - -/// Insert a another `Block` into a `Block`'s children -/// - Return 200 and `Block`'s data if insert successful. -/// - Return 404 Not Found if `Workspace` or `Block` not exists. -#[utoipa::path( - post, - tag = "Blocks", - context_path = "/api/block", - path = "/{workspace_id}/{block_id}/children", - params( - ("workspace_id", description = "workspace id"), - ("block_id", description = "block id"), - ), - request_body( - content = InsertChildren, - description = "json", - content_type = "application/json" - ), - responses( - (status = 200, description = "Block inserted", body = Block), - (status = 404, description = "Workspace or block not found"), - (status = 500, description = "Failed to insert block") - ) -)] -pub async fn insert_block_children( - Extension(context): Extension>, - Path(params): Path<(String, String)>, - Json(payload): Json, -) -> Response { - let (ws_id, block) = params; - info!("insert_block: {}, {}", ws_id, block); - if let Ok(space) = context - .get_workspace(&ws_id) - .await - .and_then(|mut ws| Ok(ws.get_blocks()?)) - { - if let Some(mut block) = space.get(block) { - match payload { - InsertChildren::Push(block_id) => { - if let Some(mut child) = space.get(block_id) { - if let Err(e) = block.push_children(&mut child) { - error!("failed to insert block: {:?}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - } - } - InsertChildren::InsertBefore { id, before } => { - if let Some(mut child) = space.get(id) { - if let Err(e) = block.insert_children_before(&mut child, &before) { - error!("failed to insert children before: {:?}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - } - } - InsertChildren::InsertAfter { id, after } => { - if let Some(mut child) = space.get(id) { - if let Err(e) = block.insert_children_after(&mut child, &after) { - // TODO: handle error correctly - error!("failed to insert children after: {:?}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - } - } - InsertChildren::InsertAt { id, pos } => { - if let Some(mut child) = space.get(id) { - if let Err(e) = block.insert_children_at(&mut child, pos) { - // TODO: handle error correctly - error!("failed to insert children at: {:?}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - } - } - }; - - // response block content - return Json(block).into_response(); - } - } - StatusCode::NOT_FOUND.into_response() -} - -/// Remove children in `Block` -/// - Return 200 and `Block`'s data if remove successful. -/// - Return 404 Not Found if `Workspace` or `Block` not exists. -#[utoipa::path( - delete, - tag = "Blocks", - context_path = "/api/block", - path = "/{workspace_id}/{block_id}/children/{children}", - params( - ("workspace_id", description = "workspace id"), - ("block_id", description = "block id"), - ), - responses( - (status = 200, description = "Block children removed", body = Block), - (status = 404, description = "Workspace or block not found"), - ) -)] -pub async fn remove_block_children( - Extension(context): Extension>, - Path(params): Path<(String, String, String)>, -) -> Response { - let (ws_id, block, child_id) = params; - info!("insert_block: {}, {}", ws_id, block); - if let Ok(space) = context - .get_workspace(&ws_id) - .await - .and_then(|mut ws| Ok(ws.get_blocks()?)) - { - if let Some(mut block) = space.get(&block) { - if block.children_exists(&child_id) { - if let Some(mut child) = space.get(&child_id) { - if let Err(e) = block.remove_children(&mut child) { - error!("failed to remove block: {:?}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } else { - // response block content - return Json(block).into_response(); - } - } - } - } - } - StatusCode::NOT_FOUND.into_response() -} diff --git a/apps/keck-core/src/server/api/blocks/mod.rs b/apps/keck-core/src/server/api/blocks/mod.rs deleted file mode 100644 index 70e41d739..000000000 --- a/apps/keck-core/src/server/api/blocks/mod.rs +++ /dev/null @@ -1,142 +0,0 @@ -pub mod block; -pub mod schema; -pub mod workspace; - -pub use block::{delete_block, get_block, get_block_history, insert_block_children, remove_block_children, set_block}; -use schema::InsertChildren; -pub use schema::SubscribeWorkspace; -pub use workspace::{delete_workspace, get_workspace, set_workspace, subscribe_workspace, workspace_client}; - -use super::*; - -fn block_apis(router: Router) -> Router { - let block_operation = Router::new() - .route("/history", get(block::get_block_history)) - .route( - "/children", - get(block::get_block_children).post(block::insert_block_children), - ) - .route("/children/:children", delete(block::remove_block_children)); - - router.nest("/block/:workspace/:block/", block_operation).route( - "/block/:workspace/:block", - get(block::get_block).post(block::set_block).delete(block::delete_block), - ) -} - -fn workspace_apis(router: Router) -> Router { - router - .route("/block/:workspace/client", get(workspace::workspace_client)) - // .route("/block/:workspace/history", get(workspace::history_workspace_clients)) - // .route("/block/:workspace/history/:client", get(workspace::history_workspace)) - .route( - "/block/:workspace", - get(workspace::get_workspace) - .post(workspace::set_workspace) - .delete(workspace::delete_workspace), - ) - .route("/block/:workspace/flavour/:flavour", get(block::get_block_by_flavour)) - .route("/block/:workspace/blocks", get(workspace::get_workspace_block)) - // .route("/search/:workspace", get(workspace::workspace_search)) - // .route( - // "/search/:workspace/index", - // get(workspace::get_search_index).post(workspace::set_search_index), - // ) - .route("/subscribe", post(subscribe_workspace)) -} - -pub fn blocks_apis(router: Router) -> Router { - workspace_apis(block_apis(router)) -} - -#[cfg(test)] -mod tests { - use axum_test_helper::TestClient; - use serde_json::{from_str, json, to_string, Value}; - - use super::*; - - #[tokio::test] - async fn test_workspace_apis() { - let client = Arc::new(reqwest::Client::builder().no_proxy().build().unwrap()); - let runtime = Arc::new( - runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_time() - .enable_io() - .build() - .expect("Failed to create runtime"), - ); - let hook_endpoint = Arc::new(RwLock::new(String::new())); - - let ctx = Arc::new( - Context::new( - JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) - .await - .ok(), - ) - .await, - ); - let client = TestClient::new( - workspace_apis(Router::new()) - .layer(Extension(ctx.clone())) - .layer(Extension(client.clone())) - .layer(Extension(runtime.clone())) - .layer(Extension(hook_endpoint.clone())), - ); - - // basic workspace apis - let resp = client.get("/block/test").send().await; - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - let resp = client.post("/block/test").send().await; - assert_eq!(resp.status(), StatusCode::OK); - let resp = client.get("/block/test/client").send().await; - assert_eq!( - resp.text().await.parse::().unwrap(), - ctx.storage.get_workspace("test").await.unwrap().client_id() - ); - let resp = client.get("/block/test/history").send().await; - assert_eq!(resp.json::>().await, Vec::::new()); - let resp = client.get("/block/test").send().await; - assert_eq!(resp.status(), StatusCode::OK); - let resp = client.delete("/block/test").send().await; - assert_eq!(resp.status(), StatusCode::NO_CONTENT); - let resp = client.get("/block/test").send().await; - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - // workspace history apis - let resp = client.post("/block/test").send().await; - assert_eq!(resp.status(), StatusCode::OK); - - let resp = client.get("/search/test/index").send().await; - assert_eq!(resp.status(), StatusCode::OK); - let index = resp.json::>().await; - assert_eq!(index, vec!["title".to_owned(), "text".to_owned()]); - - let body = to_string(&json!(["test"])).unwrap(); - let resp = client - .post("/search/test/index") - .header("content-type", "application/json") - .body(body) - .send() - .await; - assert_eq!(resp.status(), StatusCode::OK); - - let resp = client.get("/search/test/index").send().await; - assert_eq!(resp.status(), StatusCode::OK); - let index = resp.json::>().await; - assert_eq!(index, vec!["test".to_owned()]); - - let body = json!({ - "hookEndpoint": "localhost:3000/api/hook" - }) - .to_string(); - let resp = client - .post("/subscribe") - .header("content-type", "application/json") - .body(body) - .send() - .await; - assert_eq!(resp.status(), StatusCode::OK); - } -} diff --git a/apps/keck-core/src/server/api/blocks/schema.rs b/apps/keck-core/src/server/api/blocks/schema.rs deleted file mode 100644 index c12ecf1c9..000000000 --- a/apps/keck-core/src/server/api/blocks/schema.rs +++ /dev/null @@ -1,51 +0,0 @@ -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; -use utoipa::ToSchema; - -#[derive(Default, Deserialize, PartialEq, Debug, ToSchema)] -pub struct Workspace { - pub(super) blocks: HashMap, - pub(super) updated: HashMap, -} - -#[derive(Deserialize, PartialEq, Debug, ToSchema)] -#[schema(example = json!({ - "sys_id": "0", - "sys:flavour": "affine:text", - "sys:created": 946684800000_u64, - "sys:children": ["block1", "block2"], - "prop:text": "123", - "prop:color": "#ff0000", -}))] -pub struct Block { - #[serde(rename = "sys:flavour")] - flavour: String, - #[serde(rename = "sys:created")] - created: u64, - #[serde(rename = "sys:children")] - children: Vec, -} - -#[derive(Deserialize, PartialEq, Debug, ToSchema)] -#[schema(example = json!([12345, 946684800000_u64, "add"]))] -pub struct BlockRawHistory(u64, u64, String); - -#[derive(Deserialize, ToSchema)] -#[schema(example = json!({"Push": "jwstRf4rMzua7E"}))] - -pub enum InsertChildren { - Push(String), - InsertBefore { id: String, before: String }, - InsertAfter { id: String, after: String }, - InsertAt { id: String, pos: u64 }, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -#[schema(example=json!({ - "hookEndpoint": "http://localhost:3000/api/hooks" -}))] -pub struct SubscribeWorkspace { - #[serde(rename = "hookEndpoint")] - pub hook_endpoint: String, -} diff --git a/apps/keck-core/src/server/api/blocks/workspace.rs b/apps/keck-core/src/server/api/blocks/workspace.rs deleted file mode 100644 index 505ccaa4b..000000000 --- a/apps/keck-core/src/server/api/blocks/workspace.rs +++ /dev/null @@ -1,372 +0,0 @@ -use axum::{ - extract::{Path, Query}, - response::Response, -}; -use jwst_core::DocStorage; -use utoipa::IntoParams; - -use super::*; -use crate::server::api::blocks::SubscribeWorkspace; - -/// Get a exists `Workspace` by id -/// - Return 200 Ok and `Workspace`'s data if `Workspace` is exists. -/// - Return 404 Not Found if `Workspace` not exists. -#[utoipa::path( - get, - tag = "Workspace", - context_path = "/api/block", - path = "/{workspace}", - params( - ("workspace", description = "workspace id"), - ), - responses( - (status = 200, description = "Get workspace data", body = Workspace), - (status = 404, description = "Workspace not found") - ) -)] -pub async fn get_workspace(Extension(context): Extension>, Path(workspace): Path) -> Response { - info!("get_workspace: {}", workspace); - if let Ok(workspace) = context.get_workspace(&workspace).await { - Json(workspace).into_response() - } else { - (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() - } -} - -/// Create a `Workspace` by id -/// - Return 200 Ok and `Workspace`'s data if init success or `Workspace` is -/// exists. -/// - Return 500 Internal Server Error if init failed. -#[utoipa::path( - post, - tag = "Workspace", - context_path = "/api/block", - path = "/{workspace}", - params( - ("workspace", description = "workspace id"), - ), - responses( - (status = 200, description = "Return workspace data", body = Workspace), - (status = 500, description = "Failed to init a workspace") - ) -)] -pub async fn set_workspace(Extension(context): Extension>, Path(workspace): Path) -> Response { - info!("set_workspace: {}", workspace); - match context.create_workspace(workspace).await { - Ok(workspace) => Json(workspace).into_response(), - Err(e) => { - error!("Failed to init doc: {:?}", e); - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } - } -} - -/// Delete a exists `Workspace` by id -/// - Return 204 No Content if delete successful. -/// - Return 404 Not Found if `Workspace` not exists. -/// - Return 500 Internal Server Error if delete failed. -#[utoipa::path( - delete, - tag = "Workspace", - context_path = "/api/block", - path = "/{workspace}", - params( - ("workspace", description = "workspace id"), - ), - responses( - (status = 204, description = "Workspace data deleted"), - (status = 404, description = "Workspace not exists"), - (status = 500, description = "Failed to delete workspace") - ) -)] -pub async fn delete_workspace(Extension(context): Extension>, Path(workspace): Path) -> Response { - info!("delete_workspace: {}", workspace); - if context.storage.docs().delete_workspace(&workspace).await.is_err() { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - }; - - StatusCode::NO_CONTENT.into_response() -} - -/// Get current client id of server -/// -/// When the server initializes or get the `Workspace`, a `Client` will be -/// created. This `Client` will not be destroyed until the server restarts. -/// Therefore, the `Client ID` in the history generated by modifying `Block` -/// through HTTP API will remain unchanged until the server restarts. -/// -/// This interface return the client id that server will used. -#[utoipa::path( - get, - tag = "Workspace", - context_path = "/api/block", - path = "/{workspace}/client", - params( - ("workspace", description = "workspace id"), - ), - responses( - (status = 200, description = "Get workspace client id", body = u64), - (status = 404, description = "Workspace not found") - ) -)] -pub async fn workspace_client(Extension(context): Extension>, Path(workspace): Path) -> Response { - if let Ok(workspace) = context.get_workspace(&workspace).await { - Json(workspace.client_id()).into_response() - } else { - (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() - } -} - -/// Block search query -// See doc for using utoipa search queries example here: https://github.com/juhaku/utoipa/blob/6c7f6a2d/examples/todo-axum/src/main.rs#L124-L130 -#[derive(Deserialize, IntoParams)] -pub struct BlockSearchQuery { - /// Search by title and text. - query: String, -} - -/// Search workspace blocks of server -/// -/// This will return back a list of relevant blocks. -// #[utoipa::path( -// get, -// tag = "Workspace", -// context_path = "/api/search", -// path = "/{workspace}", -// params( -// ("workspace", description = "workspace id"), -// BlockSearchQuery, -// ), -// responses( -// (status = 200, description = "Search results", body = SearchResults), -// ) -// )] -// pub async fn workspace_search( -// Extension(context): Extension>, -// Path(workspace): Path, -// query: Query, -// ) -> Response { let query_text = &query.query; let ws_id = workspace; info!("workspace_search: {ws_id:?} query = -// {query_text:?}"); if let Ok(workspace) = context.get_workspace(&ws_id).await { match workspace.search(query_text) { -// Ok(list) => { debug!("workspace_search: {ws_id:?} query = {query_text:?}; {list:#?}"); Json(list).into_response() } -// Err(err) => { error!("Internal server error calling workspace_search: {err:?}"); -// StatusCode::INTERNAL_SERVER_ERROR.into_response() } } } else { (StatusCode::NOT_FOUND, -// format!("Workspace({ws_id:?}) not found")).into_response() } -// } - -// #[utoipa::path( -// get, -// tag = "Workspace", -// context_path = "/api/search", -// path = "/{workspace}/index", -// params( -// ("workspace", description = "workspace id"), -// ), -// responses( -// (status = 200, description = "result", body = Vec), -// (status = 404, description = "Workspace not found") -// ) -// )] -// pub async fn get_search_index(Extension(context): Extension>, Path(workspace): Path) -> Response -// { info!("get_search_index: {workspace:?}"); - -// if let Ok(workspace) = context.get_workspace(&workspace).await { -// Json(workspace.metadata().search_index).into_response() -// } else { -// (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() -// } -// } - -// #[utoipa::path( -// post, -// tag = "Workspace", -// context_path = "/api/search", -// path = "/{workspace}/index", -// params( -// ("workspace", description = "workspace id"), -// ), -// responses( -// (status = 200, description = "success"), -// (status = 400, description = "Bad Request"), -// (status = 404, description = "Workspace not found") -// ) -// )] -// pub async fn set_search_index( -// Extension(context): Extension>, -// Path(workspace): Path, -// Json(fields): Json>, -// ) -> Response { info!("set_search_index: {workspace:?} fields = {fields:?}"); - -// if let Ok(workspace) = context.get_workspace(&workspace).await { -// if let Ok(true) = workspace.set_search_index(fields) { -// StatusCode::OK.into_response() -// } else { -// StatusCode::BAD_REQUEST.into_response() -// } -// } else { -// (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() -// } -// } - -/// Get `Block` in `Workspace` -/// - Return 200 and `Block`'s ID. -/// - Return 404 Not Found if `Workspace` or `Block` not exists. -#[utoipa::path( - get, - tag = "Workspace", - context_path = "/api/block", - path = "/{workspace}/blocks", - params( - ("workspace", description = "workspace id"), - Pagination - ), - responses( - (status = 200, description = "Get Blocks", body = PageData<[Block]>), - (status = 404, description = "Workspace or block not found"), - ) -)] -pub async fn get_workspace_block( - Extension(context): Extension>, - Path(workspace): Path, - Query(pagination): Query, -) -> Response { - let Pagination { offset, limit } = pagination; - info!("get_workspace_block: {workspace:?}"); - if let Ok(space) = context - .get_workspace(&workspace) - .await - .and_then(|mut ws| Ok(ws.get_blocks()?)) - { - let total = space.block_count() as usize; - let data = space.blocks(|blocks| blocks.skip(offset).take(limit).collect::>()); - - let status = if data.is_empty() { - StatusCode::NOT_FOUND - } else { - StatusCode::OK - }; - - (status, Json(PageData { total, data })).into_response() - } else { - (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() - } -} - -/// Get all client ids of the `Workspace` -/// -/// This interface returns all `Client IDs` that includes history in the -/// `Workspace` -/// -/// Every client write something into a `Workspace` will has a unique id. -/// -/// For example: -/// - A user writes a new `Block` to a `Workspace` through `Client` on the -/// front end, which will generate a series of histories. A `Client ID` -/// contained in these histories will be randomly generated by the `Client` -/// and will remain unchanged until the Client instance is destroyed -/// - When the server initializes or get the `Workspace`, a `Client` will be -/// created. This `Client` will not be destroyed until the server restarts. -/// Therefore, the `Client ID` in the history generated by modifying `Block` -/// through HTTP API will remain unchanged until the server restarts. -// #[utoipa::path( -// get, -// tag = "Workspace", -// context_path = "/api/block", -// path = "/{workspace}/history", -// params( -// ("workspace", description = "workspace id"), -// ), -// responses( -// (status = 200, description = "Get workspace history client ids", body = [u64]), -// (status = 500, description = "Failed to get workspace history") -// ) -// )] -// pub async fn history_workspace_clients( -// Extension(context): Extension>, -// Path(workspace): Path, -// ) -> Response { if let Ok(workspace) = context.get_workspace(&workspace).await { if let Some(history) = -// parse_history_client(&workspace.doc()) { Json(history).into_response() } else { -// StatusCode::INTERNAL_SERVER_ERROR.into_response() } } else { (StatusCode::NOT_FOUND, -// format!("Workspace({workspace:?}) not found")).into_response() } -// } - -/// Get the history generated by a specific `Client ID` of the `Workspace` -/// -/// If client id set to 0, return all history of the `Workspace`. -// #[utoipa::path( -// get, -// tag = "Workspace", -// context_path = "/api/block", -// path = "/{workspace}/history/{client}", -// params( -// ("workspace", description = "workspace id"), -// ("client", description = "client id, is give 0 then return all clients histories"), -// ), -// responses( -// (status = 200, description = "Get workspace history", body = [RawHistory]), -// (status = 400, description = "Client id invalid"), -// (status = 500, description = "Failed to get workspace history") -// ) -// )] -// pub async fn history_workspace( -// Extension(context): Extension>, -// Path(params): Path<(String, String)>, -// ) -> Response { let (ws_id, client) = params; if let Ok(workspace) = context.get_workspace(&ws_id).await { if let -// Ok(client) = client.parse::() { if let Some(json) = parse_history(&workspace.doc(), client).and_then(|history| -// serde_json::to_string(&history).ok()) { ([(header::CONTENT_TYPE, "application/json")], json).into_response() } else -// { StatusCode::INTERNAL_SERVER_ERROR.into_response() } } else { StatusCode::BAD_REQUEST.into_response() } } else { -// (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() } -// } - -/// Register a webhook for all block changes from all workspace changes -#[utoipa::path( - post, - tag = "Workspace", - context_path = "/api/subscribe", - path = "", - request_body( - content_type = "application/json", - content = SubscribeWorkspace, - description = "Provide endpoint of webhook server", - ), - responses( - (status = 200, description = "Subscribe workspace succeed"), - (status = 500, description = "Internal Server Error") - ) -)] -pub async fn subscribe_workspace( - Extension(hook_endpoint): Extension>>, - Json(payload): Json, -) -> Response { - info!("subscribe all workspaces, hook endpoint: {}", payload.hook_endpoint); - - let mut write_guard = hook_endpoint.write().await; - *write_guard = payload.hook_endpoint.clone(); - info!("successfully subscribed all workspaces"); - StatusCode::OK.into_response() -} - -#[cfg(all(test, feature = "sqlite"))] -mod test { - use super::*; - - #[tokio::test] - async fn workspace() { - use axum_test_helper::TestClient; - - let pool = DbPool::init_memory_pool().await.unwrap(); - let context = Arc::new(Context::new(Some(pool)).await); - - let app = super::workspace_apis(Router::new()).layer(Extension(context)); - - let client = TestClient::new(app); - - let resp = client.post("/block/test").send().await; - - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.json::().await, schema::Workspace::default()); - - let resp = client.get("/block/test").send().await; - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.json::().await, schema::Workspace::default()); - } -} diff --git a/apps/keck-core/src/server/api/mod.rs b/apps/keck-core/src/server/api/mod.rs deleted file mode 100644 index e4b9c1246..000000000 --- a/apps/keck-core/src/server/api/mod.rs +++ /dev/null @@ -1,105 +0,0 @@ -#[cfg(feature = "api")] -mod blobs; -#[cfg(feature = "api")] -mod blocks; - -use std::collections::HashMap; - -use axum::Router; -#[cfg(feature = "api")] -use axum::{ - extract::{Json, Path}, - http::StatusCode, - response::IntoResponse, - routing::{delete, get, head, post}, -}; -use jwst_core_rpc::{BroadcastChannels, RpcContextImpl}; -use jwst_core_storage::{BlobStorageType, JwstStorage, JwstStorageResult}; -use tokio::sync::RwLock; - -use super::*; - -#[derive(Deserialize)] -#[cfg_attr(feature = "api", derive(utoipa::IntoParams))] -pub struct Pagination { - #[serde(default)] - offset: usize, - #[serde(default = "default_limit")] - limit: usize, -} - -fn default_limit() -> usize { - usize::MAX -} - -#[derive(Serialize)] -pub struct PageData { - total: usize, - data: T, -} - -pub struct Context { - channel: BroadcastChannels, - storage: JwstStorage, -} - -impl Context { - pub async fn new(storage: Option) -> Self { - let blob_storage_type = BlobStorageType::DB; - - let storage = if let Some(storage) = storage { - info!("use external storage instance: {}", storage.database()); - Ok(storage) - } else if dotenvy::var("USE_MEMORY_SQLITE").is_ok() { - info!("use memory sqlite database"); - JwstStorage::new_with_migration("sqlite::memory:", blob_storage_type).await - } else if let Ok(database_url) = dotenvy::var("DATABASE_URL") { - info!("use external database: {}", database_url); - JwstStorage::new_with_migration(&database_url, blob_storage_type).await - } else { - info!("use sqlite database: jwst.db"); - JwstStorage::new_with_sqlite("jwst", blob_storage_type).await - } - .expect("Cannot create database"); - - Context { - channel: RwLock::new(HashMap::new()), - storage, - } - } - - pub async fn get_workspace(&self, workspace_id: S) -> JwstStorageResult - where - S: AsRef, - { - self.storage.get_workspace(workspace_id).await - } - - pub async fn create_workspace(&self, workspace_id: S) -> JwstStorageResult - where - S: AsRef, - { - self.storage.create_workspace(workspace_id).await - } -} - -impl RpcContextImpl<'_> for Context { - fn get_storage(&self) -> &JwstStorage { - &self.storage - } - - fn get_channel(&self) -> &BroadcastChannels { - &self.channel - } -} - -pub fn api_handler(router: Router) -> Router { - #[cfg(feature = "api")] - { - router.nest("/api", blobs::blobs_apis(blocks::blocks_apis(Router::new()))) - } - #[cfg(not(feature = "api"))] - { - router - } -} diff --git a/apps/keck-core/src/server/mod.rs b/apps/keck-core/src/server/mod.rs deleted file mode 100644 index f11540ae5..000000000 --- a/apps/keck-core/src/server/mod.rs +++ /dev/null @@ -1,96 +0,0 @@ -mod api; -mod sync; -mod utils; - -use std::{net::SocketAddr, sync::Arc}; - -use api::Context; -use axum::{http::Method, Extension, Router, Server}; -use jwst_core::Workspace; -use tokio::{runtime, signal, sync::RwLock}; -use tower_http::cors::{Any, CorsLayer}; -pub use utils::*; - -async fn shutdown_signal() { - let ctrl_c = async { - signal::ctrl_c().await.expect("failed to install Ctrl+C handler"); - }; - - #[cfg(unix)] - let terminate = async { - signal::unix::signal(signal::unix::SignalKind::terminate()) - .expect("failed to install signal handler") - .recv() - .await; - }; - - #[cfg(not(unix))] - let terminate = std::future::pending::<()>(); - - tokio::select! { - _ = ctrl_c => {}, - _ = terminate => {}, - } - - info!("Shutdown signal received, starting graceful shutdown"); -} - -pub async fn start_server() { - let origins = [ - "http://localhost:4200".parse().unwrap(), - "http://127.0.0.1:4200".parse().unwrap(), - "http://localhost:3000".parse().unwrap(), - "http://127.0.0.1:3000".parse().unwrap(), - "http://localhost:5173".parse().unwrap(), - "http://127.0.0.1:5173".parse().unwrap(), - ]; - - let cors = CorsLayer::new() - // allow `GET` and `POST` when accessing the resource - .allow_methods(vec![Method::GET, Method::POST, Method::DELETE, Method::OPTIONS]) - // allow requests from any origin - .allow_origin(origins) - .allow_headers(Any); - - let client = Arc::new(reqwest::Client::builder().no_proxy().build().unwrap()); - let runtime = Arc::new( - runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_time() - .enable_io() - .build() - .expect("Failed to create runtime"), - ); - let hook_endpoint = Arc::new(RwLock::new(dotenvy::var("HOOK_ENDPOINT").unwrap_or_default())); - - let context = Arc::new(Context::new(None).await); - - let app = sync::sync_handler(api::api_handler(Router::new())) - .layer(cors) - .layer(Extension(context.clone())) - .layer(Extension(client)) - .layer(Extension(runtime)) - .layer(Extension(hook_endpoint)); - - let addr = SocketAddr::from(( - [0, 0, 0, 0], - dotenvy::var("KECK_PORT") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(3000), - )); - info!("listening on {}", addr); - - if let Err(e) = Server::bind(&addr) - .serve(app.into_make_service()) - .with_graceful_shutdown(shutdown_signal()) - .await - { - error!("Server shutdown due to error: {}", e); - } - - // context.docs.close().await; - // context.blobs.close().await; - - info!("Server shutdown complete"); -} diff --git a/apps/keck-core/src/server/sync/blobs.rs b/apps/keck-core/src/server/sync/blobs.rs deleted file mode 100644 index f893ef4a3..000000000 --- a/apps/keck-core/src/server/sync/blobs.rs +++ /dev/null @@ -1,135 +0,0 @@ -use std::sync::Arc; - -use axum::{ - extract::{BodyStream, Path}, - headers::ContentLength, - http::{ - header::{CACHE_CONTROL, CONTENT_LENGTH, CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED}, - HeaderMap, HeaderValue, StatusCode, - }, - response::{IntoResponse, Response}, - Json, TypedHeader, -}; -use futures::{future, StreamExt}; -use jwst_core::BlobStorage; -use jwst_core_rpc::RpcContextImpl; -use time::{format_description::well_known::Rfc2822, OffsetDateTime}; - -use super::*; - -#[derive(Serialize)] -struct BlobStatus { - exists: bool, - id: String, -} - -impl Context { - async fn get_blob(&self, workspace: Option, id: String, method: Method, headers: HeaderMap) -> Response { - if let Some(etag) = headers.get(IF_NONE_MATCH).and_then(|h| h.to_str().ok()) { - if etag == id { - return StatusCode::NOT_MODIFIED.into_response(); - } - } - - let Ok(meta) = self - .get_storage() - .blobs() - .get_metadata(workspace.clone(), id.clone(), None) - .await - else { - return StatusCode::NOT_FOUND.into_response(); - }; - - if let Some(modified_since) = headers - .get(IF_MODIFIED_SINCE) - .and_then(|h| h.to_str().ok()) - .and_then(|s| OffsetDateTime::parse(s, &Rfc2822).ok()) - { - if meta.last_modified.timestamp() <= modified_since.unix_timestamp() { - return StatusCode::NOT_MODIFIED.into_response(); - } - } - - let mut header = HeaderMap::with_capacity(5); - header.insert(ETAG, HeaderValue::from_str(&id).unwrap()); - header.insert(CONTENT_TYPE, HeaderValue::from_static("application/octet-stream")); - header.insert( - LAST_MODIFIED, - HeaderValue::from_str( - &OffsetDateTime::from_unix_timestamp(meta.last_modified.timestamp()) - .unwrap() - .format(&Rfc2822) - .unwrap(), - ) - .unwrap(), - ); - header.insert(CONTENT_LENGTH, HeaderValue::from_str(&meta.size.to_string()).unwrap()); - header.insert( - CACHE_CONTROL, - HeaderValue::from_str("public, immutable, max-age=31536000").unwrap(), - ); - - if method == Method::HEAD { - return header.into_response(); - }; - - let Ok(file) = self.get_storage().blobs().get_blob(workspace, id, None).await else { - return StatusCode::NOT_FOUND.into_response(); - }; - - if meta.size != file.len() as i64 { - header.insert(CONTENT_LENGTH, HeaderValue::from_str(&file.len().to_string()).unwrap()); - } - - (header, file).into_response() - } - - async fn upload_blob(&self, stream: BodyStream, workspace: Option) -> Response { - // TODO: cancel - let mut has_error = false; - let stream = stream - .take_while(|x| { - has_error = x.is_err(); - future::ready(x.is_ok()) - }) - .filter_map(|data| future::ready(data.ok())); - - if let Ok(id) = self - .get_storage() - .blobs() - .put_blob_stream(workspace.clone(), stream) - .await - { - if has_error { - let _ = self.get_storage().blobs().delete_blob(workspace, id).await; - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } else { - Json(BlobStatus { id, exists: true }).into_response() - } - } else { - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } - } -} - -pub async fn get_blob_in_workspace( - Extension(ctx): Extension>, - Path((workspace_id, id)): Path<(String, String)>, - method: Method, - headers: HeaderMap, -) -> Response { - ctx.get_blob(Some(workspace_id), id, method, headers).await -} - -pub async fn upload_blob_in_workspace( - Extension(ctx): Extension>, - Path(workspace_id): Path, - TypedHeader(length): TypedHeader, - stream: BodyStream, -) -> Response { - if length.0 > 10 * 1024 * 1024 { - return StatusCode::PAYLOAD_TOO_LARGE.into_response(); - } - - ctx.upload_blob(stream, Some(workspace_id)).await -} diff --git a/apps/keck-core/src/server/sync/collaboration.rs b/apps/keck-core/src/server/sync/collaboration.rs deleted file mode 100644 index a28e0d4e0..000000000 --- a/apps/keck-core/src/server/sync/collaboration.rs +++ /dev/null @@ -1,288 +0,0 @@ -use std::sync::Arc; - -use axum::{ - extract::{ws::WebSocketUpgrade, Path}, - response::Response, - Json, -}; -use futures::FutureExt; -use jwst_core_rpc::{axum_socket_connector, handle_connector}; -use serde::Serialize; - -use super::*; - -#[derive(Serialize)] -pub struct WebSocketAuthentication { - protocol: String, -} - -pub async fn auth_handler(Path(workspace_id): Path) -> Json { - info!("auth: {}", workspace_id); - Json(WebSocketAuthentication { - protocol: "AFFiNE".to_owned(), - }) -} - -pub async fn upgrade_handler( - Extension(context): Extension>, - Path(workspace): Path, - ws: WebSocketUpgrade, -) -> Response { - let identifier = nanoid!(); - if let Err(e) = context.create_workspace(workspace.clone()).await { - error!("create workspace failed: {:?}", e); - } - ws.protocols(["AFFiNE"]).on_upgrade(move |socket| { - handle_connector(context.clone(), workspace.clone(), identifier, move || { - axum_socket_connector(socket, &workspace) - }) - .map(|_| ()) - }) -} - -#[cfg(test)] -mod test { - use std::{ - ffi::c_int, - io::{BufRead, BufReader}, - process::{Child, Command, Stdio}, - string::String, - sync::Arc, - thread::sleep, - time::Duration, - }; - - use jwst_core::{Block, DocStorage, Workspace}; - use jwst_core_rpc::{start_websocket_client_sync, BroadcastChannels, RpcContextImpl}; - use jwst_core_storage::{BlobStorageType, JwstStorage}; - use jwst_logger::info; - use libc::{kill, SIGTERM}; - use rand::{thread_rng, Rng}; - use tokio::runtime::Runtime; - - struct TestContext { - storage: Arc, - channel: Arc, - } - - impl TestContext { - fn new(storage: Arc) -> Self { - Self { - storage, - channel: Arc::default(), - } - } - } - - impl RpcContextImpl<'_> for TestContext { - fn get_storage(&self) -> &JwstStorage { - &self.storage - } - - fn get_channel(&self) -> &BroadcastChannels { - &self.channel - } - } - - #[test] - #[ignore = "not needed in ci"] - fn client_collaboration_with_server() { - if dotenvy::var("KECK_DEBUG").is_ok() { - jwst_logger::init_logger("keck-core"); - } - - let server_port = thread_rng().gen_range(10000..=30000); - let child = start_collaboration_server(server_port); - - let rt = Arc::new(Runtime::new().unwrap()); - let (workspace_id, mut workspace) = { - let workspace_id = "1"; - let context = rt.block_on(async move { - Arc::new(TestContext::new(Arc::new( - JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) - .await - .expect("get storage: memory sqlite failed"), - ))) - }); - let remote = format!("ws://localhost:{server_port}/collaboration/1"); - - start_websocket_client_sync( - rt.clone(), - context.clone(), - Arc::default(), - remote, - workspace_id.to_owned(), - ); - - ( - workspace_id.to_owned(), - rt.block_on(async move { context.get_workspace(workspace_id).await.unwrap() }), - ) - }; - - for block_id in 0..3 { - let block = create_block(&mut workspace, block_id.to_string(), "list".to_string()); - info!("from client, create a block:{:?}", serde_json::to_string(&block)); - } - - sleep(Duration::from_secs(1)); - info!("------------------after sync------------------"); - - for block_id in 0..3 { - let ret = get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port); - info!("get block {block_id} from server: {ret}"); - assert!(!ret.is_empty()); - } - - let space = workspace.get_space("blocks").unwrap(); - let blocks = space.get_blocks_by_flavour("list"); - let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); - assert_eq!(ids.sort(), vec!["7", "8", "9"].sort()); - info!("blocks from local storage:"); - for block in blocks { - info!("block: {:?}", block); - } - - close_collaboration_server(child); - } - - #[test] - #[ignore = "not needed in ci"] - fn client_collaboration_with_server_with_poor_connection() { - let server_port = thread_rng().gen_range(30001..=65535); - let child = start_collaboration_server(server_port); - - let rt = Runtime::new().unwrap(); - let workspace_id = String::from("1"); - let (storage, mut workspace) = rt.block_on(async { - let storage: Arc = Arc::new( - JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) - .await - .expect("get storage: memory sqlite failed"), - ); - let workspace = storage - .docs() - .get_or_create_workspace(workspace_id.clone()) - .await - .expect("get workspace: {workspace_id} failed"); - (storage, workspace) - }); - - // simulate creating a block in offline environment - let block = create_block(&mut workspace, "0".to_string(), "list".to_string()); - info!("from client, create a block: {:?}", block); - info!( - "get block 0 from server: {}", - get_block_from_server(workspace_id.clone(), "0".to_string(), server_port) - ); - assert!(get_block_from_server(workspace_id.clone(), "0".to_string(), server_port).is_empty()); - - let rt = Arc::new(Runtime::new().unwrap()); - let (workspace_id, mut workspace) = { - let workspace_id = "1"; - let context = Arc::new(TestContext::new(storage)); - let remote = format!("ws://localhost:{server_port}/collaboration/1"); - - start_websocket_client_sync( - rt.clone(), - context.clone(), - Arc::default(), - remote, - workspace_id.to_owned(), - ); - - ( - workspace_id.to_owned(), - rt.block_on(async move { context.get_workspace(workspace_id).await.unwrap() }), - ) - }; - - info!("----------------start syncing from start_sync_thread()----------------"); - - for block_id in 1..3 { - let block = create_block(&mut workspace, block_id.to_string(), "list".to_string()); - info!("from client, create a block: {:?}", serde_json::to_string(&block)); - } - - let space = workspace.get_blocks().unwrap(); - let blocks = space.get_blocks_by_flavour("list"); - let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); - assert_eq!(ids.sort(), vec!["0", "1", "2"].sort()); - info!("blocks from local storage:"); - for block in blocks { - info!("block: {:?}", block); - } - - sleep(Duration::from_secs(1)); - info!("------------------after sync------------------"); - - for block_id in 0..3 { - let ret = get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port); - info!("get block {block_id} from server: {}", ret); - assert!(!ret.is_empty()); - } - - let space = workspace.get_blocks().unwrap(); - let blocks = space.get_blocks_by_flavour("list"); - let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); - assert_eq!(ids.sort(), vec!["0", "1", "2"].sort()); - info!("blocks from local storage:"); - for block in blocks { - info!("block: {:?}", block); - } - - close_collaboration_server(child); - } - - fn get_block_from_server(workspace_id: String, block_id: String, server_port: u16) -> String { - let rt = Runtime::new().unwrap(); - rt.block_on(async { - let client = reqwest::Client::new(); - let resp = client - .get(format!( - "http://localhost:{server_port}/api/block/{}/{}", - workspace_id, block_id - )) - .send() - .await - .unwrap(); - resp.text().await.unwrap() - }) - } - - fn create_block(workspace: &mut Workspace, block_id: String, block_flavour: String) -> Block { - let mut space = workspace.get_space("blocks").unwrap(); - space.create(block_id, block_flavour).expect("failed to create block") - } - - fn start_collaboration_server(port: u16) -> Child { - let mut child = Command::new("cargo") - .args(&["run", "-p", "keck-core"]) - .env("KECK_PORT", port.to_string()) - .env("USE_MEMORY_SQLITE", "true") - .env("KECK_CORE_LOG", "debug") - .stdout(Stdio::piped()) - .spawn() - .expect("Failed to run command"); - - if let Some(ref mut stdout) = child.stdout { - let reader = BufReader::new(stdout); - - for line in reader.lines() { - let line = line.expect("Failed to read line"); - info!("{}", line); - - if line.contains("listening on 0.0.0.0:") { - info!("Keck server started"); - break; - } - } - } - - child - } - - fn close_collaboration_server(child: Child) { - unsafe { kill(child.id() as c_int, SIGTERM) }; - } -} diff --git a/apps/keck-core/src/server/sync/mod.rs b/apps/keck-core/src/server/sync/mod.rs deleted file mode 100644 index fc9f94641..000000000 --- a/apps/keck-core/src/server/sync/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -mod blobs; -mod collaboration; - -use axum::routing::{get, post, put}; - -use super::*; - -pub fn sync_handler(router: Router) -> Router { - let router = if cfg!(feature = "api") { - router - } else { - router.nest( - "/api", - Router::new() - .route("/workspace/:id/blob", put(blobs::upload_blob_in_workspace)) - .route("/workspace/:id/blob/:name", get(blobs::get_blob_in_workspace)), - ) - } - .nest_service( - "/collaboration/:workspace", - post(collaboration::auth_handler).get(collaboration::upgrade_handler), - ); - - router -} diff --git a/apps/keck-core/src/server/utils.rs b/apps/keck-core/src/server/utils.rs deleted file mode 100644 index 596fa35a0..000000000 --- a/apps/keck-core/src/server/utils.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub use jwst_logger::{debug, error, info, warn}; -pub use nanoid::nanoid; -pub use serde::{Deserialize, Serialize}; diff --git a/apps/keck/Cargo.toml b/apps/keck/Cargo.toml index ad9d547ac..9f65afd96 100644 --- a/apps/keck/Cargo.toml +++ b/apps/keck/Cargo.toml @@ -7,20 +7,20 @@ license = "AGPL-3.0-only" [features] default = ["jwst"] -affine = ["jwst-storage/sqlite"] +affine = ["jwst-core-storage/sqlite"] jwst = [ "api", "schema", - "jwst-storage/sqlite", - "jwst-storage/mysql", - "jwst-storage/postgres", + "jwst-core-storage/sqlite", + "jwst-core-storage/mysql", + "jwst-core-storage/postgres", ] api = ["utoipa"] schema = ["utoipa-swagger-ui"] [dependencies] anyhow = "1.0.70" -axum = { version = "0.6.16", features = ["headers", "ws"] } +axum = { version = "0.6.20", features = ["headers", "ws"] } cfg-if = "1.0.0" futures = "0.3.28" lib0 = { version = "0.16.5", features = ["lib0-serde"] } @@ -43,27 +43,26 @@ tower = "0.4.13" tower-http = { version = "0.4.0", features = ["cors"] } thiserror = "1.0.40" time = "0.3.20" -tokio = { version = "1.27.0", features = [ +tokio = { version = "=1.28.0", features = [ "macros", "rt-multi-thread", "signal", ] } -utoipa = { version = "3.3.0", features = ["axum_extras"], optional = true } -utoipa-swagger-ui = { version = "3.1.3", features = ["axum"], optional = true } +utoipa = { version = "3.5.0", features = ["axum_extras"], optional = true } +utoipa-swagger-ui = { version = "3.1.5", optional = true } yrs = "0.16.5" -libc = "0.2.141" +libc = "0.2.147" rand = "0.8.5" -reqwest = { version = "0.11.14", default-features = false, features = [ +reqwest = { version = "0.11.19", default-features = false, features = [ "json", "rustls-tls", ] } # ======= workspace dependencies ======= -cloud-infra = { path = "../../libs/cloud-infra" } -jwst = { workspace = true, features = ["workspace-auto-subscribe"] } +jwst-core = { workspace = true } jwst-logger = { workspace = true } -jwst-rpc = { workspace = true } -jwst-storage = { workspace = true } +jwst-core-rpc = { workspace = true } +jwst-core-storage = { workspace = true } [dev-dependencies] -axum-test-helper = "0.2.0" +axum-test-helper = "0.3.0" diff --git a/apps/keck/build.rs b/apps/keck/build.rs deleted file mode 100644 index 2a52aa8be..000000000 --- a/apps/keck/build.rs +++ /dev/null @@ -1,17 +0,0 @@ -use std::{fs::create_dir_all, process::Command}; - -fn main() { - if !Command::new("pnpm") - .args(["i"]) - .status() - .map(|s| s.success()) - .unwrap_or(false) - || !Command::new("pnpm") - .args(["run", "build"]) - .status() - .map(|s| s.success()) - .unwrap_or(false) - { - create_dir_all("../homepage/out").unwrap(); - } -} diff --git a/apps/keck/src/main.rs b/apps/keck/src/main.rs index 277f1e0d8..53ea10e9b 100644 --- a/apps/keck/src/main.rs +++ b/apps/keck/src/main.rs @@ -12,6 +12,5 @@ async fn main() { // ignore load error when missing env file let _ = dotenvy::dotenv(); init_logger(PKG_NAME); - jwst::print_versions(PKG_NAME, env!("CARGO_PKG_VERSION")); server::start_server().await; } diff --git a/apps/keck/src/server/api/blobs.rs b/apps/keck/src/server/api/blobs.rs index 3672421c1..6fae433ad 100644 --- a/apps/keck/src/server/api/blobs.rs +++ b/apps/keck/src/server/api/blobs.rs @@ -1,6 +1,6 @@ use axum::{extract::BodyStream, response::Response, routing::post}; use futures::{future, StreamExt}; -use jwst::BlobStorage; +use jwst_core::BlobStorage; use utoipa::ToSchema; use super::*; @@ -178,7 +178,7 @@ mod tests { use axum::body::{Body, Bytes}; use axum_test_helper::TestClient; use futures::stream; - use jwst_storage::BlobStorageType; + use jwst_core_storage::BlobStorageType; use super::*; @@ -188,7 +188,6 @@ mod tests { JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) .await .ok(), - None, ) .await; let client = TestClient::new(blobs_apis(Router::new()).layer(Extension(Arc::new(ctx)))); diff --git a/apps/keck/src/server/api/blocks/block.rs b/apps/keck/src/server/api/blocks/block.rs index 2b8e97a6c..4b52caf22 100644 --- a/apps/keck/src/server/api/blocks/block.rs +++ b/apps/keck/src/server/api/blocks/block.rs @@ -1,6 +1,5 @@ use axum::{extract::Query, response::Response}; -use jwst::{constants, DocStorage}; -use lib0::any::Any; +use jwst_core::{constants, Any}; use serde_json::Value as JsonValue; use super::*; @@ -25,8 +24,12 @@ use super::*; pub async fn get_block(Extension(context): Extension>, Path(params): Path<(String, String)>) -> Response { let (ws_id, block) = params; info!("get_block: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(ws_id).await { - if let Some(block) = workspace.with_trx(|mut t| t.get_blocks().get(&t.trx, block)) { + if let Ok(space) = context + .get_workspace(ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if let Some(block) = space.get(block) { Json(block).into_response() } else { StatusCode::NOT_FOUND.into_response() @@ -68,66 +71,46 @@ pub async fn set_block( ) -> Response { let (ws_id, block_id) = params; info!("set_block: {}, {}", ws_id, block_id); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - let mut update = None; - if let Some(block) = workspace.with_trx(|mut t| { - let flavour = if let Some(query_map) = query_param { - query_map - .get("flavour") - .map_or_else(|| String::from("text"), |v| v.clone()) - } else { - String::from("text") - }; + if let Ok(mut space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + let flavour = if let Some(query_map) = query_param { + query_map + .get("flavour") + .map_or_else(|| String::from("text"), |v| v.clone()) + } else { + String::from("text") + }; - if let Ok(block) = t - .get_blocks() - .create(&mut t.trx, &block_id, flavour) - .map_err(|e| error!("failed to create block: {:?}", e)) - { - // set block content - if let Some(block_content) = payload.as_object() { - let mut changed = false; - for (key, value) in block_content.iter() { - if key == constants::sys::FLAVOUR { - continue; - } - changed = true; - if let Ok(value) = serde_json::from_value::(value.clone()) { - if let Err(e) = block.set(&mut t.trx, key, value.clone()) { - error!("failed to set block {} content: {}, {}, {:?}", block_id, key, value, e); - } - } + if let Ok(mut block) = space + .create(&block_id, flavour) + .map_err(|e| error!("failed to create block: {:?}", e)) + { + // set block content + if let Some(block_content) = payload.as_object() { + for (key, value) in block_content.iter() { + if key == constants::sys::FLAVOUR { + continue; } - if changed { - update = t.trx.encode_update_v1().ok(); + if let Ok(value) = serde_json::from_value::(value.clone()) { + if let Err(e) = block.set(key, value.clone()) { + error!( + "failed to set block {} content: {}, {:?}, {:?}", + block_id, key, value, e + ); + } } } - - Some(block) - } else { - None - } - }) { - if let Some(update) = update { - if let Err(e) = context - .storage - .docs() - .update_doc(ws_id, workspace.doc_guid().to_string(), &update) - .await - { - error!("db write error: {:?}", e); - } } // response block content - Json(block).into_response() - } else { - StatusCode::INTERNAL_SERVER_ERROR.into_response() + return Json(block).into_response(); } - } else { - StatusCode::NOT_FOUND.into_response() } + StatusCode::NOT_FOUND.into_response() } /// Get exists `Blocks` in certain `Workspace` by flavour @@ -154,15 +137,12 @@ pub async fn get_block_by_flavour( ) -> Response { let (ws_id, flavour) = params; info!("get_block_by_flavour: ws_id, {}, flavour, {}", ws_id, flavour); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - match workspace.try_with_trx(|mut trx| trx.get_blocks().get_blocks_by_flavour(&trx.trx, &flavour)) { - Some(blocks) => Json(blocks).into_response(), - None => ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Workspace({ws_id:?}) get transaction error"), - ) - .into_response(), - } + if let Ok(space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + Json(space.get_blocks_by_flavour(&flavour)).into_response() } else { (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() } @@ -191,14 +171,16 @@ pub async fn get_block_history( ) -> Response { let (ws_id, block) = params; info!("get_block_history: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - workspace.with_trx(|mut t| { - if let Some(block) = t.get_blocks().get(&t.trx, block) { - Json(&block.history(&t.trx)).into_response() - } else { - StatusCode::NOT_FOUND.into_response() - } - }) + if let Ok(space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if let Some(block) = space.get(block) { + Json(&block.history()).into_response() + } else { + StatusCode::NOT_FOUND.into_response() + } } else { StatusCode::NOT_FOUND.into_response() } @@ -227,22 +209,12 @@ pub async fn delete_block( ) -> StatusCode { let (ws_id, block) = params; info!("delete_block: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - if let Some(update) = workspace.with_trx(|mut t| { - if t.get_blocks().remove(&mut t.trx, &block) { - t.trx.encode_update_v1().ok() - } else { - None - } - }) { - if let Err(e) = context - .storage - .docs() - .update_doc(ws_id, workspace.doc_guid().to_string(), &update) - .await - { - error!("db write error: {:?}", e); - } + if let Ok(mut space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if space.remove(&block) { return StatusCode::NO_CONTENT; } } @@ -275,8 +247,12 @@ pub async fn get_block_children( let (ws_id, block) = params; let Pagination { offset, limit } = pagination; info!("get_block_children: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(ws_id).await { - if let Some(block) = workspace.with_trx(|mut t| t.get_blocks().get(&t.trx, &block)) { + if let Ok(space) = context + .get_workspace(ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if let Some(block) = space.get(&block) { let data: Vec = block.children_iter(|children| children.skip(offset).take(limit).collect()); let status = if data.is_empty() { @@ -285,20 +261,17 @@ pub async fn get_block_children( StatusCode::OK }; - ( + return ( status, Json(PageData { total: block.children_len() as usize, data, }), ) - .into_response() - } else { - StatusCode::NOT_FOUND.into_response() + .into_response(); } - } else { - StatusCode::NOT_FOUND.into_response() } + StatusCode::NOT_FOUND.into_response() } /// Insert a another `Block` into a `Block`'s children @@ -331,84 +304,54 @@ pub async fn insert_block_children( ) -> Response { let (ws_id, block) = params; info!("insert_block: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - let mut update = None; - - if let Some(block) = workspace.with_trx(|mut t| t.get_blocks().get(&t.trx, block)) { - if let Some(block) = workspace.with_trx(|mut t| { - let space = t.get_blocks(); - let mut changed = false; - match payload { - InsertChildren::Push(block_id) => { - if let Some(child) = space.get(&t.trx, block_id) { - changed = true; - if let Err(e) = block.push_children(&mut t.trx, &child) { - // TODO: handle error correctly - error!("failed to insert block: {:?}", e); - return None; - } + if let Ok(space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if let Some(mut block) = space.get(block) { + match payload { + InsertChildren::Push(block_id) => { + if let Some(mut child) = space.get(block_id) { + if let Err(e) = block.push_children(&mut child) { + error!("failed to insert block: {:?}", e); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } } - InsertChildren::InsertBefore { id, before } => { - if let Some(child) = space.get(&t.trx, id) { - changed = true; - if let Err(e) = block.insert_children_before(&mut t.trx, &child, &before) { - // TODO: handle error correctly - error!("failed to insert children before: {:?}", e); - return None; - } - } - } - InsertChildren::InsertAfter { id, after } => { - if let Some(child) = space.get(&t.trx, id) { - changed = true; - if let Err(e) = block.insert_children_after(&mut t.trx, &child, &after) { - // TODO: handle error correctly - error!("failed to insert children after: {:?}", e); - return None; - } + } + InsertChildren::InsertBefore { id, before } => { + if let Some(mut child) = space.get(id) { + if let Err(e) = block.insert_children_before(&mut child, &before) { + error!("failed to insert children before: {:?}", e); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } } - InsertChildren::InsertAt { id, pos } => { - if let Some(child) = space.get(&t.trx, id) { - changed = true; - if let Err(e) = block.insert_children_at(&mut t.trx, &child, pos) { - // TODO: handle error correctly - error!("failed to insert children at: {:?}", e); - return None; - } + } + InsertChildren::InsertAfter { id, after } => { + if let Some(mut child) = space.get(id) { + if let Err(e) = block.insert_children_after(&mut child, &after) { + // TODO: handle error correctly + error!("failed to insert children after: {:?}", e); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } } - }; - - if changed { - update = t.trx.encode_update_v1().ok(); } - - Some(block) - }) { - if let Some(update) = update { - if let Err(e) = context - .storage - .docs() - .update_doc(ws_id, workspace.doc_guid().to_string(), &update) - .await - { - error!("db write error: {:?}", e); + InsertChildren::InsertAt { id, pos } => { + if let Some(mut child) = space.get(id) { + if let Err(e) = block.insert_children_at(&mut child, pos) { + // TODO: handle error correctly + error!("failed to insert children at: {:?}", e); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } } } + }; - // response block content - Json(block).into_response() - } else { - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } - } else { - StatusCode::NOT_FOUND.into_response() + // response block content + return Json(block).into_response(); } - } else { - StatusCode::NOT_FOUND.into_response() } + StatusCode::NOT_FOUND.into_response() } /// Remove children in `Block` @@ -434,35 +377,24 @@ pub async fn remove_block_children( ) -> Response { let (ws_id, block, child_id) = params; info!("insert_block: {}, {}", ws_id, block); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - if let Some(update) = workspace.with_trx(|mut t| { - let space = t.get_blocks(); - if let Some(block) = space.get(&t.trx, &block) { - if block.children_exists(&t.trx, &child_id) { - if let Some(child) = space.get(&t.trx, &child_id) { - return block - .remove_children(&mut t.trx, &child) - .and_then(|_| Ok(t.trx.encode_update_v1()?)) - .ok(); + if let Ok(space) = context + .get_workspace(&ws_id) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + if let Some(mut block) = space.get(&block) { + if block.children_exists(&child_id) { + if let Some(mut child) = space.get(&child_id) { + if let Err(e) = block.remove_children(&mut child) { + error!("failed to remove block: {:?}", e); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } else { + // response block content + return Json(block).into_response(); } } } - None - }) { - if let Err(e) = context - .storage - .docs() - .update_doc(ws_id, workspace.doc_guid().to_string(), &update) - .await - { - error!("db write error: {:?}", e); - } - // response block content - Json(block).into_response() - } else { - StatusCode::NOT_FOUND.into_response() } - } else { - StatusCode::NOT_FOUND.into_response() } + StatusCode::NOT_FOUND.into_response() } diff --git a/apps/keck/src/server/api/blocks/mod.rs b/apps/keck/src/server/api/blocks/mod.rs index 9f7d22d29..8ace5264c 100644 --- a/apps/keck/src/server/api/blocks/mod.rs +++ b/apps/keck/src/server/api/blocks/mod.rs @@ -5,10 +5,7 @@ pub mod workspace; pub use block::{delete_block, get_block, get_block_history, insert_block_children, remove_block_children, set_block}; use schema::InsertChildren; pub use schema::SubscribeWorkspace; -pub use workspace::{ - delete_workspace, get_workspace, history_workspace, history_workspace_clients, set_workspace, subscribe_workspace, - workspace_client, -}; +pub use workspace::{delete_workspace, get_workspace, set_workspace, subscribe_workspace, workspace_client}; use super::*; @@ -21,19 +18,17 @@ fn block_apis(router: Router) -> Router { ) .route("/children/:children", delete(block::remove_block_children)); - doc_apis(router) - .nest("/block/:workspace/:block/", block_operation) - .route( - "/block/:workspace/:block", - get(block::get_block).post(block::set_block).delete(block::delete_block), - ) + doc_apis(router).nest("/block/:workspace/:block/", block_operation).route( + "/block/:workspace/:block", + get(block::get_block).post(block::set_block).delete(block::delete_block), + ) } fn workspace_apis(router: Router) -> Router { router .route("/block/:workspace/client", get(workspace::workspace_client)) - .route("/block/:workspace/history", get(workspace::history_workspace_clients)) - .route("/block/:workspace/history/:client", get(workspace::history_workspace)) + // .route("/block/:workspace/history", get(workspace::history_workspace_clients)) + // .route("/block/:workspace/history/:client", get(workspace::history_workspace)) .route( "/block/:workspace", get(workspace::get_workspace) @@ -42,11 +37,11 @@ fn workspace_apis(router: Router) -> Router { ) .route("/block/:workspace/flavour/:flavour", get(block::get_block_by_flavour)) .route("/block/:workspace/blocks", get(workspace::get_workspace_block)) - .route("/search/:workspace", get(workspace::workspace_search)) - .route( - "/search/:workspace/index", - get(workspace::get_search_index).post(workspace::set_search_index), - ) + // .route("/search/:workspace", get(workspace::workspace_search)) + // .route( + // "/search/:workspace/index", + // get(workspace::get_search_index).post(workspace::set_search_index), + // ) .route("/subscribe", post(subscribe_workspace)) } @@ -86,21 +81,13 @@ mod tests { .build() .expect("Failed to create runtime"), ); - let workspace_changed_blocks = Arc::new(RwLock::new(HashMap::::new())); let hook_endpoint = Arc::new(RwLock::new(String::new())); - let cb: WorkspaceRetrievalCallback = { - let workspace_changed_blocks = workspace_changed_blocks.clone(); - let runtime = runtime.clone(); - Some(Arc::new(Box::new(move |workspace: &Workspace| { - workspace.set_callback(generate_ws_callback(&workspace_changed_blocks, &runtime)); - }))) - }; + let ctx = Arc::new( Context::new( JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) .await .ok(), - cb, ) .await, ); @@ -109,7 +96,6 @@ mod tests { .layer(Extension(ctx.clone())) .layer(Extension(client.clone())) .layer(Extension(runtime.clone())) - .layer(Extension(workspace_changed_blocks.clone())) .layer(Extension(hook_endpoint.clone())), ); diff --git a/apps/keck/src/server/api/blocks/schema.rs b/apps/keck/src/server/api/blocks/schema.rs index c6507c771..c12ecf1c9 100644 --- a/apps/keck/src/server/api/blocks/schema.rs +++ b/apps/keck/src/server/api/blocks/schema.rs @@ -38,7 +38,7 @@ pub enum InsertChildren { Push(String), InsertBefore { id: String, before: String }, InsertAfter { id: String, after: String }, - InsertAt { id: String, pos: u32 }, + InsertAt { id: String, pos: u64 }, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] diff --git a/apps/keck/src/server/api/blocks/workspace.rs b/apps/keck/src/server/api/blocks/workspace.rs index 462ee9369..505ccaa4b 100644 --- a/apps/keck/src/server/api/blocks/workspace.rs +++ b/apps/keck/src/server/api/blocks/workspace.rs @@ -1,9 +1,8 @@ use axum::{ extract::{Path, Query}, - http::header, response::Response, }; -use jwst::{parse_history, parse_history_client, DocStorage}; +use jwst_core::DocStorage; use utoipa::IntoParams; use super::*; @@ -129,97 +128,84 @@ pub struct BlockSearchQuery { /// Search workspace blocks of server /// /// This will return back a list of relevant blocks. -#[utoipa::path( - get, - tag = "Workspace", - context_path = "/api/search", - path = "/{workspace}", - params( - ("workspace", description = "workspace id"), - BlockSearchQuery, - ), - responses( - (status = 200, description = "Search results", body = SearchResults), - ) -)] -pub async fn workspace_search( - Extension(context): Extension>, - Path(workspace): Path, - query: Query, -) -> Response { - let query_text = &query.query; - let ws_id = workspace; - info!("workspace_search: {ws_id:?} query = {query_text:?}"); - if let Ok(workspace) = context.get_workspace(&ws_id).await { - match workspace.search(query_text) { - Ok(list) => { - debug!("workspace_search: {ws_id:?} query = {query_text:?}; {list:#?}"); - Json(list).into_response() - } - Err(err) => { - error!("Internal server error calling workspace_search: {err:?}"); - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } - } - } else { - (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() - } -} +// #[utoipa::path( +// get, +// tag = "Workspace", +// context_path = "/api/search", +// path = "/{workspace}", +// params( +// ("workspace", description = "workspace id"), +// BlockSearchQuery, +// ), +// responses( +// (status = 200, description = "Search results", body = SearchResults), +// ) +// )] +// pub async fn workspace_search( +// Extension(context): Extension>, +// Path(workspace): Path, +// query: Query, +// ) -> Response { let query_text = &query.query; let ws_id = workspace; info!("workspace_search: {ws_id:?} query = +// {query_text:?}"); if let Ok(workspace) = context.get_workspace(&ws_id).await { match workspace.search(query_text) { +// Ok(list) => { debug!("workspace_search: {ws_id:?} query = {query_text:?}; {list:#?}"); Json(list).into_response() } +// Err(err) => { error!("Internal server error calling workspace_search: {err:?}"); +// StatusCode::INTERNAL_SERVER_ERROR.into_response() } } } else { (StatusCode::NOT_FOUND, +// format!("Workspace({ws_id:?}) not found")).into_response() } +// } -#[utoipa::path( - get, - tag = "Workspace", - context_path = "/api/search", - path = "/{workspace}/index", - params( - ("workspace", description = "workspace id"), - ), - responses( - (status = 200, description = "result", body = Vec), - (status = 404, description = "Workspace not found") - ) -)] -pub async fn get_search_index(Extension(context): Extension>, Path(workspace): Path) -> Response { - info!("get_search_index: {workspace:?}"); +// #[utoipa::path( +// get, +// tag = "Workspace", +// context_path = "/api/search", +// path = "/{workspace}/index", +// params( +// ("workspace", description = "workspace id"), +// ), +// responses( +// (status = 200, description = "result", body = Vec), +// (status = 404, description = "Workspace not found") +// ) +// )] +// pub async fn get_search_index(Extension(context): Extension>, Path(workspace): Path) -> Response +// { info!("get_search_index: {workspace:?}"); - if let Ok(workspace) = context.get_workspace(&workspace).await { - Json(workspace.metadata().search_index).into_response() - } else { - (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() - } -} +// if let Ok(workspace) = context.get_workspace(&workspace).await { +// Json(workspace.metadata().search_index).into_response() +// } else { +// (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() +// } +// } -#[utoipa::path( - post, - tag = "Workspace", - context_path = "/api/search", - path = "/{workspace}/index", - params( - ("workspace", description = "workspace id"), - ), - responses( - (status = 200, description = "success"), - (status = 400, description = "Bad Request"), - (status = 404, description = "Workspace not found") - ) -)] -pub async fn set_search_index( - Extension(context): Extension>, - Path(workspace): Path, - Json(fields): Json>, -) -> Response { - info!("set_search_index: {workspace:?} fields = {fields:?}"); +// #[utoipa::path( +// post, +// tag = "Workspace", +// context_path = "/api/search", +// path = "/{workspace}/index", +// params( +// ("workspace", description = "workspace id"), +// ), +// responses( +// (status = 200, description = "success"), +// (status = 400, description = "Bad Request"), +// (status = 404, description = "Workspace not found") +// ) +// )] +// pub async fn set_search_index( +// Extension(context): Extension>, +// Path(workspace): Path, +// Json(fields): Json>, +// ) -> Response { info!("set_search_index: {workspace:?} fields = {fields:?}"); - if let Ok(workspace) = context.get_workspace(&workspace).await { - if let Ok(true) = workspace.set_search_index(fields) { - StatusCode::OK.into_response() - } else { - StatusCode::BAD_REQUEST.into_response() - } - } else { - (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() - } -} +// if let Ok(workspace) = context.get_workspace(&workspace).await { +// if let Ok(true) = workspace.set_search_index(fields) { +// StatusCode::OK.into_response() +// } else { +// StatusCode::BAD_REQUEST.into_response() +// } +// } else { +// (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() +// } +// } /// Get `Block` in `Workspace` /// - Return 200 and `Block`'s ID. @@ -245,15 +231,13 @@ pub async fn get_workspace_block( ) -> Response { let Pagination { offset, limit } = pagination; info!("get_workspace_block: {workspace:?}"); - if let Ok(workspace) = context.get_workspace(&workspace).await { - let (total, data) = workspace.with_trx(|mut t| { - let space = t.get_blocks(); - - let total = space.block_count() as usize; - let data = space.blocks(&t.trx, |blocks| blocks.skip(offset).take(limit).collect::>()); - - (total, data) - }); + if let Ok(space) = context + .get_workspace(&workspace) + .await + .and_then(|mut ws| Ok(ws.get_blocks()?)) + { + let total = space.block_count() as usize; + let data = space.blocks(|blocks| blocks.skip(offset).take(limit).collect::>()); let status = if data.is_empty() { StatusCode::NOT_FOUND @@ -283,73 +267,55 @@ pub async fn get_workspace_block( /// created. This `Client` will not be destroyed until the server restarts. /// Therefore, the `Client ID` in the history generated by modifying `Block` /// through HTTP API will remain unchanged until the server restarts. -#[utoipa::path( - get, - tag = "Workspace", - context_path = "/api/block", - path = "/{workspace}/history", - params( - ("workspace", description = "workspace id"), - ), - responses( - (status = 200, description = "Get workspace history client ids", body = [u64]), - (status = 500, description = "Failed to get workspace history") - ) -)] -pub async fn history_workspace_clients( - Extension(context): Extension>, - Path(workspace): Path, -) -> Response { - if let Ok(workspace) = context.get_workspace(&workspace).await { - if let Some(history) = parse_history_client(&workspace.doc()) { - Json(history).into_response() - } else { - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } - } else { - (StatusCode::NOT_FOUND, format!("Workspace({workspace:?}) not found")).into_response() - } -} +// #[utoipa::path( +// get, +// tag = "Workspace", +// context_path = "/api/block", +// path = "/{workspace}/history", +// params( +// ("workspace", description = "workspace id"), +// ), +// responses( +// (status = 200, description = "Get workspace history client ids", body = [u64]), +// (status = 500, description = "Failed to get workspace history") +// ) +// )] +// pub async fn history_workspace_clients( +// Extension(context): Extension>, +// Path(workspace): Path, +// ) -> Response { if let Ok(workspace) = context.get_workspace(&workspace).await { if let Some(history) = +// parse_history_client(&workspace.doc()) { Json(history).into_response() } else { +// StatusCode::INTERNAL_SERVER_ERROR.into_response() } } else { (StatusCode::NOT_FOUND, +// format!("Workspace({workspace:?}) not found")).into_response() } +// } /// Get the history generated by a specific `Client ID` of the `Workspace` /// /// If client id set to 0, return all history of the `Workspace`. -#[utoipa::path( - get, - tag = "Workspace", - context_path = "/api/block", - path = "/{workspace}/history/{client}", - params( - ("workspace", description = "workspace id"), - ("client", description = "client id, is give 0 then return all clients histories"), - ), - responses( - (status = 200, description = "Get workspace history", body = [RawHistory]), - (status = 400, description = "Client id invalid"), - (status = 500, description = "Failed to get workspace history") - ) -)] -pub async fn history_workspace( - Extension(context): Extension>, - Path(params): Path<(String, String)>, -) -> Response { - let (ws_id, client) = params; - if let Ok(workspace) = context.get_workspace(&ws_id).await { - if let Ok(client) = client.parse::() { - if let Some(json) = - parse_history(&workspace.doc(), client).and_then(|history| serde_json::to_string(&history).ok()) - { - ([(header::CONTENT_TYPE, "application/json")], json).into_response() - } else { - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } - } else { - StatusCode::BAD_REQUEST.into_response() - } - } else { - (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() - } -} +// #[utoipa::path( +// get, +// tag = "Workspace", +// context_path = "/api/block", +// path = "/{workspace}/history/{client}", +// params( +// ("workspace", description = "workspace id"), +// ("client", description = "client id, is give 0 then return all clients histories"), +// ), +// responses( +// (status = 200, description = "Get workspace history", body = [RawHistory]), +// (status = 400, description = "Client id invalid"), +// (status = 500, description = "Failed to get workspace history") +// ) +// )] +// pub async fn history_workspace( +// Extension(context): Extension>, +// Path(params): Path<(String, String)>, +// ) -> Response { let (ws_id, client) = params; if let Ok(workspace) = context.get_workspace(&ws_id).await { if let +// Ok(client) = client.parse::() { if let Some(json) = parse_history(&workspace.doc(), client).and_then(|history| +// serde_json::to_string(&history).ok()) { ([(header::CONTENT_TYPE, "application/json")], json).into_response() } else +// { StatusCode::INTERNAL_SERVER_ERROR.into_response() } } else { StatusCode::BAD_REQUEST.into_response() } } else { +// (StatusCode::NOT_FOUND, format!("Workspace({ws_id:?}) not found")).into_response() } +// } /// Register a webhook for all block changes from all workspace changes #[utoipa::path( diff --git a/apps/keck/src/server/api/doc.rs b/apps/keck/src/server/api/doc.rs index 45d9d48df..ada123c65 100644 --- a/apps/keck/src/server/api/doc.rs +++ b/apps/keck/src/server/api/doc.rs @@ -1,5 +1,6 @@ -use cloud_infra::with_api_doc; -use utoipa::OpenApi; +use utoipa::{openapi::License, OpenApi}; +#[cfg(feature = "schema")] +use utoipa_swagger_ui::{serve, Config, Url}; use super::{ blobs, @@ -14,12 +15,12 @@ use super::{ workspace::set_workspace, workspace::delete_workspace, workspace::workspace_client, - workspace::history_workspace_clients, - workspace::history_workspace, + // workspace::history_workspace_clients, + // workspace::history_workspace, workspace::get_workspace_block, - workspace::workspace_search, - workspace::set_search_index, - workspace::get_search_index, + // workspace::workspace_search, + // workspace::set_search_index, + // workspace::get_search_index, workspace::subscribe_workspace, block::get_block, block::get_block_by_flavour, @@ -37,9 +38,9 @@ use super::{ components( schemas( blobs::BlobStatus, schema::InsertChildren, - schema::Workspace, schema::Block, schema::BlockRawHistory, schema::SubscribeWorkspace, - jwst::BlockHistory, jwst::HistoryOperation, jwst::RawHistory, - jwst::SearchResults, jwst::SearchResult, + schema::Workspace, schema::Block, schema::BlockRawHistory, schema::SubscribeWorkspace + // jwst::BlockHistory, jwst::HistoryOperation, jwst::RawHistory, + // jwst::SearchResults, jwst::SearchResult, ) ), tags( @@ -52,12 +53,41 @@ struct ApiDoc; const README: &str = include_str!("../../../../homepage/pages/docs/introduction.md"); const DISTINCTIVE_FEATURES: &str = include_str!("../../../../homepage/pages/docs/overview/distinctive_features.md"); +#[cfg(feature = "schema")] +async fn serve_swagger_ui( + tail: Option>, + Extension(state): Extension>>, +) -> impl IntoResponse { + match serve(&tail.map(|p| p.to_string()).unwrap_or("".into()), state) { + Ok(file) => file + .map(|file| (StatusCode::OK, [("Content-Type", file.content_type)], file.bytes).into_response()) + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()), + Err(error) => (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(), + } +} + pub fn doc_apis(router: Router) -> Router { - if cfg!(feature = "schema") { + #[cfg(feature = "schema")] + { let mut openapi = ApiDoc::openapi(); openapi.info.description = Some(vec![README, DISTINCTIVE_FEATURES].join("\n")); - with_api_doc(router, openapi, "jwst") - } else { + + let name = "jwst"; + if cfg!(debug_assertions) || std::env::var("JWST_DEV").is_ok() { + let config = Url::from(format!("/api/{name}.json")); + let config = Config::new(vec![config]); + openapi.info.license = Some(License::new(env!("CARGO_PKG_LICENSE"))); + router + .route(&format!("/{name}.json"), get(move || async { Json(openapi) })) + .route("/docs/", get(serve_swagger_ui)) + .route("/docs/*tail", get(serve_swagger_ui)) + .layer(Extension(Arc::new(config))) + } else { + router + } + } + #[cfg(not(feature = "schema"))] + { router } } diff --git a/apps/keck/src/server/api/mod.rs b/apps/keck/src/server/api/mod.rs index a770acb39..99e0acb8a 100644 --- a/apps/keck/src/server/api/mod.rs +++ b/apps/keck/src/server/api/mod.rs @@ -6,7 +6,6 @@ mod doc; use std::collections::HashMap; -use anyhow::Context as AnyhowContext; use axum::Router; #[cfg(feature = "api")] use axum::{ @@ -16,8 +15,8 @@ use axum::{ routing::{delete, get, head, post}, }; use doc::doc_apis; -use jwst_rpc::{BroadcastChannels, RpcContextImpl}; -use jwst_storage::{BlobStorageType, JwstStorage, JwstStorageResult, MixedBucketDBParam}; +use jwst_core_rpc::{BroadcastChannels, RpcContextImpl}; +use jwst_core_storage::{BlobStorageType, JwstStorage, JwstStorageResult}; use tokio::sync::RwLock; use super::*; @@ -44,24 +43,11 @@ pub struct PageData { pub struct Context { channel: BroadcastChannels, storage: JwstStorage, - callback: WorkspaceRetrievalCallback, } impl Context { - pub async fn new(storage: Option, cb: WorkspaceRetrievalCallback) -> Self { - let use_bucket_storage = dotenvy::var("ENABLE_BUCKET_STORAGE").map_or(false, |v| v.eq("true")); - - let blob_storage_type = if use_bucket_storage { - info!("use database and s3 bucket as blob storage"); - BlobStorageType::MixedBucketDB( - MixedBucketDBParam::new_from_env() - .context("failed to load bucket param from env") - .unwrap(), - ) - } else { - info!("use database as blob storage"); - BlobStorageType::DB - }; + pub async fn new(storage: Option) -> Self { + let blob_storage_type = BlobStorageType::DB; let storage = if let Some(storage) = storage { info!("use external storage instance: {}", storage.database()); @@ -81,7 +67,6 @@ impl Context { Context { channel: RwLock::new(HashMap::new()), storage, - callback: cb, } } @@ -89,24 +74,14 @@ impl Context { where S: AsRef, { - let workspace = self.storage.get_workspace(workspace_id).await?; - if let Some(cb) = self.callback.clone() { - cb(&workspace); - } - - Ok(workspace) + self.storage.get_workspace(workspace_id).await } pub async fn create_workspace(&self, workspace_id: S) -> JwstStorageResult where S: AsRef, { - let workspace = self.storage.create_workspace(workspace_id).await?; - if let Some(cb) = self.callback.clone() { - cb(&workspace); - } - - Ok(workspace) + self.storage.create_workspace(workspace_id).await } } diff --git a/apps/keck/src/server/files.rs b/apps/keck/src/server/files.rs deleted file mode 100644 index 31e230547..000000000 --- a/apps/keck/src/server/files.rs +++ /dev/null @@ -1,30 +0,0 @@ -use axum::{ - body::BoxBody, - http::{Response, Uri}, - response::IntoResponse, - routing::get, -}; -use cloud_infra::{fetch_static_response, rust_embed, RustEmbed}; - -use super::*; - -#[derive(RustEmbed)] -#[folder = "../homepage/out/"] -#[include = "*"] -#[exclude = "*.txt"] -#[exclude = "*.map"] -struct Frontend; - -async fn frontend_handler(uri: Uri) -> Response { - fetch_static_response(uri.clone(), true, Some(Frontend::get)) - .await - .into_response() -} - -pub fn static_files(router: Router) -> Router { - if cfg!(debug_assertions) { - router - } else { - router.fallback_service(get(frontend_handler)) - } -} diff --git a/apps/keck/src/server/mod.rs b/apps/keck/src/server/mod.rs index 4d8ba28f8..f11540ae5 100644 --- a/apps/keck/src/server/mod.rs +++ b/apps/keck/src/server/mod.rs @@ -1,15 +1,12 @@ mod api; -mod files; -mod subscribe; mod sync; mod utils; -use std::{collections::HashMap, net::SocketAddr, sync::Arc, thread::sleep}; +use std::{net::SocketAddr, sync::Arc}; use api::Context; use axum::{http::Method, Extension, Router, Server}; -use jwst::Workspace; -pub use subscribe::*; +use jwst_core::Workspace; use tokio::{runtime, signal, sync::RwLock}; use tower_http::cors::{Any, CorsLayer}; pub use utils::*; @@ -38,29 +35,6 @@ async fn shutdown_signal() { info!("Shutdown signal received, starting graceful shutdown"); } -type WorkspaceRetrievalCallback = Option>>; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WorkspaceChangedBlocks { - #[serde(rename(serialize = "workspaceId"))] - pub workspace_id: String, - #[serde(rename(serialize = "blockIds"))] - pub block_ids: Vec, -} - -impl WorkspaceChangedBlocks { - pub fn new(workspace_id: String) -> WorkspaceChangedBlocks { - WorkspaceChangedBlocks { - workspace_id, - block_ids: Vec::new(), - } - } - - pub fn insert_block_ids(&mut self, mut updated_block_ids: Vec) { - self.block_ids.append(&mut updated_block_ids); - } -} - pub async fn start_server() { let origins = [ "http://localhost:4200".parse().unwrap(), @@ -87,30 +61,15 @@ pub async fn start_server() { .build() .expect("Failed to create runtime"), ); - let workspace_changed_blocks = Arc::new(RwLock::new(HashMap::::new())); let hook_endpoint = Arc::new(RwLock::new(dotenvy::var("HOOK_ENDPOINT").unwrap_or_default())); - let cb: WorkspaceRetrievalCallback = { - let workspace_changed_blocks = workspace_changed_blocks.clone(); - let runtime = runtime.clone(); - Some(Arc::new(Box::new(move |workspace: &Workspace| { - workspace.set_callback(generate_ws_callback(&workspace_changed_blocks, &runtime)); - }))) - }; - let context = Arc::new(Context::new(None, cb).await); - start_handling_observed_blocks( - runtime.clone(), - workspace_changed_blocks.clone(), - hook_endpoint.clone(), - client.clone(), - ); + let context = Arc::new(Context::new(None).await); - let app = files::static_files(sync::sync_handler(api::api_handler(Router::new()))) + let app = sync::sync_handler(api::api_handler(Router::new())) .layer(cors) .layer(Extension(context.clone())) .layer(Extension(client)) .layer(Extension(runtime)) - .layer(Extension(workspace_changed_blocks)) .layer(Extension(hook_endpoint)); let addr = SocketAddr::from(( diff --git a/apps/keck/src/server/subscribe.rs b/apps/keck/src/server/subscribe.rs deleted file mode 100644 index 766d48e0c..000000000 --- a/apps/keck/src/server/subscribe.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::{collections::HashMap, sync::Arc, thread, time::Duration}; - -use reqwest::Client; -use tokio::{runtime::Runtime, sync::RwLock}; - -use super::*; -use crate::server::WorkspaceChangedBlocks; - -pub fn generate_ws_callback( - workspace_changed_blocks: &Arc>>, - runtime: &Arc, -) -> Arc) + Send + Sync>> { - let workspace_changed_blocks = workspace_changed_blocks.clone(); - let runtime = runtime.clone(); - Arc::new(Box::new(move |workspace_id, block_ids| { - let workspace_changed_blocks = workspace_changed_blocks.clone(); - runtime.spawn(async move { - let mut write_guard = workspace_changed_blocks.write().await; - write_guard - .entry(workspace_id.clone()) - .or_insert(WorkspaceChangedBlocks::new(workspace_id.clone())) - .insert_block_ids(block_ids.clone()); - }); - })) -} - -pub fn start_handling_observed_blocks( - runtime: Arc, - workspace_changed_blocks: Arc>>, - hook_endpoint: Arc>, - client: Arc, -) { - thread::spawn(move || loop { - let workspace_changed_blocks = workspace_changed_blocks.clone(); - let hook_endpoint = hook_endpoint.clone(); - let runtime_cloned = runtime.clone(); - let client = client.clone(); - runtime.spawn(async move { - let read_guard = workspace_changed_blocks.read().await; - if !read_guard.is_empty() { - let endpoint = hook_endpoint.read().await; - if !endpoint.is_empty() { - { - let workspace_changed_blocks_clone = read_guard.clone(); - let post_body = workspace_changed_blocks_clone - .into_values() - .collect::>(); - let endpoint = endpoint.clone(); - runtime_cloned.spawn(async move { - let response = client.post(endpoint.to_string()).json(&post_body).send().await; - match response { - Ok(response) => info!( - "notified hook endpoint, endpoint response status: {}", - response.status() - ), - Err(e) => error!("Failed to send notify: {}", e), - } - }); - } - } - drop(read_guard); - let mut write_guard = workspace_changed_blocks.write().await; - info!("workspace_changed_blocks: {:?}", write_guard); - write_guard.clear(); - } - }); - sleep(Duration::from_millis(200)); - }); -} diff --git a/apps/keck/src/server/sync/blobs.rs b/apps/keck/src/server/sync/blobs.rs index 7d66a2bf5..f893ef4a3 100644 --- a/apps/keck/src/server/sync/blobs.rs +++ b/apps/keck/src/server/sync/blobs.rs @@ -11,8 +11,8 @@ use axum::{ Json, TypedHeader, }; use futures::{future, StreamExt}; -use jwst::BlobStorage; -use jwst_rpc::RpcContextImpl; +use jwst_core::BlobStorage; +use jwst_core_rpc::RpcContextImpl; use time::{format_description::well_known::Rfc2822, OffsetDateTime}; use super::*; diff --git a/apps/keck/src/server/sync/collaboration.rs b/apps/keck/src/server/sync/collaboration.rs index 2dbb5e097..a56a8c568 100644 --- a/apps/keck/src/server/sync/collaboration.rs +++ b/apps/keck/src/server/sync/collaboration.rs @@ -1,19 +1,13 @@ use std::sync::Arc; -#[cfg(feature = "websocket")] -use axum::extract; use axum::{ extract::{ws::WebSocketUpgrade, Path}, response::Response, Json, }; use futures::FutureExt; -use jwst_rpc::{axum_socket_connector, handle_connector}; -#[cfg(feature = "websocket")] -use jwst_rpc::{webrtc_datachannel_server_connector, RTCSessionDescription}; +use jwst_core_rpc::{axum_socket_connector, handle_connector}; use serde::Serialize; -#[cfg(feature = "websocket")] -use tokio::sync::mpsc::channel; use super::*; @@ -46,37 +40,6 @@ pub async fn upgrade_handler( }) } -#[cfg(feature = "webrtc")] -pub async fn webrtc_handler( - Extension(context): Extension>, - Path(workspace): Path, - extract::Json(offer): extract::Json, -) -> Json { - let (answer, tx, rx, _) = webrtc_datachannel_server_connector(offer).await; - - let (first_init_tx, mut first_init_rx) = channel::(10); - let workspace_id = workspace.to_owned(); - tokio::spawn(async move { - while let Some(time) = first_init_rx.recv().await { - if time > 0 { - info!("socket sync success: {}", workspace_id); - } else { - error!("socket sync failed: {}", workspace_id); - } - } - }); - - let identifier = nanoid!(); - tokio::spawn(async move { - handle_connector(context.clone(), workspace.clone(), identifier, move || { - (tx, rx, first_init_tx) - }) - .await; - }); - - Json(answer) -} - #[cfg(test)] mod test { use std::{ @@ -85,14 +48,14 @@ mod test { process::{Child, Command, Stdio}, string::String, sync::Arc, + thread::sleep, + time::Duration, }; - use jwst::{Block, DocStorage, Workspace}; + use jwst_core::{Block, DocStorage, Workspace}; + use jwst_core_rpc::{start_websocket_client_sync, BroadcastChannels, RpcContextImpl}; + use jwst_core_storage::{BlobStorageType, JwstStorage}; use jwst_logger::info; - #[cfg(feature = "websocket")] - use jwst_rpc::start_webrtc_client_sync; - use jwst_rpc::{start_websocket_client_sync, BroadcastChannels, RpcContextImpl}; - use jwst_storage::{BlobStorageType, JwstStorage}; use libc::{kill, SIGTERM}; use rand::{thread_rng, Rng}; use tokio::runtime::Runtime; @@ -120,76 +83,6 @@ mod test { &self.channel } } - #[test] - #[ignore = "not needed in ci"] - #[cfg(feature = "webrtc")] - fn client_collaboration_with_webrtc_server() { - if dotenvy::var("KECK_DEBUG").is_ok() { - jwst_logger::init_logger("keck"); - } - - // TODO: - // there has a weird question: - // use new terminal run keck is pass - // use `start_collaboration_server` funtion is high probability block - - //let server_port = 3000; - let server_port = thread_rng().gen_range(10000..=30000); - let child = start_collaboration_server(server_port); - - let rt = Runtime::new().unwrap(); - let (workspace_id, workspace) = rt.block_on(async move { - let workspace_id = "1"; - let context = Arc::new(TestContext::new(Arc::new( - JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) - .await - .expect("get storage: memory sqlite failed"), - ))); - let remote = format!("http://localhost:{server_port}/webrtc-sdp/1"); - - start_webrtc_client_sync( - Arc::new(Runtime::new().unwrap()), - context.clone(), - Arc::default(), - remote, - workspace_id.to_owned(), - ); - - ( - workspace_id.to_owned(), - context.get_workspace(workspace_id).await.unwrap(), - ) - }); - - for block_id in 0..3 { - let block = create_block(&workspace, block_id.to_string(), "list".to_string()); - info!("from client, create a block: {:?}", block); - } - - info!("------------------after sync------------------"); - std::thread::sleep(std::time::Duration::from_secs(1)); - - for block_id in 0..3 { - info!( - "get block {block_id} from server: {}", - get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port) - ); - assert!(!get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port).is_empty()); - } - - workspace.with_trx(|mut trx| { - let space = trx.get_space("blocks"); - let blocks = space.get_blocks_by_flavour(&trx.trx, "list"); - let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); - assert_eq!(ids.sort(), vec!["7", "8", "9"].sort()); - info!("blocks from local storage:"); - for block in blocks { - info!("block: {:?}", block); - } - }); - - close_collaboration_server(child); - } #[test] #[ignore = "not needed in ci"] @@ -201,18 +94,20 @@ mod test { let server_port = thread_rng().gen_range(10000..=30000); let child = start_collaboration_server(server_port); - let rt = Runtime::new().unwrap(); - let (workspace_id, workspace) = rt.block_on(async move { + let rt = Arc::new(Runtime::new().unwrap()); + let (workspace_id, mut workspace) = { let workspace_id = "1"; - let context = Arc::new(TestContext::new(Arc::new( - JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) - .await - .expect("get storage: memory sqlite failed"), - ))); + let context = rt.block_on(async move { + Arc::new(TestContext::new(Arc::new( + JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) + .await + .expect("get storage: memory sqlite failed"), + ))) + }); let remote = format!("ws://localhost:{server_port}/collaboration/1"); start_websocket_client_sync( - Arc::new(Runtime::new().unwrap()), + rt.clone(), context.clone(), Arc::default(), remote, @@ -221,35 +116,32 @@ mod test { ( workspace_id.to_owned(), - context.get_workspace(workspace_id).await.unwrap(), + rt.block_on(async move { context.get_workspace(workspace_id).await.unwrap() }), ) - }); + }; for block_id in 0..3 { - let block = create_block(&workspace, block_id.to_string(), "list".to_string()); - info!("from client, create a block: {:?}", block); + let block = create_block(&mut workspace, block_id.to_string(), "list".to_string()); + info!("from client, create a block:{:?}", serde_json::to_string(&block)); } + sleep(Duration::from_secs(1)); info!("------------------after sync------------------"); for block_id in 0..3 { - info!( - "get block {block_id} from server: {}", - get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port) - ); - assert!(!get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port).is_empty()); + let ret = get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port); + info!("get block {block_id} from server: {ret}"); + assert!(!ret.is_empty()); } - workspace.with_trx(|mut trx| { - let space = trx.get_space("blocks"); - let blocks = space.get_blocks_by_flavour(&trx.trx, "list"); - let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); - assert_eq!(ids.sort(), vec!["7", "8", "9"].sort()); - info!("blocks from local storage:"); - for block in blocks { - info!("block: {:?}", block); - } - }); + let space = workspace.get_space("blocks").unwrap(); + let blocks = space.get_blocks_by_flavour("list"); + let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); + assert_eq!(ids.sort(), vec!["7", "8", "9"].sort()); + info!("blocks from local storage:"); + for block in blocks { + info!("block: {:?}", block); + } close_collaboration_server(child); } @@ -262,7 +154,7 @@ mod test { let rt = Runtime::new().unwrap(); let workspace_id = String::from("1"); - let (storage, workspace) = rt.block_on(async { + let (storage, mut workspace) = rt.block_on(async { let storage: Arc = Arc::new( JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB) .await @@ -277,7 +169,7 @@ mod test { }); // simulate creating a block in offline environment - let block = create_block(&workspace, "0".to_string(), "list".to_string()); + let block = create_block(&mut workspace, "0".to_string(), "list".to_string()); info!("from client, create a block: {:?}", block); info!( "get block 0 from server: {}", @@ -285,14 +177,14 @@ mod test { ); assert!(get_block_from_server(workspace_id.clone(), "0".to_string(), server_port).is_empty()); - let rt = Runtime::new().unwrap(); - let (workspace_id, workspace) = rt.block_on(async move { + let rt = Arc::new(Runtime::new().unwrap()); + let (workspace_id, mut workspace) = { let workspace_id = "1"; let context = Arc::new(TestContext::new(storage)); let remote = format!("ws://localhost:{server_port}/collaboration/1"); start_websocket_client_sync( - Arc::new(Runtime::new().unwrap()), + rt.clone(), context.clone(), Arc::default(), remote, @@ -301,53 +193,43 @@ mod test { ( workspace_id.to_owned(), - context.get_workspace(workspace_id).await.unwrap(), + rt.block_on(async move { context.get_workspace(workspace_id).await.unwrap() }), ) - }); + }; info!("----------------start syncing from start_sync_thread()----------------"); for block_id in 1..3 { - let block = create_block(&workspace, block_id.to_string(), "list".to_string()); - info!("from client, create a block: {:?}", block); - info!( - "get block {block_id} from server: {}", - get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port) - ); - assert!(!get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port).is_empty()); + let block = create_block(&mut workspace, block_id.to_string(), "list".to_string()); + info!("from client, create a block: {:?}", serde_json::to_string(&block)); } - workspace.with_trx(|mut trx| { - let space = trx.get_space("blocks"); - let blocks = space.get_blocks_by_flavour(&trx.trx, "list"); - let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); - assert_eq!(ids.sort(), vec!["0", "1", "2"].sort()); - info!("blocks from local storage:"); - for block in blocks { - info!("block: {:?}", block); - } - }); + let space = workspace.get_blocks().unwrap(); + let blocks = space.get_blocks_by_flavour("list"); + let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); + assert_eq!(ids.sort(), vec!["0", "1", "2"].sort()); + info!("blocks from local storage:"); + for block in blocks { + info!("block: {:?}", block); + } + sleep(Duration::from_secs(1)); info!("------------------after sync------------------"); for block_id in 0..3 { - info!( - "get block {block_id} from server: {}", - get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port) - ); - assert!(!get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port).is_empty()); + let ret = get_block_from_server(workspace_id.clone(), block_id.to_string(), server_port); + info!("get block {block_id} from server: {}", ret); + assert!(!ret.is_empty()); } - workspace.with_trx(|mut trx| { - let space = trx.get_space("blocks"); - let blocks = space.get_blocks_by_flavour(&trx.trx, "list"); - let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); - assert_eq!(ids.sort(), vec!["0", "1", "2"].sort()); - info!("blocks from local storage:"); - for block in blocks { - info!("block: {:?}", block); - } - }); + let space = workspace.get_blocks().unwrap(); + let blocks = space.get_blocks_by_flavour("list"); + let mut ids: Vec<_> = blocks.iter().map(|block| block.block_id()).collect(); + assert_eq!(ids.sort(), vec!["0", "1", "2"].sort()); + info!("blocks from local storage:"); + for block in blocks { + info!("block: {:?}", block); + } close_collaboration_server(child); } @@ -368,13 +250,9 @@ mod test { }) } - fn create_block(workspace: &Workspace, block_id: String, block_flavour: String) -> Block { - workspace.with_trx(|mut trx| { - let space = trx.get_space("blocks"); - space - .create(&mut trx.trx, block_id, block_flavour) - .expect("failed to create block") - }) + fn create_block(workspace: &mut Workspace, block_id: String, block_flavour: String) -> Block { + let mut space = workspace.get_space("blocks").unwrap(); + space.create(block_id, block_flavour).expect("failed to create block") } fn start_collaboration_server(port: u16) -> Child { @@ -382,7 +260,7 @@ mod test { .args(&["run", "-p", "keck"]) .env("KECK_PORT", port.to_string()) .env("USE_MEMORY_SQLITE", "true") - .env("KECK_LOG", "INFO") + .env("KECK_LOG", "debug") .stdout(Stdio::piped()) .spawn() .expect("Failed to run command"); diff --git a/apps/keck/src/server/sync/mod.rs b/apps/keck/src/server/sync/mod.rs index 92cd59598..fc9f94641 100644 --- a/apps/keck/src/server/sync/mod.rs +++ b/apps/keck/src/server/sync/mod.rs @@ -21,11 +21,5 @@ pub fn sync_handler(router: Router) -> Router { post(collaboration::auth_handler).get(collaboration::upgrade_handler), ); - #[cfg(feature = "webrtc")] - { - router.nest_service("/webrtc-sdp/:workspace", post(collaboration::webrtc_handler)) - } - - #[cfg(not(feature = "webrtc"))] router } diff --git a/y-octo-utils/bin/bench_result_render.rs b/y-octo-utils/bin/bench_result_render.rs deleted file mode 100644 index dad0195d5..000000000 --- a/y-octo-utils/bin/bench_result_render.rs +++ /dev/null @@ -1,112 +0,0 @@ -#![allow(unused)] - -use std::collections::HashMap; - -fn process_duration(duration: &str) -> (f64, f64) { - let dur_split: Vec = duration.split("±").map(String::from).collect(); - let units = dur_split[1].chars().skip_while(|c| c.is_digit(10)).collect::(); - let dur_secs = convert_dur_to_seconds(dur_split[0].parse::().unwrap(), &units); - let error_secs = convert_dur_to_seconds( - dur_split[1] - .chars() - .take_while(|c| c.is_digit(10)) - .collect::() - .parse::() - .unwrap(), - &units, - ); - (dur_secs, error_secs) -} - -fn convert_dur_to_seconds(dur: f64, units: &str) -> f64 { - let factors = [ - ("s", 1.0), - ("ms", 1.0 / 1000.0), - ("µs", 1.0 / 1_000_000.0), - ("ns", 1.0 / 1_000_000_000.0), - ] - .iter() - .cloned() - .collect::>(); - dur * factors.get(units).unwrap_or(&1.0) -} - -#[allow(dead_code)] -fn is_significant(changes_dur: f64, changes_err: f64, base_dur: f64, base_err: f64) -> bool { - if changes_dur < base_dur { - changes_dur + changes_err < base_dur || base_dur - base_err > changes_dur - } else { - changes_dur - changes_err > base_dur || base_dur + base_err < changes_dur - } -} - -#[cfg(feature = "bench")] -fn convert_to_markdown() -> impl Iterator { - let re = regex::Regex::new(r"\s{2,}").unwrap(); - std::io::stdin() - .lines() - .skip(2) - .map(|l| l.ok()) - .map(move |row| { - if let Some(row) = row { - let columns = re.split(&row).collect::>(); - let name = columns.get(0)?; - let base_duration = columns.get(2)?; - let changes_duration = columns.get(5)?; - Some(( - name.to_string(), - base_duration.to_string(), - changes_duration.to_string(), - )) - } else { - None - } - }) - .flatten() - .map(|(name, base_duration, changes_duration)| { - let mut difference = "N/A".to_string(); - let base_undefined = base_duration == "?"; - let changes_undefined = changes_duration == "?"; - - if !base_undefined && !changes_undefined { - let (base_dur_secs, base_err_secs) = process_duration(&base_duration); - let (changes_dur_secs, changes_err_secs) = process_duration(&changes_duration); - - let diff = -(1.0 - changes_dur_secs / base_dur_secs) * 100.0; - difference = format!("{:+.2}%", diff); - - if is_significant(changes_dur_secs, changes_err_secs, base_dur_secs, base_err_secs) { - difference = format!("**{}**", difference); - } - } - - format!( - "| {} | {} | {} | {} |", - name.replace("|", "\\|"), - if base_undefined { "N/A" } else { &base_duration }, - if changes_undefined { "N/A" } else { &changes_duration }, - difference - ) - }) -} - -fn main() { - let platform = std::env::args().into_iter().skip(1).next().unwrap(); - - #[cfg(feature = "bench")] - for ret in [ - &format!("## Benchmark for {platform}"), - "
", - " Click to view benchmark", - "", - "| Test | Base | PR | % |", - "| --- | --- | --- | --- |", - ] - .into_iter() - .map(|s| s.to_owned()) - .chain(convert_to_markdown()) - .chain(["
".to_owned()]) - { - println!("{}", ret); - } -} From 600d539aa20fb06f95de48d65201c225f63454c0 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 28 Aug 2023 16:06:56 +0800 Subject: [PATCH 26/49] chore: adjust debug config --- .vscode/launch.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 37d715ec8..26d4b6e5f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -26,11 +26,11 @@ { "type": "lldb", "request": "launch", - "name": "Debug executable 'keck-core'", + "name": "Debug executable 'keck'", "cargo": { - "args": ["build", "--bin=keck-core", "--package=keck-core"], + "args": ["build", "--bin=keck", "--package=keck"], "filter": { - "name": "keck-core", + "name": "keck", "kind": "bin" } }, From e80fdc5432aeac4dec8b92a7650530f952f244ad Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 28 Aug 2023 16:13:40 +0800 Subject: [PATCH 27/49] fix: keck test --- apps/keck/src/server/api/blocks/mod.rs | 50 +++++++++++++------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/apps/keck/src/server/api/blocks/mod.rs b/apps/keck/src/server/api/blocks/mod.rs index 8ace5264c..8941268e2 100644 --- a/apps/keck/src/server/api/blocks/mod.rs +++ b/apps/keck/src/server/api/blocks/mod.rs @@ -18,10 +18,12 @@ fn block_apis(router: Router) -> Router { ) .route("/children/:children", delete(block::remove_block_children)); - doc_apis(router).nest("/block/:workspace/:block/", block_operation).route( - "/block/:workspace/:block", - get(block::get_block).post(block::set_block).delete(block::delete_block), - ) + doc_apis(router) + .nest("/block/:workspace/:block/", block_operation) + .route( + "/block/:workspace/:block", + get(block::get_block).post(block::set_block).delete(block::delete_block), + ) } fn workspace_apis(router: Router) -> Router { @@ -109,8 +111,8 @@ mod tests { resp.text().await.parse::().unwrap(), ctx.storage.get_workspace("test").await.unwrap().client_id() ); - let resp = client.get("/block/test/history").send().await; - assert_eq!(resp.json::>().await, Vec::::new()); + // let resp = client.get("/block/test/history").send().await; + // assert_eq!(resp.json::>().await, Vec::::new()); let resp = client.get("/block/test").send().await; assert_eq!(resp.status(), StatusCode::OK); let resp = client.delete("/block/test").send().await; @@ -122,24 +124,24 @@ mod tests { let resp = client.post("/block/test").send().await; assert_eq!(resp.status(), StatusCode::OK); - let resp = client.get("/search/test/index").send().await; - assert_eq!(resp.status(), StatusCode::OK); - let index = resp.json::>().await; - assert_eq!(index, vec!["title".to_owned(), "text".to_owned()]); - - let body = to_string(&json!(["test"])).unwrap(); - let resp = client - .post("/search/test/index") - .header("content-type", "application/json") - .body(body) - .send() - .await; - assert_eq!(resp.status(), StatusCode::OK); - - let resp = client.get("/search/test/index").send().await; - assert_eq!(resp.status(), StatusCode::OK); - let index = resp.json::>().await; - assert_eq!(index, vec!["test".to_owned()]); + // let resp = client.get("/search/test/index").send().await; + // assert_eq!(resp.status(), StatusCode::OK); + // let index = resp.json::>().await; + // assert_eq!(index, vec!["title".to_owned(), "text".to_owned()]); + + // let body = to_string(&json!(["test"])).unwrap(); + // let resp = client + // .post("/search/test/index") + // .header("content-type", "application/json") + // .body(body) + // .send() + // .await; + // assert_eq!(resp.status(), StatusCode::OK); + + // let resp = client.get("/search/test/index").send().await; + // assert_eq!(resp.status(), StatusCode::OK); + // let index = resp.json::>().await; + // assert_eq!(index, vec!["test".to_owned()]); let body = json!({ "hookEndpoint": "localhost:3000/api/hook" From e5d5c1bed54591a1e23599ddc7f7d3997c714a0b Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 28 Aug 2023 16:21:28 +0800 Subject: [PATCH 28/49] chore: fix lint --- Cargo.lock | 338 +----------------- Cargo.toml | 3 - libs/jwst-binding/jwst-swift/src/storage.rs | 2 +- libs/jwst-binding/jwst-swift/src/workspace.rs | 4 +- libs/jwst-core-rpc/src/handler.rs | 2 +- 5 files changed, 5 insertions(+), 344 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ddfcd65f..e180bd1b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,12 +18,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "adler32" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" - [[package]] name = "aead" version = "0.3.2" @@ -121,14 +115,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "affine-cloud-migration" -version = "0.1.0" -dependencies = [ - "sea-orm-migration", - "tokio", -] - [[package]] name = "ahash" version = "0.7.6" @@ -701,16 +687,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "bstr" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "bumpalo" version = "3.13.0" @@ -934,55 +910,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" -[[package]] -name = "cloud-database" -version = "0.1.0" -dependencies = [ - "affine-cloud-migration", - "anyhow", - "async-trait", - "chrono", - "jwst", - "jwst-logger", - "nanoid", - "schemars", - "sea-orm", - "sea-orm-migration", - "serde", - "serde_repr", - "sqlx", - "tokio", - "yrs", -] - -[[package]] -name = "cloud-infra" -version = "0.1.0" -dependencies = [ - "aes-gcm", - "axum", - "chrono", - "cloud-database", - "dotenvy", - "handlebars", - "jsonwebtoken", - "jwst", - "lettre", - "nanoid", - "pem", - "rand 0.8.5", - "reqwest", - "rust-embed", - "serde", - "sha2", - "thiserror", - "url", - "url-escape", - "utoipa", - "utoipa-swagger-ui", - "x509-parser 0.15.1", -] - [[package]] name = "color_quant" version = "1.1.0" @@ -1587,22 +1514,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "email-encoding" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbfb21b9878cf7a348dcb8559109aabc0ec40d69924bd706fa5149846c4fef75" -dependencies = [ - "base64 0.21.2", - "memchr", -] - -[[package]] -name = "email_address" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2153bd83ebc09db15bcbdc3e2194d901804952e3dc96967e1cd3b0c5c32d112" - [[package]] name = "encode_unicode" version = "0.3.6" @@ -2080,19 +1991,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "globset" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" -dependencies = [ - "aho-corasick 1.0.4", - "bstr", - "fnv", - "log", - "regex", -] - [[package]] name = "governor" version = "0.5.1" @@ -2174,20 +2072,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "handlebars" -version = "4.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c3372087601b532857d332f5957cbae686da52bb7810bf038c3e3c3cc2fa0d" -dependencies = [ - "log", - "pest", - "pest_derive", - "serde", - "serde_json", - "thiserror", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -2411,16 +2295,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.4.0" @@ -2451,40 +2325,6 @@ dependencies = [ "webp", ] -[[package]] -name = "include-flate" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e11569346406931d20276cc460215ee2826e7cad43aa986999cb244dd7adb0" -dependencies = [ - "include-flate-codegen-exports", - "lazy_static", - "libflate", -] - -[[package]] -name = "include-flate-codegen" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a7d6e1419fa3129eb0802b4c99603c0d425c79fb5d76191d5a20d0ab0d664e8" -dependencies = [ - "libflate", - "proc-macro-hack", - "proc-macro2 1.0.66", - "quote 1.0.33", - "syn 1.0.109", -] - -[[package]] -name = "include-flate-codegen-exports" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75657043ffe3d8280f1cb8aef0f505532b392ed7758e0baeac22edadcee31a03" -dependencies = [ - "include-flate-codegen", - "proc-macro-hack", -] - [[package]] name = "indexmap" version = "1.9.3" @@ -2650,20 +2490,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jsonwebtoken" -version = "8.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" -dependencies = [ - "base64 0.21.2", - "pem", - "ring", - "serde", - "serde_json", - "simple_asn1", -] - [[package]] name = "jwst" version = "0.1.1" @@ -3001,33 +2827,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" -[[package]] -name = "lettre" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bd09637ae3ec7bd605b8e135e757980b3968430ff2b1a4a94fb7769e50166d" -dependencies = [ - "async-trait", - "base64 0.21.2", - "email-encoding", - "email_address", - "fastrand 1.9.0", - "futures-io", - "futures-util", - "httpdate", - "idna 0.3.0", - "mime", - "nom", - "once_cell", - "quoted_printable", - "rustls 0.21.6", - "rustls-pemfile", - "socket2", - "tokio", - "tokio-rustls", - "webpki-roots 0.23.1", -] - [[package]] name = "levenshtein_automata" version = "0.2.1" @@ -3050,26 +2849,6 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" -[[package]] -name = "libflate" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18" -dependencies = [ - "adler32", - "crc32fast", - "libflate_lz77", -] - -[[package]] -name = "libflate_lz77" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" -dependencies = [ - "rle-decode-fast", -] - [[package]] name = "libgit2-sys" version = "0.14.2+1.5.1" @@ -3743,50 +3522,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" -[[package]] -name = "pest" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" -dependencies = [ - "thiserror", - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2 1.0.66", - "quote 1.0.33", - "syn 2.0.29", -] - -[[package]] -name = "pest_meta" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" -dependencies = [ - "once_cell", - "pest", - "sha2", -] - [[package]] name = "petgraph" version = "0.6.4" @@ -4172,12 +3907,6 @@ dependencies = [ "proc-macro2 1.0.66", ] -[[package]] -name = "quoted_printable" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3866219251662ec3b26fc217e3e05bf9c4f84325234dfb96bf0bf840889e49" - [[package]] name = "radium" version = "0.7.0" @@ -4538,12 +4267,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "rle-decode-fast" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" - [[package]] name = "rsa" version = "0.9.2" @@ -4596,7 +4319,6 @@ version = "6.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a36224c3276f8c4ebc8c20f158eca7ca4359c8db89991c4925132aaaf6702661" dependencies = [ - "include-flate", "rust-embed-impl", "rust-embed-utils", "walkdir", @@ -4622,8 +4344,6 @@ version = "7.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d38ff6bf570dc3bb7100fce9f7b60c33fa71d80e88da3f2580df4ff2bdded74" dependencies = [ - "globset", - "mime_guess", "sha2", "walkdir", ] @@ -5138,17 +4858,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_repr" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" -dependencies = [ - "proc-macro2 1.0.66", - "quote 1.0.33", - "syn 2.0.29", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5242,18 +4951,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" -[[package]] -name = "simple_asn1" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror", - "time 0.3.27", -] - [[package]] name = "siphasher" version = "0.3.11" @@ -6270,12 +5967,6 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" -[[package]] -name = "ucd-trie" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" - [[package]] name = "unarray" version = "0.1.4" @@ -6365,16 +6056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", - "idna 0.4.0", - "percent-encoding", -] - -[[package]] -name = "url-escape" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44e0ce4d1246d075ca5abec4b41d33e87a6054d08e2366b63205665e950db218" -dependencies = [ + "idna", "percent-encoding", ] @@ -6427,7 +6109,6 @@ version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84614caa239fb25b2bb373a52859ffd94605ceb256eeb1d63436325cf81e3653" dependencies = [ - "axum", "mime_guess", "regex", "rust-embed", @@ -7141,23 +6822,6 @@ dependencies = [ "time 0.3.27", ] -[[package]] -name = "x509-parser" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" -dependencies = [ - "asn1-rs 0.5.2", - "data-encoding", - "der-parser 8.2.0", - "lazy_static", - "nom", - "oid-registry 0.6.1", - "rusticata-macros", - "thiserror", - "time 0.3.27", -] - [[package]] name = "y-sync" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index e18a15a48..38ec573f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,9 +3,6 @@ members = [ "apps/doc_merger", "apps/keck", - "libs/cloud-database", - "libs/cloud-database/migration", - "libs/cloud-infra", "libs/jwst", # "libs/jwst-binding/jwst-ffi", "libs/jwst-binding/jwst-jni", diff --git a/libs/jwst-binding/jwst-swift/src/storage.rs b/libs/jwst-binding/jwst-swift/src/storage.rs index 615a355c4..8b727925b 100644 --- a/libs/jwst-binding/jwst-swift/src/storage.rs +++ b/libs/jwst-binding/jwst-swift/src/storage.rs @@ -133,7 +133,7 @@ impl Storage { ); } - Ok(Workspace { workspace, rt }) + Ok(Workspace { workspace, _rt: rt }) } Err(e) => Err(e), } diff --git a/libs/jwst-binding/jwst-swift/src/workspace.rs b/libs/jwst-binding/jwst-swift/src/workspace.rs index 147450401..881ac2683 100644 --- a/libs/jwst-binding/jwst-swift/src/workspace.rs +++ b/libs/jwst-binding/jwst-swift/src/workspace.rs @@ -7,14 +7,14 @@ use super::*; pub struct Workspace { pub(crate) workspace: JwstWorkspace, - pub(crate) rt: Arc, + pub(crate) _rt: Arc, } impl Workspace { pub fn new(id: String) -> Self { Self { workspace: JwstWorkspace::new(&id).unwrap(), - rt: Arc::new(Runtime::new().unwrap()), + _rt: Arc::new(Runtime::new().unwrap()), } } diff --git a/libs/jwst-core-rpc/src/handler.rs b/libs/jwst-core-rpc/src/handler.rs index bc2207d64..8b6dc5eea 100644 --- a/libs/jwst-core-rpc/src/handler.rs +++ b/libs/jwst-core-rpc/src/handler.rs @@ -338,7 +338,7 @@ mod test { // close connection after doc1 is broadcasted let block_id = format!("block{}", i); { - let block_id = block_id.clone(); + // let block_id = block_id.clone(); let doc_tx = doc_tx.clone(); doc.doc().subscribe(move |_u| { // TODO: support changed block record From eafb42bd5d345a73fd3b1a0ea540bf69d8fb81b2 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 28 Aug 2023 16:30:18 +0800 Subject: [PATCH 29/49] chore: merge jwst-rpc --- Cargo.lock | 230 +-------- Cargo.toml | 5 +- apps/keck/Cargo.toml | 2 +- apps/keck/src/server/api/mod.rs | 2 +- apps/keck/src/server/sync/blobs.rs | 2 +- apps/keck/src/server/sync/collaboration.rs | 4 +- libs/jwst-binding/jwst-swift/Cargo.toml | 2 +- libs/jwst-binding/jwst-swift/src/storage.rs | 2 +- libs/jwst-core-rpc/Cargo.toml | 50 -- libs/jwst-core-rpc/src/broadcast.rs | 93 ---- libs/jwst-core-rpc/src/client/mod.rs | 101 ---- libs/jwst-core-rpc/src/client/webrtc.rs | 90 ---- libs/jwst-core-rpc/src/client/websocket.rs | 97 ---- .../src/connector/axum_socket.rs | 80 ---- libs/jwst-core-rpc/src/connector/memory.rs | 118 ----- libs/jwst-core-rpc/src/connector/mod.rs | 21 - .../src/connector/tungstenite_socket.rs | 71 --- libs/jwst-core-rpc/src/connector/webrtc.rs | 179 ------- libs/jwst-core-rpc/src/context.rs | 214 --------- libs/jwst-core-rpc/src/handler.rs | 447 ------------------ libs/jwst-core-rpc/src/lib.rs | 52 -- libs/jwst-core-rpc/src/types.rs | 20 - .../src/utils/memory_workspace.rs | 55 --- libs/jwst-core-rpc/src/utils/mod.rs | 7 - .../jwst-core-rpc/src/utils/server_context.rs | 75 --- libs/jwst-rpc/Cargo.toml | 13 +- libs/jwst-rpc/src/broadcast.rs | 26 +- libs/jwst-rpc/src/connector/webrtc.rs | 2 +- libs/jwst-rpc/src/context.rs | 35 +- libs/jwst-rpc/src/handler.rs | 142 +++--- libs/jwst-rpc/src/lib.rs | 4 +- libs/jwst-rpc/src/types.rs | 4 +- libs/jwst-rpc/src/utils/compare.rs | 64 --- libs/jwst-rpc/src/utils/mod.rs | 2 - libs/jwst-rpc/src/utils/server_context.rs | 7 +- 35 files changed, 100 insertions(+), 2218 deletions(-) delete mode 100644 libs/jwst-core-rpc/Cargo.toml delete mode 100644 libs/jwst-core-rpc/src/broadcast.rs delete mode 100644 libs/jwst-core-rpc/src/client/mod.rs delete mode 100644 libs/jwst-core-rpc/src/client/webrtc.rs delete mode 100644 libs/jwst-core-rpc/src/client/websocket.rs delete mode 100644 libs/jwst-core-rpc/src/connector/axum_socket.rs delete mode 100644 libs/jwst-core-rpc/src/connector/memory.rs delete mode 100644 libs/jwst-core-rpc/src/connector/mod.rs delete mode 100644 libs/jwst-core-rpc/src/connector/tungstenite_socket.rs delete mode 100644 libs/jwst-core-rpc/src/connector/webrtc.rs delete mode 100644 libs/jwst-core-rpc/src/context.rs delete mode 100644 libs/jwst-core-rpc/src/handler.rs delete mode 100644 libs/jwst-core-rpc/src/lib.rs delete mode 100644 libs/jwst-core-rpc/src/types.rs delete mode 100644 libs/jwst-core-rpc/src/utils/memory_workspace.rs delete mode 100644 libs/jwst-core-rpc/src/utils/mod.rs delete mode 100644 libs/jwst-core-rpc/src/utils/server_context.rs delete mode 100644 libs/jwst-rpc/src/utils/compare.rs diff --git a/Cargo.lock b/Cargo.lock index e180bd1b5..71717919b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,24 +174,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" -[[package]] -name = "android_log-sys" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ecc8056bf6ab9892dcd53216c83d1597487d7dacac16c8df6b877d127df9937" - -[[package]] -name = "android_logger" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c494134f746c14dc653a35a4ea5aca24ac368529da5370ecf41fe0341c35772f" -dependencies = [ - "android_log-sys", - "env_logger", - "log", - "once_cell", -] - [[package]] name = "android_system_properties" version = "0.1.5" @@ -1330,17 +1312,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "derive-new" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" -dependencies = [ - "proc-macro2 1.0.66", - "quote 1.0.33", - "syn 1.0.109", -] - [[package]] name = "derive_arbitrary" version = "1.3.1" @@ -1549,16 +1520,6 @@ dependencies = [ "syn 2.0.29", ] -[[package]] -name = "env_logger" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" -dependencies = [ - "log", - "regex", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -1690,38 +1651,12 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "flagset" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda653ca797810c02f7ca4b804b40b8b95ae046eb989d356bce17919a8c25499" -[[package]] -name = "flapigen" -version = "0.6.0-pre13" -source = "git+https://github.com/Dushistov/flapigen-rs?rev=7d343c6#7d343c60afcc094f7f07741174e6777fb8ef113c" -dependencies = [ - "bitflags 1.3.2", - "heck", - "lazy_static", - "log", - "petgraph", - "proc-macro2 1.0.66", - "quote 1.0.33", - "rustc-hash", - "smallvec", - "smol_str 0.1.24", - "strum 0.24.1", - "syn 1.0.109", - "which", -] - [[package]] name = "flate2" version = "1.0.27" @@ -2457,12 +2392,6 @@ dependencies = [ "regex", ] -[[package]] -name = "jni-sys" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" - [[package]] name = "jobserver" version = "0.1.26" @@ -2575,36 +2504,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "jwst-core-rpc" -version = "0.1.0" -dependencies = [ - "anyhow", - "assert-json-diff", - "async-trait", - "axum", - "byteorder", - "bytes", - "chrono", - "futures", - "indicatif", - "jwst-codec", - "jwst-core", - "jwst-core-storage", - "jwst-logger", - "nanoid", - "rand 0.8.5", - "reqwest", - "serde", - "serde_json", - "tempfile", - "thiserror", - "tokio", - "tokio-tungstenite", - "url", - "webrtc", -] - [[package]] name = "jwst-core-storage" version = "0.1.0" @@ -2641,31 +2540,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "jwst-jni" -version = "0.1.0" -dependencies = [ - "android_logger", - "chrono", - "flapigen", - "futures", - "jni-sys", - "jwst", - "jwst-core", - "jwst-rpc", - "jwst-storage", - "lib0", - "log-panics", - "nanoid", - "rifgen", - "serde", - "serde_json", - "sqlx", - "tokio", - "tracing", - "yrs", -] - [[package]] name = "jwst-logger" version = "0.1.0" @@ -2691,15 +2565,11 @@ dependencies = [ "chrono", "futures", "indicatif", - "jwst", "jwst-codec", "jwst-core", + "jwst-core-storage", "jwst-logger", - "jwst-storage", - "lib0", - "lru_time_cache", "nanoid", - "nom", "rand 0.8.5", "reqwest", "serde", @@ -2710,7 +2580,6 @@ dependencies = [ "tokio-tungstenite", "url", "webrtc", - "yrs", ] [[package]] @@ -2758,9 +2627,9 @@ dependencies = [ "chrono", "futures", "jwst-core", - "jwst-core-rpc", "jwst-core-storage", "jwst-logger", + "jwst-rpc", "nanoid", "regex", "reqwest", @@ -2789,9 +2658,9 @@ dependencies = [ "dotenvy", "futures", "jwst-core", - "jwst-core-rpc", "jwst-core-storage", "jwst-logger", + "jwst-rpc", "lib0", "libc", "log", @@ -2932,15 +2801,6 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "log-panics" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f9dd8546191c1850ecf67d22f5ff00a935b890d0e84713159a55495cc2ac5f" -dependencies = [ - "log", -] - [[package]] name = "loom" version = "0.5.6" @@ -2979,12 +2839,6 @@ dependencies = [ "hashbrown 0.12.3", ] -[[package]] -name = "lru_time_cache" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" - [[package]] name = "lz4_flex" version = "0.9.5" @@ -3522,16 +3376,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" -[[package]] -name = "petgraph" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" -dependencies = [ - "fixedbitset", - "indexmap 2.0.0", -] - [[package]] name = "phf" version = "0.11.2" @@ -4202,28 +4046,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rifgen" -version = "0.1.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba139f9f8c0d72232525b60bf83a81c6801df932d80ad6b4d0c8c0ccee9fa116" -dependencies = [ - "Inflector", - "derive-new", - "rifgen_attr", - "syn 1.0.109", -] - -[[package]] -name = "rifgen_attr" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3df05e0adb96c94ddb287013558ba7ff67f097219f6afa3c789506472e71272" -dependencies = [ - "quote 1.0.33", - "syn 1.0.109", -] - [[package]] name = "ring" version = "0.16.20" @@ -4631,7 +4453,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "strum 0.25.0", + "strum", "thiserror", "time 0.3.27", "tracing", @@ -4981,15 +4803,6 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" -[[package]] -name = "smol_str" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad6c857cbab2627dcf01ec85a623ca4e7dcb5691cbaa3d7fb7653671f0d09c9" -dependencies = [ - "serde", -] - [[package]] name = "smol_str" version = "0.2.0" @@ -5300,34 +5113,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "strum" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" -dependencies = [ - "strum_macros", -] - [[package]] name = "strum" version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" -[[package]] -name = "strum_macros" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" -dependencies = [ - "heck", - "proc-macro2 1.0.66", - "quote 1.0.33", - "rustversion", - "syn 1.0.109", -] - [[package]] name = "stun" version = "0.4.4" @@ -6375,7 +6166,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "smol_str 0.2.0", + "smol_str", "stun", "thiserror", "time 0.3.27", @@ -6565,17 +6356,6 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" -[[package]] -name = "which" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" -dependencies = [ - "either", - "libc", - "once_cell", -] - [[package]] name = "whoami" version = "1.4.1" diff --git a/Cargo.toml b/Cargo.toml index 38ec573f3..87ebde659 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,16 +5,14 @@ members = [ "apps/keck", "libs/jwst", # "libs/jwst-binding/jwst-ffi", - "libs/jwst-binding/jwst-jni", + # "libs/jwst-binding/jwst-jni", # "libs/jwst-binding/jwst-py", "libs/jwst-binding/jwst-swift", "libs/jwst-binding/jwst-swift/jwst-swift-integrate", # "libs/jwst-binding/jwst-wasm", "libs/jwst-codec", "libs/jwst-codec-util", - #"libs/jwst-codec/fuzz", "libs/jwst-core", - "libs/jwst-core-rpc", "libs/jwst-core-storage", "libs/jwst-core-storage/src/migration", "libs/jwst-logger", @@ -28,7 +26,6 @@ resolver = "2" jwst = { workspace = true, path = "libs/jwst" } jwst-codec = { workspace = true, path = "libs/jwst-codec" } jwst-core = { workspace = true, path = "libs/jwst-core" } -jwst-core-rpc = { workspace = true, path = "libs/jwst-core-rpc" } jwst-core-storage = { workspace = true, path = "libs/jwst-core-storage" } jwst-logger = { workspace = true, path = "libs/jwst-logger" } jwst-rpc = { workspace = true, path = "libs/jwst-rpc" } diff --git a/apps/keck/Cargo.toml b/apps/keck/Cargo.toml index 9f65afd96..fcb71e023 100644 --- a/apps/keck/Cargo.toml +++ b/apps/keck/Cargo.toml @@ -61,7 +61,7 @@ reqwest = { version = "0.11.19", default-features = false, features = [ # ======= workspace dependencies ======= jwst-core = { workspace = true } jwst-logger = { workspace = true } -jwst-core-rpc = { workspace = true } +jwst-rpc = { workspace = true } jwst-core-storage = { workspace = true } [dev-dependencies] diff --git a/apps/keck/src/server/api/mod.rs b/apps/keck/src/server/api/mod.rs index 99e0acb8a..36037c6f7 100644 --- a/apps/keck/src/server/api/mod.rs +++ b/apps/keck/src/server/api/mod.rs @@ -15,8 +15,8 @@ use axum::{ routing::{delete, get, head, post}, }; use doc::doc_apis; -use jwst_core_rpc::{BroadcastChannels, RpcContextImpl}; use jwst_core_storage::{BlobStorageType, JwstStorage, JwstStorageResult}; +use jwst_rpc::{BroadcastChannels, RpcContextImpl}; use tokio::sync::RwLock; use super::*; diff --git a/apps/keck/src/server/sync/blobs.rs b/apps/keck/src/server/sync/blobs.rs index f893ef4a3..383d88f49 100644 --- a/apps/keck/src/server/sync/blobs.rs +++ b/apps/keck/src/server/sync/blobs.rs @@ -12,7 +12,7 @@ use axum::{ }; use futures::{future, StreamExt}; use jwst_core::BlobStorage; -use jwst_core_rpc::RpcContextImpl; +use jwst_rpc::RpcContextImpl; use time::{format_description::well_known::Rfc2822, OffsetDateTime}; use super::*; diff --git a/apps/keck/src/server/sync/collaboration.rs b/apps/keck/src/server/sync/collaboration.rs index a56a8c568..3ed935e8a 100644 --- a/apps/keck/src/server/sync/collaboration.rs +++ b/apps/keck/src/server/sync/collaboration.rs @@ -6,7 +6,7 @@ use axum::{ Json, }; use futures::FutureExt; -use jwst_core_rpc::{axum_socket_connector, handle_connector}; +use jwst_rpc::{axum_socket_connector, handle_connector}; use serde::Serialize; use super::*; @@ -53,9 +53,9 @@ mod test { }; use jwst_core::{Block, DocStorage, Workspace}; - use jwst_core_rpc::{start_websocket_client_sync, BroadcastChannels, RpcContextImpl}; use jwst_core_storage::{BlobStorageType, JwstStorage}; use jwst_logger::info; + use jwst_rpc::{start_websocket_client_sync, BroadcastChannels, RpcContextImpl}; use libc::{kill, SIGTERM}; use rand::{thread_rng, Rng}; use tokio::runtime::Runtime; diff --git a/libs/jwst-binding/jwst-swift/Cargo.toml b/libs/jwst-binding/jwst-swift/Cargo.toml index 2b449eb36..ae3a89e8c 100644 --- a/libs/jwst-binding/jwst-swift/Cargo.toml +++ b/libs/jwst-binding/jwst-swift/Cargo.toml @@ -16,7 +16,7 @@ serde_json = "1.0.104" # ======= workspace dependencies ======= jwst-core = { workspace = true } -jwst-core-rpc = { workspace = true } +jwst-rpc = { workspace = true } jwst-core-storage = { workspace = true, features = ["sqlite"] } jwst-logger = { workspace = true } diff --git a/libs/jwst-binding/jwst-swift/src/storage.rs b/libs/jwst-binding/jwst-swift/src/storage.rs index 8b727925b..2f4add053 100644 --- a/libs/jwst-binding/jwst-swift/src/storage.rs +++ b/libs/jwst-binding/jwst-swift/src/storage.rs @@ -1,8 +1,8 @@ use std::sync::{Arc, RwLock}; use futures::TryFutureExt; -use jwst_core_rpc::{start_websocket_client_sync, BroadcastChannels, CachedLastSynced, RpcContextImpl, SyncState}; use jwst_core_storage::{BlobStorageType, JwstStorage as AutoStorage, JwstStorageResult}; +use jwst_rpc::{start_websocket_client_sync, BroadcastChannels, CachedLastSynced, RpcContextImpl, SyncState}; use nanoid::nanoid; use tokio::{ runtime::{Builder, Runtime}, diff --git a/libs/jwst-core-rpc/Cargo.toml b/libs/jwst-core-rpc/Cargo.toml deleted file mode 100644 index c2d3a88ab..000000000 --- a/libs/jwst-core-rpc/Cargo.toml +++ /dev/null @@ -1,50 +0,0 @@ -[package] -name = "jwst-core-rpc" -version = "0.1.0" -edition = "2021" -authors = ["DarkSky "] -license = "AGPL-3.0-only" - -[features] -default = ["websocket"] -websocket = ["axum", "tokio-tungstenite", "url"] -webrtc = ["bytes", "reqwest", "webrtcrs"] - -[dependencies] -anyhow = "1.0.70" -assert-json-diff = "2.0.2" -async-trait = "0.1.68" -byteorder = "1.4.3" -chrono = "0.4.26" -futures = "0.3.28" -nanoid = "0.4.0" -rand = "0.8.5" -serde = "1.0.183" -serde_json = "1.0.104" -thiserror = "1.0.40" -tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] } - -# ======== websocket dependencies ======== -axum = { version = "0.6.16", features = ["ws"], optional = true } -tokio-tungstenite = { version = "0.20.0", features = [ - "rustls-tls-webpki-roots", -], optional = true } -url = { version = "2.3.1", optional = true } - -# ======== webrtc dependencies ======== -bytes = { version = "1.4", optional = true } -reqwest = { version = "0.11.18", default-features = false, features = [ - "json", - "rustls-tls", -], optional = true } -webrtcrs = { package = "webrtc", version = "0.8.0", optional = true } - -# ======= workspace dependencies ======= -jwst-codec = { workspace = true } -jwst-core = { workspace = true } -jwst-core-storage = { workspace = true } - -[dev-dependencies] -indicatif = "0.17.3" -jwst-logger = { path = "../jwst-logger" } -tempfile = "3.4.0" diff --git a/libs/jwst-core-rpc/src/broadcast.rs b/libs/jwst-core-rpc/src/broadcast.rs deleted file mode 100644 index 8e4581613..000000000 --- a/libs/jwst-core-rpc/src/broadcast.rs +++ /dev/null @@ -1,93 +0,0 @@ -use std::collections::HashMap; - -use jwst_codec::{encode_update_as_message, encode_update_with_guid, write_sync_message, SyncMessage}; -use jwst_core::Workspace; -use tokio::sync::{broadcast::Sender, RwLock}; - -use super::*; - -#[derive(Clone)] -pub enum BroadcastType { - BroadcastAwareness(Vec), - BroadcastContent(Vec), - BroadcastRawContent(Vec), - CloseUser(String), - CloseAll, -} - -type Broadcast = Sender; -pub type BroadcastChannels = RwLock>; - -pub async fn subscribe(workspace: &mut Workspace, identifier: String, sender: Broadcast) { - { - let sender = sender.clone(); - let workspace_id = workspace.id(); - - workspace - .on_awareness_update(move |awareness, e| { - let mut buffer = Vec::new(); - if let Err(e) = write_sync_message( - &mut buffer, - &SyncMessage::Awareness(e.get_updated(awareness.get_states())), - ) { - error!("failed to write awareness update: {}", e); - return; - } - - if sender.send(BroadcastType::BroadcastAwareness(buffer.clone())).is_err() { - debug!("broadcast channel {workspace_id} has been closed",) - } - }) - .await; - } - { - let sender = sender.clone(); - let workspace_id = workspace.id(); - workspace.doc().subscribe(move |update| { - debug!("workspace {} changed: {}bytes", workspace_id, update.len()); - - match encode_update_with_guid(update.to_vec(), workspace_id.clone()) - .and_then(|update_with_guid| encode_update_as_message(update.to_vec()).map(|u| (update_with_guid, u))) - { - Ok((broadcast_update, sendable_update)) => { - if sender - .send(BroadcastType::BroadcastRawContent(broadcast_update)) - .is_err() - { - debug!("broadcast channel {workspace_id} has been closed",) - } - - if sender.send(BroadcastType::BroadcastContent(sendable_update)).is_err() { - debug!("broadcast channel {workspace_id} has been closed",) - } - } - Err(e) => { - debug!("failed to encode update: {}", e); - } - } - }); - }; - - let workspace_id = workspace.id(); - tokio::spawn(async move { - let mut rx = sender.subscribe(); - loop { - tokio::select! { - Ok(msg) = rx.recv()=> { - match msg { - BroadcastType::CloseUser(user) if user == identifier => break, - BroadcastType::CloseAll => break, - _ => {} - } - }, - _ = sleep(Duration::from_millis(100)) => { - let count = sender.receiver_count(); - if count < 1 { - break; - } - } - } - } - debug!("broadcast channel {workspace_id} has been closed"); - }); -} diff --git a/libs/jwst-core-rpc/src/client/mod.rs b/libs/jwst-core-rpc/src/client/mod.rs deleted file mode 100644 index fb40f4f47..000000000 --- a/libs/jwst-core-rpc/src/client/mod.rs +++ /dev/null @@ -1,101 +0,0 @@ -#[cfg(feature = "webrtc")] -mod webrtc; -#[cfg(feature = "websocket")] -mod websocket; - -use std::sync::Mutex; - -use chrono::Utc; -use tokio::{runtime::Runtime, task::JoinHandle}; -#[cfg(feature = "webrtc")] -pub use webrtc::start_webrtc_client_sync; -#[cfg(feature = "websocket")] -pub use websocket::start_websocket_client_sync; - -use super::*; - -#[derive(Clone, Default)] -pub struct CachedLastSynced { - synced: Arc>>, - _threads: Arc>>>, -} - -impl CachedLastSynced { - pub fn add_receiver_wait_first_update(&self, rt: Arc, recv: Receiver) { - self.add_receiver(rt, recv); - while self.synced.lock().unwrap().is_empty() { - std::thread::sleep(Duration::from_millis(100)); - } - } - - pub fn add_receiver(&self, rt: Arc, mut recv: Receiver) { - let synced = self.synced.clone(); - rt.spawn(async move { - while let Some(last_synced) = recv.recv().await { - let mut synced = synced.lock().unwrap(); - if synced.is_empty() || *synced.last().unwrap() != last_synced { - synced.push(last_synced); - } - } - }); - } - - pub fn pop(&self) -> Vec { - let mut synced = self.synced.lock().unwrap(); - let ret = synced.clone(); - synced.clear(); - ret - } -} - -#[cfg(test)] -mod tests { - use std::thread::spawn; - - use tokio::sync::mpsc::channel; - - use super::*; - - #[test] - fn test_synced() { - let synced = CachedLastSynced::default(); - let (tx, rx) = channel::(1); - let rt = Arc::new(Runtime::new().unwrap()); - - synced.add_receiver(rt.clone(), rx); - { - let tx = tx.clone(); - rt.block_on(async { - tx.send(1).await.unwrap(); - tx.send(2).await.unwrap(); - tx.send(3).await.unwrap(); - sleep(Duration::from_millis(100)).await; - }); - } - { - let synced = synced.clone(); - spawn(move || { - assert_eq!(synced.pop(), vec![1, 2, 3]); - }) - .join() - .unwrap(); - } - - { - let tx = tx.clone(); - rt.block_on(async { - tx.send(4).await.unwrap(); - tx.send(5).await.unwrap(); - sleep(Duration::from_millis(100)).await; - }); - } - { - let synced = synced.clone(); - spawn(move || { - assert_eq!(synced.pop(), vec![4, 5]); - }) - .join() - .unwrap(); - } - } -} diff --git a/libs/jwst-core-rpc/src/client/webrtc.rs b/libs/jwst-core-rpc/src/client/webrtc.rs deleted file mode 100644 index 872452205..000000000 --- a/libs/jwst-core-rpc/src/client/webrtc.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::sync::RwLock; - -use nanoid::nanoid; -use reqwest::Client; -use tokio::{runtime::Runtime, sync::mpsc::channel}; - -use super::*; - -async fn webrtc_connection(remote: &str) -> (Sender, Receiver>) { - warn!("webrtc_connection start"); - let (offer, pc, tx, rx, mut s) = webrtc_datachannel_client_begin().await; - let client = Client::new(); - - match client.post(remote).json(&offer).send().await { - Ok(res) => { - webrtc_datachannel_client_commit(res.json::().await.unwrap(), pc).await; - s.recv().await.ok(); - warn!("client already connected"); - } - Err(e) => { - error!("failed to http post: {}", e); - } - } - - (tx, rx) -} - -pub fn start_webrtc_client_sync( - rt: Arc, - context: Arc + Send + Sync + 'static>, - sync_state: Arc>, - remote: String, - workspace_id: String, -) -> CachedLastSynced { - debug!("spawn sync thread"); - let (last_synced_tx, last_synced_rx) = channel::(128); - - let runtime = rt.clone(); - std::thread::spawn(move || { - runtime.block_on(async move { - let workspace = match context.get_workspace(&workspace_id).await { - Ok(workspace) => workspace, - Err(e) => { - error!("failed to create workspace: {:?}", e); - return; - } - }; - if !workspace.is_empty() { - info!("Workspace not empty, starting async remote connection"); - last_synced_tx.send(Utc::now().timestamp_millis()).await.unwrap(); - } else { - info!("Workspace empty, starting sync remote connection"); - } - - loop { - let (tx, rx) = webrtc_connection(&remote).await; - *sync_state.write().unwrap() = SyncState::Connected; - - let ret = { - let identifier = nanoid!(); - let workspace_id = workspace_id.clone(); - let last_synced_tx = last_synced_tx.clone(); - handle_connector(context.clone(), workspace_id, identifier, move || { - (tx, rx, last_synced_tx.clone()) - }) - .await - }; - - { - last_synced_tx.send(0).await.unwrap(); - let mut state = sync_state.write().unwrap(); - if ret { - debug!("sync thread finished"); - *state = SyncState::Finished; - } else { - *state = SyncState::Error("Remote sync connection disconnected".to_string()); - } - } - - warn!("remote sync connection disconnected, try again in 2 seconds"); - sleep(Duration::from_secs(2)).await; - } - }); - }); - - let timeline = CachedLastSynced::default(); - timeline.add_receiver(rt, last_synced_rx); - - timeline -} diff --git a/libs/jwst-core-rpc/src/client/websocket.rs b/libs/jwst-core-rpc/src/client/websocket.rs deleted file mode 100644 index f7f9a7305..000000000 --- a/libs/jwst-core-rpc/src/client/websocket.rs +++ /dev/null @@ -1,97 +0,0 @@ -use std::sync::RwLock; - -use nanoid::nanoid; -use tokio::{net::TcpStream, runtime::Runtime, sync::mpsc::channel}; -use tokio_tungstenite::{ - connect_async, - tungstenite::{client::IntoClientRequest, http::HeaderValue}, - MaybeTlsStream, WebSocketStream, -}; -use url::Url; - -use super::{types::JwstRpcResult, *}; - -type Socket = WebSocketStream>; - -async fn prepare_connection(remote: &str) -> JwstRpcResult { - debug!("generate remote config"); - let uri = Url::parse(remote)?; - - let mut req = uri.into_client_request()?; - req.headers_mut() - .append("Sec-WebSocket-Protocol", HeaderValue::from_static("AFFiNE")); - - debug!("connect to remote: {}", req.uri()); - Ok(connect_async(req).await?.0) -} - -pub fn start_websocket_client_sync( - rt: Arc, - context: Arc + Send + Sync + 'static>, - sync_state: Arc>, - remote: String, - workspace_id: String, -) -> CachedLastSynced { - debug!("spawn sync thread"); - let (last_synced_tx, last_synced_rx) = channel::(128); - - let runtime = rt.clone(); - runtime.spawn(async move { - debug!("start sync thread"); - let workspace = match context.get_workspace(&workspace_id).await { - Ok(workspace) => workspace, - Err(e) => { - warn!("failed to create workspace: {:?}", e); - return; - } - }; - if !workspace.is_empty() { - info!("Workspace not empty, starting async remote connection"); - last_synced_tx.send(Utc::now().timestamp_millis()).await.unwrap(); - } else { - info!("Workspace empty, starting sync remote connection"); - } - - loop { - let socket = match prepare_connection(&remote).await { - Ok(socket) => socket, - Err(e) => { - warn!("failed to connect to remote, try again in 2 seconds: {}", e); - sleep(Duration::from_secs(2)).await; - continue; - } - }; - *sync_state.write().unwrap() = SyncState::Connected; - - let ret = { - let identifier = nanoid!(); - let workspace_id = workspace_id.clone(); - let last_synced_tx = last_synced_tx.clone(); - handle_connector(context.clone(), workspace_id.clone(), identifier, move || { - let (tx, rx) = tungstenite_socket_connector(socket, &workspace_id); - (tx, rx, last_synced_tx) - }) - .await - }; - - { - last_synced_tx.send(0).await.unwrap(); - let mut state = sync_state.write().unwrap(); - if ret { - debug!("sync thread finished"); - *state = SyncState::Finished; - } else { - *state = SyncState::Error("remote sync connection disconnected".to_string()); - } - } - - warn!("Remote sync connection disconnected, try again in 2 seconds"); - sleep(Duration::from_secs(2)).await; - } - }); - - let timeline = CachedLastSynced::default(); - timeline.add_receiver_wait_first_update(rt, last_synced_rx); - - timeline -} diff --git a/libs/jwst-core-rpc/src/connector/axum_socket.rs b/libs/jwst-core-rpc/src/connector/axum_socket.rs deleted file mode 100644 index 6cab797bf..000000000 --- a/libs/jwst-core-rpc/src/connector/axum_socket.rs +++ /dev/null @@ -1,80 +0,0 @@ -use axum::extract::ws::{Message as WebSocketMessage, WebSocket}; -use futures::{sink::SinkExt, stream::StreamExt}; -use tokio::sync::mpsc::{channel, Receiver, Sender}; -use tokio_tungstenite::tungstenite::Error as SocketError; - -use super::*; - -impl From for WebSocketMessage { - fn from(value: Message) -> Self { - match value { - Message::Binary(data) => WebSocketMessage::Binary(data), - Message::Close => WebSocketMessage::Close(None), - Message::Ping => WebSocketMessage::Ping(vec![]), - } - } -} - -pub fn axum_socket_connector( - socket: WebSocket, - workspace_id: &str, -) -> (Sender, Receiver>, Sender) { - let (mut socket_tx, mut socket_rx) = socket.split(); - - // send to remote pipeline - let (local_sender, mut local_receiver) = channel::(100); - { - // socket send thread - let workspace_id = workspace_id.to_owned(); - tokio::spawn(async move { - while let Some(msg) = local_receiver.recv().await { - if let Err(e) = socket_tx.send(msg.into()).await { - let error = e.to_string(); - if !e.into_inner().downcast::().map_or_else( - |_| false, - |e| matches!(e.as_ref(), SocketError::ConnectionClosed | SocketError::AlreadyClosed), - ) { - error!("socket send error: {}", error); - } - break; - } - } - info!("socket send final: {}", workspace_id); - }); - } - - let (remote_sender, remote_receiver) = channel::>(512); - { - // socket recv thread - let workspace_id = workspace_id.to_owned(); - tokio::spawn(async move { - while let Some(msg) = socket_rx.next().await { - if let Ok(WebSocketMessage::Binary(binary)) = msg { - trace!("recv from remote: {}bytes", binary.len()); - if remote_sender.send(binary).await.is_err() { - // pipeline was closed - break; - } - } - } - info!("socket recv final: {}", workspace_id); - }); - } - - let (first_init_tx, mut first_init_rx) = channel::(10); - { - // init notify thread - let workspace_id = workspace_id.to_owned(); - tokio::spawn(async move { - while let Some(time) = first_init_rx.recv().await { - if time > 0 { - info!("socket sync success: {}", workspace_id); - } else { - error!("socket sync failed: {}", workspace_id); - } - } - }); - } - - (local_sender, remote_receiver, first_init_tx) -} diff --git a/libs/jwst-core-rpc/src/connector/memory.rs b/libs/jwst-core-rpc/src/connector/memory.rs deleted file mode 100644 index 6b666049f..000000000 --- a/libs/jwst-core-rpc/src/connector/memory.rs +++ /dev/null @@ -1,118 +0,0 @@ -use std::{ - sync::atomic::{AtomicBool, Ordering}, - thread::JoinHandle as StdJoinHandler, -}; - -use jwst_codec::{decode_update_with_guid, encode_update_as_message, Doc, DocMessage, SyncMessage, SyncMessageScanner}; -use tokio::{runtime::Runtime, sync::mpsc::channel, task::JoinHandle as TokioJoinHandler}; - -use super::*; - -// just for test -pub fn memory_connector( - rt: Arc, - doc: Doc, -) -> ( - Sender, - Receiver>, - TokioJoinHandler<()>, - StdJoinHandler<()>, -) { - // recv from remote pipeline - let (remote_sender, remote_receiver) = channel::>(512); - // send to remote pipeline - let (local_sender, mut local_receiver) = channel::(100); - let id = rand::random::(); - - // recv thread - let recv_handler = { - debug!("init memory recv thread"); - let finish = Arc::new(AtomicBool::new(false)); - - { - let finish = finish.clone(); - doc.subscribe(move |update| { - debug!("send change: {}", update.len()); - - match encode_update_as_message(update.to_vec()) { - Ok(buffer) => { - if rt.block_on(remote_sender.send(buffer)).is_err() { - // pipeline was closed - finish.store(true, Ordering::Release); - } - debug!("send change: {} end", update.len()); - } - Err(e) => { - error!("write sync message error: {}", e); - } - } - }); - } - - { - let local_sender = local_sender.clone(); - let doc = doc.clone(); - std::thread::spawn(move || { - while let Ok(false) | Err(false) = finish - .compare_exchange_weak(true, false, Ordering::Acquire, Ordering::Acquire) - .or_else(|_| Ok(local_sender.is_closed())) - { - std::thread::sleep(Duration::from_millis(100)); - } - doc.unsubscribe_all(); - debug!("recv final: {}", id); - }) - } - }; - - // send thread - let send_handler = { - debug!("init memory send thread"); - tokio::spawn(async move { - while let Some(msg) = local_receiver.recv().await { - match msg { - Message::Binary(data) => { - info!("msg:binary"); - let mut doc = doc.clone(); - tokio::task::spawn_blocking(move || { - trace!("recv change: {:?}", data.len()); - for update in SyncMessageScanner::new(&data).filter_map(|m| { - m.ok().and_then(|m| { - if let SyncMessage::Doc(DocMessage::Update(update)) = m { - Some(update) - } else { - None - } - }) - }) { - match decode_update_with_guid(update.clone()) { - Ok((_, update1)) => { - if let Err(e) = doc.apply_update_from_binary(update1) { - error!("failed to decode update1: {}, update: {:?}", e, update); - } - } - Err(e) => { - error!("failed to decode update2: {}, update: {:?}", e, update); - } - } - } - - trace!("recv change: {} end", data.len()); - }); - } - Message::Close => { - info!("msg:close"); - break; - } - Message::Ping => { - info!("msg:ping"); - continue; - } - } - } - debug!("send final: {}", id); - }) - }; - - (local_sender, remote_receiver, send_handler, recv_handler) -} diff --git a/libs/jwst-core-rpc/src/connector/mod.rs b/libs/jwst-core-rpc/src/connector/mod.rs deleted file mode 100644 index ca5db388c..000000000 --- a/libs/jwst-core-rpc/src/connector/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -#[cfg(feature = "websocket")] -mod axum_socket; -mod memory; -#[cfg(feature = "websocket")] -mod tungstenite_socket; -#[cfg(feature = "webrtc")] -mod webrtc; - -#[cfg(feature = "websocket")] -pub use axum_socket::axum_socket_connector; -pub use memory::memory_connector; -#[cfg(feature = "websocket")] -pub use tungstenite_socket::tungstenite_socket_connector; -#[cfg(feature = "webrtc")] -pub use webrtc::webrtc_datachannel_client_begin; -#[cfg(feature = "webrtc")] -pub use webrtc::webrtc_datachannel_client_commit; -#[cfg(feature = "webrtc")] -pub use webrtc::webrtc_datachannel_server_connector; - -use super::*; diff --git a/libs/jwst-core-rpc/src/connector/tungstenite_socket.rs b/libs/jwst-core-rpc/src/connector/tungstenite_socket.rs deleted file mode 100644 index 498a66ea2..000000000 --- a/libs/jwst-core-rpc/src/connector/tungstenite_socket.rs +++ /dev/null @@ -1,71 +0,0 @@ -use futures::{sink::SinkExt, stream::StreamExt}; -use tokio::{ - net::TcpStream, - sync::mpsc::{channel, Receiver, Sender}, -}; -use tokio_tungstenite::{ - tungstenite::{Error as SocketError, Message as WebSocketMessage}, - MaybeTlsStream, WebSocketStream, -}; - -use super::*; - -type WebSocket = WebSocketStream>; - -impl From for WebSocketMessage { - fn from(value: Message) -> Self { - match value { - Message::Binary(data) => WebSocketMessage::Binary(data), - Message::Close => WebSocketMessage::Close(None), - Message::Ping => WebSocketMessage::Ping(vec![]), - } - } -} - -pub fn tungstenite_socket_connector(socket: WebSocket, workspace_id: &str) -> (Sender, Receiver>) { - let (mut socket_tx, mut socket_rx) = socket.split(); - - // send to remote pipeline - let (local_sender, mut local_receiver) = channel::(100); - { - // socket send thread - let workspace_id = workspace_id.to_owned(); - tokio::spawn(async move { - let mut retry = 5; - while let Some(msg) = local_receiver.recv().await { - if let Err(e) = socket_tx.send(msg.into()).await { - let error = e.to_string(); - if matches!(e, SocketError::ConnectionClosed | SocketError::AlreadyClosed) || retry == 0 { - break; - } else { - retry -= 1; - error!("socket send error: {}", error); - } - } else { - retry = 5; - } - } - info!("socket send final: {}", workspace_id); - }); - } - - let (remote_sender, remote_receiver) = channel::>(512); - { - // socket recv thread - let workspace_id = workspace_id.to_owned(); - tokio::spawn(async move { - while let Some(msg) = socket_rx.next().await { - if let Ok(WebSocketMessage::Binary(binary)) = msg { - trace!("recv from remote: {}bytes", binary.len()); - if remote_sender.send(binary).await.is_err() { - // pipeline was closed - break; - } - } - } - info!("socket recv final: {}", workspace_id); - }); - } - - (local_sender, remote_receiver) -} diff --git a/libs/jwst-core-rpc/src/connector/webrtc.rs b/libs/jwst-core-rpc/src/connector/webrtc.rs deleted file mode 100644 index 5c49ea0ad..000000000 --- a/libs/jwst-core-rpc/src/connector/webrtc.rs +++ /dev/null @@ -1,179 +0,0 @@ -use std::sync::Arc; - -use bytes::Bytes; -use jwst_core::{debug, error, info, trace, warn}; -use tokio::sync::mpsc::{channel, Receiver, Sender}; -use webrtcrs::{ - api::APIBuilder, - data_channel::{data_channel_init::RTCDataChannelInit, data_channel_message::DataChannelMessage, OnMessageHdlrFn}, - peer_connection::{ - configuration::RTCConfiguration, peer_connection_state::RTCPeerConnectionState, - sdp::session_description::RTCSessionDescription, RTCPeerConnection, - }, -}; - -use super::Message; - -const DATA_CHANNEL_ID: u16 = 42; -const DATA_CHANNEL_LABEL: &str = "affine"; - -fn create_on_message_handler(channel: Sender>) -> OnMessageHdlrFn { - Box::new(move |message: DataChannelMessage| { - let channel = channel.clone(); - Box::pin(async move { - let m = message.data.to_vec(); - trace!("WebRTC Recv: {:?}", m.clone()); - channel.send(m).await.unwrap(); - }) - }) -} - -async fn new_peer_connection() -> ( - Arc, - Sender, - Receiver>, - tokio::sync::broadcast::Receiver, -) { - let api = APIBuilder::new().build(); - let peer_connection = Arc::new( - api.new_peer_connection(RTCConfiguration { ..Default::default() }) - .await - .unwrap(), - ); - - peer_connection.on_peer_connection_state_change(Box::new(move |s: RTCPeerConnectionState| { - warn!("Client: Peer Connection State has changed: {s}"); - if s == RTCPeerConnectionState::Failed { - error!("Peer Connection has gone to failed exiting"); - } - Box::pin(async {}) - })); - - let data_channel = peer_connection - .create_data_channel( - DATA_CHANNEL_LABEL, - Some(RTCDataChannelInit { - ordered: Some(true), - negotiated: Some(DATA_CHANNEL_ID), - ..Default::default() - }), - ) - .await - .unwrap(); - - let d0 = Arc::clone(&data_channel); - let (local_sender, mut local_receiver) = channel::(100); - - let (signal_tx, mut signal_rx) = tokio::sync::broadcast::channel::(1); - let signal_tx2 = signal_tx.subscribe(); - let sender_handle = tokio::spawn(async move { - signal_rx.recv().await.ok(); - while let Some(msg) = local_receiver.recv().await { - match msg { - Message::Binary(data) => { - trace!("WebRTC Send: {:?}", data.clone()); - d0.send(&Bytes::copy_from_slice(data.as_slice())).await.unwrap(); - } - Message::Close => info!("Close"), - Message::Ping => info!("Ping"), - } - } - }); - - data_channel.on_open(Box::new(move || { - debug!("Data channel opened"); - Box::pin(async move { - signal_tx.send(true).ok(); - }) - })); - - data_channel.on_close(Box::new(move || { - debug!("Data channel closed"); - sender_handle.abort(); - Box::pin(async move {}) - })); - - let (remote_sender, remote_receiver) = channel::>(512); - data_channel.on_message(create_on_message_handler(remote_sender)); - - (peer_connection, local_sender, remote_receiver, signal_tx2) -} - -pub async fn webrtc_datachannel_client_begin() -> ( - RTCSessionDescription, - Arc, - Sender, - Receiver>, - tokio::sync::broadcast::Receiver, -) { - let (peer_connection, tx, rx, s) = new_peer_connection().await; - let offer = peer_connection.create_offer(None).await.unwrap(); - - // Create channel that is blocked until ICE Gathering is complete - let mut gather_complete = peer_connection.gathering_complete_promise().await; - - // Sets the LocalDescription, and starts our UDP listeners - match peer_connection.set_local_description(offer).await { - Ok(_) => {} - Err(e) => { - error!("set local description failed: {}", e); - } - } - - // Block until ICE Gathering is complete, disabling trickle ICE - // we do this because we only can exchange one signaling message - // in a production application you should exchange ICE Candidates via - // OnICECandidate - let _ = gather_complete.recv().await; - - let local_desc = peer_connection.local_description().await.unwrap(); - (local_desc, peer_connection, tx, rx, s) -} - -pub async fn webrtc_datachannel_client_commit(answer: RTCSessionDescription, peer_connection: Arc) { - match peer_connection.set_remote_description(answer).await { - Ok(_) => {} - Err(e) => { - error!("set remote description failed: {}", e); - } - } -} - -pub async fn webrtc_datachannel_server_connector( - offer: RTCSessionDescription, -) -> ( - RTCSessionDescription, - Sender, - Receiver>, - tokio::sync::broadcast::Receiver, -) { - let (peer_connection, tx, rx, s) = new_peer_connection().await; - - match peer_connection.set_remote_description(offer).await { - Ok(_) => {} - Err(e) => { - error!("set remote description failed: {}", e); - } - } - - let answer = peer_connection.create_answer(None).await.unwrap(); - - // Create channel that is blocked until ICE Gathering is complete - let mut gather_complete = peer_connection.gathering_complete_promise().await; - - match peer_connection.set_local_description(answer).await { - Ok(_) => {} - Err(e) => { - error!("set local description failed: {}", e); - } - } - - // Block until ICE Gathering is complete, disabling trickle ICE - // we do this because we only can exchange one signaling message - // in a production application you should exchange ICE Candidates via - // OnICECandidate - let _ = gather_complete.recv().await; - - let local_desc = peer_connection.local_description().await.unwrap(); - (local_desc, tx, rx, s) -} diff --git a/libs/jwst-core-rpc/src/context.rs b/libs/jwst-core-rpc/src/context.rs deleted file mode 100644 index e442fc3bc..000000000 --- a/libs/jwst-core-rpc/src/context.rs +++ /dev/null @@ -1,214 +0,0 @@ -use std::collections::HashMap; - -use async_trait::async_trait; -use chrono::Utc; -use jwst_codec::{CrdtReader, RawDecoder}; -use jwst_core::{DocStorage, Workspace}; -use jwst_core_storage::{JwstStorage, JwstStorageResult}; -use tokio::sync::{ - broadcast::{channel as broadcast, error::RecvError, Receiver as BroadcastReceiver, Sender as BroadcastSender}, - mpsc::{Receiver as MpscReceiver, Sender as MpscSender}, - Mutex, -}; - -use super::{ - broadcast::{subscribe, BroadcastChannels, BroadcastType}, - *, -}; - -#[async_trait] -pub trait RpcContextImpl<'a> { - fn get_storage(&self) -> &JwstStorage; - fn get_channel(&self) -> &BroadcastChannels; - - async fn get_workspace(&self, id: &str) -> JwstStorageResult { - self.get_storage().create_workspace(id).await - } - - async fn join_server_broadcast(&self, id: &str) -> BroadcastReceiver> { - let id = id.into(); - match self.get_storage().docs().remote().write().await.entry(id) { - Entry::Occupied(tx) => tx.get().subscribe(), - Entry::Vacant(v) => { - let (tx, rx) = broadcast(100); - v.insert(tx); - rx - } - } - } - - async fn join_broadcast( - &self, - workspace: &mut Workspace, - identifier: String, - last_synced: Sender, - ) -> BroadcastSender { - let id = workspace.id(); - info!("join_broadcast, {:?}", workspace.id()); - // broadcast channel - let broadcast_tx = match self.get_channel().write().await.entry(id.clone()) { - Entry::Occupied(tx) => tx.get().clone(), - Entry::Vacant(v) => { - let (tx, _) = broadcast(10240); - v.insert(tx.clone()); - tx.clone() - } - }; - - // Listen to changes of the local workspace, encode changes in awareness and - // Doc, and broadcast them. It returns the 'broadcast_rx' object to - // receive the content that was sent - subscribe(workspace, identifier.clone(), broadcast_tx.clone()).await; - - // save update thread - self.save_update(&id, identifier, broadcast_tx.subscribe(), last_synced) - .await; - - // returns the 'broadcast_tx' which can be subscribed later, to receive local - // workspace changes - broadcast_tx - } - - async fn save_update( - &self, - id: &str, - identifier: String, - mut broadcast: BroadcastReceiver, - last_synced: Sender, - ) { - let docs = self.get_storage().docs().clone(); - let id = id.to_string(); - - tokio::spawn(async move { - trace!("save update thread {id}-{identifier} started"); - let updates = Arc::new(Mutex::new(HashMap::>>::new())); - - let handler = { - let id = id.clone(); - let updates = updates.clone(); - tokio::spawn(async move { - loop { - match broadcast.recv().await { - Ok(data) => match data { - BroadcastType::BroadcastRawContent(update) => { - trace!("receive raw update: {}", update.len()); - let mut decoder = RawDecoder::new(update); - if let Ok(guid) = decoder.read_var_string() { - match updates.lock().await.entry(guid) { - Entry::Occupied(mut updates) => { - updates.get_mut().push(decoder.drain()); - } - Entry::Vacant(v) => { - v.insert(vec![decoder.drain()]); - } - }; - }; - } - BroadcastType::CloseUser(user) if user == identifier => break, - BroadcastType::CloseAll => break, - _ => {} - }, - Err(RecvError::Lagged(num)) => { - debug!("save update thread {id}-{identifier} lagged: {num}"); - } - Err(RecvError::Closed) => { - debug!("save update thread {id}-{identifier} closed"); - break; - } - } - } - trace!("save update thread {id}-{identifier} finished"); - }) - }; - - loop { - { - let mut updates = updates.lock().await; - if !updates.is_empty() { - for (guid, updates) in updates.drain() { - debug!("save {} updates from {guid}", updates.len()); - - for update in updates { - if let Err(e) = docs.update_doc(id.clone(), guid.clone(), &update).await { - error!("failed to save update of {}: {:?}", id, e); - } - } - } - last_synced.send(Utc::now().timestamp_millis()).await.unwrap(); - } else if handler.is_finished() { - break; - } - } - sleep(Duration::from_secs(1)).await; - } - }); - } - - async fn apply_change( - &self, - id: &str, - identifier: &str, - local_tx: MpscSender, - mut remote_rx: MpscReceiver>, - last_synced: Sender, - ) { - // collect messages from remote - let identifier = identifier.to_owned(); - let id = id.to_string(); - let mut workspace = self - .get_storage() - .get_workspace(&id) - .await - .expect("workspace not found"); - tokio::spawn(async move { - trace!("apply update thread {id}-{identifier} started"); - let mut updates = Vec::>::new(); - - loop { - tokio::select! { - binary = remote_rx.recv() => { - if let Some(binary) = binary { - if binary == [0, 2, 2, 0, 0] || binary == [1, 1, 0] { - // skip empty update - continue; - } - debug!("apply_change: recv binary: {:?}", binary.len()); - updates.push(binary); - } else { - break; - } - }, - _ = sleep(Duration::from_millis(100)) => { - if !updates.is_empty() { - debug!("apply {} updates for {id}", updates.len()); - - let updates = updates.drain(..).collect::>(); - let updates_len = updates.len(); - let ts = Instant::now(); - let message = workspace.sync_messages(updates).await; - if ts.elapsed().as_micros() > 50 { - debug!( - "apply {updates_len} remote update cost: {}ms", - ts.elapsed().as_micros(), - ); - } - - for reply in message { - trace!("send pipeline message by {identifier:?}: {}", reply.len()); - if local_tx.send(Message::Binary(reply.clone())).await.is_err() { - // pipeline was closed - break; - } - } - - last_synced - .send(Utc::now().timestamp_millis()) - .await - .unwrap(); - } - } - } - } - }); - } -} diff --git a/libs/jwst-core-rpc/src/handler.rs b/libs/jwst-core-rpc/src/handler.rs deleted file mode 100644 index 8b6dc5eea..000000000 --- a/libs/jwst-core-rpc/src/handler.rs +++ /dev/null @@ -1,447 +0,0 @@ -use std::{sync::Arc, time::Instant}; - -use chrono::Utc; -use jwst_core::{debug, error, info, trace, warn}; -use tokio::{ - sync::mpsc::{Receiver, Sender}, - time::{sleep, Duration}, -}; - -use super::{BroadcastType, Message, RpcContextImpl}; - -pub async fn handle_connector( - context: Arc + Send + Sync + 'static>, - workspace_id: String, - identifier: String, - get_channel: impl FnOnce() -> (Sender, Receiver>, Sender), -) -> bool { - info!("{} collaborate with workspace {}", identifier, workspace_id); - - // An abstraction of the established socket connection. Use tx to broadcast and - // rx to receive. - let (tx, rx, last_synced) = get_channel(); - - let mut ws = context - .get_workspace(&workspace_id) - .await - .expect("failed to get workspace"); - - // Continuously receive information from the remote socket, apply it to the - // local workspace, and send the encoded updates back to the remote end - // through the socket. - context - .apply_change(&workspace_id, &identifier, tx.clone(), rx, last_synced.clone()) - .await; - - // Both of broadcast_update and server_update are sent to the remote socket - // through 'tx' The 'broadcast_update' is the receiver for updates to the - // awareness and Doc of the local workspace. It uses channel, which is owned - // by the server itself and is stored in the server's memory (not persisted)." - let broadcast_tx = context - .join_broadcast(&mut ws, identifier.clone(), last_synced.clone()) - .await; - let mut broadcast_rx = broadcast_tx.subscribe(); - // Obtaining the receiver corresponding to DocAutoStorage in storage. The sender - // is used in the doc::write_update(). The remote used is the one belonging - // to DocAutoStorage and is owned by the server itself, stored in the - // server's memory (not persisted). - let mut server_rx = context.join_server_broadcast(&workspace_id).await; - - // Send initialization message. - match ws.sync_init_message().await { - Ok(init_data) => { - debug!("send init data:{:?}", init_data); - if tx.send(Message::Binary(init_data)).await.is_err() { - warn!("failed to send init message: {}", identifier); - // client disconnected - if let Err(e) = tx.send(Message::Close).await { - error!("failed to send close event: {}", e); - } - last_synced.send(0).await.unwrap(); - return false; - } - } - Err(e) => { - warn!("failed to generate {} init message: {}", identifier, e); - if let Err(e) = tx.send(Message::Close).await { - error!("failed to send close event: {}", e); - } - last_synced.send(0).await.unwrap(); - return false; - } - } - - last_synced.send(Utc::now().timestamp_millis()).await.unwrap(); - - 'sync: loop { - tokio::select! { - Ok(msg) = server_rx.recv()=> { - let ts = Instant::now(); - trace!("recv from server update: {:?}", msg); - if tx.send(Message::Binary(msg.clone())).await.is_err() { - // pipeline was closed - break 'sync; - } - if ts.elapsed().as_micros() > 100 { - debug!("process server update cost: {}ms", ts.elapsed().as_micros()); - } - }, - Ok(msg) = broadcast_rx.recv()=> { - let ts = Instant::now(); - match msg { - BroadcastType::BroadcastAwareness(data) => { - let ts = Instant::now(); - trace!( - "recv awareness update from broadcast: {:?}bytes", - data.len() - ); - if tx.send(Message::Binary(data.clone())).await.is_err() { - // pipeline was closed - break 'sync; - } - if ts.elapsed().as_micros() > 100 { - debug!( - "process broadcast awareness cost: {}ms", - ts.elapsed().as_micros() - ); - } - } - BroadcastType::BroadcastContent(data) => { - let ts = Instant::now(); - trace!("recv content update from broadcast: {:?}bytes", data.len()); - if tx.send(Message::Binary(data.clone())).await.is_err() { - // pipeline was closed - break 'sync; - } - if ts.elapsed().as_micros() > 100 { - debug!( - "process broadcast content cost: {}ms", - ts.elapsed().as_micros() - ); - } - } - BroadcastType::CloseUser(user) if user == identifier => { - let ts = Instant::now(); - if tx.send(Message::Close).await.is_err() { - // pipeline was closed - break 'sync; - } - if ts.elapsed().as_micros() > 100 { - debug!("process close user cost: {}ms", ts.elapsed().as_micros()); - } - - break; - } - BroadcastType::CloseAll => { - let ts = Instant::now(); - if tx.send(Message::Close).await.is_err() { - // pipeline was closed - break 'sync; - } - if ts.elapsed().as_micros() > 100 { - debug!("process close all cost: {}ms", ts.elapsed().as_micros()); - } - - break 'sync; - } - _ => {} - } - - if ts.elapsed().as_micros() > 100 { - debug!("process broadcast cost: {}ms", ts.elapsed().as_micros()); - } - }, - _ = sleep(Duration::from_secs(1)) => { - if tx.is_closed() || tx.send(Message::Ping).await.is_err() { - break 'sync; - } - } - } - } - - // make a final store - context - .get_storage() - .full_migrate(workspace_id.clone(), None, false) - .await; - let _ = broadcast_tx.send(BroadcastType::CloseUser(identifier.clone())); - info!("{} stop collaborate with workspace {}", identifier, workspace_id); - true -} - -#[cfg(test)] -mod test { - use std::sync::atomic::{AtomicU64, Ordering}; - - use indicatif::{MultiProgress, ProgressBar, ProgressIterator, ProgressStyle}; - use jwst_core::JwstResult; - - use super::{ - super::{connect_memory_workspace, MinimumServerContext}, - *, - }; - #[cfg(feature = "webrtc")] - use crate::{ - webrtc_datachannel_client_begin, webrtc_datachannel_client_commit, webrtc_datachannel_server_connector, - }; - - #[tokio::test] - #[ignore = "skip in ci"] - #[cfg(feature = "webrtc")] - async fn webrtc_datachannel_connector_test() { - let (offer, pc, tx1, mut rx1, _) = webrtc_datachannel_client_begin().await; - let (answer, tx2, mut rx2, _) = webrtc_datachannel_server_connector(offer).await; - webrtc_datachannel_client_commit(answer, pc).await; - - let data_a_1 = String::from("data_a_1"); - let data_a_2 = String::from("data_a_2"); - let data_a_3 = String::from("data_a_3"); - - let data_b_1 = String::from("data_b_1"); - let data_b_2 = String::from("data_b_2"); - let data_b_3 = String::from("data_b_3"); - - tx1.send(Message::Binary(data_a_1.clone().into_bytes())).await.unwrap(); - tx1.send(Message::Binary(data_a_2.clone().into_bytes())).await.unwrap(); - - tx2.send(Message::Binary(data_b_1.clone().into_bytes())).await.unwrap(); - tx2.send(Message::Binary(data_b_2.clone().into_bytes())).await.unwrap(); - - tx1.send(Message::Binary(data_a_3.clone().into_bytes())).await.unwrap(); - tx2.send(Message::Binary(data_b_3.clone().into_bytes())).await.unwrap(); - - if let Some(message) = rx2.recv().await { - assert_eq!(String::from_utf8(message).ok().unwrap(), data_a_1); - } - if let Some(message) = rx2.recv().await { - assert_eq!(String::from_utf8(message).ok().unwrap(), data_a_2); - } - - if let Some(message) = rx1.recv().await { - assert_eq!(String::from_utf8(message).ok().unwrap(), data_b_1); - } - if let Some(message) = rx1.recv().await { - assert_eq!(String::from_utf8(message).ok().unwrap(), data_b_2); - } - - if let Some(message) = rx2.recv().await { - assert_eq!(String::from_utf8(message).ok().unwrap(), data_a_3); - } - if let Some(message) = rx1.recv().await { - assert_eq!(String::from_utf8(message).ok().unwrap(), data_b_3); - } - } - - #[tokio::test] - #[ignore = "unstable, skip in ci and wait for the refactoring of the sync logic"] - async fn sync_test() -> JwstResult<()> { - let workspace_id = format!("test{}", rand::random::()); - - let (server, mut ws, init_state) = MinimumServerContext::new_with_workspace(&workspace_id).await; - - let (mut doc1, _, _, _, _) = connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; - let (doc2, tx2, tx_handler, rx_handler, rt) = - connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; - - // close connection after doc1 is broadcasted - doc2.subscribe(move |_| { - rt.block_on(async { - tx2.send(Message::Close).await.unwrap(); - }); - }); - - // collect the update from yrs's editing - let update = { - let mut ws = jwst_core::Workspace::from_binary(doc1.encode_update_v1().unwrap(), &workspace_id).unwrap(); - - let mut space = ws.get_space("space").unwrap(); - let mut block1 = space.create("block1", "flavour1").unwrap(); - block1.set("key1", "val1").unwrap(); - - ws.doc().encode_update_v1().unwrap() - }; - // apply update with jwst-codec - doc1.apply_update_from_binary(update).unwrap(); - - // await the task to make sure the doc1 is broadcasted before check doc2 - tx_handler.await.unwrap(); - rx_handler.join().unwrap(); - - // collect the update from jwst-codec and check the result - { - let mut ws = jwst_core::Workspace::from_binary(doc2.encode_update_v1().unwrap(), &workspace_id).unwrap(); - - let space = ws.get_space("space").unwrap(); - let block1 = space.get("block1").unwrap(); - - assert_eq!(block1.flavour(), "flavour1"); - assert_eq!(block1.get("key1").unwrap().to_string(), "val1"); - } - - { - let space = ws.get_space("space").unwrap(); - let block1 = space.get("block1").unwrap(); - - assert_eq!(block1.flavour(), "flavour1"); - assert_eq!(block1.get("key1").unwrap().to_string(), "val1"); - } - - Ok(()) - } - - #[ignore = "somewhat slow, only natively tested"] - #[test] - fn sync_test_cycle() -> JwstResult<()> { - jwst_logger::init_logger("jwst-rpc"); - - for _ in 0..1000 { - sync_test()?; - } - Ok(()) - } - - async fn single_sync_stress_test(mp: &MultiProgress) -> JwstResult<()> { - // jwst_logger::init_logger("jwst-rpc"); - - let style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}") - .unwrap() - .progress_chars("##-"); - - let workspace_id = format!("test{}", rand::random::()); - - let (server, mut ws, init_state) = MinimumServerContext::new_with_workspace(&workspace_id).await; - - let mut jobs = vec![]; - - let collaborator_pb = mp.add(ProgressBar::new(10)); - collaborator_pb.set_style(style.clone()); - collaborator_pb.set_message("collaborators"); - let collaborator = Arc::new(AtomicU64::new(0)); - - let pb = mp.add(ProgressBar::new(1000)); - pb.set_style(style.clone()); - pb.set_message("writing"); - for i in (0..1000).progress_with(pb) { - let collaborator = collaborator.clone(); - let collaborator_pb = collaborator_pb.clone(); - let init_state = init_state.clone(); - let server = server.clone(); - let mut ws = ws.clone(); - - collaborator.fetch_add(1, Ordering::Relaxed); - collaborator_pb.set_position(collaborator.load(Ordering::Relaxed)); - let (doc, doc_tx, tx_handler, rx_handler, _rt) = - connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; - let mut doc = jwst_core::Workspace::from_binary(doc.encode_update_v1().unwrap(), &workspace_id).unwrap(); - - let handler = std::thread::spawn(move || { - // close connection after doc1 is broadcasted - let block_id = format!("block{}", i); - { - // let block_id = block_id.clone(); - let doc_tx = doc_tx.clone(); - doc.doc().subscribe(move |_u| { - // TODO: support changed block record - // let block_changed = t - // .changed_parent_types() - // .iter() - // .filter_map(|ptr| { - // let value: yrs::types::Value = (*ptr).into(); - // value.to_ymap() - // }) - // .flat_map(|map| map.keys(t).map(|k| k.to_string()).collect::>()) - // .any(|key| key == block_id); - let block_changed = false; - - if block_changed { - if let Err(e) = futures::executor::block_on(doc_tx.send(Message::Close)) { - error!("send close message failed: {}", e); - } - } - }); - } - - { - let mut space = doc.get_space("space").unwrap(); - let mut block = space.create(block_id.clone(), format!("flavour{}", i)).unwrap(); - block.set(&format!("key{}", i), format!("val{}", i)).unwrap(); - } - - // await the task to make sure the doc1 is broadcasted before check doc2 - futures::executor::block_on(tx_handler).unwrap(); - rx_handler.join().unwrap(); - - { - let space = ws.get_space("space").unwrap(); - let block1 = space.get(format!("block{}", i)).unwrap(); - - assert_eq!(block1.flavour(), format!("flavour{}", i)); - assert_eq!( - block1.get(&format!("key{}", i)).unwrap().to_string(), - format!("val{}", i) - ); - } - - collaborator.fetch_sub(1, Ordering::Relaxed); - collaborator_pb.set_position(collaborator.load(Ordering::Relaxed)); - }); - jobs.push(handler); - } - - let pb = mp.add(ProgressBar::new(jobs.len().try_into().unwrap())); - pb.set_style(style.clone()); - pb.set_message("joining"); - for handler in jobs.into_iter().progress_with(pb) { - if let Err(e) = handler.join() { - panic!("{:?}", e.as_ref()); - } - } - - info!("check the workspace"); - - let pb = mp.add(ProgressBar::new(1000)); - pb.set_style(style.clone()); - pb.set_message("final checking"); - - { - let space = ws.get_space("space").unwrap(); - for i in (0..1000).progress_with(pb) { - let block1 = space.get(format!("block{}", i)).unwrap(); - - assert_eq!(block1.flavour(), format!("flavour{}", i)); - assert_eq!( - block1.get(&format!("key{}", i)).unwrap().to_string(), - format!("val{}", i) - ); - } - } - - Ok(()) - } - - #[ignore = "somewhat slow, only natively tested"] - #[tokio::test(flavor = "multi_thread")] - async fn sync_stress_test() -> JwstResult<()> { - let mp = MultiProgress::new(); - single_sync_stress_test(&mp).await - } - - #[ignore = "somewhat slow, only natively tested"] - #[tokio::test(flavor = "multi_thread")] - async fn sync_stress_test_cycle() -> JwstResult<()> { - // jwst_logger::init_logger("jwst-rpc"); - - let mp = MultiProgress::new(); - let style = ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}") - .unwrap() - .progress_chars("##-"); - - let pb = mp.add(ProgressBar::new(1000)); - pb.set_style(style.clone()); - pb.set_message("cycle stress test"); - for _ in (0..1000).progress_with(pb.clone()) { - single_sync_stress_test(&mp).await?; - } - Ok(()) - } -} diff --git a/libs/jwst-core-rpc/src/lib.rs b/libs/jwst-core-rpc/src/lib.rs deleted file mode 100644 index 5c3a884df..000000000 --- a/libs/jwst-core-rpc/src/lib.rs +++ /dev/null @@ -1,52 +0,0 @@ -#[forbid(unsafe_code)] -mod broadcast; -mod client; -mod connector; -mod context; -mod handler; -mod types; -mod utils; - -use std::{collections::hash_map::Entry, sync::Arc, time::Instant}; - -pub use broadcast::{BroadcastChannels, BroadcastType}; -#[cfg(feature = "webrtc")] -pub use client::start_webrtc_client_sync; -#[cfg(feature = "websocket")] -pub use client::start_websocket_client_sync; -pub use client::CachedLastSynced; -pub use connector::memory_connector; -#[cfg(feature = "webrtc")] -pub use connector::webrtc_datachannel_client_begin; -#[cfg(feature = "webrtc")] -pub use connector::webrtc_datachannel_client_commit; -#[cfg(feature = "webrtc")] -pub use connector::webrtc_datachannel_server_connector; -#[cfg(feature = "websocket")] -pub use connector::{axum_socket_connector, tungstenite_socket_connector}; -pub use context::RpcContextImpl; -pub use handler::handle_connector; -use jwst_core::{debug, error, info, trace, warn}; -use tokio::{ - sync::mpsc::{Receiver, Sender}, - time::{sleep, Duration}, -}; -pub use utils::{connect_memory_workspace, MinimumServerContext}; -#[cfg(feature = "webrtc")] -pub use webrtcrs::peer_connection::sdp::session_description::RTCSessionDescription; - -#[derive(Clone, PartialEq, Eq, Debug, Default)] -pub enum SyncState { - #[default] - Offline, - Connected, - Finished, - Error(String), -} - -#[derive(Debug)] -pub enum Message { - Binary(Vec), - Close, - Ping, -} diff --git a/libs/jwst-core-rpc/src/types.rs b/libs/jwst-core-rpc/src/types.rs deleted file mode 100644 index 3182f90c0..000000000 --- a/libs/jwst-core-rpc/src/types.rs +++ /dev/null @@ -1,20 +0,0 @@ -use thiserror::Error; -#[cfg(feature = "websocket")] -use tokio_tungstenite::tungstenite; - -#[derive(Debug, Error)] -pub enum JwstRpcError { - #[cfg(feature = "websocket")] - #[error("failed to connect websocket: {0}")] - WebsocketConnect(#[from] tungstenite::Error), - #[error("jwst error")] - Jwst(#[from] jwst_core::JwstError), - #[allow(dead_code)] - #[error("failed to encode sync message")] - ProtocolEncode(std::io::Error), - #[cfg(feature = "websocket")] - #[error("failed to parse url")] - UrlParse(#[from] url::ParseError), -} - -pub type JwstRpcResult = Result; diff --git a/libs/jwst-core-rpc/src/utils/memory_workspace.rs b/libs/jwst-core-rpc/src/utils/memory_workspace.rs deleted file mode 100644 index 9e38031c9..000000000 --- a/libs/jwst-core-rpc/src/utils/memory_workspace.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::thread::JoinHandle as StdJoinHandler; - -use jwst_codec::Doc; -use nanoid::nanoid; -use tokio::{runtime::Runtime, sync::mpsc::channel, task::JoinHandle as TokioJoinHandler}; - -use super::*; - -pub async fn connect_memory_workspace( - server: Arc, - init_state: &[u8], - id: &str, -) -> ( - Doc, - Sender, - TokioJoinHandler<()>, - StdJoinHandler<()>, - Arc, -) { - let rt = Arc::new(tokio::runtime::Runtime::new().unwrap()); - - let mut doc = Doc::default(); - doc.apply_update_from_binary(init_state.to_vec()).unwrap(); - - let (tx, rx, tx_handler, rx_handler) = memory_connector(rt.clone(), doc.clone()); - { - let (last_synced_tx, mut last_synced_rx) = channel::(128); - let tx = tx.clone(); - let workspace_id = id.to_string(); - { - let rt = rt.clone(); - std::thread::spawn(move || { - rt.block_on(handle_connector(server, workspace_id, nanoid!(), move || { - (tx, rx, last_synced_tx) - })); - }); - } - - let success = last_synced_rx.recv().await; - - if success.unwrap_or(0) > 0 { - info!("{id} first init success"); - } else { - error!("{id} first init failed"); - } - - rt.spawn(async move { - while let Some(last_synced) = last_synced_rx.recv().await { - info!("last synced: {}", last_synced); - } - }); - } - - (doc, tx, tx_handler, rx_handler, rt) -} diff --git a/libs/jwst-core-rpc/src/utils/mod.rs b/libs/jwst-core-rpc/src/utils/mod.rs deleted file mode 100644 index 90bd7951c..000000000 --- a/libs/jwst-core-rpc/src/utils/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod memory_workspace; -mod server_context; - -pub use memory_workspace::connect_memory_workspace; -pub use server_context::MinimumServerContext; - -use super::*; diff --git a/libs/jwst-core-rpc/src/utils/server_context.rs b/libs/jwst-core-rpc/src/utils/server_context.rs deleted file mode 100644 index d2e114bcb..000000000 --- a/libs/jwst-core-rpc/src/utils/server_context.rs +++ /dev/null @@ -1,75 +0,0 @@ -use std::{collections::HashMap, time::Duration}; - -use jwst_codec::StateVector; -use jwst_core::{DocStorage, Workspace}; -use jwst_core_storage::{BlobStorageType, JwstStorage}; -use tokio::{sync::RwLock, time::sleep}; - -use super::*; - -pub struct MinimumServerContext { - channel: BroadcastChannels, - storage: JwstStorage, -} - -// just for test -impl MinimumServerContext { - pub async fn new() -> Arc { - let storage = 'connect: loop { - let mut retry = 3; - match JwstStorage::new_with_migration( - &std::env::var("DATABASE_URL") - .map(|url| format!("{url}_binary")) - .unwrap_or("sqlite::memory:".into()), - BlobStorageType::DB, - ) - .await - { - Ok(storage) => break 'connect Ok(storage), - Err(e) => { - retry -= 1; - if retry > 0 { - error!("failed to connect database: {}", e); - sleep(Duration::from_secs(1)).await; - } else { - break 'connect Err(e); - } - } - } - } - .unwrap(); - - Arc::new(Self { - channel: RwLock::new(HashMap::new()), - storage, - }) - } - - pub async fn new_with_workspace(workspace_id: &str) -> (Arc, Workspace, Vec) { - let server = Self::new().await; - server - .get_storage() - .docs() - .delete_workspace(workspace_id) - .await - .unwrap(); - let ws = server.get_workspace(workspace_id).await.unwrap(); - - let init_state = ws - .doc() - .encode_state_as_update_v1(&StateVector::default()) - .expect("encode_state_as_update_v1 failed"); - - (server, ws, init_state) - } -} - -impl RpcContextImpl<'_> for MinimumServerContext { - fn get_storage(&self) -> &JwstStorage { - &self.storage - } - - fn get_channel(&self) -> &BroadcastChannels { - &self.channel - } -} diff --git a/libs/jwst-rpc/Cargo.toml b/libs/jwst-rpc/Cargo.toml index 16e188f03..7cb9fe494 100644 --- a/libs/jwst-rpc/Cargo.toml +++ b/libs/jwst-rpc/Cargo.toml @@ -17,20 +17,12 @@ async-trait = "0.1.68" byteorder = "1.4.3" chrono = "0.4.26" futures = "0.3.28" -lib0 = { version = "0.16.5", features = ["lib0-serde"] } -lru_time_cache = "0.11.11" nanoid = "0.4.0" -nom = "7.1.3" rand = "0.8.5" serde = "1.0.183" serde_json = "1.0.104" thiserror = "1.0.40" -tokio = { version = "1.27.0", features = [ - "macros", - "rt-multi-thread", - "signal", -] } -yrs = "0.16.5" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] } # ======== websocket dependencies ======== axum = { version = "0.6.16", features = ["ws"], optional = true } @@ -48,10 +40,9 @@ reqwest = { version = "0.11.18", default-features = false, features = [ webrtcrs = { package = "webrtc", version = "0.8.0", optional = true } # ======= workspace dependencies ======= -jwst = { workspace = true } jwst-codec = { workspace = true } jwst-core = { workspace = true } -jwst-storage = { workspace = true } +jwst-core-storage = { workspace = true } [dev-dependencies] indicatif = "0.17.3" diff --git a/libs/jwst-rpc/src/broadcast.rs b/libs/jwst-rpc/src/broadcast.rs index 29bf8a84e..8e4581613 100644 --- a/libs/jwst-rpc/src/broadcast.rs +++ b/libs/jwst-rpc/src/broadcast.rs @@ -1,8 +1,7 @@ -use std::{collections::HashMap, sync::Mutex}; +use std::collections::HashMap; -use jwst::Workspace; use jwst_codec::{encode_update_as_message, encode_update_with_guid, write_sync_message, SyncMessage}; -use lru_time_cache::LruCache; +use jwst_core::Workspace; use tokio::sync::{broadcast::Sender, RwLock}; use super::*; @@ -24,11 +23,6 @@ pub async fn subscribe(workspace: &mut Workspace, identifier: String, sender: Br let sender = sender.clone(); let workspace_id = workspace.id(); - let dedup_cache = Arc::new(Mutex::new(LruCache::with_expiry_duration_and_capacity( - Duration::from_micros(100), - 128, - ))); - workspace .on_awareness_update(move |awareness, e| { let mut buffer = Vec::new(); @@ -40,12 +34,8 @@ pub async fn subscribe(workspace: &mut Workspace, identifier: String, sender: Br return; } - let mut dedup_cache = dedup_cache.lock().unwrap_or_else(|e| e.into_inner()); - if !dedup_cache.contains_key(&buffer) { - if sender.send(BroadcastType::BroadcastAwareness(buffer.clone())).is_err() { - debug!("broadcast channel {workspace_id} has been closed",) - } - dedup_cache.insert(buffer, ()); + if sender.send(BroadcastType::BroadcastAwareness(buffer.clone())).is_err() { + debug!("broadcast channel {workspace_id} has been closed",) } }) .await; @@ -53,11 +43,11 @@ pub async fn subscribe(workspace: &mut Workspace, identifier: String, sender: Br { let sender = sender.clone(); let workspace_id = workspace.id(); - workspace.observe(move |_, e| { - trace!("workspace {} changed: {}bytes", workspace_id, &e.update.len()); + workspace.doc().subscribe(move |update| { + debug!("workspace {} changed: {}bytes", workspace_id, update.len()); - match encode_update_with_guid(e.update.clone(), workspace_id.clone()) - .and_then(|update| encode_update_as_message(update.clone()).map(|u| (update, u))) + match encode_update_with_guid(update.to_vec(), workspace_id.clone()) + .and_then(|update_with_guid| encode_update_as_message(update.to_vec()).map(|u| (update_with_guid, u))) { Ok((broadcast_update, sendable_update)) => { if sender diff --git a/libs/jwst-rpc/src/connector/webrtc.rs b/libs/jwst-rpc/src/connector/webrtc.rs index 839966fd7..5c49ea0ad 100644 --- a/libs/jwst-rpc/src/connector/webrtc.rs +++ b/libs/jwst-rpc/src/connector/webrtc.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use bytes::Bytes; -use jwst::{debug, error, info, trace, warn}; +use jwst_core::{debug, error, info, trace, warn}; use tokio::sync::mpsc::{channel, Receiver, Sender}; use webrtcrs::{ api::APIBuilder, diff --git a/libs/jwst-rpc/src/context.rs b/libs/jwst-rpc/src/context.rs index b8a6d5422..e442fc3bc 100644 --- a/libs/jwst-rpc/src/context.rs +++ b/libs/jwst-rpc/src/context.rs @@ -1,45 +1,21 @@ -use std::{ - collections::HashMap, - panic::{catch_unwind, AssertUnwindSafe}, -}; +use std::collections::HashMap; use async_trait::async_trait; use chrono::Utc; -use jwst::{DocStorage, Workspace}; use jwst_codec::{CrdtReader, RawDecoder}; -use jwst_storage::{JwstStorage, JwstStorageResult}; +use jwst_core::{DocStorage, Workspace}; +use jwst_core_storage::{JwstStorage, JwstStorageResult}; use tokio::sync::{ broadcast::{channel as broadcast, error::RecvError, Receiver as BroadcastReceiver, Sender as BroadcastSender}, mpsc::{Receiver as MpscReceiver, Sender as MpscSender}, Mutex, }; -use yrs::merge_updates_v1; use super::{ broadcast::{subscribe, BroadcastChannels, BroadcastType}, *, }; -fn merge_updates(id: &str, updates: Vec>) -> Vec> { - match catch_unwind(AssertUnwindSafe(move || { - match merge_updates_v1(&updates.iter().map(std::ops::Deref::deref).collect::>()) { - Ok(update) => { - info!("merge {} updates", updates.len()); - vec![update] - } - Err(e) => { - error!("failed to merge update of {}: {}", id, e); - updates - } - } - })) { - Ok(updates) => updates, - Err(e) => { - error!("failed to merge update of {}: {:?}", id, e); - vec![] - } - } -} #[async_trait] pub trait RpcContextImpl<'a> { fn get_storage(&self) -> &JwstStorage; @@ -141,6 +117,7 @@ pub trait RpcContextImpl<'a> { } } } + trace!("save update thread {id}-{identifier} finished"); }) }; @@ -151,8 +128,6 @@ pub trait RpcContextImpl<'a> { for (guid, updates) in updates.drain() { debug!("save {} updates from {guid}", updates.len()); - let updates = merge_updates(&id, updates); - for update in updates { if let Err(e) = docs.update_doc(id.clone(), guid.clone(), &update).await { error!("failed to save update of {}: {:?}", id, e); @@ -197,7 +172,7 @@ pub trait RpcContextImpl<'a> { // skip empty update continue; } - trace!("apply_change: recv binary: {:?}", binary.len()); + debug!("apply_change: recv binary: {:?}", binary.len()); updates.push(binary); } else { break; diff --git a/libs/jwst-rpc/src/handler.rs b/libs/jwst-rpc/src/handler.rs index 2c63afa3a..8b6dc5eea 100644 --- a/libs/jwst-rpc/src/handler.rs +++ b/libs/jwst-rpc/src/handler.rs @@ -1,7 +1,7 @@ use std::{sync::Arc, time::Instant}; use chrono::Utc; -use jwst::{debug, error, info, trace, warn}; +use jwst_core::{debug, error, info, trace, warn}; use tokio::{ sync::mpsc::{Receiver, Sender}, time::{sleep, Duration}, @@ -50,6 +50,7 @@ pub async fn handle_connector( // Send initialization message. match ws.sync_init_message().await { Ok(init_data) => { + debug!("send init data:{:?}", init_data); if tx.send(Message::Binary(init_data)).await.is_err() { warn!("failed to send init message: {}", identifier); // client disconnected @@ -173,8 +174,7 @@ mod test { use std::sync::atomic::{AtomicU64, Ordering}; use indicatif::{MultiProgress, ProgressBar, ProgressIterator, ProgressStyle}; - use jwst::{JwstError, JwstResult}; - use yrs::Map; + use jwst_core::JwstResult; use super::{ super::{connect_memory_workspace, MinimumServerContext}, @@ -237,7 +237,7 @@ mod test { async fn sync_test() -> JwstResult<()> { let workspace_id = format!("test{}", rand::random::()); - let (server, ws, init_state) = MinimumServerContext::new_with_workspace(&workspace_id).await; + let (server, mut ws, init_state) = MinimumServerContext::new_with_workspace(&workspace_id).await; let (mut doc1, _, _, _, _) = connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; let (doc2, tx2, tx_handler, rx_handler, rt) = @@ -251,14 +251,15 @@ mod test { }); // collect the update from yrs's editing - let update = - jwst::Workspace::from_binary(&doc1.encode_update_v1().unwrap(), &workspace_id).with_trx(|mut t| { - let space = t.get_space("space"); - let block1 = space.create(&mut t.trx, "block1", "flavour1").unwrap(); - block1.set(&mut t.trx, "key1", "val1").unwrap(); - t.commit(); - t.trx.encode_update_v1().unwrap() - }); + let update = { + let mut ws = jwst_core::Workspace::from_binary(doc1.encode_update_v1().unwrap(), &workspace_id).unwrap(); + + let mut space = ws.get_space("space").unwrap(); + let mut block1 = space.create("block1", "flavour1").unwrap(); + block1.set("key1", "val1").unwrap(); + + ws.doc().encode_update_v1().unwrap() + }; // apply update with jwst-codec doc1.apply_update_from_binary(update).unwrap(); @@ -267,27 +268,23 @@ mod test { rx_handler.join().unwrap(); // collect the update from jwst-codec and check the result - jwst::Workspace::from_binary(&doc2.encode_update_v1().unwrap(), &workspace_id).retry_with_trx( - |mut t| { - let space = t.get_space("space"); - let block1 = space.get(&mut t.trx, "block1").unwrap(); + { + let mut ws = jwst_core::Workspace::from_binary(doc2.encode_update_v1().unwrap(), &workspace_id).unwrap(); - assert_eq!(block1.flavour(&t.trx), "flavour1"); - assert_eq!(block1.get(&t.trx, "key1").unwrap().to_string(), "val1"); - }, - 10, - )?; + let space = ws.get_space("space").unwrap(); + let block1 = space.get("block1").unwrap(); - ws.retry_with_trx( - |mut t| { - let space = t.get_space("space"); - let block1 = space.get(&mut t.trx, "block1").unwrap(); + assert_eq!(block1.flavour(), "flavour1"); + assert_eq!(block1.get("key1").unwrap().to_string(), "val1"); + } - assert_eq!(block1.flavour(&t.trx), "flavour1"); - assert_eq!(block1.get(&t.trx, "key1").unwrap().to_string(), "val1"); - }, - 10, - )?; + { + let space = ws.get_space("space").unwrap(); + let block1 = space.get("block1").unwrap(); + + assert_eq!(block1.flavour(), "flavour1"); + assert_eq!(block1.get("key1").unwrap().to_string(), "val1"); + } Ok(()) } @@ -312,7 +309,7 @@ mod test { let workspace_id = format!("test{}", rand::random::()); - let (server, ws, init_state) = MinimumServerContext::new_with_workspace(&workspace_id).await; + let (server, mut ws, init_state) = MinimumServerContext::new_with_workspace(&workspace_id).await; let mut jobs = vec![]; @@ -329,30 +326,32 @@ mod test { let collaborator_pb = collaborator_pb.clone(); let init_state = init_state.clone(); let server = server.clone(); - let ws = ws.clone(); + let mut ws = ws.clone(); collaborator.fetch_add(1, Ordering::Relaxed); collaborator_pb.set_position(collaborator.load(Ordering::Relaxed)); let (doc, doc_tx, tx_handler, rx_handler, _rt) = connect_memory_workspace(server.clone(), &init_state, &workspace_id).await; - let mut doc = jwst::Workspace::from_binary(&doc.encode_update_v1().unwrap(), &workspace_id); + let mut doc = jwst_core::Workspace::from_binary(doc.encode_update_v1().unwrap(), &workspace_id).unwrap(); let handler = std::thread::spawn(move || { // close connection after doc1 is broadcasted let block_id = format!("block{}", i); { - let block_id = block_id.clone(); + // let block_id = block_id.clone(); let doc_tx = doc_tx.clone(); - doc.observe(move |t, _| { - let block_changed = t - .changed_parent_types() - .iter() - .filter_map(|ptr| { - let value: yrs::types::Value = (*ptr).into(); - value.to_ymap() - }) - .flat_map(|map| map.keys(t).map(|k| k.to_string()).collect::>()) - .any(|key| key == block_id); + doc.doc().subscribe(move |_u| { + // TODO: support changed block record + // let block_changed = t + // .changed_parent_types() + // .iter() + // .filter_map(|ptr| { + // let value: yrs::types::Value = (*ptr).into(); + // value.to_ymap() + // }) + // .flat_map(|map| map.keys(t).map(|k| k.to_string()).collect::>()) + // .any(|key| key == block_id); + let block_changed = false; if block_changed { if let Err(e) = futures::executor::block_on(doc_tx.send(Message::Close)) { @@ -362,38 +361,26 @@ mod test { }); } - doc.retry_with_trx( - |mut t| { - let space = t.get_space("space"); - let block = space.create(&mut t.trx, block_id.clone(), format!("flavour{}", i))?; - block.set(&mut t.trx, &format!("key{}", i), format!("val{}", i))?; - Ok::<_, JwstError>(()) - }, - 50, - ) - .and_then(|v| v) - .unwrap(); + { + let mut space = doc.get_space("space").unwrap(); + let mut block = space.create(block_id.clone(), format!("flavour{}", i)).unwrap(); + block.set(&format!("key{}", i), format!("val{}", i)).unwrap(); + } // await the task to make sure the doc1 is broadcasted before check doc2 futures::executor::block_on(tx_handler).unwrap(); rx_handler.join().unwrap(); - ws.retry_with_trx( - |mut t| { - let space = t.get_space("space"); - let block1 = space.get(&mut t.trx, format!("block{}", i))?; - - assert_eq!(block1.flavour(&t.trx), format!("flavour{}", i)); - assert_eq!( - block1.get(&t.trx, &format!("key{}", i))?.to_string(), - format!("val{}", i) - ); - None::<()> - }, - 50, - ) - .unwrap() - .unwrap(); + { + let space = ws.get_space("space").unwrap(); + let block1 = space.get(format!("block{}", i)).unwrap(); + + assert_eq!(block1.flavour(), format!("flavour{}", i)); + assert_eq!( + block1.get(&format!("key{}", i)).unwrap().to_string(), + format!("val{}", i) + ); + } collaborator.fetch_sub(1, Ordering::Relaxed); collaborator_pb.set_position(collaborator.load(Ordering::Relaxed)); @@ -415,18 +402,19 @@ mod test { let pb = mp.add(ProgressBar::new(1000)); pb.set_style(style.clone()); pb.set_message("final checking"); - ws.with_trx(|mut t| { - let space = t.get_space("space"); + + { + let space = ws.get_space("space").unwrap(); for i in (0..1000).progress_with(pb) { - let block1 = space.get(&mut t.trx, format!("block{}", i)).unwrap(); + let block1 = space.get(format!("block{}", i)).unwrap(); - assert_eq!(block1.flavour(&t.trx), format!("flavour{}", i)); + assert_eq!(block1.flavour(), format!("flavour{}", i)); assert_eq!( - block1.get(&t.trx, &format!("key{}", i)).unwrap().to_string(), + block1.get(&format!("key{}", i)).unwrap().to_string(), format!("val{}", i) ); } - }); + } Ok(()) } diff --git a/libs/jwst-rpc/src/lib.rs b/libs/jwst-rpc/src/lib.rs index 98a5ffdb9..5c3a884df 100644 --- a/libs/jwst-rpc/src/lib.rs +++ b/libs/jwst-rpc/src/lib.rs @@ -26,12 +26,12 @@ pub use connector::webrtc_datachannel_server_connector; pub use connector::{axum_socket_connector, tungstenite_socket_connector}; pub use context::RpcContextImpl; pub use handler::handle_connector; -use jwst::{debug, error, info, trace, warn}; +use jwst_core::{debug, error, info, trace, warn}; use tokio::{ sync::mpsc::{Receiver, Sender}, time::{sleep, Duration}, }; -pub use utils::{connect_memory_workspace, workspace_compare, MinimumServerContext}; +pub use utils::{connect_memory_workspace, MinimumServerContext}; #[cfg(feature = "webrtc")] pub use webrtcrs::peer_connection::sdp::session_description::RTCSessionDescription; diff --git a/libs/jwst-rpc/src/types.rs b/libs/jwst-rpc/src/types.rs index 31070de2a..3182f90c0 100644 --- a/libs/jwst-rpc/src/types.rs +++ b/libs/jwst-rpc/src/types.rs @@ -8,12 +8,10 @@ pub enum JwstRpcError { #[error("failed to connect websocket: {0}")] WebsocketConnect(#[from] tungstenite::Error), #[error("jwst error")] - Jwst(#[from] jwst::JwstError), + Jwst(#[from] jwst_core::JwstError), #[allow(dead_code)] #[error("failed to encode sync message")] ProtocolEncode(std::io::Error), - #[error("failed to decode sync message")] - ProtocolDecode(#[from] nom::Err>), #[cfg(feature = "websocket")] #[error("failed to parse url")] UrlParse(#[from] url::ParseError), diff --git a/libs/jwst-rpc/src/utils/compare.rs b/libs/jwst-rpc/src/utils/compare.rs deleted file mode 100644 index 50f8eb963..000000000 --- a/libs/jwst-rpc/src/utils/compare.rs +++ /dev/null @@ -1,64 +0,0 @@ -use assert_json_diff::{assert_json_matches_no_panic, CompareMode, Config, NumericMode}; -use yrs::{types::ToJson, Map, ReadTxn}; - -fn get_yrs_struct(trx: yrs::TransactionMut, block_id: Option<&str>) -> Result { - let json = trx - .get_map("space:blocks") - .ok_or_else(|| "get_yrs_struct: blocks not found".to_string()) - .and_then(|b| { - if let Some(block_id) = block_id { - b.get(&trx, block_id) - .ok_or("get_yrs_struct: block not found".into()) - .map(|b| b.to_json(&trx)) - } else { - Ok(b.to_json(&trx)) - } - })?; - drop(trx); - - serde_json::to_value(json).map_err(|e| format!("get_yrs_struct: serde_json::to_value failed: {}", e)) -} - -fn get_jwst_struct(ws: &mut jwst_core::Workspace, block_id: Option<&str>) -> Result { - match ws.get_blocks() { - Ok(blocks) => { - if let Some(block_id) = block_id { - blocks - .get(block_id) - .ok_or("get_jwst_struct: block not found".into()) - .and_then(|b| { - serde_json::to_value(&b) - .map_err(|e| format!("get_yrs_struct: serde_json::to_value failed: {}", e)) - }) - } else { - serde_json::to_value(&blocks) - .map_err(|e| format!("get_jwst_struct: serde_json::to_value failed: {}", e)) - } - } - Err(e) => Err(format!("get_jwst_struct: get_blocks failed: {}", e)), - } -} - -pub fn workspace_compare( - yrs_trx: yrs::TransactionMut, - jwst_ws: &mut jwst_core::Workspace, - block_id: Option<&str>, -) -> String { - match get_yrs_struct(yrs_trx, block_id) { - Ok(yrs_value) => match get_jwst_struct(jwst_ws, block_id) { - Ok(jwst_value) => { - if let Err(error) = assert_json_matches_no_panic( - &yrs_value, - &jwst_value, - Config::new(CompareMode::Strict).numeric_mode(NumericMode::AssumeFloat), - ) { - format!("workspace_compare: struct compare failed: {}", error) - } else { - "workspace_compare: struct compare success".into() - } - } - Err(e) => e, - }, - Err(e) => e, - } -} diff --git a/libs/jwst-rpc/src/utils/mod.rs b/libs/jwst-rpc/src/utils/mod.rs index d7fb74de4..90bd7951c 100644 --- a/libs/jwst-rpc/src/utils/mod.rs +++ b/libs/jwst-rpc/src/utils/mod.rs @@ -1,8 +1,6 @@ -mod compare; mod memory_workspace; mod server_context; -pub use compare::workspace_compare; pub use memory_workspace::connect_memory_workspace; pub use server_context::MinimumServerContext; diff --git a/libs/jwst-rpc/src/utils/server_context.rs b/libs/jwst-rpc/src/utils/server_context.rs index f2d40edef..d2e114bcb 100644 --- a/libs/jwst-rpc/src/utils/server_context.rs +++ b/libs/jwst-rpc/src/utils/server_context.rs @@ -1,9 +1,9 @@ use std::{collections::HashMap, time::Duration}; -use jwst::{DocStorage, Workspace}; -use jwst_storage::{BlobStorageType, JwstStorage}; +use jwst_codec::StateVector; +use jwst_core::{DocStorage, Workspace}; +use jwst_core_storage::{BlobStorageType, JwstStorage}; use tokio::{sync::RwLock, time::sleep}; -use yrs::{ReadTxn, StateVector, Transact}; use super::*; @@ -57,7 +57,6 @@ impl MinimumServerContext { let init_state = ws .doc() - .transact() .encode_state_as_update_v1(&StateVector::default()) .expect("encode_state_as_update_v1 failed"); From 023342d3fa7d3609d1689c06a20c87f41f0de7bc Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 28 Aug 2023 16:41:07 +0800 Subject: [PATCH 30/49] chore: merge jwst-storage --- Cargo.lock | 95 +-- Cargo.toml | 3 - apps/keck/Cargo.toml | 12 +- apps/keck/src/server/api/blobs.rs | 2 +- apps/keck/src/server/api/mod.rs | 2 +- apps/keck/src/server/sync/collaboration.rs | 2 +- libs/jwst-binding/jwst-swift/Cargo.toml | 4 +- libs/jwst-binding/jwst-swift/src/storage.rs | 2 +- libs/jwst-core-storage/Cargo.toml | 49 -- libs/jwst-core-storage/src/entities/blobs.rs | 21 - .../src/entities/bucket_blobs.rs | 19 - .../src/entities/diff_log.rs | 18 - libs/jwst-core-storage/src/entities/docs.rs | 21 - libs/jwst-core-storage/src/entities/mod.rs | 9 - .../src/entities/optimized_blobs.rs | 23 - .../jwst-core-storage/src/entities/prelude.rs | 6 - libs/jwst-core-storage/src/lib.rs | 37 -- .../src/migration/Cargo.toml | 16 - .../jwst-core-storage/src/migration/README.md | 41 -- .../src/migration/src/lib.rs | 25 - .../m20220101_000001_initial_blob_table.rs | 50 -- .../src/m20220101_000002_initial_doc_table.rs | 50 -- .../m20230321_000001_blob_optimized_table.rs | 64 -- ...230614_000001_initial_bucket_blob_table.rs | 57 -- .../src/m20230626_023319_doc_guid.rs | 51 -- ...m20230814_061223_initial_diff_log_table.rs | 41 -- .../src/migration/src/main.rs | 6 - .../src/migration/src/schemas.rs | 50 -- libs/jwst-core-storage/src/rate_limiter.rs | 79 --- .../src/storage/blobs/mod.rs | 182 ------ .../src/storage/blobs/utils.rs | 40 -- .../src/storage/docs/database.rs | 584 ------------------ .../jwst-core-storage/src/storage/docs/mod.rs | 137 ---- .../src/storage/docs/utils.rs | 25 - libs/jwst-core-storage/src/storage/mod.rs | 216 ------- libs/jwst-core-storage/src/storage/test.rs | 44 -- libs/jwst-core-storage/src/types.rs | 30 - libs/jwst-rpc/Cargo.toml | 2 +- libs/jwst-rpc/src/context.rs | 2 +- libs/jwst-rpc/src/utils/server_context.rs | 2 +- libs/jwst-storage/Cargo.toml | 32 +- libs/jwst-storage/src/lib.rs | 11 +- libs/jwst-storage/src/migration/Cargo.toml | 1 - .../blobs/auto_storage/auto_storage.rs | 0 .../src/storage/blobs/auto_storage/mod.rs | 0 .../src/storage/blobs/auto_storage/utils.rs | 0 .../src/storage/blobs/blob_storage.rs | 0 .../src/storage/blobs/bucket/bucket.rs | 0 .../src/storage/blobs/bucket/mod.rs | 0 .../src/storage/blobs/bucket/utils.rs | 0 .../src/storage/blobs/bucket_local_db.rs | 358 ----------- .../src/storage/blobs/local_db.rs | 291 --------- libs/jwst-storage/src/storage/blobs/mod.rs | 563 +++-------------- libs/jwst-storage/src/storage/blobs/utils.rs | 212 +------ libs/jwst-storage/src/storage/difflog.rs | 43 -- .../jwst-storage/src/storage/docs/database.rs | 118 ++-- libs/jwst-storage/src/storage/docs/utils.rs | 35 +- libs/jwst-storage/src/storage/mod.rs | 34 +- libs/jwst-storage/src/storage/test.rs | 3 + libs/jwst-storage/src/types.rs | 6 +- 60 files changed, 222 insertions(+), 3604 deletions(-) delete mode 100644 libs/jwst-core-storage/Cargo.toml delete mode 100644 libs/jwst-core-storage/src/entities/blobs.rs delete mode 100644 libs/jwst-core-storage/src/entities/bucket_blobs.rs delete mode 100644 libs/jwst-core-storage/src/entities/diff_log.rs delete mode 100644 libs/jwst-core-storage/src/entities/docs.rs delete mode 100644 libs/jwst-core-storage/src/entities/mod.rs delete mode 100644 libs/jwst-core-storage/src/entities/optimized_blobs.rs delete mode 100644 libs/jwst-core-storage/src/entities/prelude.rs delete mode 100644 libs/jwst-core-storage/src/lib.rs delete mode 100644 libs/jwst-core-storage/src/migration/Cargo.toml delete mode 100644 libs/jwst-core-storage/src/migration/README.md delete mode 100644 libs/jwst-core-storage/src/migration/src/lib.rs delete mode 100644 libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs delete mode 100644 libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs delete mode 100644 libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs delete mode 100644 libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs delete mode 100644 libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs delete mode 100644 libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs delete mode 100644 libs/jwst-core-storage/src/migration/src/main.rs delete mode 100644 libs/jwst-core-storage/src/migration/src/schemas.rs delete mode 100644 libs/jwst-core-storage/src/rate_limiter.rs delete mode 100644 libs/jwst-core-storage/src/storage/blobs/mod.rs delete mode 100644 libs/jwst-core-storage/src/storage/blobs/utils.rs delete mode 100644 libs/jwst-core-storage/src/storage/docs/database.rs delete mode 100644 libs/jwst-core-storage/src/storage/docs/mod.rs delete mode 100644 libs/jwst-core-storage/src/storage/docs/utils.rs delete mode 100644 libs/jwst-core-storage/src/storage/mod.rs delete mode 100644 libs/jwst-core-storage/src/storage/test.rs delete mode 100644 libs/jwst-core-storage/src/types.rs rename libs/{jwst-core-storage => jwst-storage}/src/storage/blobs/auto_storage/auto_storage.rs (100%) rename libs/{jwst-core-storage => jwst-storage}/src/storage/blobs/auto_storage/mod.rs (100%) rename libs/{jwst-core-storage => jwst-storage}/src/storage/blobs/auto_storage/utils.rs (100%) rename libs/{jwst-core-storage => jwst-storage}/src/storage/blobs/blob_storage.rs (100%) rename libs/{jwst-core-storage => jwst-storage}/src/storage/blobs/bucket/bucket.rs (100%) rename libs/{jwst-core-storage => jwst-storage}/src/storage/blobs/bucket/mod.rs (100%) rename libs/{jwst-core-storage => jwst-storage}/src/storage/blobs/bucket/utils.rs (100%) delete mode 100644 libs/jwst-storage/src/storage/blobs/bucket_local_db.rs delete mode 100644 libs/jwst-storage/src/storage/blobs/local_db.rs delete mode 100644 libs/jwst-storage/src/storage/difflog.rs diff --git a/Cargo.lock b/Cargo.lock index 71717919b..d6ac30f46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1926,24 +1926,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "governor" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c390a940a5d157878dd057c78680a33ce3415bcd05b4799509ea44210914b4d5" -dependencies = [ - "cfg-if 1.0.0", - "dashmap", - "futures", - "futures-timer", - "no-std-compat", - "nonzero_ext", - "parking_lot", - "quanta 0.9.3", - "rand 0.8.5", - "smallvec", -] - [[package]] name = "governor" version = "0.6.0" @@ -1957,7 +1939,7 @@ dependencies = [ "no-std-compat", "nonzero_ext", "parking_lot", - "quanta 0.11.1", + "quanta", "rand 0.8.5", "smallvec", ] @@ -2504,42 +2486,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "jwst-core-storage" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "bytes", - "chrono", - "dotenvy", - "futures", - "governor 0.6.0", - "image", - "jwst-codec", - "jwst-core", - "jwst-core-storage-migration", - "jwst-logger", - "opendal", - "path-ext", - "rand 0.8.5", - "sea-orm", - "sea-orm-migration", - "sha2", - "thiserror", - "tokio", - "tokio-util", - "url", -] - -[[package]] -name = "jwst-core-storage-migration" -version = "0.1.0" -dependencies = [ - "sea-orm-migration", - "tokio", -] - [[package]] name = "jwst-logger" version = "0.1.0" @@ -2567,8 +2513,8 @@ dependencies = [ "indicatif", "jwst-codec", "jwst-core", - "jwst-core-storage", "jwst-logger", + "jwst-storage", "nanoid", "rand 0.8.5", "reqwest", @@ -2592,13 +2538,12 @@ dependencies = [ "chrono", "dotenvy", "futures", - "governor 0.5.1", + "governor", "image", - "jwst", "jwst-codec", + "jwst-core", "jwst-logger", "jwst-storage-migration", - "lib0", "opendal", "path-ext", "rand 0.8.5", @@ -2609,7 +2554,6 @@ dependencies = [ "tokio", "tokio-util", "url", - "yrs", ] [[package]] @@ -2627,9 +2571,9 @@ dependencies = [ "chrono", "futures", "jwst-core", - "jwst-core-storage", "jwst-logger", "jwst-rpc", + "jwst-storage", "nanoid", "regex", "reqwest", @@ -2658,10 +2602,9 @@ dependencies = [ "dotenvy", "futures", "jwst-core", - "jwst-core-storage", "jwst-logger", "jwst-rpc", - "lib0", + "jwst-storage", "libc", "log", "mimalloc", @@ -2678,7 +2621,6 @@ dependencies = [ "tower-http", "utoipa", "utoipa-swagger-ui", - "yrs", ] [[package]] @@ -2845,15 +2787,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a8cbbb2831780bc3b9c15a41f5b49222ef756b6730a95f3decfdd15903eb5a3" -[[package]] -name = "mach" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" -dependencies = [ - "libc", -] - [[package]] name = "mach2" version = "0.4.1" @@ -3685,22 +3618,6 @@ dependencies = [ "bytemuck", ] -[[package]] -name = "quanta" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" -dependencies = [ - "crossbeam-utils", - "libc", - "mach", - "once_cell", - "raw-cpuid", - "wasi 0.10.0+wasi-snapshot-preview1", - "web-sys", - "winapi", -] - [[package]] name = "quanta" version = "0.11.1" diff --git a/Cargo.toml b/Cargo.toml index 87ebde659..4f36aee1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,8 +13,6 @@ members = [ "libs/jwst-codec", "libs/jwst-codec-util", "libs/jwst-core", - "libs/jwst-core-storage", - "libs/jwst-core-storage/src/migration", "libs/jwst-logger", "libs/jwst-rpc", "libs/jwst-storage", @@ -26,7 +24,6 @@ resolver = "2" jwst = { workspace = true, path = "libs/jwst" } jwst-codec = { workspace = true, path = "libs/jwst-codec" } jwst-core = { workspace = true, path = "libs/jwst-core" } -jwst-core-storage = { workspace = true, path = "libs/jwst-core-storage" } jwst-logger = { workspace = true, path = "libs/jwst-logger" } jwst-rpc = { workspace = true, path = "libs/jwst-rpc" } jwst-storage = { workspace = true, path = "libs/jwst-storage" } diff --git a/apps/keck/Cargo.toml b/apps/keck/Cargo.toml index fcb71e023..9353fbf60 100644 --- a/apps/keck/Cargo.toml +++ b/apps/keck/Cargo.toml @@ -7,13 +7,13 @@ license = "AGPL-3.0-only" [features] default = ["jwst"] -affine = ["jwst-core-storage/sqlite"] +affine = ["jwst-storage/sqlite"] jwst = [ "api", "schema", - "jwst-core-storage/sqlite", - "jwst-core-storage/mysql", - "jwst-core-storage/postgres", + "jwst-storage/sqlite", + "jwst-storage/mysql", + "jwst-storage/postgres", ] api = ["utoipa"] schema = ["utoipa-swagger-ui"] @@ -23,7 +23,6 @@ anyhow = "1.0.70" axum = { version = "0.6.20", features = ["headers", "ws"] } cfg-if = "1.0.0" futures = "0.3.28" -lib0 = { version = "0.16.5", features = ["lib0-serde"] } log = { version = "0.4.17", features = [ "max_level_trace", "release_max_level_info", @@ -50,7 +49,6 @@ tokio = { version = "=1.28.0", features = [ ] } utoipa = { version = "3.5.0", features = ["axum_extras"], optional = true } utoipa-swagger-ui = { version = "3.1.5", optional = true } -yrs = "0.16.5" libc = "0.2.147" rand = "0.8.5" reqwest = { version = "0.11.19", default-features = false, features = [ @@ -62,7 +60,7 @@ reqwest = { version = "0.11.19", default-features = false, features = [ jwst-core = { workspace = true } jwst-logger = { workspace = true } jwst-rpc = { workspace = true } -jwst-core-storage = { workspace = true } +jwst-storage = { workspace = true } [dev-dependencies] axum-test-helper = "0.3.0" diff --git a/apps/keck/src/server/api/blobs.rs b/apps/keck/src/server/api/blobs.rs index 6fae433ad..567f675dd 100644 --- a/apps/keck/src/server/api/blobs.rs +++ b/apps/keck/src/server/api/blobs.rs @@ -178,7 +178,7 @@ mod tests { use axum::body::{Body, Bytes}; use axum_test_helper::TestClient; use futures::stream; - use jwst_core_storage::BlobStorageType; + use jwst_storage::BlobStorageType; use super::*; diff --git a/apps/keck/src/server/api/mod.rs b/apps/keck/src/server/api/mod.rs index 36037c6f7..31551fdf1 100644 --- a/apps/keck/src/server/api/mod.rs +++ b/apps/keck/src/server/api/mod.rs @@ -15,8 +15,8 @@ use axum::{ routing::{delete, get, head, post}, }; use doc::doc_apis; -use jwst_core_storage::{BlobStorageType, JwstStorage, JwstStorageResult}; use jwst_rpc::{BroadcastChannels, RpcContextImpl}; +use jwst_storage::{BlobStorageType, JwstStorage, JwstStorageResult}; use tokio::sync::RwLock; use super::*; diff --git a/apps/keck/src/server/sync/collaboration.rs b/apps/keck/src/server/sync/collaboration.rs index 3ed935e8a..10504667b 100644 --- a/apps/keck/src/server/sync/collaboration.rs +++ b/apps/keck/src/server/sync/collaboration.rs @@ -53,9 +53,9 @@ mod test { }; use jwst_core::{Block, DocStorage, Workspace}; - use jwst_core_storage::{BlobStorageType, JwstStorage}; use jwst_logger::info; use jwst_rpc::{start_websocket_client_sync, BroadcastChannels, RpcContextImpl}; + use jwst_storage::{BlobStorageType, JwstStorage}; use libc::{kill, SIGTERM}; use rand::{thread_rng, Rng}; use tokio::runtime::Runtime; diff --git a/libs/jwst-binding/jwst-swift/Cargo.toml b/libs/jwst-binding/jwst-swift/Cargo.toml index ae3a89e8c..16e11f0d8 100644 --- a/libs/jwst-binding/jwst-swift/Cargo.toml +++ b/libs/jwst-binding/jwst-swift/Cargo.toml @@ -16,9 +16,9 @@ serde_json = "1.0.104" # ======= workspace dependencies ======= jwst-core = { workspace = true } -jwst-rpc = { workspace = true } -jwst-core-storage = { workspace = true, features = ["sqlite"] } jwst-logger = { workspace = true } +jwst-rpc = { workspace = true } +jwst-storage = { workspace = true, features = ["sqlite"] } [lib] name = "octobase" diff --git a/libs/jwst-binding/jwst-swift/src/storage.rs b/libs/jwst-binding/jwst-swift/src/storage.rs index 2f4add053..196802df0 100644 --- a/libs/jwst-binding/jwst-swift/src/storage.rs +++ b/libs/jwst-binding/jwst-swift/src/storage.rs @@ -1,8 +1,8 @@ use std::sync::{Arc, RwLock}; use futures::TryFutureExt; -use jwst_core_storage::{BlobStorageType, JwstStorage as AutoStorage, JwstStorageResult}; use jwst_rpc::{start_websocket_client_sync, BroadcastChannels, CachedLastSynced, RpcContextImpl, SyncState}; +use jwst_storage::{BlobStorageType, JwstStorage as AutoStorage, JwstStorageResult}; use nanoid::nanoid; use tokio::{ runtime::{Builder, Runtime}, diff --git a/libs/jwst-core-storage/Cargo.toml b/libs/jwst-core-storage/Cargo.toml deleted file mode 100644 index a885ca77e..000000000 --- a/libs/jwst-core-storage/Cargo.toml +++ /dev/null @@ -1,49 +0,0 @@ -[package] -name = "jwst-core-storage" -version = "0.1.0" -edition = "2021" -authors = ["DarkSky "] -license = "AGPL-3.0-only" - -[features] -default = ["sqlite"] -bucket = ["dotenvy", "opendal"] -image = ["dep:image"] -mysql = ["sea-orm/sqlx-mysql"] -postgres = ["sea-orm/sqlx-postgres"] -sqlite = ["sea-orm/sqlx-sqlite"] - -[dependencies] -anyhow = "1.0.75" -async-trait = "0.1.73" -bytes = "1.4.0" -chrono = { version = "0.4.24", features = ["serde"] } -futures = "0.3.28" -governor = "0.6.0" -path-ext = "0.1.0" -sha2 = "0.10.7" -sea-orm = { version = "0.12.2", features = ["runtime-tokio-rustls", "macros"] } -sea-orm-migration = { version = "0.12.2", default-features = false } -thiserror = "1.0.47" -tokio = { version = "1", features = ["fs", "macros", "sync"] } -tokio-util = { version = "0.7.8", features = ["io"] } -url = "2.4.0" - -# ======== bucket dependencies ======= -opendal = { version = "0.39.0", default-features = false, features = [ - "rustls", - "services-s3", -], optional = true } -dotenvy = { version = "0.15.7", optional = true } - -# ======= image dependencies ====== -image = { version = "0.24.6", features = ["webp-encoder"], optional = true } - -# ======= workspace dependencies ======= -jwst-core = { workspace = true } -jwst-codec = { workspace = true } -jwst-logger = { workspace = true } -jwst-core-storage-migration = { path = "./src/migration" } - -[dev-dependencies] -rand = "0.8.5" diff --git a/libs/jwst-core-storage/src/entities/blobs.rs b/libs/jwst-core-storage/src/entities/blobs.rs deleted file mode 100644 index 4470efff4..000000000 --- a/libs/jwst-core-storage/src/entities/blobs.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 - -use sea_orm::entity::prelude::*; - -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] -#[sea_orm(table_name = "blobs")] -pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub workspace_id: String, - #[sea_orm(primary_key, auto_increment = false)] - pub hash: String, - #[sea_orm(column_type = "Binary(BlobSize::Blob(None))")] - pub blob: Vec, - pub length: i64, - pub created_at: DateTimeWithTimeZone, -} - -#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation {} - -impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/jwst-core-storage/src/entities/bucket_blobs.rs b/libs/jwst-core-storage/src/entities/bucket_blobs.rs deleted file mode 100644 index 8b92b05e1..000000000 --- a/libs/jwst-core-storage/src/entities/bucket_blobs.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 - -use sea_orm::entity::prelude::*; - -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] -#[sea_orm(table_name = "bucket_blobs")] -pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub workspace_id: String, - #[sea_orm(primary_key, auto_increment = false)] - pub hash: String, - pub length: i64, - pub created_at: DateTimeWithTimeZone, -} - -#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation {} - -impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/jwst-core-storage/src/entities/diff_log.rs b/libs/jwst-core-storage/src/entities/diff_log.rs deleted file mode 100644 index f3e56c374..000000000 --- a/libs/jwst-core-storage/src/entities/diff_log.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 - -use sea_orm::entity::prelude::*; - -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] -#[sea_orm(table_name = "diff_log")] -pub struct Model { - #[sea_orm(primary_key)] - pub id: i32, - pub workspace: String, - pub timestamp: DateTimeWithTimeZone, - pub log: String, -} - -#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation {} - -impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/jwst-core-storage/src/entities/docs.rs b/libs/jwst-core-storage/src/entities/docs.rs deleted file mode 100644 index 2492bd5b5..000000000 --- a/libs/jwst-core-storage/src/entities/docs.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 - -use sea_orm::entity::prelude::*; - -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] -#[sea_orm(table_name = "docs")] -pub struct Model { - #[sea_orm(primary_key)] - pub id: i32, - pub workspace_id: String, - pub created_at: DateTimeWithTimeZone, - #[sea_orm(column_type = "Binary(BlobSize::Blob(None))")] - pub blob: Vec, - pub guid: String, - pub is_workspace: bool, -} - -#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation {} - -impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/jwst-core-storage/src/entities/mod.rs b/libs/jwst-core-storage/src/entities/mod.rs deleted file mode 100644 index 2425a9d77..000000000 --- a/libs/jwst-core-storage/src/entities/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 - -pub mod prelude; - -pub mod blobs; -pub mod bucket_blobs; -pub mod diff_log; -pub mod docs; -pub mod optimized_blobs; diff --git a/libs/jwst-core-storage/src/entities/optimized_blobs.rs b/libs/jwst-core-storage/src/entities/optimized_blobs.rs deleted file mode 100644 index a1b581888..000000000 --- a/libs/jwst-core-storage/src/entities/optimized_blobs.rs +++ /dev/null @@ -1,23 +0,0 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 - -use sea_orm::entity::prelude::*; - -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] -#[sea_orm(table_name = "optimized_blobs")] -pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub workspace_id: String, - #[sea_orm(primary_key, auto_increment = false)] - pub hash: String, - #[sea_orm(column_type = "Binary(BlobSize::Blob(None))")] - pub blob: Vec, - pub length: i64, - pub created_at: DateTimeWithTimeZone, - #[sea_orm(primary_key, auto_increment = false)] - pub params: String, -} - -#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation {} - -impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/jwst-core-storage/src/entities/prelude.rs b/libs/jwst-core-storage/src/entities/prelude.rs deleted file mode 100644 index aaabca6ac..000000000 --- a/libs/jwst-core-storage/src/entities/prelude.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 - -pub use super::{ - blobs::Entity as Blobs, bucket_blobs::Entity as BucketBlobs, diff_log::Entity as DiffLog, docs::Entity as Docs, - optimized_blobs::Entity as OptimizedBlobs, -}; diff --git a/libs/jwst-core-storage/src/lib.rs b/libs/jwst-core-storage/src/lib.rs deleted file mode 100644 index 438447d49..000000000 --- a/libs/jwst-core-storage/src/lib.rs +++ /dev/null @@ -1,37 +0,0 @@ -#[forbid(unsafe_code)] -mod entities; -mod rate_limiter; -mod storage; -mod types; - -use std::{path::PathBuf, sync::Arc, time::Duration}; - -use async_trait::async_trait; -use chrono::Utc; -use futures::{Future, Stream}; -use jwst_core::{DocStorage, JwstResult, Workspace}; -use jwst_logger::{debug, error, info, trace, warn}; -use path_ext::PathExt; -use rate_limiter::{get_bucket, is_sqlite, Bucket}; -use sea_orm::{prelude::*, ConnectOptions, Database, DbErr, QuerySelect, Set}; -#[cfg(feature = "bucket")] -pub use storage::blobs::MixedBucketDBParam; -pub use storage::{blobs::BlobStorageType, JwstStorage}; -pub use types::{JwstStorageError, JwstStorageResult}; - -#[inline] -async fn create_connection(database: &str, single_thread: bool) -> JwstStorageResult { - let connection = Database::connect( - ConnectOptions::from(database) - .max_connections(if single_thread { 1 } else { 50 }) - .min_connections(if single_thread { 1 } else { 10 }) - .acquire_timeout(Duration::from_secs(5)) - .connect_timeout(Duration::from_secs(5)) - .idle_timeout(Duration::from_secs(5)) - .max_lifetime(Duration::from_secs(30)) - .to_owned(), - ) - .await?; - - Ok(connection) -} diff --git a/libs/jwst-core-storage/src/migration/Cargo.toml b/libs/jwst-core-storage/src/migration/Cargo.toml deleted file mode 100644 index 38591c4a4..000000000 --- a/libs/jwst-core-storage/src/migration/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "jwst-core-storage-migration" -version = "0.1.0" -edition = "2021" -publish = false - -[lib] -name = "jwst_storage_migration" -path = "src/lib.rs" - -[dependencies] -tokio = { version = "^1", features = ["macros", "rt-multi-thread"] } - -[dependencies.sea-orm-migration] -version = "0.12.2" -features = ["runtime-tokio-rustls", "sqlx-postgres"] diff --git a/libs/jwst-core-storage/src/migration/README.md b/libs/jwst-core-storage/src/migration/README.md deleted file mode 100644 index b3ea53eb4..000000000 --- a/libs/jwst-core-storage/src/migration/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Running Migrator CLI - -- Generate a new migration file - ```sh - cargo run -- migrate generate MIGRATION_NAME - ``` -- Apply all pending migrations - ```sh - cargo run - ``` - ```sh - cargo run -- up - ``` -- Apply first 10 pending migrations - ```sh - cargo run -- up -n 10 - ``` -- Rollback last applied migrations - ```sh - cargo run -- down - ``` -- Rollback last 10 applied migrations - ```sh - cargo run -- down -n 10 - ``` -- Drop all tables from the database, then reapply all migrations - ```sh - cargo run -- fresh - ``` -- Rollback all applied migrations, then reapply all migrations - ```sh - cargo run -- refresh - ``` -- Rollback all applied migrations - ```sh - cargo run -- reset - ``` -- Check the status of all migrations - ```sh - cargo run -- status - ``` diff --git a/libs/jwst-core-storage/src/migration/src/lib.rs b/libs/jwst-core-storage/src/migration/src/lib.rs deleted file mode 100644 index 8861c8ec9..000000000 --- a/libs/jwst-core-storage/src/migration/src/lib.rs +++ /dev/null @@ -1,25 +0,0 @@ -pub use sea_orm_migration::prelude::*; - -mod m20220101_000001_initial_blob_table; -mod m20220101_000002_initial_doc_table; -mod m20230321_000001_blob_optimized_table; -mod m20230614_000001_initial_bucket_blob_table; -mod m20230626_023319_doc_guid; -mod m20230814_061223_initial_diff_log_table; -mod schemas; - -pub struct Migrator; - -#[async_trait::async_trait] -impl MigratorTrait for Migrator { - fn migrations() -> Vec> { - vec![ - Box::new(m20220101_000001_initial_blob_table::Migration), - Box::new(m20220101_000002_initial_doc_table::Migration), - Box::new(m20230321_000001_blob_optimized_table::Migration), - Box::new(m20230614_000001_initial_bucket_blob_table::Migration), - Box::new(m20230626_023319_doc_guid::Migration), - Box::new(m20230814_061223_initial_diff_log_table::Migration), - ] - } -} diff --git a/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs b/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs deleted file mode 100644 index f6b4e5185..000000000 --- a/libs/jwst-core-storage/src/migration/src/m20220101_000001_initial_blob_table.rs +++ /dev/null @@ -1,50 +0,0 @@ -use sea_orm_migration::prelude::*; - -use super::schemas::Blobs; - -pub struct Migration; - -impl MigrationName for Migration { - fn name(&self) -> &str { - "m20220101_000001_initial_blob_table" - } -} - -#[async_trait::async_trait] -impl MigrationTrait for Migration { - // Define how to apply this migration: Create the Bakery table. - async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .create_table( - Table::create() - .table(Blobs::Table) - .col(ColumnDef::new(Blobs::Workspace).string().not_null()) - .col(ColumnDef::new(Blobs::Hash).string().not_null()) - .col(ColumnDef::new(Blobs::Blob).binary().not_null()) - .col(ColumnDef::new(Blobs::Length).big_integer().not_null()) - .col(ColumnDef::new(Blobs::Timestamp).timestamp_with_time_zone().not_null()) - .primary_key(Index::create().col(Blobs::Workspace).col(Blobs::Hash)) - .to_owned(), - ) - .await?; - - manager - .create_index( - Index::create() - .name("blobs_list") - .table(Blobs::Table) - .col(Blobs::Workspace) - .to_owned(), - ) - .await?; - - Ok(()) - } - - // Define how to rollback this migration: Drop the Bakery table. - async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager.drop_index(Index::drop().name("blobs_list").to_owned()).await?; - manager.drop_table(Table::drop().table(Blobs::Table).to_owned()).await?; - Ok(()) - } -} diff --git a/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs b/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs deleted file mode 100644 index c357701e5..000000000 --- a/libs/jwst-core-storage/src/migration/src/m20220101_000002_initial_doc_table.rs +++ /dev/null @@ -1,50 +0,0 @@ -use sea_orm_migration::prelude::*; - -use super::schemas::Docs; - -pub struct Migration; - -impl MigrationName for Migration { - fn name(&self) -> &str { - "m20220101_000001_initial_doc_table" - } -} - -#[async_trait::async_trait] -impl MigrationTrait for Migration { - // Define how to apply this migration: Create the Bakery table. - async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .create_table( - Table::create() - .table(Docs::Table) - .col(ColumnDef::new(Docs::Id).integer().auto_increment().primary_key()) - .col(ColumnDef::new(Docs::Workspace).string().not_null()) - .col(ColumnDef::new(Docs::Timestamp).timestamp_with_time_zone().not_null()) - .col(ColumnDef::new(Docs::Blob).binary().not_null()) - .to_owned(), - ) - .await?; - - manager - .create_index( - Index::create() - .name("workspaces_update") - .table(Docs::Table) - .col(Docs::Workspace) - .to_owned(), - ) - .await?; - - Ok(()) - } - - // Define how to rollback this migration: Drop the Bakery table. - async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .drop_index(Index::drop().name("workspaces_update").to_owned()) - .await?; - manager.drop_table(Table::drop().table(Docs::Table).to_owned()).await?; - Ok(()) - } -} diff --git a/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs b/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs deleted file mode 100644 index 8f9676ddb..000000000 --- a/libs/jwst-core-storage/src/migration/src/m20230321_000001_blob_optimized_table.rs +++ /dev/null @@ -1,64 +0,0 @@ -use sea_orm_migration::prelude::*; - -use super::schemas::OptimizedBlobs; - -pub struct Migration; - -impl MigrationName for Migration { - fn name(&self) -> &str { - "m20230321_000001_blob_optimized_table" - } -} - -#[async_trait::async_trait] -impl MigrationTrait for Migration { - // Define how to apply this migration: Create the Bakery table. - async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .create_table( - Table::create() - .table(OptimizedBlobs::Table) - .col(ColumnDef::new(OptimizedBlobs::Workspace).string().not_null()) - .col(ColumnDef::new(OptimizedBlobs::Hash).string().not_null()) - .col(ColumnDef::new(OptimizedBlobs::Blob).binary().not_null()) - .col(ColumnDef::new(OptimizedBlobs::Length).big_integer().not_null()) - .col( - ColumnDef::new(OptimizedBlobs::Timestamp) - .timestamp_with_time_zone() - .not_null(), - ) - .col(ColumnDef::new(OptimizedBlobs::Params).string().not_null()) - .primary_key( - Index::create() - .col(OptimizedBlobs::Workspace) - .col(OptimizedBlobs::Hash) - .col(OptimizedBlobs::Params), - ) - .to_owned(), - ) - .await?; - - manager - .create_index( - Index::create() - .name("blobs_optimized_list") - .table(OptimizedBlobs::Table) - .col(OptimizedBlobs::Workspace) - .to_owned(), - ) - .await?; - - Ok(()) - } - - // Define how to rollback this migration: Drop the Bakery table. - async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .drop_index(Index::drop().name("blobs_optimized_list").to_owned()) - .await?; - manager - .drop_table(Table::drop().table(OptimizedBlobs::Table).to_owned()) - .await?; - Ok(()) - } -} diff --git a/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs b/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs deleted file mode 100644 index ceb35c575..000000000 --- a/libs/jwst-core-storage/src/migration/src/m20230614_000001_initial_bucket_blob_table.rs +++ /dev/null @@ -1,57 +0,0 @@ -use sea_orm_migration::prelude::*; - -use super::schemas::BucketBlobs; - -pub struct Migration; - -impl MigrationName for Migration { - fn name(&self) -> &str { - "m20230614_000001_initial_bucket_blob_table" - } -} - -#[async_trait::async_trait] -impl MigrationTrait for Migration { - // Define how to apply this migration: Create the Bakery table. - async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .create_table( - Table::create() - .table(BucketBlobs::Table) - .col(ColumnDef::new(BucketBlobs::Workspace).string().not_null()) - .col(ColumnDef::new(BucketBlobs::Hash).string().not_null()) - .col(ColumnDef::new(BucketBlobs::Length).big_integer().not_null()) - .col( - ColumnDef::new(BucketBlobs::Timestamp) - .timestamp_with_time_zone() - .not_null(), - ) - .primary_key(Index::create().col(BucketBlobs::Workspace).col(BucketBlobs::Hash)) - .to_owned(), - ) - .await?; - - manager - .create_index( - Index::create() - .name("bucket_blobs_list") - .table(BucketBlobs::Table) - .col(BucketBlobs::Workspace) - .to_owned(), - ) - .await?; - - Ok(()) - } - - // Define how to rollback this migration: Drop the Bakery table. - async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .drop_index(Index::drop().name("bucket_blobs_list").to_owned()) - .await?; - manager - .drop_table(Table::drop().table(BucketBlobs::Table).to_owned()) - .await?; - Ok(()) - } -} diff --git a/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs b/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs deleted file mode 100644 index ccaf1f49a..000000000 --- a/libs/jwst-core-storage/src/migration/src/m20230626_023319_doc_guid.rs +++ /dev/null @@ -1,51 +0,0 @@ -use sea_orm_migration::prelude::*; - -use crate::schemas::Docs; - -#[derive(DeriveMigrationName)] -pub struct Migration; - -#[async_trait::async_trait] -impl MigrationTrait for Migration { - async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .alter_table( - Table::alter() - .table(Docs::Table) - .add_column(ColumnDef::new(Docs::Guid).string().not_null()) - .to_owned(), - ) - .await?; - manager - .alter_table( - Table::alter() - .table(Docs::Table) - .add_column(ColumnDef::new(Docs::IsWorkspace).boolean().not_null().default(true)) - .to_owned(), - ) - .await?; - manager - .create_index( - Index::create() - .table(Docs::Table) - .name("docs_guid") - .col(Docs::Guid) - .to_owned(), - ) - .await - } - - async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager.drop_index(Index::drop().name("docs_guid").to_owned()).await?; - - manager - .alter_table( - Table::alter() - .table(Docs::Table) - .drop_column(Docs::Guid) - .drop_column(Docs::IsWorkspace) - .to_owned(), - ) - .await - } -} diff --git a/libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs b/libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs deleted file mode 100644 index 8662469b8..000000000 --- a/libs/jwst-core-storage/src/migration/src/m20230814_061223_initial_diff_log_table.rs +++ /dev/null @@ -1,41 +0,0 @@ -use sea_orm_migration::prelude::*; - -#[derive(DeriveMigrationName)] -pub struct Migration; - -#[async_trait::async_trait] -impl MigrationTrait for Migration { - async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .create_table( - Table::create() - .table(DiffLog::Table) - .if_not_exists() - .col( - ColumnDef::new(DiffLog::Id) - .integer() - .not_null() - .auto_increment() - .primary_key(), - ) - .col(ColumnDef::new(DiffLog::Workspace).string().not_null()) - .col(ColumnDef::new(DiffLog::Timestamp).timestamp_with_time_zone().not_null()) - .col(ColumnDef::new(DiffLog::Log).string().not_null()) - .to_owned(), - ) - .await - } - - async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager.drop_table(Table::drop().table(DiffLog::Table).to_owned()).await - } -} - -#[derive(DeriveIden)] -enum DiffLog { - Table, - Id, - Workspace, - Timestamp, - Log, -} diff --git a/libs/jwst-core-storage/src/migration/src/main.rs b/libs/jwst-core-storage/src/migration/src/main.rs deleted file mode 100644 index 632e03b8f..000000000 --- a/libs/jwst-core-storage/src/migration/src/main.rs +++ /dev/null @@ -1,6 +0,0 @@ -use sea_orm_migration::prelude::*; - -#[tokio::main] -async fn main() { - cli::run_cli(jwst_storage_migration::Migrator).await; -} diff --git a/libs/jwst-core-storage/src/migration/src/schemas.rs b/libs/jwst-core-storage/src/migration/src/schemas.rs deleted file mode 100644 index b3d6eeb33..000000000 --- a/libs/jwst-core-storage/src/migration/src/schemas.rs +++ /dev/null @@ -1,50 +0,0 @@ -use sea_orm_migration::prelude::*; - -#[derive(Iden)] -pub enum Blobs { - Table, - #[iden = "workspace_id"] - Workspace, - Hash, - Blob, - Length, - #[iden = "created_at"] - Timestamp, -} - -#[derive(Iden)] -pub enum Docs { - Table, - Id, - #[iden = "workspace_id"] - Workspace, - Guid, - IsWorkspace, - #[iden = "created_at"] - Timestamp, - Blob, -} - -#[derive(Iden)] -pub enum OptimizedBlobs { - Table, - #[iden = "workspace_id"] - Workspace, - Hash, - Blob, - Length, - #[iden = "created_at"] - Timestamp, - Params, -} - -#[derive(Iden)] -pub enum BucketBlobs { - Table, - #[iden = "workspace_id"] - Workspace, - Hash, - Length, - #[iden = "created_at"] - Timestamp, -} diff --git a/libs/jwst-core-storage/src/rate_limiter.rs b/libs/jwst-core-storage/src/rate_limiter.rs deleted file mode 100644 index 2ba158ff4..000000000 --- a/libs/jwst-core-storage/src/rate_limiter.rs +++ /dev/null @@ -1,79 +0,0 @@ -use std::{num::NonZeroU32, sync::Arc}; - -use governor::{ - clock::{QuantaClock, QuantaInstant}, - middleware::NoOpMiddleware, - state::{InMemoryState, NotKeyed}, - Quota, RateLimiter, -}; -use tokio::sync::{OwnedSemaphorePermit, RwLock, RwLockReadGuard, RwLockWriteGuard, Semaphore}; -use url::Url; - -pub enum BucketLocker<'a> { - Semaphore(OwnedSemaphorePermit), - ReadLock(RwLockReadGuard<'a, ()>), - WriteLock(RwLockWriteGuard<'a, ()>), -} - -enum BucketLock { - Semaphore(Arc), - RwLock(Arc>), -} - -pub struct Bucket { - bucket: Arc>>, - lock: BucketLock, -} - -impl Bucket { - fn new(bucket_size: u32, semaphore_size: usize) -> Self { - let bucket_size = NonZeroU32::new(bucket_size).unwrap_or(unsafe { NonZeroU32::new_unchecked(1) }); - - Self { - bucket: Arc::new(RateLimiter::direct( - Quota::per_second(bucket_size).allow_burst(bucket_size), - )), - lock: if semaphore_size > 1 { - BucketLock::Semaphore(Arc::new(Semaphore::new(semaphore_size))) - } else { - // use for sqlite - BucketLock::RwLock(Arc::default()) - }, - } - } - - pub async fn read(&self) -> BucketLocker { - self.bucket.until_ready().await; - match &self.lock { - BucketLock::RwLock(lock) => BucketLocker::ReadLock(lock.read().await), - BucketLock::Semaphore(semaphore) => { - BucketLocker::Semaphore(semaphore.clone().acquire_owned().await.unwrap()) - } - } - } - - pub async fn write(&self) -> BucketLocker { - self.bucket.until_ready().await; - match &self.lock { - BucketLock::RwLock(lock) => BucketLocker::WriteLock(lock.write().await), - BucketLock::Semaphore(semaphore) => { - BucketLocker::Semaphore(semaphore.clone().acquire_owned().await.unwrap()) - } - } - } -} - -#[inline] -pub fn is_sqlite(database: &str) -> bool { - Url::parse(database).map(|u| u.scheme() == "sqlite").unwrap_or(false) -} - -#[inline] -pub fn get_bucket(single_thread: bool) -> Arc { - // sqlite only support 1 writer at a time - // we use semaphore to limit the number of concurrent writers - Arc::new(Bucket::new( - if single_thread { 10 } else { 25 }, - if single_thread { 1 } else { 5 }, - )) -} diff --git a/libs/jwst-core-storage/src/storage/blobs/mod.rs b/libs/jwst-core-storage/src/storage/blobs/mod.rs deleted file mode 100644 index db57f9767..000000000 --- a/libs/jwst-core-storage/src/storage/blobs/mod.rs +++ /dev/null @@ -1,182 +0,0 @@ -#[cfg(feature = "bucket")] -mod bucket; -#[cfg(feature = "bucket")] -pub use bucket::{BlobBucketStorage, MixedBucketDBParam}; - -#[cfg(feature = "image")] -mod auto_storage; -#[cfg(feature = "image")] -pub use auto_storage::{BlobAutoStorage, ImageError, ImageParams}; - -mod blob_storage; -mod utils; - -#[cfg(test)] -pub use blob_storage::blobs_storage_test; -pub use blob_storage::BlobDBStorage; -use bytes::Bytes; -use jwst_core::{BlobMetadata, BlobStorage}; -use thiserror::Error; -use tokio::task::JoinError; -use utils::{get_hash, InternalBlobMetadata}; - -use super::{entities::prelude::*, *}; - -#[derive(Debug, Error)] -pub enum JwstBlobError { - #[error("blob not found: {0}")] - BlobNotFound(String), - #[error("database error")] - Database(#[from] DbErr), - #[cfg(feature = "image")] - #[error("failed to optimize image")] - Image(#[from] ImageError), - #[error("failed to optimize image")] - ImageThread(#[from] JoinError), - #[error("optimize params error: {0:?}")] - ImageParams(HashMap), -} -pub type JwstBlobResult = Result; - -pub enum JwstBlobStorage { - RawStorage(Arc), - #[cfg(feature = "image")] - AutoStorage(BlobAutoStorage), - #[cfg(feature = "bucket")] - BucketStorage(BlobBucketStorage), -} - -pub enum BlobStorageType { - DB, - #[cfg(feature = "bucket")] - MixedBucketDB(MixedBucketDBParam), -} - -#[async_trait] -impl BlobStorage for JwstBlobStorage { - async fn list_blobs(&self, workspace: Option) -> JwstResult, JwstStorageError> { - match self { - JwstBlobStorage::RawStorage(db) => db.list_blobs(workspace).await, - #[cfg(feature = "image")] - JwstBlobStorage::AutoStorage(db) => db.list_blobs(workspace).await, - #[cfg(feature = "bucket")] - JwstBlobStorage::BucketStorage(db) => db.list_blobs(workspace).await, - } - } - - async fn check_blob(&self, workspace: Option, id: String) -> JwstResult { - match self { - JwstBlobStorage::RawStorage(db) => db.check_blob(workspace, id).await, - #[cfg(feature = "image")] - JwstBlobStorage::AutoStorage(db) => db.check_blob(workspace, id).await, - #[cfg(feature = "bucket")] - JwstBlobStorage::BucketStorage(db) => db.check_blob(workspace, id).await, - } - } - - async fn get_blob( - &self, - workspace: Option, - id: String, - params: Option>, - ) -> JwstResult, JwstStorageError> { - match self { - JwstBlobStorage::RawStorage(db) => db.get_blob(workspace, id, params).await, - #[cfg(feature = "image")] - JwstBlobStorage::AutoStorage(db) => db.get_blob(workspace, id, params).await, - #[cfg(feature = "bucket")] - JwstBlobStorage::BucketStorage(db) => db.get_blob(workspace, id, params).await, - } - } - - async fn get_metadata( - &self, - workspace: Option, - id: String, - params: Option>, - ) -> JwstResult { - match self { - JwstBlobStorage::RawStorage(db) => db.get_metadata(workspace, id, params).await, - #[cfg(feature = "image")] - JwstBlobStorage::AutoStorage(db) => db.get_metadata(workspace, id, params).await, - #[cfg(feature = "bucket")] - JwstBlobStorage::BucketStorage(db) => db.get_metadata(workspace, id, params).await, - } - } - - async fn put_blob_stream( - &self, - workspace: Option, - stream: impl Stream + Send, - ) -> JwstResult { - match self { - JwstBlobStorage::RawStorage(db) => db.put_blob_stream(workspace, stream).await, - #[cfg(feature = "image")] - JwstBlobStorage::AutoStorage(db) => db.put_blob_stream(workspace, stream).await, - #[cfg(feature = "bucket")] - JwstBlobStorage::BucketStorage(db) => db.put_blob_stream(workspace, stream).await, - } - } - - async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstResult { - match self { - JwstBlobStorage::RawStorage(db) => db.put_blob(workspace, blob).await, - #[cfg(feature = "image")] - JwstBlobStorage::AutoStorage(db) => db.put_blob(workspace, blob).await, - #[cfg(feature = "bucket")] - JwstBlobStorage::BucketStorage(db) => db.put_blob(workspace, blob).await, - } - } - - async fn delete_blob(&self, workspace: Option, id: String) -> JwstResult { - match self { - JwstBlobStorage::RawStorage(db) => db.delete_blob(workspace, id).await, - #[cfg(feature = "image")] - JwstBlobStorage::AutoStorage(db) => db.delete_blob(workspace, id).await, - #[cfg(feature = "bucket")] - JwstBlobStorage::BucketStorage(db) => db.delete_blob(workspace, id).await, - } - } - - async fn delete_workspace(&self, workspace_id: String) -> JwstResult<(), JwstStorageError> { - match self { - JwstBlobStorage::RawStorage(db) => db.delete_workspace(workspace_id).await, - #[cfg(feature = "image")] - JwstBlobStorage::AutoStorage(db) => db.delete_workspace(workspace_id).await, - #[cfg(feature = "bucket")] - JwstBlobStorage::BucketStorage(db) => db.delete_workspace(workspace_id).await, - } - } - - async fn get_blobs_size(&self, workspace_id: String) -> JwstResult { - match self { - JwstBlobStorage::RawStorage(db) => Ok(db.get_blobs_size(&workspace_id).await?.unwrap_or(0)), - #[cfg(feature = "image")] - JwstBlobStorage::AutoStorage(db) => db.get_blobs_size(workspace_id).await, - #[cfg(feature = "bucket")] - JwstBlobStorage::BucketStorage(db) => db.get_blobs_size(workspace_id).await, - } - } -} - -impl JwstBlobStorage { - pub fn get_blob_db(&self) -> Option> { - match self { - JwstBlobStorage::RawStorage(db) => Some(db.clone()), - #[cfg(feature = "image")] - JwstBlobStorage::AutoStorage(db) => Some(db.db.clone()), - #[cfg(feature = "bucket")] - JwstBlobStorage::BucketStorage(_) => None, - } - } - - #[cfg(feature = "bucket")] - pub fn get_mixed_bucket_db(&self) -> Option { - match self { - JwstBlobStorage::RawStorage(_) => None, - #[cfg(feature = "image")] - JwstBlobStorage::AutoStorage(_) => None, - JwstBlobStorage::BucketStorage(db) => Some(db.clone()), - } - } -} diff --git a/libs/jwst-core-storage/src/storage/blobs/utils.rs b/libs/jwst-core-storage/src/storage/blobs/utils.rs deleted file mode 100644 index b129fb3af..000000000 --- a/libs/jwst-core-storage/src/storage/blobs/utils.rs +++ /dev/null @@ -1,40 +0,0 @@ -use bytes::Bytes; -use chrono::{DateTime, Utc}; -use futures::{ - stream::{iter, StreamExt}, - Stream, -}; -use jwst_core::{Base64Engine, BlobMetadata, URL_SAFE_ENGINE}; -use sea_orm::FromQueryResult; -use sha2::{Digest, Sha256}; - -pub async fn get_hash(stream: impl Stream + Send) -> (String, Vec) { - let mut hasher = Sha256::new(); - - let buffer = stream - .flat_map(|buffer| { - hasher.update(&buffer); - iter(buffer) - }) - .collect() - .await; - - let hash = URL_SAFE_ENGINE.encode(hasher.finalize()); - (hash, buffer) -} - -#[derive(FromQueryResult)] -pub(super) struct InternalBlobMetadata { - pub(super) size: i64, - pub(super) created_at: DateTime, -} - -impl From for BlobMetadata { - fn from(val: InternalBlobMetadata) -> Self { - BlobMetadata { - content_type: "application/octet-stream".into(), - last_modified: val.created_at.naive_local(), - size: val.size, - } - } -} diff --git a/libs/jwst-core-storage/src/storage/docs/database.rs b/libs/jwst-core-storage/src/storage/docs/database.rs deleted file mode 100644 index bb93e8dba..000000000 --- a/libs/jwst-core-storage/src/storage/docs/database.rs +++ /dev/null @@ -1,584 +0,0 @@ -use std::collections::hash_map::Entry; - -use jwst_codec::{encode_update_as_message, CrdtReader, Doc, DocOptions, RawDecoder, StateVector}; -use jwst_core::{DocStorage, Workspace}; -use sea_orm::Condition; - -use super::{entities::prelude::*, *}; -use crate::types::JwstStorageResult; - -const MAX_TRIM_UPDATE_LIMIT: u64 = 500; - -type DocsModel = ::Model; -type DocsActiveModel = super::entities::docs::ActiveModel; -type DocsColumn = ::Column; - -pub struct DocDBStorage { - bucket: Arc, - pub(super) pool: DatabaseConnection, - workspaces: RwLock>, // memory cache - remote: RwLock>>>, -} - -impl DocDBStorage { - pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { - Ok(Self { - bucket, - pool, - workspaces: RwLock::new(HashMap::new()), - remote: RwLock::new(HashMap::new()), - }) - } - - pub async fn init_pool(database: &str) -> JwstStorageResult { - let is_sqlite = is_sqlite(database); - let pool = create_connection(database, is_sqlite).await?; - - Self::init_with_pool(pool, get_bucket(is_sqlite)).await - } - - pub fn remote(&self) -> &RwLock>>> { - &self.remote - } - - /// warn: records of the same workspace may belong to different doc - #[allow(unused)] - async fn workspace_all(conn: &C, workspace: &str) -> JwstStorageResult> - where - C: ConnectionTrait, - { - trace!("start scan all records of workspace: {workspace}"); - let models = Docs::find() - .filter(DocsColumn::WorkspaceId.eq(workspace)) - .all(conn) - .await?; - trace!("end scan all records of workspace: {workspace}, {}", models.len()); - Ok(models) - } - - async fn doc_all(conn: &C, guid: &str) -> JwstStorageResult> - where - C: ConnectionTrait, - { - trace!("start scan all records with guid: {guid}"); - let models = Docs::find().filter(DocsColumn::Guid.eq(guid)).all(conn).await?; - trace!("end scan all: {guid}, {}", models.len()); - Ok(models) - } - - async fn count(conn: &C, guid: &str) -> JwstStorageResult - where - C: ConnectionTrait, - { - trace!("start count: {guid}"); - let count = Docs::find().filter(DocsColumn::Guid.eq(guid)).count(conn).await?; - trace!("end count: {guid}, {count}"); - Ok(count) - } - - async fn workspace_count(conn: &C, workspace: &str) -> JwstStorageResult - where - C: ConnectionTrait, - { - trace!("start count: {workspace}"); - let count = Docs::find() - .filter(DocsColumn::WorkspaceId.eq(workspace)) - .count(conn) - .await?; - trace!("end count: {workspace}, {count}"); - Ok(count) - } - - async fn workspace_guid(conn: &C, workspace: &str) -> JwstStorageResult> - where - C: ConnectionTrait, - { - let record = Docs::find() - .filter( - Condition::all() - .add(DocsColumn::WorkspaceId.eq(workspace)) - .add(DocsColumn::IsWorkspace.eq(true)), - ) - .one(conn) - .await?; - - Ok(record.map(|r| r.guid)) - } - - async fn is_workspace(conn: &C, guid: &str) -> JwstStorageResult - where - C: ConnectionTrait, - { - let record = Docs::find().filter(DocsColumn::Guid.eq(guid)).one(conn).await?; - - Ok(record.map_or(false, |r| r.is_workspace)) - } - - async fn insert(conn: &C, workspace: &str, guid: &str, blob: &[u8]) -> JwstStorageResult<()> - where - C: ConnectionTrait, - { - let workspace_guid = Self::workspace_guid(conn, workspace).await?; - trace!("start insert: {workspace}"); - Docs::insert(DocsActiveModel { - workspace_id: Set(workspace.into()), - guid: Set(guid.into()), - blob: Set(blob.into()), - created_at: Set(Utc::now().into()), - // if not workspace guid found, the current insertion should be workspace init data. - is_workspace: Set(workspace_guid.map_or(true, |g| g == guid)), - ..Default::default() - }) - .exec(conn) - .await?; - trace!("end insert: {workspace}"); - Ok(()) - } - - async fn replace_with(conn: &C, workspace: &str, guid: &str, blob: Vec) -> JwstStorageResult<()> - where - C: ConnectionTrait, - { - trace!("start replace: {guid}"); - - let is_workspace = Self::is_workspace(conn, guid).await?; - - Docs::delete_many().filter(DocsColumn::Guid.eq(guid)).exec(conn).await?; - Docs::insert(DocsActiveModel { - workspace_id: Set(workspace.into()), - guid: Set(guid.into()), - blob: Set(blob), - is_workspace: Set(is_workspace), - created_at: Set(Utc::now().into()), - ..Default::default() - }) - .exec(conn) - .await?; - trace!("end replace: {workspace}"); - Ok(()) - } - - pub async fn delete(conn: &C, guid: &str) -> JwstStorageResult<()> - where - C: ConnectionTrait, - { - trace!("start drop: {guid}"); - Docs::delete_many().filter(DocsColumn::Guid.eq(guid)).exec(conn).await?; - trace!("end drop: {guid}"); - Ok(()) - } - - pub async fn delete_workspace(conn: &C, workspace_id: &str) -> JwstStorageResult<()> - where - C: ConnectionTrait, - { - trace!("start drop workspace: {workspace_id}"); - Docs::delete_many() - .filter(DocsColumn::WorkspaceId.eq(workspace_id)) - .exec(conn) - .await?; - trace!("end drop workspace: {workspace_id}"); - Ok(()) - } - - async fn update(&self, conn: &C, workspace: &str, guid: &str, blob: Vec) -> JwstStorageResult<()> - where - C: ConnectionTrait, - { - trace!("start update: {guid}"); - let update_size = Self::count(conn, guid).await?; - if update_size > MAX_TRIM_UPDATE_LIMIT - 1 { - trace!("full migrate update: {guid}, {update_size}"); - let doc_records = Self::doc_all(conn, guid).await?; - - let data = tokio::task::spawn_blocking(move || utils::merge_doc_records(doc_records)) - .await - .map_err(JwstStorageError::DocMerge)??; - - Self::replace_with(conn, workspace, guid, data).await?; - } - - trace!("insert update: {guid}, {update_size}"); - Self::insert(conn, workspace, guid, &blob).await?; - trace!("end update: {guid}"); - - trace!("update {}bytes to {}", blob.len(), guid); - if let Entry::Occupied(remote) = self.remote.write().await.entry(guid.into()) { - let broadcast = &remote.get(); - if broadcast.send(encode_update_as_message(blob)?).is_err() { - // broadcast failures are not fatal errors, only warnings are required - warn!("send {guid} update to pipeline failed"); - } - } - trace!("end update broadcast: {guid}"); - - Ok(()) - } - - async fn full_migrate(&self, workspace: &str, guid: &str, blob: Vec) -> JwstStorageResult<()> { - trace!("start full migrate: {guid}"); - Self::replace_with(&self.pool, workspace, guid, blob).await?; - trace!("end full migrate: {guid}"); - Ok(()) - } - - async fn init_workspace(conn: &C, workspace: &str) -> JwstStorageResult - where - C: ConnectionTrait, - { - trace!("start create doc in workspace: {workspace}"); - let all_data = Docs::find() - .filter( - Condition::all() - .add(DocsColumn::WorkspaceId.eq(workspace)) - .add(DocsColumn::IsWorkspace.eq(true)), - ) - .all(conn) - .await?; - - let ws = if all_data.is_empty() { - trace!("create workspace: {workspace}"); - // keep workspace root doc's guid the same as workspaceId - let doc = Doc::with_options(DocOptions { - guid: Some(workspace.into()), - ..Default::default() - }); - let ws = Workspace::from_doc(doc.clone(), workspace)?; - - let update = doc.encode_state_as_update_v1(&StateVector::default())?; - - Self::insert(conn, workspace, doc.guid(), &update).await?; - ws - } else { - trace!("migrate workspace: {workspace}"); - let doc = Doc::with_options(DocOptions { - guid: all_data.first().unwrap().guid.clone().into(), - ..Default::default() - }); - - let can_merge = all_data.len() > 1; - - let doc = utils::migrate_update(all_data, doc)?; - - if can_merge { - let update = doc.encode_state_as_update_v1(&StateVector::default())?; - Self::replace_with(conn, workspace, doc.guid(), update).await?; - } - - Workspace::from_doc(doc, workspace)? - }; - - trace!("end create doc in workspace: {workspace}"); - - Ok(ws) - } -} - -#[async_trait] -impl DocStorage for DocDBStorage { - async fn detect_workspace(&self, workspace_id: &str) -> JwstStorageResult { - trace!("check workspace exists: get lock"); - let _lock = self.bucket.read().await; - - Ok(self.workspaces.read().await.contains_key(workspace_id) - || Self::workspace_count(&self.pool, workspace_id).await.map(|c| c > 0)?) - } - - async fn get_or_create_workspace(&self, workspace_id: String) -> JwstStorageResult { - trace!("get workspace enter: {workspace_id}"); - match self.workspaces.write().await.entry(workspace_id.clone()) { - Entry::Occupied(ws) => { - trace!("get workspace cache: {workspace_id}"); - Ok(ws.get().clone()) - } - Entry::Vacant(v) => { - debug!("init workspace cache: get lock"); - let _lock = self.bucket.write().await; - info!("init workspace cache: {workspace_id}"); - let ws = Self::init_workspace(&self.pool, &workspace_id).await?; - - Ok(v.insert(ws).clone()) - } - } - } - - async fn flush_workspace(&self, workspace_id: String, data: Vec) -> JwstStorageResult { - trace!("create workspace: get lock"); - let _lock = self.bucket.write().await; - - let ws = Self::init_workspace(&self.pool, &workspace_id).await?; - - self.full_migrate(&workspace_id, ws.doc_guid(), data).await?; - - debug_assert_eq!(Self::workspace_count(&self.pool, &workspace_id).await?, 1u64); - - Ok(ws) - } - - async fn delete_workspace(&self, workspace_id: &str) -> JwstStorageResult<()> { - debug!("delete workspace: get lock"); - let _lock = self.bucket.write().await; - - debug!("delete workspace cache: {workspace_id}"); - self.workspaces.write().await.remove(workspace_id); - DocDBStorage::delete_workspace(&self.pool, workspace_id).await?; - - Ok(()) - } - - async fn detect_doc(&self, guid: &str) -> JwstStorageResult { - trace!("check doc exists: get lock"); - let _lock = self.bucket.read().await; - - Ok(Self::count(&self.pool, guid).await? > 0) - } - - async fn get_doc(&self, guid: String) -> JwstStorageResult> { - let records = Docs::find().filter(DocsColumn::Guid.eq(guid)).all(&self.pool).await?; - - if records.is_empty() { - return Ok(None); - } - - let doc = utils::migrate_update(records, Doc::default())?; - - Ok(Some(doc)) - } - - async fn update_doc(&self, workspace_id: String, guid: String, data: &[u8]) -> JwstStorageResult<()> { - debug!("write_update: get lock"); - let _lock = self.bucket.write().await; - - trace!("write_update: {:?}", data); - self.update(&self.pool, &workspace_id, &guid, data.into()).await?; - - Ok(()) - } - - async fn update_doc_with_guid(&self, workspace_id: String, data: &[u8]) -> JwstStorageResult<()> { - debug!("write_update: get lock"); - let _lock = self.bucket.write().await; - - trace!("write_update: {:?}", data); - let mut decoder = RawDecoder::new(data.to_vec()); - let guid = decoder.read_var_string()?; - - self.update(&self.pool, &workspace_id, &guid, decoder.drain()).await?; - - Ok(()) - } - - async fn delete_doc(&self, guid: &str) -> JwstStorageResult<()> { - trace!("start drop doc: {guid}"); - - Docs::delete_many() - .filter(DocsColumn::Guid.eq(guid)) - .exec(&self.pool) - .await?; - - trace!("end drop doc: {guid}"); - Ok(()) - } -} - -#[cfg(test)] -pub async fn docs_storage_test(pool: &DocDBStorage) -> anyhow::Result<()> { - let conn = &pool.pool; - - DocDBStorage::delete_workspace(conn, "basic").await?; - - // empty table - assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 0); - - // first insert - DocDBStorage::insert(conn, "basic", "1", &[1, 2, 3, 4]).await?; - DocDBStorage::insert(conn, "basic", "1", &[2, 2, 3, 4]).await?; - assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 2); - - // second insert - DocDBStorage::replace_with(conn, "basic", "1", vec![3, 2, 3, 4]).await?; - - let all = DocDBStorage::workspace_all(conn, "basic").await?; - assert_eq!( - all, - vec![DocsModel { - id: all.get(0).unwrap().id, - guid: all.get(0).unwrap().guid.clone(), - workspace_id: "basic".into(), - created_at: all.get(0).unwrap().created_at, - is_workspace: true, - blob: vec![3, 2, 3, 4] - }] - ); - assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 1); - - DocDBStorage::delete_workspace(conn, "basic").await?; - - DocDBStorage::insert(conn, "basic", "1", &[1, 2, 3, 4]).await?; - - let all = DocDBStorage::workspace_all(conn, "basic").await?; - assert_eq!( - all, - vec![DocsModel { - id: all.get(0).unwrap().id, - workspace_id: "basic".into(), - guid: all.get(0).unwrap().guid.clone(), - created_at: all.get(0).unwrap().created_at, - is_workspace: true, - blob: vec![1, 2, 3, 4] - }] - ); - assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 1); - - // no cache - { - pool.workspaces.write().await.clear(); - let workspace = DocDBStorage::init_workspace(conn, "basic").await?; - assert_eq!(workspace.doc_guid(), "1"); - } - - Ok(()) -} - -#[cfg(test)] -#[cfg(feature = "postgres")] -pub async fn full_migration_stress_test(pool: &DocDBStorage) -> anyhow::Result<()> { - let final_bytes: Vec = (0..1024 * 100).map(|_| rand::random::()).collect(); - for i in 0..=50 { - let random_bytes: Vec = if i == 50 { - final_bytes.clone() - } else { - (0..1024 * 100).map(|_| rand::random::()).collect() - }; - let (r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15) = tokio::join!( - pool.flush_workspace("full_migration_1".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_2".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_3".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_4".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_5".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_6".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_7".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_8".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_9".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_10".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_11".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_12".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_13".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_14".to_owned(), random_bytes.clone()), - pool.flush_workspace("full_migration_15".to_owned(), random_bytes.clone()) - ); - r1?; - r2?; - r3?; - r4?; - r5?; - r6?; - r7?; - r8?; - r9?; - r10?; - r11?; - r12?; - r13?; - r14?; - r15?; - } - - assert_eq!( - DocDBStorage::workspace_all(&pool.pool, "full_migration_1") - .await? - .into_iter() - .map(|d| d.blob) - .collect::>(), - vec![final_bytes.clone()] - ); - - assert_eq!( - DocDBStorage::workspace_all(&pool.pool, "full_migration_2") - .await? - .into_iter() - .map(|d| d.blob) - .collect::>(), - vec![final_bytes] - ); - - Ok(()) -} - -#[cfg(test)] -pub async fn docs_storage_partial_test(pool: &DocDBStorage) -> anyhow::Result<()> { - use tokio::sync::mpsc::channel; - - let conn = &pool.pool; - - DocDBStorage::delete_workspace(conn, "basic").await?; - - // empty table - assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 0); - - { - let mut ws = pool.get_or_create_workspace("basic".into()).await.unwrap(); - let guid = ws.doc_guid().to_string(); - let (tx, mut rx) = channel(100); - - ws.doc().subscribe(move |update| { - futures::executor::block_on(async { - tx.send(update.to_vec()).await.unwrap(); - }); - }); - - { - let mut space = ws.get_space("test").unwrap(); - let mut block = space.create("block1", "text").unwrap(); - block.set("test1", "value1").unwrap(); - } - - { - let space = ws.get_space("test").unwrap(); - let mut block = space.get("block1").unwrap(); - block.set("test2", "value2").unwrap(); - } - - { - let mut space = ws.get_space("test").unwrap(); - let mut block = space.create("block2", "block2").unwrap(); - block.set("test3", "value3").unwrap(); - } - - loop { - tokio::select! { - Some(update) = rx.recv() => { - info!("recv: {}", update.len()); - pool.update_doc("basic".into(), guid.clone(), &update) - .await - .unwrap(); - } - _ = tokio::time::sleep(std::time::Duration::from_millis(5000)) => { - break; - } - } - } - - assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 2); - } - - // clear memory cache - pool.workspaces.write().await.clear(); - - { - // memory cache empty, retrieve data from db - let mut ws = pool.get_or_create_workspace("basic".into()).await.unwrap(); - let space = ws.get_space("test").unwrap(); - - let block = space.get("block1").unwrap(); - assert_eq!(block.flavour(), "text"); - assert_eq!(block.get("test1"), Some("value1".into())); - assert_eq!(block.get("test2"), Some("value2".into())); - - let block = space.get("block2").unwrap(); - assert_eq!(block.flavour(), "block2"); - assert_eq!(block.get("test3"), Some("value3".into())); - } - - Ok(()) -} diff --git a/libs/jwst-core-storage/src/storage/docs/mod.rs b/libs/jwst-core-storage/src/storage/docs/mod.rs deleted file mode 100644 index 50ac6a891..000000000 --- a/libs/jwst-core-storage/src/storage/docs/mod.rs +++ /dev/null @@ -1,137 +0,0 @@ -mod database; -mod utils; - -use std::ops::Deref; - -#[cfg(test)] -#[cfg(feature = "postgres")] -pub(super) use database::full_migration_stress_test; -use database::DocDBStorage; -#[cfg(test)] -pub(super) use database::{docs_storage_partial_test, docs_storage_test}; -use tokio::sync::{broadcast::Sender, RwLock}; - -use super::*; - -#[derive(Clone)] -pub struct SharedDocDBStorage(pub(super) Arc); - -impl SharedDocDBStorage { - pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { - Ok(Self(Arc::new(DocDBStorage::init_with_pool(pool, bucket).await?))) - } - - pub async fn init_pool(database: &str) -> JwstStorageResult { - Ok(Self(Arc::new(DocDBStorage::init_pool(database).await?))) - } - - pub fn remote(&self) -> &RwLock>>> { - self.0.remote() - } -} - -impl Deref for SharedDocDBStorage { - type Target = DocDBStorage; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -#[cfg(test)] -mod test { - use std::collections::HashSet; - - use jwst_storage_migration::Migrator; - use rand::random; - use sea_orm_migration::MigratorTrait; - use tokio::task::JoinSet; - - use super::{error, info, DocStorage, SharedDocDBStorage}; - use crate::{JwstStorageError, JwstStorageResult}; - - async fn create_workspace_stress_test(storage: SharedDocDBStorage, range: usize) -> JwstStorageResult<()> { - let mut join_set = JoinSet::new(); - let mut set = HashSet::new(); - - for _ in 0..range { - let id = random::().to_string(); - set.insert(id.clone()); - let storage = storage.clone(); - - join_set.spawn(async move { - info!("create workspace: {}", id); - let workspace = storage.get_or_create_workspace(id.clone()).await?; - info!("create workspace finish: {}", id); - assert_eq!(workspace.id(), id); - Ok::<_, JwstStorageError>(()) - }); - } - - while let Some(ret) = join_set.join_next().await { - if let Err(e) = ret.map_err(JwstStorageError::DocMerge)? { - error!("failed to execute creator: {e}"); - } - } - - for (i, id) in set.iter().enumerate() { - let storage = storage.clone(); - info!("check {i}: {id}"); - let id = id.clone(); - tokio::spawn(async move { - info!("get workspace: {}", id); - let workspace = storage.get_or_create_workspace(id.clone()).await?; - assert_eq!(workspace.id(), id); - Ok::<_, JwstStorageError>(()) - }); - } - - while let Some(ret) = join_set.join_next().await { - if let Err(e) = ret.map_err(JwstStorageError::DocMerge)? { - error!("failed to execute: {e}"); - } - } - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread")] - async fn sqlite_create_workspace_stress_test_faster() -> anyhow::Result<()> { - jwst_logger::init_logger("jwst-storage"); - let storage = SharedDocDBStorage::init_pool("sqlite::memory:").await?; - Migrator::up(&storage.0.pool, None).await.unwrap(); - create_workspace_stress_test(storage.clone(), 100).await?; - - Ok(()) - } - - #[ignore = "for stress testing"] - #[tokio::test(flavor = "multi_thread")] - async fn sqlite_create_workspace_stress_test() -> anyhow::Result<()> { - jwst_logger::init_logger("jwst-storage"); - let storage = SharedDocDBStorage::init_pool("sqlite::memory:").await?; - create_workspace_stress_test(storage.clone(), 10000).await?; - - Ok(()) - } - - #[ignore = "for stress testing"] - #[tokio::test(flavor = "multi_thread", worker_threads = 1)] - async fn postgres_create_workspace_stress_test() -> anyhow::Result<()> { - jwst_logger::init_logger("jwst-storage"); - let storage = SharedDocDBStorage::init_pool("postgresql://affine:affine@localhost:5432/affine_binary").await?; - create_workspace_stress_test(storage.clone(), 10000).await?; - - Ok(()) - } - - #[ignore = "for stress testing"] - #[tokio::test(flavor = "multi_thread", worker_threads = 1)] - async fn mysql_create_workspace_stress_test() -> anyhow::Result<()> { - // jwst_logger::init_logger(); - let storage = SharedDocDBStorage::init_pool("mysql://affine:affine@localhost:3306/affine_binary").await?; - create_workspace_stress_test(storage.clone(), 10000).await?; - - Ok(()) - } -} diff --git a/libs/jwst-core-storage/src/storage/docs/utils.rs b/libs/jwst-core-storage/src/storage/docs/utils.rs deleted file mode 100644 index 3a05346fe..000000000 --- a/libs/jwst-core-storage/src/storage/docs/utils.rs +++ /dev/null @@ -1,25 +0,0 @@ -use jwst_codec::{Doc, StateVector}; - -use super::{entities::prelude::*, types::JwstStorageResult, *}; - -// apply all updates to the given doc -pub fn migrate_update(update_records: Vec<::Model>, mut doc: Doc) -> JwstResult { - // stop update dispatch before apply updates - doc.publisher.stop(); - for record in update_records { - let id = record.created_at; - if let Err(e) = doc.apply_update_from_binary(record.blob) { - warn!("update {} merge failed, skip it: {:?}", id, e); - } - } - doc.publisher.start(); - - Ok(doc) -} - -pub fn merge_doc_records(update_records: Vec<::Model>) -> JwstStorageResult> { - let doc = migrate_update(update_records, Doc::default())?; - let state_vector = doc.encode_state_as_update_v1(&StateVector::default())?; - - Ok(state_vector) -} diff --git a/libs/jwst-core-storage/src/storage/mod.rs b/libs/jwst-core-storage/src/storage/mod.rs deleted file mode 100644 index 99a1c09f1..000000000 --- a/libs/jwst-core-storage/src/storage/mod.rs +++ /dev/null @@ -1,216 +0,0 @@ -pub(crate) mod blobs; -mod docs; -mod test; - -use std::{collections::HashMap, time::Instant}; - -#[cfg(feature = "image")] -use blobs::BlobAutoStorage; -#[cfg(feature = "bucket")] -use blobs::BlobBucketStorage; -use blobs::{BlobDBStorage, BlobStorageType, JwstBlobStorage}; -use docs::SharedDocDBStorage; -use jwst_storage_migration::{Migrator, MigratorTrait}; -use tokio::sync::Mutex; - -use super::*; - -pub struct JwstStorage { - pool: DatabaseConnection, - blobs: JwstBlobStorage, - docs: SharedDocDBStorage, - last_migrate: Mutex>, -} - -impl JwstStorage { - pub async fn new(database: &str, blob_storage_type: BlobStorageType) -> JwstStorageResult { - let is_sqlite = is_sqlite(database); - let pool = create_connection(database, is_sqlite).await?; - let bucket = get_bucket(is_sqlite); - - if is_sqlite { - pool.execute_unprepared("PRAGMA journal_mode=WAL;").await.unwrap(); - } - - let blobs = match blob_storage_type { - BlobStorageType::DB => { - #[cfg(feature = "image")] - { - JwstBlobStorage::AutoStorage(BlobAutoStorage::init_with_pool(pool.clone(), bucket.clone()).await?) - } - #[cfg(not(feature = "image"))] - { - let db = BlobDBStorage::init_with_pool(pool.clone(), bucket.clone()).await?; - JwstBlobStorage::RawStorage(Arc::new(db)) - } - } - #[cfg(feature = "bucket")] - BlobStorageType::MixedBucketDB(param) => JwstBlobStorage::BucketStorage( - BlobBucketStorage::init_with_pool(pool.clone(), bucket.clone(), Some(param.try_into()?)).await?, - ), - }; - let docs = SharedDocDBStorage::init_with_pool(pool.clone(), bucket.clone()).await?; - - Ok(Self { - pool, - blobs, - docs, - last_migrate: Mutex::new(HashMap::new()), - }) - } - - pub async fn new_with_migration(database: &str, blob_storage_type: BlobStorageType) -> JwstStorageResult { - let storage = Self::new(database, blob_storage_type).await?; - - storage.db_migrate().await?; - - Ok(storage) - } - - async fn db_migrate(&self) -> JwstStorageResult<()> { - Migrator::up(&self.pool, None).await?; - Ok(()) - } - - pub async fn new_with_sqlite(file: &str, blob_storage_type: BlobStorageType) -> JwstStorageResult { - use std::fs::create_dir; - - let data = PathBuf::from("./data"); - if !data.exists() { - create_dir(&data).map_err(JwstStorageError::CreateDataFolder)?; - } - - Self::new_with_migration( - &format!( - "sqlite:{}?mode=rwc", - data.join(PathBuf::from(file).name_str()).with_extension("db").display() - ), - blob_storage_type, - ) - .await - } - - pub fn database(&self) -> String { - format!("{:?}", self.pool) - } - - pub fn blobs(&self) -> &JwstBlobStorage { - &self.blobs - } - - pub fn docs(&self) -> &SharedDocDBStorage { - &self.docs - } - - pub async fn with_pool(&self, func: F) -> JwstStorageResult - where - F: Fn(DatabaseConnection) -> Fut, - Fut: Future>, - { - func(self.pool.clone()).await - } - - pub async fn create_workspace(&self, workspace_id: S) -> JwstStorageResult - where - S: AsRef, - { - info!("create_workspace: {}", workspace_id.as_ref()); - - self.docs - .get_or_create_workspace(workspace_id.as_ref().into()) - .await - .map_err(|_err| JwstStorageError::Crud(format!("Failed to create workspace {}", workspace_id.as_ref()))) - } - - pub async fn get_workspace(&self, workspace_id: S) -> JwstStorageResult - where - S: AsRef, - { - trace!("get_workspace: {}", workspace_id.as_ref()); - if self - .docs - .detect_workspace(workspace_id.as_ref()) - .await - .map_err(|_err| JwstStorageError::Crud(format!("failed to check workspace {}", workspace_id.as_ref())))? - { - Ok(self - .docs - .get_or_create_workspace(workspace_id.as_ref().into()) - .await - .map_err(|_err| JwstStorageError::Crud(format!("failed to get workspace {}", workspace_id.as_ref())))?) - } else { - Err(JwstStorageError::WorkspaceNotFound(workspace_id.as_ref().into())) - } - } - - pub async fn full_migrate(&self, workspace_id: String, update: Option>, force: bool) -> bool { - let mut map = self.last_migrate.lock().await; - let ts = map.entry(workspace_id.clone()).or_insert(Instant::now()); - - if ts.elapsed().as_secs() > 5 || force { - debug!("full migrate: {workspace_id}"); - match self.docs.get_or_create_workspace(workspace_id.clone()).await { - Ok(workspace) => { - let update = if let Some(update) = update { - if let Err(e) = self.docs.delete_workspace(&workspace_id).await { - error!("full_migrate write error: {}", e.to_string()); - return false; - }; - Some(update) - } else { - workspace.sync_migration().ok() - }; - - let Some(update) = update else { - error!("full migrate failed: wait transact timeout"); - return false; - }; - if let Err(e) = self.docs.flush_workspace(workspace_id.clone(), update).await { - error!("db write error: {}", e.to_string()); - return false; - } - - *ts = Instant::now(); - - info!("full migrate final: {workspace_id}"); - true - } - Err(e) => { - warn!("workspace {workspace_id} not exists in cache: {e:?}"); - false - } - } - } else { - true - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_sqlite_storage() { - let storage = JwstStorage::new_with_sqlite(":memory:", BlobStorageType::DB) - .await - .unwrap(); - assert_eq!(storage.database(), "SqlxSqlitePoolConnection"); - } - - #[tokio::test] - #[cfg(feature = "bucket")] - #[ignore = "need to config bucket auth"] - async fn test_bucket_storage() { - let bucket_params = MixedBucketDBParam { - access_key: dotenvy::var("BUCKET_ACCESS_TOKEN").unwrap().to_string(), - secret_access_key: dotenvy::var("BUCKET_SECRET_TOKEN").unwrap().to_string(), - endpoint: dotenvy::var("BUCKET_ENDPOINT").unwrap().to_string(), - bucket: Some(dotenvy::var("BUCKET_NAME").unwrap()), - root: Some(dotenvy::var("BUCKET_ROOT").unwrap()), - }; - let _storage = JwstStorage::new_with_sqlite(":memory:", BlobStorageType::MixedBucketDB(bucket_params)) - .await - .unwrap(); - } -} diff --git a/libs/jwst-core-storage/src/storage/test.rs b/libs/jwst-core-storage/src/storage/test.rs deleted file mode 100644 index 33b6aab36..000000000 --- a/libs/jwst-core-storage/src/storage/test.rs +++ /dev/null @@ -1,44 +0,0 @@ -#[cfg(test)] -use super::{ - blobs::blobs_storage_test, - docs::{docs_storage_partial_test, docs_storage_test}, - *, -}; - -#[tokio::test] -async fn sqlite_storage_test() -> anyhow::Result<()> { - let storage = JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB).await?; - - info!("blobs_storage_test"); - blobs_storage_test(&storage.blobs().get_blob_db().unwrap()).await?; - info!("docs_storage_test"); - docs_storage_test(&storage.docs().0).await?; - info!("docs_storage_partial_test"); - docs_storage_partial_test(&storage.docs().0).await?; - - Ok(()) -} - -#[ignore = "need postgres server"] -#[cfg(feature = "postgres")] -#[tokio::test] -async fn postgres_storage_test() -> anyhow::Result<()> { - use test::docs::full_migration_stress_test; - - let db = "postgresql://affine:affine@localhost:5432/affine_binary"; - let storage = JwstStorage::new_with_migration(db, BlobStorageType::DB).await?; - let blob_db = storage.blobs().get_blob_db().unwrap(); - let (r1, r2, r3, r4) = tokio::join!( - blobs_storage_test(&blob_db), - docs_storage_test(&storage.docs().0), - docs_storage_partial_test(&storage.docs().0), - full_migration_stress_test(&storage.docs().0), - ); - - r1?; - r2?; - r3?; - r4?; - - Ok(()) -} diff --git a/libs/jwst-core-storage/src/types.rs b/libs/jwst-core-storage/src/types.rs deleted file mode 100644 index 2b2259e9a..000000000 --- a/libs/jwst-core-storage/src/types.rs +++ /dev/null @@ -1,30 +0,0 @@ -use jwst_codec::JwstCodecError; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum JwstStorageError { - #[error("failed to create data directory")] - CreateDataFolder(std::io::Error), - #[error("db manipulate error: {0}")] - Crud(String), - #[error("db error")] - Db(#[from] sea_orm::DbErr), - #[error("doc codec error")] - DocJwstCodec(#[from] JwstCodecError), - #[error("merge thread panic")] - DocMerge(tokio::task::JoinError), - #[error("workspace {0} not found")] - WorkspaceNotFound(String), - #[error("jwst error")] - Jwst(#[from] jwst_core::JwstError), - #[error("failed to process blob")] - JwstBlob(#[from] crate::storage::blobs::JwstBlobError), - #[cfg(feature = "bucket")] - #[error("bucket error")] - JwstBucket(#[from] opendal::Error), - #[cfg(feature = "bucket")] - #[error("env variables read error")] - DotEnvy(#[from] dotenvy::Error), -} - -pub type JwstStorageResult = Result; diff --git a/libs/jwst-rpc/Cargo.toml b/libs/jwst-rpc/Cargo.toml index 7cb9fe494..71069feb1 100644 --- a/libs/jwst-rpc/Cargo.toml +++ b/libs/jwst-rpc/Cargo.toml @@ -42,7 +42,7 @@ webrtcrs = { package = "webrtc", version = "0.8.0", optional = true } # ======= workspace dependencies ======= jwst-codec = { workspace = true } jwst-core = { workspace = true } -jwst-core-storage = { workspace = true } +jwst-storage = { workspace = true } [dev-dependencies] indicatif = "0.17.3" diff --git a/libs/jwst-rpc/src/context.rs b/libs/jwst-rpc/src/context.rs index e442fc3bc..789fda34f 100644 --- a/libs/jwst-rpc/src/context.rs +++ b/libs/jwst-rpc/src/context.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use chrono::Utc; use jwst_codec::{CrdtReader, RawDecoder}; use jwst_core::{DocStorage, Workspace}; -use jwst_core_storage::{JwstStorage, JwstStorageResult}; +use jwst_storage::{JwstStorage, JwstStorageResult}; use tokio::sync::{ broadcast::{channel as broadcast, error::RecvError, Receiver as BroadcastReceiver, Sender as BroadcastSender}, mpsc::{Receiver as MpscReceiver, Sender as MpscSender}, diff --git a/libs/jwst-rpc/src/utils/server_context.rs b/libs/jwst-rpc/src/utils/server_context.rs index d2e114bcb..f9f9dc76e 100644 --- a/libs/jwst-rpc/src/utils/server_context.rs +++ b/libs/jwst-rpc/src/utils/server_context.rs @@ -2,7 +2,7 @@ use std::{collections::HashMap, time::Duration}; use jwst_codec::StateVector; use jwst_core::{DocStorage, Workspace}; -use jwst_core_storage::{BlobStorageType, JwstStorage}; +use jwst_storage::{BlobStorageType, JwstStorage}; use tokio::{sync::RwLock, time::sleep}; use super::*; diff --git a/libs/jwst-storage/Cargo.toml b/libs/jwst-storage/Cargo.toml index 286df9281..b9722e2e2 100644 --- a/libs/jwst-storage/Cargo.toml +++ b/libs/jwst-storage/Cargo.toml @@ -7,36 +7,40 @@ license = "AGPL-3.0-only" [features] default = ["sqlite"] +bucket = ["dotenvy", "opendal"] +image = ["dep:image"] mysql = ["sea-orm/sqlx-mysql"] postgres = ["sea-orm/sqlx-postgres"] sqlite = ["sea-orm/sqlx-sqlite"] [dependencies] -anyhow = "1.0.70" -async-trait = "0.1.68" +anyhow = "1.0.75" +async-trait = "0.1.73" bytes = "1.4.0" chrono = { version = "0.4.24", features = ["serde"] } futures = "0.3.28" -governor = "0.5.1" -image = { version = "0.24.6", features = ["webp-encoder"] } -lib0 = "0.16.5" +governor = "0.6.0" path-ext = "0.1.0" -sha2 = "0.10.6" +sha2 = "0.10.7" sea-orm = { version = "0.12.2", features = ["runtime-tokio-rustls", "macros"] } sea-orm-migration = { version = "0.12.2", default-features = false } -thiserror = "1.0.40" -tokio = { version = "1.27.0", features = ["fs", "macros", "sync"] } -tokio-util = { version = "0.7.7", features = ["io"] } -url = "2.3.1" -yrs = "0.16.5" +thiserror = "1.0.47" +tokio = { version = "1", features = ["fs", "macros", "sync"] } +tokio-util = { version = "0.7.8", features = ["io"] } +url = "2.4.0" + +# ======== bucket dependencies ======= opendal = { version = "0.39.0", default-features = false, features = [ "rustls", "services-s3", -] } -dotenvy = "0.15.7" +], optional = true } +dotenvy = { version = "0.15.7", optional = true } + +# ======= image dependencies ====== +image = { version = "0.24.6", features = ["webp-encoder"], optional = true } # ======= workspace dependencies ======= -jwst = { workspace = true } +jwst-core = { workspace = true } jwst-codec = { workspace = true } jwst-logger = { workspace = true } jwst-storage-migration = { path = "./src/migration" } diff --git a/libs/jwst-storage/src/lib.rs b/libs/jwst-storage/src/lib.rs index 647d8b19d..438447d49 100644 --- a/libs/jwst-storage/src/lib.rs +++ b/libs/jwst-storage/src/lib.rs @@ -7,17 +7,16 @@ mod types; use std::{path::PathBuf, sync::Arc, time::Duration}; use async_trait::async_trait; -use chrono::{DateTime, Utc}; +use chrono::Utc; use futures::{Future, Stream}; -use jwst::{DocStorage, JwstResult, Workspace}; +use jwst_core::{DocStorage, JwstResult, Workspace}; use jwst_logger::{debug, error, info, trace, warn}; use path_ext::PathExt; use rate_limiter::{get_bucket, is_sqlite, Bucket}; use sea_orm::{prelude::*, ConnectOptions, Database, DbErr, QuerySelect, Set}; -pub use storage::{ - blobs::{BlobStorageType, MixedBucketDBParam}, - JwstStorage, -}; +#[cfg(feature = "bucket")] +pub use storage::blobs::MixedBucketDBParam; +pub use storage::{blobs::BlobStorageType, JwstStorage}; pub use types::{JwstStorageError, JwstStorageResult}; #[inline] diff --git a/libs/jwst-storage/src/migration/Cargo.toml b/libs/jwst-storage/src/migration/Cargo.toml index 5b8bba191..6c740b997 100644 --- a/libs/jwst-storage/src/migration/Cargo.toml +++ b/libs/jwst-storage/src/migration/Cargo.toml @@ -13,5 +13,4 @@ tokio = { version = "^1", features = ["macros", "rt-multi-thread"] } [dependencies.sea-orm-migration] version = "0.12.2" -default-features = false features = ["runtime-tokio-rustls", "sqlx-postgres"] diff --git a/libs/jwst-core-storage/src/storage/blobs/auto_storage/auto_storage.rs b/libs/jwst-storage/src/storage/blobs/auto_storage/auto_storage.rs similarity index 100% rename from libs/jwst-core-storage/src/storage/blobs/auto_storage/auto_storage.rs rename to libs/jwst-storage/src/storage/blobs/auto_storage/auto_storage.rs diff --git a/libs/jwst-core-storage/src/storage/blobs/auto_storage/mod.rs b/libs/jwst-storage/src/storage/blobs/auto_storage/mod.rs similarity index 100% rename from libs/jwst-core-storage/src/storage/blobs/auto_storage/mod.rs rename to libs/jwst-storage/src/storage/blobs/auto_storage/mod.rs diff --git a/libs/jwst-core-storage/src/storage/blobs/auto_storage/utils.rs b/libs/jwst-storage/src/storage/blobs/auto_storage/utils.rs similarity index 100% rename from libs/jwst-core-storage/src/storage/blobs/auto_storage/utils.rs rename to libs/jwst-storage/src/storage/blobs/auto_storage/utils.rs diff --git a/libs/jwst-core-storage/src/storage/blobs/blob_storage.rs b/libs/jwst-storage/src/storage/blobs/blob_storage.rs similarity index 100% rename from libs/jwst-core-storage/src/storage/blobs/blob_storage.rs rename to libs/jwst-storage/src/storage/blobs/blob_storage.rs diff --git a/libs/jwst-core-storage/src/storage/blobs/bucket/bucket.rs b/libs/jwst-storage/src/storage/blobs/bucket/bucket.rs similarity index 100% rename from libs/jwst-core-storage/src/storage/blobs/bucket/bucket.rs rename to libs/jwst-storage/src/storage/blobs/bucket/bucket.rs diff --git a/libs/jwst-core-storage/src/storage/blobs/bucket/mod.rs b/libs/jwst-storage/src/storage/blobs/bucket/mod.rs similarity index 100% rename from libs/jwst-core-storage/src/storage/blobs/bucket/mod.rs rename to libs/jwst-storage/src/storage/blobs/bucket/mod.rs diff --git a/libs/jwst-core-storage/src/storage/blobs/bucket/utils.rs b/libs/jwst-storage/src/storage/blobs/bucket/utils.rs similarity index 100% rename from libs/jwst-core-storage/src/storage/blobs/bucket/utils.rs rename to libs/jwst-storage/src/storage/blobs/bucket/utils.rs diff --git a/libs/jwst-storage/src/storage/blobs/bucket_local_db.rs b/libs/jwst-storage/src/storage/blobs/bucket_local_db.rs deleted file mode 100644 index 4b3e0a84b..000000000 --- a/libs/jwst-storage/src/storage/blobs/bucket_local_db.rs +++ /dev/null @@ -1,358 +0,0 @@ -use std::{collections::HashMap, sync::Arc}; - -use bytes::Bytes; -use futures::Stream; -use jwst::{BlobMetadata, BlobStorage, BucketBlobStorage, JwstResult}; -use jwst_storage_migration::Migrator; -use opendal::{services::S3, Operator}; -use sea_orm::{DatabaseConnection, EntityTrait}; -use sea_orm_migration::MigratorTrait; - -use super::{ - utils::{calculate_hash, get_hash}, - *, -}; -use crate::{rate_limiter::Bucket, JwstStorageError}; - -pub(super) type BucketBlobModel = ::Model; -type BucketBlobActiveModel = entities::bucket_blobs::ActiveModel; -type BucketBlobColumn = ::Column; - -#[derive(Clone)] -pub struct BlobBucketDBStorage { - bucket: Arc, - pub(super) pool: DatabaseConnection, - pub(super) bucket_storage: BucketStorage, -} - -impl AsRef for BlobBucketDBStorage { - fn as_ref(&self) -> &DatabaseConnection { - &self.pool - } -} - -impl BlobBucketDBStorage { - pub async fn init_with_pool( - pool: DatabaseConnection, - bucket: Arc, - bucket_storage: Option, - ) -> JwstStorageResult { - Migrator::up(&pool, None).await?; - Ok(Self { - bucket, - pool, - bucket_storage: bucket_storage.unwrap_or(BucketStorage::new()?), - }) - } - - #[allow(unused)] - pub async fn init_pool(database: &str, bucket_storage: Option) -> JwstStorageResult { - let is_sqlite = is_sqlite(database); - let pool = create_connection(database, is_sqlite).await?; - - Self::init_with_pool(pool, get_bucket(is_sqlite), bucket_storage).await - } - - #[allow(unused)] - async fn all(&self, workspace: &str) -> Result, DbErr> { - BucketBlobs::find() - .filter(BucketBlobColumn::WorkspaceId.eq(workspace)) - .all(&self.pool) - .await - } - - async fn keys(&self, workspace: &str) -> Result, DbErr> { - BucketBlobs::find() - .filter(BucketBlobColumn::WorkspaceId.eq(workspace)) - .column_as(BucketBlobColumn::Hash, "hash") - .all(&self.pool) - .await - .map(|r| r.into_iter().map(|f| f.hash).collect()) - } - - #[allow(unused)] - async fn count(&self, workspace: &str) -> Result { - BucketBlobs::find() - .filter(BucketBlobColumn::WorkspaceId.eq(workspace)) - .count(&self.pool) - .await - } - - async fn exists(&self, workspace: &str, hash: &str) -> Result { - BucketBlobs::find_by_id((workspace.into(), hash.into())) - .count(&self.pool) - .await - .map(|c| c > 0) - } - - pub(super) async fn metadata(&self, workspace: &str, hash: &str) -> JwstBlobResult { - BucketBlobs::find_by_id((workspace.into(), hash.into())) - .select_only() - .column_as(BucketBlobColumn::Length, "size") - .column_as(BucketBlobColumn::CreatedAt, "created_at") - .into_model::() - .one(&self.pool) - .await - .map_err(|e| e.into()) - .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) - } - - async fn get_blobs_size(&self, workspace: &str) -> Result, DbErr> { - BucketBlobs::find() - .filter(BucketBlobColumn::WorkspaceId.eq(workspace)) - .column_as(BucketBlobColumn::Length, "size") - .column_as(BucketBlobColumn::CreatedAt, "created_at") - .into_model::() - .all(&self.pool) - .await - .map(|r| r.into_iter().map(|f| f.size).reduce(|a, b| a + b)) - } - - async fn insert(&self, workspace: &str, hash: &str, blob: &[u8]) -> Result<(), DbErr> { - if !self.exists(workspace, hash).await? { - BucketBlobs::insert(BucketBlobActiveModel { - workspace_id: Set(workspace.into()), - hash: Set(hash.into()), - length: Set(blob.len().try_into().unwrap()), - created_at: Set(Utc::now().into()), - }) - .exec(&self.pool) - .await?; - } - - Ok(()) - } - - async fn delete(&self, workspace: &str, hash: &str) -> Result { - BucketBlobs::delete_by_id((workspace.into(), hash.into())) - .exec(&self.pool) - .await - .map(|r| r.rows_affected == 1) - } - - async fn drop(&self, workspace: &str) -> Result<(), DbErr> { - BucketBlobs::delete_many() - .filter(BucketBlobColumn::WorkspaceId.eq(workspace)) - .exec(&self.pool) - .await?; - - Ok(()) - } -} - -#[derive(Clone)] -pub struct BucketStorage { - pub(super) op: Operator, -} - -impl BucketStorage { - #[allow(unused)] - pub fn new() -> JwstStorageResult { - let access_key = dotenvy::var("BUCKET_ACCESS_TOKEN")?; - let secret_access_key = dotenvy::var("BUCKET_SECRET_TOKEN")?; - let endpoint = dotenvy::var("BUCKET_ENDPOINT")?; - let bucket = dotenvy::var("BUCKET_NAME"); - let root = dotenvy::var("BUCKET_ROOT"); - - let mut builder = S3::default(); - - builder.bucket(bucket.unwrap_or("__default_bucket__".to_string()).as_str()); - builder.root(root.unwrap_or("__default_root__".to_string()).as_str()); - builder.endpoint(endpoint.as_str()); - builder.access_key_id(access_key.as_str()); - builder.secret_access_key(secret_access_key.as_str()); - - Ok(Self { - op: Operator::new(builder)?.finish(), - }) - } -} - -#[async_trait] -impl BucketBlobStorage for BucketStorage { - async fn get_blob(&self, workspace: Option, id: String) -> JwstResult, JwstStorageError> { - let workspace = get_workspace(workspace); - let key = build_key(workspace, id); - let bs = self.op.read(&key).await?; - - Ok(bs) - } - - async fn put_blob( - &self, - workspace: Option, - hash: String, - blob: Vec, - ) -> JwstResult<(), JwstStorageError> { - let workspace = get_workspace(workspace); - let key = build_key(workspace, hash); - let _ = self.op.write(&key, blob).await?; - - Ok(()) - } - - async fn delete_blob(&self, workspace: Option, id: String) -> JwstResult { - let workspace = get_workspace(workspace); - let key = build_key(workspace, id); - - match self.op.delete(&key).await { - Ok(_) => Ok(true), - Err(e) => Err(JwstStorageError::from(e)), - } - } - - async fn delete_workspace(&self, workspace_id: String) -> JwstResult<(), JwstStorageError> { - self.op.remove_all(&workspace_id).await?; - - Ok(()) - } -} - -#[async_trait] -impl BlobStorage for BlobBucketDBStorage { - async fn list_blobs(&self, workspace: Option) -> JwstResult, JwstStorageError> { - let _lock = self.bucket.read().await; - let workspace = workspace.unwrap_or("__default__".into()); - if let Ok(keys) = self.keys(&workspace).await { - return Ok(keys); - } - - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - - async fn check_blob(&self, workspace: Option, id: String) -> JwstResult { - let _lock = self.bucket.read().await; - let workspace = workspace.unwrap_or("__default__".into()); - if let Ok(exists) = self.exists(&workspace, &id).await { - return Ok(exists); - } - - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - - async fn get_blob( - &self, - workspace: Option, - id: String, - _params: Option>, - ) -> JwstResult, JwstStorageError> { - self.bucket_storage.get_blob(workspace, id).await - } - - async fn get_metadata( - &self, - workspace: Option, - id: String, - _params: Option>, - ) -> JwstResult { - let _lock = self.bucket.read().await; - let workspace = get_workspace(workspace); - if let Ok(metadata) = self.metadata(&workspace, &id).await { - Ok(metadata.into()) - } else { - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - } - - async fn put_blob_stream( - &self, - workspace: Option, - stream: impl Stream + Send, - ) -> JwstResult { - let (hash, blob) = get_hash(stream).await; - self.bucket_storage - .put_blob(workspace.clone(), hash.clone(), blob.clone()) - .await?; - let _lock = self.bucket.write().await; - let workspace = get_workspace(workspace); - - if self.insert(&workspace, &hash, &blob).await.is_ok() { - Ok(hash) - } else { - self.bucket_storage.delete_blob(Some(workspace.clone()), hash).await?; - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - } - - async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstResult { - let hash = calculate_hash(&blob); - self.bucket_storage - .put_blob(workspace.clone(), hash.clone(), blob.clone()) - .await?; - - let _lock = self.bucket.write().await; - let workspace = get_workspace(workspace); - - if self.insert(&workspace, &hash, &blob).await.is_ok() { - Ok(hash) - } else { - self.bucket_storage.delete_blob(Some(workspace.clone()), hash).await?; - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - } - - async fn delete_blob(&self, workspace: Option, id: String) -> JwstResult { - self.bucket_storage.delete_blob(workspace.clone(), id.clone()).await?; - let _lock = self.bucket.write().await; - let workspace = get_workspace(workspace); - if let Ok(success) = self.delete(&workspace, &id).await { - Ok(success) - } else { - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - } - - async fn delete_workspace(&self, workspace_id: String) -> JwstResult<(), JwstStorageError> { - self.bucket_storage.delete_workspace(workspace_id.clone()).await?; - let _lock = self.bucket.write().await; - if self.drop(&workspace_id).await.is_ok() { - Ok(()) - } else { - Err(JwstStorageError::WorkspaceNotFound(workspace_id)) - } - } - - async fn get_blobs_size(&self, workspace_id: String) -> JwstResult { - let _lock = self.bucket.read().await; - let size = self.get_blobs_size(&workspace_id).await?; - return Ok(size.unwrap_or(0)); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::storage::blobs::utils::BucketStorageBuilder; - - #[tokio::test] - #[ignore = "need to config bucket auth"] - async fn test_init_bucket_storage() { - let bucket_storage = BucketStorageBuilder::new() - .endpoint("ENDPOINT") - .access_key("ACCESS_KEY") - .secret_access_key("SECRET_ACCESS_KEY") - .bucket("__default_bucket__") - .root("__default_root__") - .build() - .unwrap(); - - BlobBucketDBStorage::init_pool("sqlite::memory:", Some(bucket_storage)) - .await - .unwrap(); - } -} - -/// get_workspace will get the workspace name from the input. -/// -/// If the input is None, it will return the default workspace name. -fn get_workspace(workspace: Option) -> String { - match workspace { - Some(w) => w, - None => "__default__".into(), - } -} - -/// build_key will build the request key for the bucket storage. -fn build_key(workspace: String, id: String) -> String { - format!("{}/{}", workspace, id) -} diff --git a/libs/jwst-storage/src/storage/blobs/local_db.rs b/libs/jwst-storage/src/storage/blobs/local_db.rs deleted file mode 100644 index 49c4ef363..000000000 --- a/libs/jwst-storage/src/storage/blobs/local_db.rs +++ /dev/null @@ -1,291 +0,0 @@ -use jwst::{Base64Engine, URL_SAFE_ENGINE}; -use sha2::{Digest, Sha256}; - -use super::{utils::get_hash, *}; -use crate::types::JwstStorageResult; -pub(super) type BlobModel = ::Model; -type BlobActiveModel = super::entities::blobs::ActiveModel; -type BlobColumn = ::Column; - -#[derive(Clone)] -pub struct BlobDBStorage { - bucket: Arc, - pub(super) pool: DatabaseConnection, -} - -impl AsRef for BlobDBStorage { - fn as_ref(&self) -> &DatabaseConnection { - &self.pool - } -} - -impl BlobDBStorage { - pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { - Ok(Self { bucket, pool }) - } - - pub async fn init_pool(database: &str) -> JwstStorageResult { - let is_sqlite = is_sqlite(database); - let pool = create_connection(database, is_sqlite).await?; - - Self::init_with_pool(pool, get_bucket(is_sqlite)).await - } - - #[allow(unused)] - async fn all(&self, workspace: &str) -> Result, DbErr> { - Blobs::find() - .filter(BlobColumn::WorkspaceId.eq(workspace)) - .all(&self.pool) - .await - } - - async fn keys(&self, workspace: &str) -> Result, DbErr> { - Blobs::find() - .filter(BlobColumn::WorkspaceId.eq(workspace)) - .column(BlobColumn::Hash) - .all(&self.pool) - .await - .map(|r| r.into_iter().map(|f| f.hash).collect()) - } - - #[allow(unused)] - async fn count(&self, workspace: &str) -> Result { - Blobs::find() - .filter(BlobColumn::WorkspaceId.eq(workspace)) - .count(&self.pool) - .await - } - - async fn exists(&self, workspace: &str, hash: &str) -> Result { - Blobs::find_by_id((workspace.into(), hash.into())) - .count(&self.pool) - .await - .map(|c| c > 0) - } - - pub(super) async fn metadata(&self, workspace: &str, hash: &str) -> JwstBlobResult { - Blobs::find_by_id((workspace.into(), hash.into())) - .select_only() - .column_as(BlobColumn::Length, "size") - .column_as(BlobColumn::CreatedAt, "created_at") - .into_model::() - .one(&self.pool) - .await - .map_err(|e| e.into()) - .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) - } - - pub(super) async fn get_blobs_size(&self, workspace: &str) -> Result, DbErr> { - Blobs::find() - .filter(BlobColumn::WorkspaceId.eq(workspace)) - .column_as(BlobColumn::Length, "size") - .column_as(BlobColumn::CreatedAt, "created_at") - .into_model::() - .all(&self.pool) - .await - .map(|r| r.into_iter().map(|f| f.size).reduce(|a, b| a + b)) - } - - async fn insert(&self, workspace: &str, hash: &str, blob: &[u8]) -> Result<(), DbErr> { - if !self.exists(workspace, hash).await? { - Blobs::insert(BlobActiveModel { - workspace_id: Set(workspace.into()), - hash: Set(hash.into()), - blob: Set(blob.into()), - length: Set(blob.len().try_into().unwrap()), - created_at: Set(Utc::now().into()), - }) - .exec(&self.pool) - .await?; - } - - Ok(()) - } - - pub(super) async fn get(&self, workspace: &str, hash: &str) -> JwstBlobResult { - Blobs::find_by_id((workspace.into(), hash.into())) - .one(&self.pool) - .await - .map_err(|e| e.into()) - .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) - } - - async fn delete(&self, workspace: &str, hash: &str) -> Result { - Blobs::delete_by_id((workspace.into(), hash.into())) - .exec(&self.pool) - .await - .map(|r| r.rows_affected == 1) - } - - async fn drop(&self, workspace: &str) -> Result<(), DbErr> { - Blobs::delete_many() - .filter(BlobColumn::WorkspaceId.eq(workspace)) - .exec(&self.pool) - .await?; - - Ok(()) - } -} - -#[async_trait] -impl BlobStorage for BlobDBStorage { - async fn list_blobs(&self, workspace: Option) -> JwstStorageResult> { - let _lock = self.bucket.read().await; - let workspace = workspace.unwrap_or("__default__".into()); - if let Ok(keys) = self.keys(&workspace).await { - return Ok(keys); - } - - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - - async fn check_blob(&self, workspace: Option, id: String) -> JwstStorageResult { - let _lock = self.bucket.read().await; - let workspace = workspace.unwrap_or("__default__".into()); - if let Ok(exists) = self.exists(&workspace, &id).await { - return Ok(exists); - } - - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - - async fn get_blob( - &self, - workspace: Option, - id: String, - _params: Option>, - ) -> JwstStorageResult> { - let _lock = self.bucket.read().await; - let workspace = workspace.unwrap_or("__default__".into()); - if let Ok(blob) = self.get(&workspace, &id).await { - return Ok(blob.blob); - } - - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - - async fn get_metadata( - &self, - workspace: Option, - id: String, - _params: Option>, - ) -> JwstStorageResult { - let _lock = self.bucket.read().await; - let workspace = workspace.unwrap_or("__default__".into()); - if let Ok(metadata) = self.metadata(&workspace, &id).await { - Ok(metadata.into()) - } else { - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - } - - async fn put_blob_stream( - &self, - workspace: Option, - stream: impl Stream + Send, - ) -> JwstStorageResult { - let _lock = self.bucket.write().await; - let workspace = workspace.unwrap_or("__default__".into()); - - let (hash, blob) = get_hash(stream).await; - - if self.insert(&workspace, &hash, &blob).await.is_ok() { - Ok(hash) - } else { - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - } - - async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstStorageResult { - let _lock = self.bucket.write().await; - let workspace = workspace.unwrap_or("__default__".into()); - let mut hasher = Sha256::new(); - - hasher.update(&blob); - let hash = URL_SAFE_ENGINE.encode(hasher.finalize()); - - if self.insert(&workspace, &hash, &blob).await.is_ok() { - Ok(hash) - } else { - Err(JwstStorageError::WorkspaceNotFound(workspace)) - } - } - - async fn delete_blob(&self, workspace_id: Option, id: String) -> JwstStorageResult { - let _lock = self.bucket.write().await; - let workspace_id = workspace_id.unwrap_or("__default__".into()); - if let Ok(success) = self.delete(&workspace_id, &id).await { - Ok(success) - } else { - Err(JwstStorageError::WorkspaceNotFound(workspace_id)) - } - } - - async fn delete_workspace(&self, workspace_id: String) -> JwstStorageResult<()> { - let _lock = self.bucket.write().await; - if self.drop(&workspace_id).await.is_ok() { - Ok(()) - } else { - Err(JwstStorageError::WorkspaceNotFound(workspace_id)) - } - } - - async fn get_blobs_size(&self, workspace_id: String) -> JwstStorageResult { - let _lock = self.bucket.read().await; - let size = self.get_blobs_size(&workspace_id).await?; - return Ok(size.unwrap_or(0)); - } -} - -#[cfg(test)] -pub async fn blobs_storage_test(pool: &BlobDBStorage) -> anyhow::Result<()> { - // empty table - assert_eq!(pool.count("basic").await?, 0); - - // first insert - pool.insert("basic", "test", &[1, 2, 3, 4]).await?; - assert_eq!(pool.count("basic").await?, 1); - - let all = pool.all("basic").await?; - assert_eq!( - all, - vec![BlobModel { - workspace_id: "basic".into(), - hash: "test".into(), - blob: vec![1, 2, 3, 4], - length: 4, - created_at: all.get(0).unwrap().created_at - }] - ); - assert_eq!(pool.count("basic").await?, 1); - assert_eq!(pool.keys("basic").await?, vec!["test"]); - - pool.drop("basic").await?; - assert_eq!(pool.count("basic").await?, 0); - assert_eq!(pool.keys("basic").await?, Vec::::new()); - - pool.insert("basic", "test1", &[1, 2, 3, 4]).await?; - - let all = pool.all("basic").await?; - assert_eq!( - all, - vec![BlobModel { - workspace_id: "basic".into(), - hash: "test1".into(), - blob: vec![1, 2, 3, 4], - length: 4, - created_at: all.get(0).unwrap().created_at - }] - ); - assert_eq!(pool.count("basic").await?, 1); - assert_eq!(pool.keys("basic").await?, vec!["test1"]); - - let metadata = pool.metadata("basic", "test1").await?; - - assert_eq!(metadata.size, 4); - assert!((metadata.created_at.timestamp() - Utc::now().timestamp()).abs() < 2); - - pool.drop("basic").await?; - - Ok(()) -} diff --git a/libs/jwst-storage/src/storage/blobs/mod.rs b/libs/jwst-storage/src/storage/blobs/mod.rs index 90320bbc8..db57f9767 100644 --- a/libs/jwst-storage/src/storage/blobs/mod.rs +++ b/libs/jwst-storage/src/storage/blobs/mod.rs @@ -1,18 +1,24 @@ -mod bucket_local_db; -mod local_db; +#[cfg(feature = "bucket")] +mod bucket; +#[cfg(feature = "bucket")] +pub use bucket::{BlobBucketStorage, MixedBucketDBParam}; + +#[cfg(feature = "image")] +mod auto_storage; +#[cfg(feature = "image")] +pub use auto_storage::{BlobAutoStorage, ImageError, ImageParams}; + +mod blob_storage; mod utils; -pub use bucket_local_db::BlobBucketDBStorage; -use bytes::Bytes; -use image::ImageError; -use jwst::{BlobMetadata, BlobStorage}; #[cfg(test)] -pub use local_db::blobs_storage_test; -use local_db::BlobDBStorage; +pub use blob_storage::blobs_storage_test; +pub use blob_storage::BlobDBStorage; +use bytes::Bytes; +use jwst_core::{BlobMetadata, BlobStorage}; use thiserror::Error; use tokio::task::JoinError; -pub use utils::BucketStorageBuilder; -use utils::{ImageParams, InternalBlobMetadata}; +use utils::{get_hash, InternalBlobMetadata}; use super::{entities::prelude::*, *}; @@ -22,6 +28,7 @@ pub enum JwstBlobError { BlobNotFound(String), #[error("database error")] Database(#[from] DbErr), + #[cfg(feature = "image")] #[error("failed to optimize image")] Image(#[from] ImageError), #[error("failed to optimize image")] @@ -31,294 +38,39 @@ pub enum JwstBlobError { } pub type JwstBlobResult = Result; -pub(super) type OptimizedBlobModel = ::Model; -type OptimizedBlobActiveModel = super::entities::optimized_blobs::ActiveModel; -type OptimizedBlobColumn = ::Column; - -#[derive(Clone)] -pub struct BlobAutoStorage { - pub(super) db: Arc, - pool: DatabaseConnection, -} - pub enum JwstBlobStorage { - DB(BlobAutoStorage), - MixedBucketDB(BlobBucketDBStorage), + RawStorage(Arc), + #[cfg(feature = "image")] + AutoStorage(BlobAutoStorage), + #[cfg(feature = "bucket")] + BucketStorage(BlobBucketStorage), } pub enum BlobStorageType { DB, + #[cfg(feature = "bucket")] MixedBucketDB(MixedBucketDBParam), } -pub struct MixedBucketDBParam { - pub(crate) access_key: String, - pub(crate) secret_access_key: String, - pub(crate) endpoint: String, - pub(crate) bucket: Option, - pub(crate) root: Option, -} - -impl MixedBucketDBParam { - pub fn new_from_env() -> JwstResult { - Ok(MixedBucketDBParam { - access_key: dotenvy::var("BUCKET_ACCESS_TOKEN")?, - secret_access_key: dotenvy::var("BUCKET_SECRET_TOKEN")?, - endpoint: dotenvy::var("BUCKET_ENDPOINT")?, - bucket: dotenvy::var("BUCKET_NAME").ok(), - root: dotenvy::var("BUCKET_ROOT").ok(), - }) - } - - pub fn new( - access_key: String, - secret_access_key: String, - endpoint: String, - bucket: Option, - root: Option, - ) -> Self { - MixedBucketDBParam { - access_key, - secret_access_key, - endpoint, - bucket, - root, - } - } -} - -impl BlobAutoStorage { - pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { - let db = Arc::new(BlobDBStorage::init_with_pool(pool, bucket).await?); - let pool = db.pool.clone(); - Ok(Self { db, pool }) - } - - pub async fn init_pool(database: &str) -> JwstStorageResult { - let db = Arc::new(BlobDBStorage::init_pool(database).await?); - let pool = db.pool.clone(); - Ok(Self { db, pool }) - } - - async fn exists(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { - Ok(OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) - .count(&self.pool) - .await - .map(|c| c > 0)?) - } - - async fn insert(&self, table: &str, hash: &str, params: &str, blob: &[u8]) -> JwstBlobResult<()> { - if !self.exists(table, hash, params).await? { - OptimizedBlobs::insert(OptimizedBlobActiveModel { - workspace_id: Set(table.into()), - hash: Set(hash.into()), - blob: Set(blob.into()), - length: Set(blob.len().try_into().unwrap()), - params: Set(params.into()), - created_at: Set(Utc::now().into()), - }) - .exec(&self.pool) - .await?; - } - - Ok(()) - } - - async fn get(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { - OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) - .one(&self.pool) - .await - .map_err(|e| e.into()) - .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) - } - - async fn metadata(&self, table: &str, hash: &str, params: &str) -> JwstBlobResult { - OptimizedBlobs::find_by_id((table.into(), hash.into(), params.into())) - .select_only() - .column_as(OptimizedBlobColumn::Length, "size") - .column_as(OptimizedBlobColumn::CreatedAt, "created_at") - .into_model::() - .one(&self.pool) - .await - .map_err(|e| e.into()) - .and_then(|r| r.ok_or(JwstBlobError::BlobNotFound(hash.into()))) - } - - async fn get_metadata_auto( - &self, - workspace: Option, - id: String, - params: Option>, - ) -> JwstBlobResult { - let workspace_id = workspace.as_deref().unwrap_or("__default__"); - if let Some(params) = params { - if let Ok(params) = ImageParams::try_from(¶ms) { - let params_token = params.to_string(); - if self.exists(workspace_id, &id, ¶ms_token).await? { - let metadata = self.metadata(workspace_id, &id, ¶ms_token).await?; - Ok(BlobMetadata { - content_type: format!("image/{}", params.format()), - ..metadata.into() - }) - } else { - self.db.metadata(workspace_id, &id).await.map(Into::into) - } - } else { - Err(JwstBlobError::ImageParams(params)) - } - } else { - self.db.metadata(workspace_id, &id).await.map(Into::into) - } - } - - async fn get_auto( - &self, - workspace: Option, - id: String, - params: Option>, - ) -> JwstBlobResult> { - let workspace_id = workspace.as_deref().unwrap_or("__default__"); - if let Some(params) = params { - if let Ok(params) = ImageParams::try_from(¶ms) { - let params_token = params.to_string(); - if let Ok(blob) = self.get(workspace_id, &id, ¶ms_token).await { - info!( - "exists optimized image: {} {} {}, {}bytes", - workspace_id, - id, - params_token, - blob.blob.len() - ); - Ok(blob.blob) - } else { - // TODO: need ddos mitigation - let blob = self.db.get(workspace_id, &id).await?; - let blob_len = blob.blob.len(); - let image = tokio::task::spawn_blocking(move || params.optimize_image(&blob.blob)).await??; - self.insert(workspace_id, &id, ¶ms_token, &image).await?; - info!( - "optimized image: {} {} {}, {}bytes -> {}bytes", - workspace_id, - id, - params_token, - blob_len, - image.len() - ); - Ok(image) - } - } else { - Err(JwstBlobError::ImageParams(params)) - } - } else { - self.db.get(workspace_id, &id).await.map(|m| m.blob) - } - } - - async fn delete(&self, table: &str, hash: &str) -> JwstBlobResult { - Ok(OptimizedBlobs::delete_many() - .filter( - OptimizedBlobColumn::WorkspaceId - .eq(table) - .and(OptimizedBlobColumn::Hash.eq(hash)), - ) - .exec(&self.pool) - .await - .map(|r| r.rows_affected)?) - } - - async fn drop(&self, table: &str) -> Result<(), DbErr> { - OptimizedBlobs::delete_many() - .filter(OptimizedBlobColumn::WorkspaceId.eq(table)) - .exec(&self.pool) - .await?; - - Ok(()) - } -} - -#[async_trait] -impl BlobStorage for BlobAutoStorage { - async fn list_blobs(&self, workspace: Option) -> JwstStorageResult> { - self.db.list_blobs(workspace).await - } - - async fn check_blob(&self, workspace: Option, id: String) -> JwstStorageResult { - self.db.check_blob(workspace, id).await - } - - async fn get_blob( - &self, - workspace: Option, - id: String, - params: Option>, - ) -> JwstStorageResult> { - let blob = self.get_auto(workspace, id, params).await?; - Ok(blob) - } - - async fn get_metadata( - &self, - workspace: Option, - id: String, - params: Option>, - ) -> JwstStorageResult { - let metadata = self.get_metadata_auto(workspace, id, params).await?; - Ok(metadata) - } - - async fn put_blob_stream( - &self, - workspace: Option, - stream: impl Stream + Send, - ) -> JwstStorageResult { - self.db.put_blob_stream(workspace, stream).await - } - - async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstStorageResult { - self.db.put_blob(workspace, blob).await - } - - async fn delete_blob(&self, workspace_id: Option, id: String) -> JwstStorageResult { - // delete origin blobs - let success = self.db.delete_blob(workspace_id.clone(), id.clone()).await?; - if success { - // delete optimized blobs - let workspace_id = workspace_id.unwrap_or("__default__".into()); - self.delete(&workspace_id, &id).await?; - } - Ok(success) - } - - async fn delete_workspace(&self, workspace_id: String) -> JwstStorageResult<()> { - // delete origin blobs - self.db.delete_workspace(workspace_id.clone()).await?; - - // delete optimized blobs - self.drop(&workspace_id).await?; - - Ok(()) - } - - async fn get_blobs_size(&self, workspace_id: String) -> JwstStorageResult { - let size = self.db.get_blobs_size(&workspace_id).await?; - - return Ok(size.unwrap_or(0)); - } -} - #[async_trait] impl BlobStorage for JwstBlobStorage { async fn list_blobs(&self, workspace: Option) -> JwstResult, JwstStorageError> { match self { - JwstBlobStorage::DB(db) => db.list_blobs(workspace).await, - JwstBlobStorage::MixedBucketDB(db) => db.list_blobs(workspace).await, + JwstBlobStorage::RawStorage(db) => db.list_blobs(workspace).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.list_blobs(workspace).await, + #[cfg(feature = "bucket")] + JwstBlobStorage::BucketStorage(db) => db.list_blobs(workspace).await, } } async fn check_blob(&self, workspace: Option, id: String) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.check_blob(workspace, id).await, - JwstBlobStorage::MixedBucketDB(db) => db.check_blob(workspace, id).await, + JwstBlobStorage::RawStorage(db) => db.check_blob(workspace, id).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.check_blob(workspace, id).await, + #[cfg(feature = "bucket")] + JwstBlobStorage::BucketStorage(db) => db.check_blob(workspace, id).await, } } @@ -329,8 +81,11 @@ impl BlobStorage for JwstBlobStorage { params: Option>, ) -> JwstResult, JwstStorageError> { match self { - JwstBlobStorage::DB(db) => db.get_blob(workspace, id, params).await, - JwstBlobStorage::MixedBucketDB(db) => db.get_blob(workspace, id, params).await, + JwstBlobStorage::RawStorage(db) => db.get_blob(workspace, id, params).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.get_blob(workspace, id, params).await, + #[cfg(feature = "bucket")] + JwstBlobStorage::BucketStorage(db) => db.get_blob(workspace, id, params).await, } } @@ -341,8 +96,11 @@ impl BlobStorage for JwstBlobStorage { params: Option>, ) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.get_metadata(workspace, id, params).await, - JwstBlobStorage::MixedBucketDB(db) => db.get_metadata(workspace, id, params).await, + JwstBlobStorage::RawStorage(db) => db.get_metadata(workspace, id, params).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.get_metadata(workspace, id, params).await, + #[cfg(feature = "bucket")] + JwstBlobStorage::BucketStorage(db) => db.get_metadata(workspace, id, params).await, } } @@ -352,36 +110,51 @@ impl BlobStorage for JwstBlobStorage { stream: impl Stream + Send, ) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.put_blob_stream(workspace, stream).await, - JwstBlobStorage::MixedBucketDB(db) => db.put_blob_stream(workspace, stream).await, + JwstBlobStorage::RawStorage(db) => db.put_blob_stream(workspace, stream).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.put_blob_stream(workspace, stream).await, + #[cfg(feature = "bucket")] + JwstBlobStorage::BucketStorage(db) => db.put_blob_stream(workspace, stream).await, } } async fn put_blob(&self, workspace: Option, blob: Vec) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.put_blob(workspace, blob).await, - JwstBlobStorage::MixedBucketDB(db) => db.put_blob(workspace, blob).await, + JwstBlobStorage::RawStorage(db) => db.put_blob(workspace, blob).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.put_blob(workspace, blob).await, + #[cfg(feature = "bucket")] + JwstBlobStorage::BucketStorage(db) => db.put_blob(workspace, blob).await, } } async fn delete_blob(&self, workspace: Option, id: String) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.delete_blob(workspace, id).await, - JwstBlobStorage::MixedBucketDB(db) => db.delete_blob(workspace, id).await, + JwstBlobStorage::RawStorage(db) => db.delete_blob(workspace, id).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.delete_blob(workspace, id).await, + #[cfg(feature = "bucket")] + JwstBlobStorage::BucketStorage(db) => db.delete_blob(workspace, id).await, } } async fn delete_workspace(&self, workspace_id: String) -> JwstResult<(), JwstStorageError> { match self { - JwstBlobStorage::DB(db) => db.delete_workspace(workspace_id).await, - JwstBlobStorage::MixedBucketDB(db) => db.delete_workspace(workspace_id).await, + JwstBlobStorage::RawStorage(db) => db.delete_workspace(workspace_id).await, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.delete_workspace(workspace_id).await, + #[cfg(feature = "bucket")] + JwstBlobStorage::BucketStorage(db) => db.delete_workspace(workspace_id).await, } } async fn get_blobs_size(&self, workspace_id: String) -> JwstResult { match self { - JwstBlobStorage::DB(db) => db.get_blobs_size(workspace_id).await, - JwstBlobStorage::MixedBucketDB(db) => db.get_blobs_size(workspace_id).await, + JwstBlobStorage::RawStorage(db) => Ok(db.get_blobs_size(&workspace_id).await?.unwrap_or(0)), + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => db.get_blobs_size(workspace_id).await, + #[cfg(feature = "bucket")] + JwstBlobStorage::BucketStorage(db) => db.get_blobs_size(workspace_id).await, } } } @@ -389,199 +162,21 @@ impl BlobStorage for JwstBlobStorage { impl JwstBlobStorage { pub fn get_blob_db(&self) -> Option> { match self { - JwstBlobStorage::DB(db) => Some(db.db.clone()), - JwstBlobStorage::MixedBucketDB(_) => None, + JwstBlobStorage::RawStorage(db) => Some(db.clone()), + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(db) => Some(db.db.clone()), + #[cfg(feature = "bucket")] + JwstBlobStorage::BucketStorage(_) => None, } } - pub fn get_mixed_bucket_db(&self) -> Option { + #[cfg(feature = "bucket")] + pub fn get_mixed_bucket_db(&self) -> Option { match self { - JwstBlobStorage::DB(_) => None, - JwstBlobStorage::MixedBucketDB(db) => Some(db.clone()), + JwstBlobStorage::RawStorage(_) => None, + #[cfg(feature = "image")] + JwstBlobStorage::AutoStorage(_) => None, + JwstBlobStorage::BucketStorage(db) => Some(db.clone()), } } } - -#[cfg(test)] -mod tests { - use std::io::Cursor; - - use futures::FutureExt; - use image::{DynamicImage, ImageOutputFormat}; - - use super::*; - - #[tokio::test] - async fn test_blob_auto_storage() { - let storage = BlobAutoStorage::init_pool("sqlite::memory:").await.unwrap(); - Migrator::up(&storage.pool, None).await.unwrap(); - - let blob = Vec::from_iter((0..100).map(|_| rand::random())); - - let stream = async { Bytes::from(blob.clone()) }.into_stream(); - let hash1 = storage.put_blob_stream(Some("blob".into()), stream).await.unwrap(); - - // check origin blob result - assert_eq!( - storage - .get_blob(Some("blob".into()), hash1.clone(), None) - .await - .unwrap(), - blob - ); - assert_eq!( - storage - .get_metadata(Some("blob".into()), hash1.clone(), None) - .await - .unwrap() - .size as usize, - blob.len() - ); - - // optimize must failed if blob not supported - assert!(storage - .get_blob( - Some("blob".into()), - hash1.clone(), - Some(HashMap::from([("format".into(), "jpeg".into())])) - ) - .await - .is_err()); - - // generate image - let image = { - let mut image = Cursor::new(vec![]); - DynamicImage::new_rgba8(32, 32) - .write_to(&mut image, ImageOutputFormat::Png) - .unwrap(); - image.into_inner() - }; - let stream = async { Bytes::from(image.clone()) }.into_stream(); - let hash2 = storage.put_blob_stream(Some("blob".into()), stream).await.unwrap(); - - // check origin blob result - assert_eq!( - storage - .get_blob(Some("blob".into()), hash2.clone(), None) - .await - .unwrap(), - image - ); - assert_eq!( - storage - .get_metadata(Some("blob".into()), hash2.clone(), None) - .await - .unwrap() - .size as usize, - image.len() - ); - - // check optimized jpeg result - let jpeg_params = HashMap::from([("format".into(), "jpeg".into())]); - let jpeg = storage - .get_blob(Some("blob".into()), hash2.clone(), Some(jpeg_params.clone())) - .await - .unwrap(); - - assert!(jpeg.starts_with(&[0xff, 0xd8, 0xff])); - assert_eq!( - storage - .get_metadata(Some("blob".into()), hash2.clone(), Some(jpeg_params)) - .await - .unwrap() - .size as usize, - jpeg.len() - ); - - // check optimized webp result - let webp_params = HashMap::from([("format".into(), "webp".into())]); - let webp = storage - .get_blob(Some("blob".into()), hash2.clone(), Some(webp_params.clone())) - .await - .unwrap(); - - assert!(webp.starts_with(b"RIFF")); - assert_eq!( - storage - .get_metadata(Some("blob".into()), hash2.clone(), Some(webp_params.clone())) - .await - .unwrap() - .size as usize, - webp.len() - ); - - // optimize must failed if image params error - assert!(storage - .get_blob( - Some("blob".into()), - hash2.clone(), - Some(HashMap::from([("format".into(), "error_value".into()),])) - ) - .await - .is_err()); - assert!(storage - .get_blob( - Some("blob".into()), - hash2.clone(), - Some(HashMap::from([ - ("format".into(), "webp".into()), - ("size".into(), "error_value".into()) - ])) - ) - .await - .is_err()); - assert!(storage - .get_blob( - Some("blob".into()), - hash2.clone(), - Some(HashMap::from([ - ("format".into(), "webp".into()), - ("width".into(), "111".into()) - ])) - ) - .await - .is_err()); - assert!(storage - .get_blob( - Some("blob".into()), - hash2.clone(), - Some(HashMap::from([ - ("format".into(), "webp".into()), - ("height".into(), "111".into()) - ])) - ) - .await - .is_err()); - - assert_eq!( - storage.get_blobs_size("blob".into()).await.unwrap() as usize, - 100 + image.len() - ); - - assert!(storage.delete_blob(Some("blob".into()), hash2.clone()).await.unwrap()); - assert_eq!( - storage.check_blob(Some("blob".into()), hash2.clone()).await.unwrap(), - false - ); - assert!(storage - .get_blob(Some("blob".into()), hash2.clone(), None) - .await - .is_err()); - assert!(storage - .get_metadata(Some("blob".into()), hash2.clone(), None) - .await - .is_err()); - assert!(storage - .get_metadata(Some("blob".into()), hash2.clone(), Some(webp_params)) - .await - .is_err()); - - assert_eq!(storage.get_blobs_size("blob".into()).await.unwrap() as usize, 100); - - assert_eq!(storage.list_blobs(Some("blob".into())).await.unwrap(), vec![hash1]); - assert_eq!( - storage.list_blobs(Some("not_exists_workspace".into())).await.unwrap(), - Vec::::new() - ); - } -} diff --git a/libs/jwst-storage/src/storage/blobs/utils.rs b/libs/jwst-storage/src/storage/blobs/utils.rs index f47caab58..b129fb3af 100644 --- a/libs/jwst-storage/src/storage/blobs/utils.rs +++ b/libs/jwst-storage/src/storage/blobs/utils.rs @@ -1,117 +1,13 @@ -use std::{collections::HashMap, io::Cursor}; - use bytes::Bytes; use chrono::{DateTime, Utc}; use futures::{ stream::{iter, StreamExt}, Stream, }; -use image::{load_from_memory, ImageOutputFormat, ImageResult}; -use jwst::{Base64Engine, BlobMetadata, URL_SAFE_ENGINE}; -use opendal::{services::S3, Operator}; +use jwst_core::{Base64Engine, BlobMetadata, URL_SAFE_ENGINE}; use sea_orm::FromQueryResult; use sha2::{Digest, Sha256}; -use crate::{ - storage::blobs::{bucket_local_db::BucketStorage, MixedBucketDBParam}, - JwstStorageError, JwstStorageResult, -}; - -enum ImageFormat { - Jpeg, - WebP, -} - -pub struct ImageParams { - format: ImageFormat, - width: Option, - height: Option, -} - -impl ImageParams { - #[inline] - fn check_size(w: Option, h: Option) -> bool { - if let Some(w) = w { - if w % 320 != 0 || w > 1920 { - return false; - } - } - if let Some(h) = h { - if h % 180 != 0 || h > 1080 { - return false; - } - } - true - } - - pub(super) fn format(&self) -> String { - match self.format { - ImageFormat::Jpeg => "jpeg".to_string(), - ImageFormat::WebP => "webp".to_string(), - } - } - - fn output_format(&self) -> ImageOutputFormat { - match self.format { - ImageFormat::Jpeg => ImageOutputFormat::Jpeg(80), - ImageFormat::WebP => ImageOutputFormat::WebP, - } - } - - pub fn optimize_image(&self, data: &[u8]) -> ImageResult> { - let mut buffer = Cursor::new(vec![]); - let image = load_from_memory(data)?; - image.write_to(&mut buffer, self.output_format())?; - Ok(buffer.into_inner()) - } -} - -impl TryFrom<&HashMap> for ImageParams { - type Error = (); - - fn try_from(value: &HashMap) -> Result { - let mut format = None; - let mut width = None; - let mut height = None; - for (key, value) in value { - match key.as_str() { - "format" => { - format = match value.as_str() { - "jpeg" => Some(ImageFormat::Jpeg), - "webp" => Some(ImageFormat::WebP), - _ => return Err(()), - } - } - "width" => width = value.parse().ok(), - "height" => height = value.parse().ok(), - _ => return Err(()), - } - } - - if let Some(format) = format { - if Self::check_size(width, height) { - return Ok(Self { format, width, height }); - } - } - Err(()) - } -} - -impl ToString for ImageParams { - fn to_string(&self) -> String { - let mut params = String::new(); - - params.push_str(&format!("format={}", self.format())); - if let Some(width) = &self.width { - params.push_str(&format!("width={}", width)); - } - if let Some(height) = &self.height { - params.push_str(&format!("height={}", height)); - } - params - } -} - pub async fn get_hash(stream: impl Stream + Send) -> (String, Vec) { let mut hasher = Sha256::new(); @@ -127,13 +23,6 @@ pub async fn get_hash(stream: impl Stream + Send) -> (String, Vec< (hash, buffer) } -/// Calculate sha256 hash for given blob -pub fn calculate_hash(blob: &[u8]) -> String { - let mut hasher = Sha256::new(); - hasher.update(blob); - URL_SAFE_ENGINE.encode(hasher.finalize()) -} - #[derive(FromQueryResult)] pub(super) struct InternalBlobMetadata { pub(super) size: i64, @@ -149,102 +38,3 @@ impl From for BlobMetadata { } } } - -impl TryFrom> for BucketStorage { - type Error = JwstStorageError; - - fn try_from(map: HashMap) -> Result { - let mut builder = BucketStorageBuilder::new(); - let access_token = map.get("BUCKET_ACCESS_TOKEN"); - let secret_access_key = map.get("BUCKET_SECRET_TOKEN"); - let endpoint = map.get("BUCKET_ENDPOINT"); - let bucket = map.get("BUCKET_NAME"); - let root = map.get("BUCKET_ROOT"); - - if let Some(access_token) = access_token { - builder = builder.access_key(access_token); - } - if let Some(secret_access_key) = secret_access_key { - builder = builder.secret_access_key(secret_access_key); - } - if let Some(endpoint) = endpoint { - builder = builder.endpoint(endpoint); - } - if let Some(bucket) = bucket { - builder = builder.bucket(bucket); - } - if let Some(root) = root { - builder = builder.root(root); - } - - builder.build() - } -} - -impl TryFrom for BucketStorage { - type Error = JwstStorageError; - - fn try_from(value: MixedBucketDBParam) -> Result { - let mut builder = BucketStorageBuilder::new(); - builder = builder.access_key(&value.access_key); - builder = builder.secret_access_key(&value.secret_access_key); - builder = builder.endpoint(&value.endpoint); - builder = builder.bucket(&value.bucket.unwrap_or("__default_bucket__".to_string())); - builder = builder.root(&value.root.unwrap_or("__default_root__".to_string())); - builder.build() - } -} - -#[derive(Default)] -pub struct BucketStorageBuilder { - access_key: String, - secret_access_key: String, - endpoint: String, - bucket: String, - root: String, -} - -impl BucketStorageBuilder { - pub fn new() -> Self { - Self::default() - } - - pub fn access_key(mut self, access_key: &str) -> Self { - self.access_key = access_key.to_string(); - self - } - - pub fn secret_access_key(mut self, secret_access_key: &str) -> Self { - self.secret_access_key = secret_access_key.to_string(); - self - } - - pub fn endpoint(mut self, endpoint: &str) -> Self { - self.endpoint = endpoint.to_string(); - self - } - - pub fn bucket(mut self, bucket: &str) -> Self { - self.bucket = bucket.to_string(); - self - } - - pub fn root(mut self, root: &str) -> Self { - self.root = root.to_string(); - self - } - - pub fn build(self) -> JwstStorageResult { - let mut builder = S3::default(); - - builder.bucket(self.bucket.as_str()); - builder.root(self.root.as_str()); - builder.endpoint(self.endpoint.as_str()); - builder.access_key_id(self.access_key.as_str()); - builder.secret_access_key(self.secret_access_key.as_str()); - - Ok(BucketStorage { - op: Operator::new(builder)?.finish(), - }) - } -} diff --git a/libs/jwst-storage/src/storage/difflog.rs b/libs/jwst-storage/src/storage/difflog.rs deleted file mode 100644 index 4c7ec4bfd..000000000 --- a/libs/jwst-storage/src/storage/difflog.rs +++ /dev/null @@ -1,43 +0,0 @@ -use super::{entities::prelude::*, *}; -use crate::types::JwstStorageResult; - -// type DiffLogModel = ::Model; -type DiffLogActiveModel = super::entities::diff_log::ActiveModel; -// type DiffLogColumn = ::Column; - -pub struct DiffLogRecord { - bucket: Arc, - pool: DatabaseConnection, -} - -impl DiffLogRecord { - pub async fn init_with_pool(pool: DatabaseConnection, bucket: Arc) -> JwstStorageResult { - Ok(Self { bucket, pool }) - } - - pub async fn init_pool(database: &str) -> JwstStorageResult { - let is_sqlite = is_sqlite(database); - let pool = create_connection(database, is_sqlite).await?; - - Self::init_with_pool(pool, get_bucket(is_sqlite)).await - } - - pub async fn insert(&self, workspace: String, ts: DateTime, log: String) -> JwstStorageResult<()> { - let _lock = self.bucket.write().await; - DiffLog::insert(DiffLogActiveModel { - workspace: Set(workspace), - timestamp: Set(ts.into()), - log: Set(log), - ..Default::default() - }) - .exec(&self.pool) - .await?; - Ok(()) - } - - pub async fn count(&self) -> JwstStorageResult { - let _lock = self.bucket.read().await; - let count = DiffLog::find().count(&self.pool).await?; - Ok(count) - } -} diff --git a/libs/jwst-storage/src/storage/docs/database.rs b/libs/jwst-storage/src/storage/docs/database.rs index 1122f28e8..bb93e8dba 100644 --- a/libs/jwst-storage/src/storage/docs/database.rs +++ b/libs/jwst-storage/src/storage/docs/database.rs @@ -1,9 +1,8 @@ use std::collections::hash_map::Entry; -use jwst::{sync_encode_update, DocStorage, Workspace}; -use jwst_codec::{CrdtReader, RawDecoder}; +use jwst_codec::{encode_update_as_message, CrdtReader, Doc, DocOptions, RawDecoder, StateVector}; +use jwst_core::{DocStorage, Workspace}; use sea_orm::Condition; -use yrs::{Doc, Options, ReadTxn, StateVector, Transact}; use super::{entities::prelude::*, *}; use crate::types::JwstStorageResult; @@ -206,7 +205,7 @@ impl DocDBStorage { trace!("update {}bytes to {}", blob.len(), guid); if let Entry::Occupied(remote) = self.remote.write().await.entry(guid.into()) { let broadcast = &remote.get(); - if broadcast.send(sync_encode_update(&blob)).is_err() { + if broadcast.send(encode_update_as_message(blob)?).is_err() { // broadcast failures are not fatal errors, only warnings are required warn!("send {guid} update to pipeline failed"); } @@ -238,24 +237,35 @@ impl DocDBStorage { .await?; let ws = if all_data.is_empty() { + trace!("create workspace: {workspace}"); // keep workspace root doc's guid the same as workspaceId - let doc = Doc::with_options(Options { - guid: yrs::Uuid::from(workspace), + let doc = Doc::with_options(DocOptions { + guid: Some(workspace.into()), ..Default::default() }); - let ws = Workspace::from_doc(doc.clone(), workspace); + let ws = Workspace::from_doc(doc.clone(), workspace)?; - let update = doc.transact().encode_state_as_update_v1(&StateVector::default())?; + let update = doc.encode_state_as_update_v1(&StateVector::default())?; Self::insert(conn, workspace, doc.guid(), &update).await?; ws } else { - let doc = Doc::with_options(Options { + trace!("migrate workspace: {workspace}"); + let doc = Doc::with_options(DocOptions { guid: all_data.first().unwrap().guid.clone().into(), ..Default::default() }); + + let can_merge = all_data.len() > 1; + let doc = utils::migrate_update(all_data, doc)?; - Workspace::from_doc(doc, workspace) + + if can_merge { + let update = doc.encode_state_as_update_v1(&StateVector::default())?; + Self::replace_with(conn, workspace, doc.guid(), update).await?; + } + + Workspace::from_doc(doc, workspace)? }; trace!("end create doc in workspace: {workspace}"); @@ -330,7 +340,7 @@ impl DocStorage for DocDBStorage { return Ok(None); } - let doc = utils::migrate_update(records, Doc::new())?; + let doc = utils::migrate_update(records, Doc::default())?; Ok(Some(doc)) } @@ -507,45 +517,49 @@ pub async fn docs_storage_partial_test(pool: &DocDBStorage) -> anyhow::Result<() assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 0); { - let ws = pool.get_or_create_workspace("basic".into()).await.unwrap(); + let mut ws = pool.get_or_create_workspace("basic".into()).await.unwrap(); let guid = ws.doc_guid().to_string(); let (tx, mut rx) = channel(100); - let sub = ws - .doc() - .observe_update_v1(move |_, e| { - futures::executor::block_on(async { - tx.send(e.update.clone()).await.unwrap(); - }); - }) - .unwrap(); - - ws.with_trx(|mut t| { - let space = t.get_space("test"); - let block = space.create(&mut t.trx, "block1", "text").unwrap(); - block.set(&mut t.trx, "test1", "value1").unwrap(); + ws.doc().subscribe(move |update| { + futures::executor::block_on(async { + tx.send(update.to_vec()).await.unwrap(); + }); }); - ws.with_trx(|mut t| { - let space = t.get_space("test"); - let block = space.get(&mut t.trx, "block1").unwrap(); - block.set(&mut t.trx, "test2", "value2").unwrap(); - }); + { + let mut space = ws.get_space("test").unwrap(); + let mut block = space.create("block1", "text").unwrap(); + block.set("test1", "value1").unwrap(); + } - ws.with_trx(|mut t| { - let space = t.get_space("test"); - let block = space.create(&mut t.trx, "block2", "block2").unwrap(); - block.set(&mut t.trx, "test3", "value3").unwrap(); - }); + { + let space = ws.get_space("test").unwrap(); + let mut block = space.get("block1").unwrap(); + block.set("test2", "value2").unwrap(); + } - drop(sub); + { + let mut space = ws.get_space("test").unwrap(); + let mut block = space.create("block2", "block2").unwrap(); + block.set("test3", "value3").unwrap(); + } - while let Some(update) = rx.recv().await { - info!("recv: {}", update.len()); - pool.update_doc("basic".into(), guid.clone(), &update).await.unwrap(); + loop { + tokio::select! { + Some(update) = rx.recv() => { + info!("recv: {}", update.len()); + pool.update_doc("basic".into(), guid.clone(), &update) + .await + .unwrap(); + } + _ = tokio::time::sleep(std::time::Duration::from_millis(5000)) => { + break; + } + } } - assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 4); + assert_eq!(DocDBStorage::workspace_count(conn, "basic").await?, 2); } // clear memory cache @@ -553,19 +567,17 @@ pub async fn docs_storage_partial_test(pool: &DocDBStorage) -> anyhow::Result<() { // memory cache empty, retrieve data from db - let ws = pool.get_or_create_workspace("basic".into()).await.unwrap(); - ws.with_trx(|mut t| { - let space = t.get_space("test"); - - let block = space.get(&mut t.trx, "block1").unwrap(); - assert_eq!(block.flavour(&t.trx), "text"); - assert_eq!(block.get(&t.trx, "test1"), Some("value1".into())); - assert_eq!(block.get(&t.trx, "test2"), Some("value2".into())); - - let block = space.get(&mut t.trx, "block2").unwrap(); - assert_eq!(block.flavour(&t.trx), "block2"); - assert_eq!(block.get(&t.trx, "test3"), Some("value3".into())); - }); + let mut ws = pool.get_or_create_workspace("basic".into()).await.unwrap(); + let space = ws.get_space("test").unwrap(); + + let block = space.get("block1").unwrap(); + assert_eq!(block.flavour(), "text"); + assert_eq!(block.get("test1"), Some("value1".into())); + assert_eq!(block.get("test2"), Some("value2".into())); + + let block = space.get("block2").unwrap(); + assert_eq!(block.flavour(), "block2"); + assert_eq!(block.get("test3"), Some("value3".into())); } Ok(()) diff --git a/libs/jwst-storage/src/storage/docs/utils.rs b/libs/jwst-storage/src/storage/docs/utils.rs index 4a8148be5..3a05346fe 100644 --- a/libs/jwst-storage/src/storage/docs/utils.rs +++ b/libs/jwst-storage/src/storage/docs/utils.rs @@ -1,38 +1,25 @@ -use std::panic::{catch_unwind, AssertUnwindSafe}; - -use yrs::{updates::decoder::Decode, Doc, ReadTxn, StateVector, Transact, Update}; +use jwst_codec::{Doc, StateVector}; use super::{entities::prelude::*, types::JwstStorageResult, *}; // apply all updates to the given doc -pub fn migrate_update(update_records: Vec<::Model>, doc: Doc) -> JwstResult { - { - let mut trx = doc.transact_mut(); - for record in update_records { - let id = record.created_at; - match Update::decode_v1(&record.blob) { - Ok(update) => { - if let Err(e) = catch_unwind(AssertUnwindSafe(|| trx.apply_update(update))) { - warn!("update {} merge failed, skip it: {:?}", id, e); - } - } - Err(err) => warn!("failed to decode update: {:?}", err), - } +pub fn migrate_update(update_records: Vec<::Model>, mut doc: Doc) -> JwstResult { + // stop update dispatch before apply updates + doc.publisher.stop(); + for record in update_records { + let id = record.created_at; + if let Err(e) = doc.apply_update_from_binary(record.blob) { + warn!("update {} merge failed, skip it: {:?}", id, e); } - trx.commit(); } - - trace!( - "migrate_update: {:?}", - doc.transact().encode_state_as_update_v1(&StateVector::default())? - ); + doc.publisher.start(); Ok(doc) } pub fn merge_doc_records(update_records: Vec<::Model>) -> JwstStorageResult> { - let state_vector = migrate_update(update_records, Doc::default()) - .and_then(|doc| Ok(doc.transact().encode_state_as_update_v1(&StateVector::default())?))?; + let doc = migrate_update(update_records, Doc::default())?; + let state_vector = doc.encode_state_as_update_v1(&StateVector::default())?; Ok(state_vector) } diff --git a/libs/jwst-storage/src/storage/mod.rs b/libs/jwst-storage/src/storage/mod.rs index 8bc6fdb0f..99a1c09f1 100644 --- a/libs/jwst-storage/src/storage/mod.rs +++ b/libs/jwst-storage/src/storage/mod.rs @@ -1,27 +1,24 @@ pub(crate) mod blobs; -mod difflog; mod docs; mod test; use std::{collections::HashMap, time::Instant}; +#[cfg(feature = "image")] use blobs::BlobAutoStorage; +#[cfg(feature = "bucket")] +use blobs::BlobBucketStorage; +use blobs::{BlobDBStorage, BlobStorageType, JwstBlobStorage}; use docs::SharedDocDBStorage; use jwst_storage_migration::{Migrator, MigratorTrait}; use tokio::sync::Mutex; -use self::difflog::DiffLogRecord; use super::*; -use crate::{ - storage::blobs::{BlobBucketDBStorage, BlobStorageType, JwstBlobStorage}, - types::JwstStorageError, -}; pub struct JwstStorage { pool: DatabaseConnection, blobs: JwstBlobStorage, docs: SharedDocDBStorage, - difflog: DiffLogRecord, last_migrate: Mutex>, } @@ -37,20 +34,27 @@ impl JwstStorage { let blobs = match blob_storage_type { BlobStorageType::DB => { - JwstBlobStorage::DB(BlobAutoStorage::init_with_pool(pool.clone(), bucket.clone()).await?) + #[cfg(feature = "image")] + { + JwstBlobStorage::AutoStorage(BlobAutoStorage::init_with_pool(pool.clone(), bucket.clone()).await?) + } + #[cfg(not(feature = "image"))] + { + let db = BlobDBStorage::init_with_pool(pool.clone(), bucket.clone()).await?; + JwstBlobStorage::RawStorage(Arc::new(db)) + } } - BlobStorageType::MixedBucketDB(param) => JwstBlobStorage::MixedBucketDB( - BlobBucketDBStorage::init_with_pool(pool.clone(), bucket.clone(), Some(param.try_into()?)).await?, + #[cfg(feature = "bucket")] + BlobStorageType::MixedBucketDB(param) => JwstBlobStorage::BucketStorage( + BlobBucketStorage::init_with_pool(pool.clone(), bucket.clone(), Some(param.try_into()?)).await?, ), }; let docs = SharedDocDBStorage::init_with_pool(pool.clone(), bucket.clone()).await?; - let difflog = DiffLogRecord::init_with_pool(pool.clone(), bucket).await?; Ok(Self { pool, blobs, docs, - difflog, last_migrate: Mutex::new(HashMap::new()), }) } @@ -98,10 +102,6 @@ impl JwstStorage { &self.docs } - pub fn difflog(&self) -> &DiffLogRecord { - &self.difflog - } - pub async fn with_pool(&self, func: F) -> JwstStorageResult where F: Fn(DatabaseConnection) -> Fut, @@ -189,7 +189,6 @@ impl JwstStorage { #[cfg(test)] mod tests { use super::*; - use crate::storage::blobs::MixedBucketDBParam; #[tokio::test] async fn test_sqlite_storage() { @@ -200,6 +199,7 @@ mod tests { } #[tokio::test] + #[cfg(feature = "bucket")] #[ignore = "need to config bucket auth"] async fn test_bucket_storage() { let bucket_params = MixedBucketDBParam { diff --git a/libs/jwst-storage/src/storage/test.rs b/libs/jwst-storage/src/storage/test.rs index 03c8dddf9..33b6aab36 100644 --- a/libs/jwst-storage/src/storage/test.rs +++ b/libs/jwst-storage/src/storage/test.rs @@ -9,8 +9,11 @@ use super::{ async fn sqlite_storage_test() -> anyhow::Result<()> { let storage = JwstStorage::new_with_migration("sqlite::memory:", BlobStorageType::DB).await?; + info!("blobs_storage_test"); blobs_storage_test(&storage.blobs().get_blob_db().unwrap()).await?; + info!("docs_storage_test"); docs_storage_test(&storage.docs().0).await?; + info!("docs_storage_partial_test"); docs_storage_partial_test(&storage.docs().0).await?; Ok(()) diff --git a/libs/jwst-storage/src/types.rs b/libs/jwst-storage/src/types.rs index 1792e055d..2b2259e9a 100644 --- a/libs/jwst-storage/src/types.rs +++ b/libs/jwst-storage/src/types.rs @@ -9,8 +9,6 @@ pub enum JwstStorageError { Crud(String), #[error("db error")] Db(#[from] sea_orm::DbErr), - #[error("failed to process doc")] - DocCodec(#[from] lib0::error::Error), #[error("doc codec error")] DocJwstCodec(#[from] JwstCodecError), #[error("merge thread panic")] @@ -18,11 +16,13 @@ pub enum JwstStorageError { #[error("workspace {0} not found")] WorkspaceNotFound(String), #[error("jwst error")] - Jwst(#[from] jwst::JwstError), + Jwst(#[from] jwst_core::JwstError), #[error("failed to process blob")] JwstBlob(#[from] crate::storage::blobs::JwstBlobError), + #[cfg(feature = "bucket")] #[error("bucket error")] JwstBucket(#[from] opendal::Error), + #[cfg(feature = "bucket")] #[error("env variables read error")] DotEnvy(#[from] dotenvy::Error), } From e0e89405711f3e0f390493fa7416f37b22f2df48 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 28 Aug 2023 17:28:34 +0800 Subject: [PATCH 31/49] fix: data storage in offline env --- apps/keck/src/server/api/blocks/mod.rs | 10 +--------- apps/keck/src/server/mod.rs | 11 +---------- .../RustXcframework.xcframework/Info.plist | 14 +++++++------- libs/jwst-rpc/src/client/mod.rs | 1 - libs/jwst-rpc/src/client/websocket.rs | 7 ++++++- 5 files changed, 15 insertions(+), 28 deletions(-) diff --git a/apps/keck/src/server/api/blocks/mod.rs b/apps/keck/src/server/api/blocks/mod.rs index 8941268e2..7e0b8ace8 100644 --- a/apps/keck/src/server/api/blocks/mod.rs +++ b/apps/keck/src/server/api/blocks/mod.rs @@ -75,14 +75,7 @@ mod tests { #[tokio::test] async fn test_workspace_apis() { let client = Arc::new(reqwest::Client::builder().no_proxy().build().unwrap()); - let runtime = Arc::new( - runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_time() - .enable_io() - .build() - .expect("Failed to create runtime"), - ); + let hook_endpoint = Arc::new(RwLock::new(String::new())); let ctx = Arc::new( @@ -97,7 +90,6 @@ mod tests { workspace_apis(Router::new()) .layer(Extension(ctx.clone())) .layer(Extension(client.clone())) - .layer(Extension(runtime.clone())) .layer(Extension(hook_endpoint.clone())), ); diff --git a/apps/keck/src/server/mod.rs b/apps/keck/src/server/mod.rs index f11540ae5..432e6331b 100644 --- a/apps/keck/src/server/mod.rs +++ b/apps/keck/src/server/mod.rs @@ -7,7 +7,7 @@ use std::{net::SocketAddr, sync::Arc}; use api::Context; use axum::{http::Method, Extension, Router, Server}; use jwst_core::Workspace; -use tokio::{runtime, signal, sync::RwLock}; +use tokio::{signal, sync::RwLock}; use tower_http::cors::{Any, CorsLayer}; pub use utils::*; @@ -53,14 +53,6 @@ pub async fn start_server() { .allow_headers(Any); let client = Arc::new(reqwest::Client::builder().no_proxy().build().unwrap()); - let runtime = Arc::new( - runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_time() - .enable_io() - .build() - .expect("Failed to create runtime"), - ); let hook_endpoint = Arc::new(RwLock::new(dotenvy::var("HOOK_ENDPOINT").unwrap_or_default())); let context = Arc::new(Context::new(None).await); @@ -69,7 +61,6 @@ pub async fn start_server() { .layer(cors) .layer(Extension(context.clone())) .layer(Extension(client)) - .layer(Extension(runtime)) .layer(Extension(hook_endpoint)); let addr = SocketAddr::from(( diff --git a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist index b3d1aee06..067a2b752 100644 --- a/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist +++ b/apps/swift/OctoBaseSwift/RustXcframework.xcframework/Info.plist @@ -8,7 +8,7 @@ HeadersPath Headers LibraryIdentifier - ios-arm64-simulator + macos-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -16,15 +16,13 @@ arm64 SupportedPlatform - ios - SupportedPlatformVariant - simulator + macos HeadersPath Headers LibraryIdentifier - macos-arm64 + ios-arm64 LibraryPath liboctobase.a SupportedArchitectures @@ -32,13 +30,13 @@ arm64 SupportedPlatform - macos + ios HeadersPath Headers LibraryIdentifier - ios-arm64 + ios-arm64-simulator LibraryPath liboctobase.a SupportedArchitectures @@ -47,6 +45,8 @@ SupportedPlatform ios + SupportedPlatformVariant + simulator CFBundlePackageType diff --git a/libs/jwst-rpc/src/client/mod.rs b/libs/jwst-rpc/src/client/mod.rs index fb40f4f47..d27a1978e 100644 --- a/libs/jwst-rpc/src/client/mod.rs +++ b/libs/jwst-rpc/src/client/mod.rs @@ -5,7 +5,6 @@ mod websocket; use std::sync::Mutex; -use chrono::Utc; use tokio::{runtime::Runtime, task::JoinHandle}; #[cfg(feature = "webrtc")] pub use webrtc::start_webrtc_client_sync; diff --git a/libs/jwst-rpc/src/client/websocket.rs b/libs/jwst-rpc/src/client/websocket.rs index f7f9a7305..8b14aea2a 100644 --- a/libs/jwst-rpc/src/client/websocket.rs +++ b/libs/jwst-rpc/src/client/websocket.rs @@ -1,5 +1,6 @@ use std::sync::RwLock; +use chrono::Utc; use nanoid::nanoid; use tokio::{net::TcpStream, runtime::Runtime, sync::mpsc::channel}; use tokio_tungstenite::{ @@ -38,7 +39,7 @@ pub fn start_websocket_client_sync( let runtime = rt.clone(); runtime.spawn(async move { debug!("start sync thread"); - let workspace = match context.get_workspace(&workspace_id).await { + let mut workspace = match context.get_workspace(&workspace_id).await { Ok(workspace) => workspace, Err(e) => { warn!("failed to create workspace: {:?}", e); @@ -47,6 +48,10 @@ pub fn start_websocket_client_sync( }; if !workspace.is_empty() { info!("Workspace not empty, starting async remote connection"); + let identifier = nanoid!(); + context + .join_broadcast(&mut workspace, identifier, last_synced_tx.clone()) + .await; last_synced_tx.send(Utc::now().timestamp_millis()).await.unwrap(); } else { info!("Workspace empty, starting sync remote connection"); From 3645a701981075941d1e18a6eb68e312a8933148 Mon Sep 17 00:00:00 2001 From: DarkSky Date: Mon, 28 Aug 2023 19:07:15 +0800 Subject: [PATCH 32/49] feat: move jni binding to jwst --- Cargo.lock | 191 ++++++++++++- Cargo.toml | 5 +- apps/android/.idea/codeStyles/Project.xml | 1 + apps/android/.idea/misc.xml | 1 - .../com/example/jwst_demo/MainActivity.kt | 114 ++++---- libs/jwst-binding/jwst-jni/Cargo.toml | 10 +- .../jwst-jni/android/build.gradle | 2 +- .../main/java/com/toeverything/jwst/JWST.kt | 164 +++++------ .../java/com/toeverything/jwst/lib/Block.java | 229 ++++++--------- .../toeverything/jwst/lib/BlockObserver.java | 10 - .../jwst/lib/OnWorkspaceTransaction.java | 10 - .../com/toeverything/jwst/lib/Workspace.java | 55 ++-- .../jwst/lib/WorkspaceTransaction.java | 53 ---- libs/jwst-binding/jwst-jni/build.rs | 23 -- libs/jwst-binding/jwst-jni/src/block.rs | 263 +++++------------- .../jwst-jni/src/block_observer.rs | 22 -- libs/jwst-binding/jwst-jni/src/difflog.rs | 64 ----- .../jwst-binding/jwst-jni/src/java_glue.rs.in | 82 ++---- libs/jwst-binding/jwst-jni/src/lib.rs | 11 +- libs/jwst-binding/jwst-jni/src/storage.rs | 53 +--- libs/jwst-binding/jwst-jni/src/transaction.rs | 54 ---- libs/jwst-binding/jwst-jni/src/workspace.rs | 128 +++------ libs/jwst-storage/src/types.rs | 3 + 23 files changed, 580 insertions(+), 968 deletions(-) delete mode 100644 libs/jwst-binding/jwst-jni/android/src/main/java/com/toeverything/jwst/lib/BlockObserver.java delete mode 100644 libs/jwst-binding/jwst-jni/android/src/main/java/com/toeverything/jwst/lib/OnWorkspaceTransaction.java delete mode 100644 libs/jwst-binding/jwst-jni/android/src/main/java/com/toeverything/jwst/lib/WorkspaceTransaction.java delete mode 100644 libs/jwst-binding/jwst-jni/src/block_observer.rs delete mode 100644 libs/jwst-binding/jwst-jni/src/difflog.rs delete mode 100644 libs/jwst-binding/jwst-jni/src/transaction.rs diff --git a/Cargo.lock b/Cargo.lock index d6ac30f46..9da6438a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,6 +174,24 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" +[[package]] +name = "android_log-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ecc8056bf6ab9892dcd53216c83d1597487d7dacac16c8df6b877d127df9937" + +[[package]] +name = "android_logger" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c494134f746c14dc653a35a4ea5aca24ac368529da5370ecf41fe0341c35772f" +dependencies = [ + "android_log-sys", + "env_logger", + "log", + "once_cell", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -1312,6 +1330,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive-new" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "derive_arbitrary" version = "1.3.1" @@ -1520,6 +1549,16 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "env_logger" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +dependencies = [ + "log", + "regex", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -1651,12 +1690,38 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flagset" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda653ca797810c02f7ca4b804b40b8b95ae046eb989d356bce17919a8c25499" +[[package]] +name = "flapigen" +version = "0.6.0-pre13" +source = "git+https://github.com/Dushistov/flapigen-rs?rev=7d343c6#7d343c60afcc094f7f07741174e6777fb8ef113c" +dependencies = [ + "bitflags 1.3.2", + "heck", + "lazy_static", + "log", + "petgraph", + "proc-macro2 1.0.66", + "quote 1.0.33", + "rustc-hash", + "smallvec", + "smol_str 0.1.24", + "strum 0.24.1", + "syn 1.0.109", + "which", +] + [[package]] name = "flate2" version = "1.0.27" @@ -2374,6 +2439,12 @@ dependencies = [ "regex", ] +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.26" @@ -2486,6 +2557,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "jwst-jni" +version = "0.1.0" +dependencies = [ + "android_logger", + "chrono", + "flapigen", + "futures", + "jni-sys", + "jwst-codec", + "jwst-core", + "jwst-rpc", + "jwst-storage", + "lib0", + "log-panics", + "nanoid", + "rifgen", + "serde", + "serde_json", + "sqlx", + "tokio", + "tracing", + "yrs", +] + [[package]] name = "jwst-logger" version = "0.1.0" @@ -2743,6 +2839,15 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "log-panics" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f9dd8546191c1850ecf67d22f5ff00a935b890d0e84713159a55495cc2ac5f" +dependencies = [ + "log", +] + [[package]] name = "loom" version = "0.5.6" @@ -3309,6 +3414,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.0.0", +] + [[package]] name = "phf" version = "0.11.2" @@ -3963,6 +4078,28 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rifgen" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba139f9f8c0d72232525b60bf83a81c6801df932d80ad6b4d0c8c0ccee9fa116" +dependencies = [ + "Inflector", + "derive-new", + "rifgen_attr", + "syn 1.0.109", +] + +[[package]] +name = "rifgen_attr" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3df05e0adb96c94ddb287013558ba7ff67f097219f6afa3c789506472e71272" +dependencies = [ + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "ring" version = "0.16.20" @@ -4370,7 +4507,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "strum", + "strum 0.25.0", "thiserror", "time 0.3.27", "tracing", @@ -4547,18 +4684,18 @@ checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.185" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.185" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc59dfdcbad1437773485e0367fea4b090a2e0a16d9ffc46af47764536a298ec" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2 1.0.66", "quote 1.0.33", @@ -4720,6 +4857,15 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +[[package]] +name = "smol_str" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad6c857cbab2627dcf01ec85a623ca4e7dcb5691cbaa3d7fb7653671f0d09c9" +dependencies = [ + "serde", +] + [[package]] name = "smol_str" version = "0.2.0" @@ -5030,12 +5176,34 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + [[package]] name = "strum" version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2 1.0.66", + "quote 1.0.33", + "rustversion", + "syn 1.0.109", +] + [[package]] name = "stun" version = "0.4.4" @@ -6083,7 +6251,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "smol_str", + "smol_str 0.2.0", "stun", "thiserror", "time 0.3.27", @@ -6273,6 +6441,17 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" +[[package]] +name = "which" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +dependencies = [ + "either", + "libc", + "once_cell", +] + [[package]] name = "whoami" version = "1.4.1" diff --git a/Cargo.toml b/Cargo.toml index 4f36aee1f..500a82aad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = [ "apps/keck", "libs/jwst", # "libs/jwst-binding/jwst-ffi", - # "libs/jwst-binding/jwst-jni", + "libs/jwst-binding/jwst-jni", # "libs/jwst-binding/jwst-py", "libs/jwst-binding/jwst-swift", "libs/jwst-binding/jwst-swift/jwst-swift-integrate", @@ -42,9 +42,6 @@ lto = false inherits = "fast-release" debug = true -# [profile.release.package.jwst-codec-fuzz] -# debug = 1 - [profile.dev.package.image] opt-level = 3 diff --git a/apps/android/.idea/codeStyles/Project.xml b/apps/android/.idea/codeStyles/Project.xml index 7643783a8..a88fc36d7 100644 --- a/apps/android/.idea/codeStyles/Project.xml +++ b/apps/android/.idea/codeStyles/Project.xml @@ -1,5 +1,6 @@ +