From 12e58fe98d1cebb880a193b2ee8f2c72f9a309f8 Mon Sep 17 00:00:00 2001 From: marcus-daily <111281783+marcus-daily@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:12:32 +0100 Subject: [PATCH 1/3] RTVI 0.2.0 --- CHANGELOG.md | 25 +++ gradle/libs.versions.toml | 6 +- rtvi-client-android/build.gradle.kts | 2 +- .../src/main/java/ai/rtvi/client/Legacy.kt | 11 ++ .../client/{VoiceClient.kt => RTVIClient.kt} | 168 ++++++++++++------ ...eClientOptions.kt => RTVIClientOptions.kt} | 15 +- .../java/ai/rtvi/client/RTVIClientParams.kt | 36 ++++ ...ventCallbacks.kt => RTVIEventCallbacks.kt} | 27 ++- .../java/ai/rtvi/client/helper/LLMHelper.kt | 18 +- .../main/java/ai/rtvi/client/helper/Legacy.kt | 7 + ...iceClientHelper.kt => RTVIClientHelper.kt} | 32 ++-- .../main/java/ai/rtvi/client/result/Future.kt | 6 +- .../java/ai/rtvi/client/result/HttpError.kt | 20 ++- .../main/java/ai/rtvi/client/result/Legacy.kt | 7 + .../result/{VoiceError.kt => RTVIError.kt} | 24 +-- .../ai/rtvi/client/result/RTVIException.kt | 12 ++ .../main/java/ai/rtvi/client/result/Result.kt | 4 +- .../ai/rtvi/client/result/VoiceException.kt | 12 -- .../client/transport/MsgServerToClient.kt | 31 +++- .../ai/rtvi/client/transport/Transport.kt | 22 +-- .../rtvi/client/transport/TransportContext.kt | 8 +- .../ai/rtvi/client/types/RTVIURLEndpoints.kt | 6 + .../ai/rtvi/client/types/RegisteredHelper.kt | 4 +- .../ai/rtvi/client/types/TransportState.kt | 3 +- .../ai/rtvi/client/utils/ConnectionBundle.kt | 3 +- .../main/java/ai/rtvi/client/utils/Http.kt | 52 +++++- .../main/java/ai/rtvi/client/utils/Json.kt | 9 +- 27 files changed, 416 insertions(+), 154 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 rtvi-client-android/src/main/java/ai/rtvi/client/Legacy.kt rename rtvi-client-android/src/main/java/ai/rtvi/client/{VoiceClient.kt => RTVIClient.kt} (70%) rename rtvi-client-android/src/main/java/ai/rtvi/client/{VoiceClientOptions.kt => RTVIClientOptions.kt} (71%) create mode 100644 rtvi-client-android/src/main/java/ai/rtvi/client/RTVIClientParams.kt rename rtvi-client-android/src/main/java/ai/rtvi/client/{VoiceEventCallbacks.kt => RTVIEventCallbacks.kt} (88%) create mode 100644 rtvi-client-android/src/main/java/ai/rtvi/client/helper/Legacy.kt rename rtvi-client-android/src/main/java/ai/rtvi/client/helper/{VoiceClientHelper.kt => RTVIClientHelper.kt} (61%) create mode 100644 rtvi-client-android/src/main/java/ai/rtvi/client/result/Legacy.kt rename rtvi-client-android/src/main/java/ai/rtvi/client/result/{VoiceError.kt => RTVIError.kt} (76%) create mode 100644 rtvi-client-android/src/main/java/ai/rtvi/client/result/RTVIException.kt delete mode 100644 rtvi-client-android/src/main/java/ai/rtvi/client/result/VoiceException.kt create mode 100644 rtvi-client-android/src/main/java/ai/rtvi/client/types/RTVIURLEndpoints.kt diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..85cbbe2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,25 @@ +# 0.2.0 + +- Renamed: + - `VoiceClient` to `RTVIClient` + - `VoiceClientOptions` to `RTVIClientOptions` + - `VoiceEventCallbacks` to `RTVIEventCallbacks` + - `VoiceError` to `RTVIError` + - `VoiceException` to `RTVIException` + - `VoiceClientHelper` to `RTVIClientHelper` + - `RegisteredVoiceClient` to `RegisteredRTVIClient` + - `FailedToFetchAuthBundle` to `HttpError` +- `RTVIClient()` constructor parameter changes + - `options` is now mandatory + - `baseUrl` has been moved to `options.params.baseUrl` + - `baseUrl` and `endpoints` are now separate, and the endpoint names are appended to the `baseUrl` +- Moved `RTVIClientOptions.config` to `RTVIClientOptions.params.config` +- Moved `RTVIClientOptions.customHeaders` to `RTVIClientOptions.params.headers` +- Moved `RTVIClientOptions.customBodyParams` to `RTVIClientOptions.params.requestData` +- `TransportState` changes + - Removed `Idle` state, replaced with `Disconnected` + - Added `Disconnecting` state +- Added callbacks + - `onBotLLMText()` + - `onBotTTSText()` + - `onStorageItemStored()` \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 835b803..3c3063e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,12 +1,12 @@ [versions] -agp = "8.5.1" -kotlin = "2.0.0" +agp = "8.5.2" +kotlin = "2.0.20" coreKtx = "1.13.1" junit = "4.13.2" junitVersion = "1.2.1" appcompat = "1.7.0" kotlinxSerializationJson = "1.7.1" -kotlinxSerializationPlugin = "2.0.0" +kotlinxSerializationPlugin = "2.0.20" okhttp = "4.12.0" dokka = "1.9.20" diff --git a/rtvi-client-android/build.gradle.kts b/rtvi-client-android/build.gradle.kts index 6402d85..e352607 100644 --- a/rtvi-client-android/build.gradle.kts +++ b/rtvi-client-android/build.gradle.kts @@ -60,7 +60,7 @@ publishing { register("release") { groupId = "ai.rtvi" artifactId = "client" - version = "0.1.4" + version = "0.2.0" pom { name.set("RTVI Client") diff --git a/rtvi-client-android/src/main/java/ai/rtvi/client/Legacy.kt b/rtvi-client-android/src/main/java/ai/rtvi/client/Legacy.kt new file mode 100644 index 0000000..5d637b9 --- /dev/null +++ b/rtvi-client-android/src/main/java/ai/rtvi/client/Legacy.kt @@ -0,0 +1,11 @@ +package ai.rtvi.client + + +@Deprecated("VoiceClient is renamed to RTVIClient") +typealias VoiceClient = RTVIClient + +@Deprecated("VoiceClientOptions is renamed to RTVIClientOptions") +typealias VoiceClientOptions = RTVIClientOptions + +@Deprecated("VoiceEventCallbacks is renamed to RTVIEventCallbacks") +typealias VoiceEventCallbacks = RTVIEventCallbacks diff --git a/rtvi-client-android/src/main/java/ai/rtvi/client/VoiceClient.kt b/rtvi-client-android/src/main/java/ai/rtvi/client/RTVIClient.kt similarity index 70% rename from rtvi-client-android/src/main/java/ai/rtvi/client/VoiceClient.kt rename to rtvi-client-android/src/main/java/ai/rtvi/client/RTVIClient.kt index 4a6e4f4..a0b87c6 100644 --- a/rtvi-client-android/src/main/java/ai/rtvi/client/VoiceClient.kt +++ b/rtvi-client-android/src/main/java/ai/rtvi/client/RTVIClient.kt @@ -1,12 +1,12 @@ package ai.rtvi.client -import ai.rtvi.client.helper.RegisteredVoiceClient -import ai.rtvi.client.helper.VoiceClientHelper +import ai.rtvi.client.helper.RTVIClientHelper +import ai.rtvi.client.helper.RegisteredRTVIClient import ai.rtvi.client.result.Future import ai.rtvi.client.result.Promise +import ai.rtvi.client.result.RTVIError +import ai.rtvi.client.result.RTVIException import ai.rtvi.client.result.Result -import ai.rtvi.client.result.VoiceError -import ai.rtvi.client.result.VoiceException import ai.rtvi.client.result.resolvedPromiseErr import ai.rtvi.client.result.withPromise import ai.rtvi.client.result.withTimeout @@ -29,7 +29,9 @@ import ai.rtvi.client.types.Value import ai.rtvi.client.utils.ConnectionBundle import ai.rtvi.client.utils.JSON_INSTANCE import ai.rtvi.client.utils.ThreadRef +import ai.rtvi.client.utils.parseServerSentEvents import ai.rtvi.client.utils.post +import ai.rtvi.client.utils.valueFrom import android.util.Log import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonPrimitive @@ -44,17 +46,15 @@ import okhttp3.RequestBody.Companion.toRequestBody * * The client must be cleaned up using the [release] method when it is no longer required. * - * @param baseUrl URL of the RTVI backend. * @param transport Transport for media streaming. * @param callbacks Callbacks invoked when changes occur in the voice session. * @param options Additional options for configuring the client and backend. */ @Suppress("unused") -open class VoiceClient( - private val baseUrl: String, +open class RTVIClient( transport: TransportFactory, - callbacks: VoiceEventCallbacks, - private var options: VoiceClientOptions = VoiceClientOptions() + callbacks: RTVIEventCallbacks, + private var options: RTVIClientOptions, ) { companion object { private const val TAG = "VoiceClient" @@ -65,12 +65,12 @@ open class VoiceClient( */ val thread = ThreadRef.forCurrent() - private val callbacks = CallbackInterceptor(object : VoiceEventCallbacks() { + private val callbacks = CallbackInterceptor(object : RTVIEventCallbacks() { override fun onBackendError(message: String) {} override fun onDisconnected() { discardWaitingResponses() - connection?.ready?.resolveErr(VoiceError.OperationCancelled) + connection?.ready?.resolveErr(RTVIError.OperationCancelled) connection = null } }, callbacks) @@ -78,11 +78,11 @@ open class VoiceClient( private val helpers = mutableMapOf() private val awaitingServerResponse = - mutableMapOf) -> Unit>() + mutableMapOf) -> Unit>() private inline fun handleResponse( msg: MsgServerToClient, - action: ((Result) -> Unit) -> Unit + action: ((Result) -> Unit) -> Unit ) { val id = msg.id ?: throw Exception("${msg.type} missing ID") @@ -95,17 +95,15 @@ open class VoiceClient( private val transportCtx = object : TransportContext { override val options - get() = this@VoiceClient.options + get() = this@RTVIClient.options override val callbacks - get() = this@VoiceClient.callbacks + get() = this@RTVIClient.callbacks - override val thread = this@VoiceClient.thread + override val thread = this@RTVIClient.thread override fun onMessage(msg: MsgServerToClient) = thread.runOnThread { - Log.i(TAG, "onMessage($msg)") - try { when (msg.type) { MsgServerToClient.Type.BotReady -> { @@ -113,7 +111,7 @@ open class VoiceClient( val data = JSON_INSTANCE.decodeFromJsonElement(msg.data) - this@VoiceClient.transport.setState(TransportState.Ready) + this@RTVIClient.transport.setState(TransportState.Ready) connection?.ready?.resolveOk(Unit) @@ -138,7 +136,7 @@ open class VoiceClient( try { handleResponse(msg) { respondTo -> - respondTo(Result.Err(VoiceError.ErrorResponse(data.error))) + respondTo(Result.Err(RTVIError.ErrorResponse(data.error))) } } catch (e: Exception) { Log.e(TAG, "Got exception handling error response", e) @@ -160,7 +158,8 @@ open class VoiceClient( callbacks.onUserTranscript(data) } - MsgServerToClient.Type.BotTranscription -> { + MsgServerToClient.Type.BotTranscription, + MsgServerToClient.Type.BotTranscriptionLegacy -> { val text = (msg.data.jsonObject.get("text") as JsonPrimitive).content callbacks.onBotTranscript(text) } @@ -181,6 +180,27 @@ open class VoiceClient( callbacks.onBotStoppedSpeaking() } + MsgServerToClient.Type.BotLlmText -> { + val data: MsgServerToClient.Data.BotLLMTextData = + JSON_INSTANCE.decodeFromJsonElement(msg.data) + + callbacks.onBotLLMText(data) + } + + MsgServerToClient.Type.BotTtsText -> { + val data: MsgServerToClient.Data.BotTTSTextData = + JSON_INSTANCE.decodeFromJsonElement(msg.data) + + callbacks.onBotTTSText(data) + } + + MsgServerToClient.Type.StorageItemStored -> { + val data: MsgServerToClient.Data.StorageItemStoredData = + JSON_INSTANCE.decodeFromJsonElement(msg.data) + + callbacks.onStorageItemStored(data) + } + else -> { var match = false @@ -208,7 +228,7 @@ open class VoiceClient( private val transport: Transport = transport.createTransport(transportCtx) private inner class Connection { - val ready = Promise(thread) + val ready = Promise(thread) } private var connection: Connection? = null @@ -218,17 +238,20 @@ open class VoiceClient( * * @return A Future, representing the asynchronous result of this operation. */ - fun initDevices(): Future = transport.initDevices() + fun initDevices(): Future = transport.initDevices() + + @Deprecated("start() renamed to connect()") + fun start() = connect() /** * Initiate an RTVI session, connecting to the backend. */ - fun start(): Future = thread.runOnThreadReturningFuture { + fun connect(): Future = thread.runOnThreadReturningFuture { if (connection != null) { return@runOnThreadReturningFuture resolvedPromiseErr( thread, - VoiceError.PreviousConnectionStillActive + RTVIError.PreviousConnectionStillActive ) } @@ -237,28 +260,28 @@ open class VoiceClient( // Send POST request to the provided base_url to connect and start the bot val body = ConnectionBundle( - services = options.services.associate { it.service to it.value }, - config = options.config + services = options.services?.associate { it.service to it.value }, + config = options.config + options.params.config ) - .serializeWithCustomParams(options.customBodyParams) + .serializeWithCustomParams(options.customBodyParams + options.params.requestData) .toRequestBody("application/json".toMediaType()) val currentConnection = Connection().apply { connection = this } return@runOnThreadReturningFuture post( thread = thread, - url = baseUrl, + url = options.params.baseUrl + options.params.endpoints.connect, body = body, - customHeaders = options.customHeaders + customHeaders = options.customHeaders + options.params.headers ) - .mapError { - VoiceError.FailedToFetchAuthBundle(it) + .mapError { + RTVIError.HttpError(it) } .chain { authBundle -> if (currentConnection == connection) { transport.connect(AuthBundle(authBundle)) } else { - resolvedPromiseErr(thread, VoiceError.OperationCancelled) + resolvedPromiseErr(thread, RTVIError.OperationCancelled) } } .chain { currentConnection.ready } @@ -273,7 +296,7 @@ open class VoiceClient( * * @return A Future, representing the asynchronous result of this operation. */ - fun disconnect(): Future { + fun disconnect(): Future { return transport.disconnect() } @@ -287,7 +310,7 @@ open class VoiceClient( thread.assertCurrent() awaitingServerResponse.values.forEach { - it(Result.Err(VoiceError.OperationCancelled)) + it(Result.Err(RTVIError.OperationCancelled)) } awaitingServerResponse.clear() @@ -299,16 +322,16 @@ open class VoiceClient( * @param service Target service for this helper * @param helper Helper instance */ - @Throws(VoiceException::class) - fun registerHelper(service: String, helper: E): E { + @Throws(RTVIException::class) + fun registerHelper(service: String, helper: E): E { thread.assertCurrent() if (helpers.containsKey(service)) { - throw VoiceException(VoiceError.OtherError("Helper targeting service '$service' already registered")) + throw RTVIException(RTVIError.OtherError("Helper targeting service '$service' already registered")) } - helper.registerVoiceClient(RegisteredVoiceClient(this, service)) + helper.registerVoiceClient(RegisteredRTVIClient(this, service)) val entry = RegisteredHelper( helper = helper, @@ -323,21 +346,22 @@ open class VoiceClient( /** * Unregisters a helper from the client. */ - @Throws(VoiceException::class) + @Throws(RTVIException::class) fun unregisterHelper(service: String) { thread.assertCurrent() val entry = helpers.remove(service) - ?: throw VoiceException(VoiceError.OtherError("Helper targeting service '$service' not found")) + ?: throw RTVIException(RTVIError.OtherError("Helper targeting service '$service' not found")) entry.helper.unregisterVoiceClient() } private inline fun sendWithResponse( msg: MsgClientToServer, + allowSingleTurn: Boolean = false, crossinline filter: (M) -> R - ): Future = withPromise(thread) { promise -> + ) = withPromise(thread) { promise -> thread.runOnThread { awaitingServerResponse[msg.id] = { result -> @@ -350,12 +374,47 @@ open class VoiceClient( } } - transport.sendMessage(msg).withErrorCallback { - awaitingServerResponse.remove(msg.id) - promise.resolveErr(it) + when (transport.state()) { + TransportState.Connected, TransportState.Ready -> { + transport.sendMessage(msg) + .withTimeout(10000) + .withErrorCallback { + awaitingServerResponse.remove(msg.id) + promise.resolveErr(it) + } + } + + else -> if (allowSingleTurn) { + post( + thread = thread, + url = options.params.baseUrl + options.params.endpoints.action, + body = JSON_INSTANCE.encodeToString( + Value.serializer(), Value.Object( + (options.customBodyParams + options.params.requestData + listOf( + "actions" to Value.Array( + valueFrom(MsgClientToServer.serializer(), msg) + ) + )).toMap() + ) + ).toRequestBody("application/json".toMediaType()), + customHeaders = options.customHeaders + options.params.headers, + responseHandler = { inputStream -> + inputStream.parseServerSentEvents { msg -> + transportCtx.onMessage(JSON_INSTANCE.decodeFromString(msg)) + } + } + ).withCallback { + promise.resolveErr( + when (it) { + is Result.Err -> RTVIError.HttpError(it.error) + is Result.Ok -> RTVIError.OtherError("Connection ended before result received") + } + ) + } + } } } - }.withTimeout(10000) + } /** * Instruct a backend service to perform an action. @@ -364,18 +423,19 @@ open class VoiceClient( service: String, action: String, arguments: List