diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 5eab9d7..0eabdd3 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,7 +1,7 @@ { "name": "@localai/desktop", "private": true, - "version": "0.5.1", + "version": "0.5.2", "scripts": { "dev:next": "next dev -p 1470", "build:next": "next build", diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lock index de69237..e896be0 100644 --- a/apps/desktop/src-tauri/Cargo.lock +++ b/apps/desktop/src-tauri/Cargo.lock @@ -840,6 +840,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.16.2" @@ -1112,7 +1121,7 @@ version = "0.99.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ - "convert_case", + "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version 0.4.0", @@ -2483,6 +2492,7 @@ dependencies = [ "blake3", "chrono", "cocoa", + "convert_case 0.6.0", "diesel", "digest", "flume", diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 6278227..efd687b 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -15,7 +15,7 @@ tauri-build = { version = "1.4.0", features = [] } [dependencies] llm = { git = "https://github.com/rustformers/llm", branch = "main", package = "llm", features = [ "default", - "cublas", + # "cublas", ] } # llm = { git = "https://github.com/RedBoxing/llm.git", branch = "hf-tokenizer", package = "llm" } @@ -34,6 +34,7 @@ tauri = { version = "1.4.0", features = [ "window-show", ] } +convert_case = "0.6.0" paste = "1.0" sys-info = "0.9.1" num_cpus = "1.15.0" diff --git a/apps/desktop/src-tauri/src/inference/gpu.rs b/apps/desktop/src-tauri/src/inference/gpu.rs new file mode 100644 index 0000000..be9046b --- /dev/null +++ b/apps/desktop/src-tauri/src/inference/gpu.rs @@ -0,0 +1,9 @@ +#[tauri::command] +pub async fn check_gpu() -> Result { + // TODO: actually check if Metal is available in the future (?) + if cfg!(all(target_os = "macos", target_arch = "aarch64")) { + Ok(true) + } else { + Ok(false) + } +} diff --git a/apps/desktop/src-tauri/src/inference/mod.rs b/apps/desktop/src-tauri/src/inference/mod.rs index f6dc283..dd7c99f 100644 --- a/apps/desktop/src-tauri/src/inference/mod.rs +++ b/apps/desktop/src-tauri/src/inference/mod.rs @@ -1,4 +1,5 @@ pub mod completion; +pub mod gpu; pub mod process; pub mod server; pub mod server_config; diff --git a/apps/desktop/src-tauri/src/main.rs b/apps/desktop/src-tauri/src/main.rs index 3cbda20..6c504cf 100644 --- a/apps/desktop/src-tauri/src/main.rs +++ b/apps/desktop/src-tauri/src/main.rs @@ -38,8 +38,7 @@ fn main() { Ok(()) }) - // NOTE: New cmd should be added to src/invoke/_shared.ts - // TODO: a middleware to convert this into the ts enum would be nice + // NOTE: When adding new commands, make sure to run the generate_ts_enums test to update the TS enum .invoke_handler(tauri::generate_handler![ config::get_config, path::read_directory, @@ -66,14 +65,75 @@ fn main() { model::stats::get_model_stats, model::config::get_model_config, model::config::set_model_config, + model::pool::load_model, inference::server::start_server, inference::server::stop_server, inference::server_config::get_server_config, inference::server_config::set_server_config, - model::pool::load_model, - test::test_model, + inference::gpu::check_gpu, utils::fs::open_directory, + test::test_model, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); } + +#[test] +/** + * Generate the enum for the client side invocation + * Based on https://matklad.github.io/2022/03/26/self-modifying-code.html#Minimalist-Solution + */ +fn generate_ts_cmd_enums() { + use convert_case::{Case, Casing}; + + fn split_twice<'a>( + text: &'a str, + start_marker: &str, + end_marker: &str, + ) -> Option<(&'a str, &'a str, &'a str)> { + let (prefix, rest) = text.split_once(start_marker)?; + let (mid, suffix) = rest.split_once(end_marker)?; + Some((prefix, mid, suffix)) + } + + let main_rs_text = std::fs::read_to_string(file!()).unwrap(); + + let (_, tauri_cmds, _) = split_twice( + &main_rs_text, + ".invoke_handler(tauri::generate_handler![\n", + "])", + ) + .unwrap(); + + let arms = tauri_cmds + .lines() + .map(|line| { + line + .trim() + .trim_end_matches(',') + .split("::") + .last() + .unwrap() + }) + .enumerate() + // filter only non-empty string + .filter(|(_, cmd)| !cmd.is_empty()) + .map(|(_, cmd)| format!(" {} = \"{cmd}\"", cmd.to_case(Case::Pascal))) + .collect::>() + .join(",\n"); + + let ts_enum_path = std::path::Path::new("../src/features/invoke/_shared.ts"); + + let ts_original_text = std::fs::read_to_string(ts_enum_path).unwrap(); + + let new_text = { + let start_marker = " //#region GENERATED\n"; + let end_marker = "\n //#endregion\n"; + + let (prefix, _, suffix) = + split_twice(&ts_original_text, start_marker, end_marker).unwrap(); + format!("{prefix}{start_marker}{arms}{end_marker}{suffix}") + }; + + std::fs::write(ts_enum_path, new_text).unwrap(); +} diff --git a/apps/desktop/src/features/inference-server/server-config.tsx b/apps/desktop/src/features/inference-server/server-config.tsx index 4382628..c8d1fd8 100644 --- a/apps/desktop/src/features/inference-server/server-config.tsx +++ b/apps/desktop/src/features/inference-server/server-config.tsx @@ -4,6 +4,8 @@ import { IntInput } from "@lab/ui/int-input" import { Switch } from "@lab/ui/switch" import { useState } from "react" +import { InitState, useInit } from "~features/inference-server/use-init" +import { InvokeCommand, invoke } from "~features/invoke" import { useGlobal } from "~providers/global" export const ServerConfig = () => { @@ -15,6 +17,16 @@ export const ServerConfig = () => { } = useGlobal() const [isLoading, setIsLoading] = useState(false) + + const [hasGpu, setHasGpu] = useState(false) + const gpuCheck = useInit(async () => { + const _hasGpu = await invoke(InvokeCommand.CheckGpu) + if (!_hasGpu) { + serverConfig.update({ useGpu: false }) + } + setHasGpu(_hasGpu) + }) + return (
{/*