diff --git a/package-lock.json b/package-lock.json index a5dfa41f..99ad90a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@pixiv/three-vrm": "^2.0.7", "@ricky0123/vad-react": "^0.0.17", "@supabase/supabase-js": "^2.39.0", + "@tabler/icons-react": "^3.11.0", "@tailwindcss/forms": "^0.5.6", "@tailwindcss/line-clamp": "^0.4.4", "@xenova/transformers": "^2.7.0", @@ -3596,6 +3597,30 @@ "tslib": "^2.4.0" } }, + "node_modules/@tabler/icons": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/@tabler/icons/-/icons-3.11.0.tgz", + "integrity": "sha512-/vZinJNvCYhdAB+RUsyCpanSPuOEKHHIZi4Uu0Bw7ilewHnQhCWUPrT704uHCRli2ROl7spADPmWzAqOganA5A==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/codecalm" + } + }, + "node_modules/@tabler/icons-react": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/@tabler/icons-react/-/icons-react-3.11.0.tgz", + "integrity": "sha512-xHNBi9mns1slvqos+7LkP3ube4CjWrANMbxMaorzwzO9J/+y1sAEG/sN8CV8FmtpYW/9/gDR+OWCjjLLg0RmAw==", + "dependencies": { + "@tabler/icons": "3.11.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/codecalm" + }, + "peerDependencies": { + "react": ">= 16" + } + }, "node_modules/@tailwindcss/forms": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/@tailwindcss/forms/-/forms-0.5.6.tgz", @@ -16527,6 +16552,19 @@ "tslib": "^2.4.0" } }, + "@tabler/icons": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/@tabler/icons/-/icons-3.11.0.tgz", + "integrity": "sha512-/vZinJNvCYhdAB+RUsyCpanSPuOEKHHIZi4Uu0Bw7ilewHnQhCWUPrT704uHCRli2ROl7spADPmWzAqOganA5A==" + }, + "@tabler/icons-react": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/@tabler/icons-react/-/icons-react-3.11.0.tgz", + "integrity": "sha512-xHNBi9mns1slvqos+7LkP3ube4CjWrANMbxMaorzwzO9J/+y1sAEG/sN8CV8FmtpYW/9/gDR+OWCjjLLg0RmAw==", + "requires": { + "@tabler/icons": "3.11.0" + } + }, "@tailwindcss/forms": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/@tailwindcss/forms/-/forms-0.5.6.tgz", diff --git a/package.json b/package.json index 7b80c338..25504ece 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "@pixiv/three-vrm": "^2.0.7", "@ricky0123/vad-react": "^0.0.17", "@supabase/supabase-js": "^2.39.0", + "@tabler/icons-react": "^3.11.0", "@tailwindcss/forms": "^0.5.6", "@tailwindcss/line-clamp": "^0.4.4", "@xenova/transformers": "^2.7.0", diff --git a/public/animations/dance.vrma b/public/animations/dance.vrma new file mode 100644 index 00000000..9bc75f78 Binary files /dev/null and b/public/animations/dance.vrma differ diff --git a/public/animations/greeting.vrma b/public/animations/greeting.vrma new file mode 100644 index 00000000..422d37fe Binary files /dev/null and b/public/animations/greeting.vrma differ diff --git a/public/animations/modelPose.vrma b/public/animations/modelPose.vrma new file mode 100644 index 00000000..cc9754ff Binary files /dev/null and b/public/animations/modelPose.vrma differ diff --git a/public/animations/peaceSign.vrma b/public/animations/peaceSign.vrma new file mode 100644 index 00000000..b1e06458 Binary files /dev/null and b/public/animations/peaceSign.vrma differ diff --git a/public/animations/shoot.vrma b/public/animations/shoot.vrma new file mode 100644 index 00000000..662576c4 Binary files /dev/null and b/public/animations/shoot.vrma differ diff --git a/public/animations/showFullBody.vrma b/public/animations/showFullBody.vrma new file mode 100644 index 00000000..5a49326f Binary files /dev/null and b/public/animations/showFullBody.vrma differ diff --git a/public/animations/spin.vrma b/public/animations/spin.vrma new file mode 100644 index 00000000..8c8f978d Binary files /dev/null and b/public/animations/spin.vrma differ diff --git a/public/animations/squat.vrma b/public/animations/squat.vrma new file mode 100644 index 00000000..eab9834a Binary files /dev/null and b/public/animations/squat.vrma differ diff --git a/public/vrm/AvatarSample_A.vrm b/public/vrm/AvatarSample_A.vrm index 4ed141f5..f85bd08c 100644 Binary files a/public/vrm/AvatarSample_A.vrm and b/public/vrm/AvatarSample_A.vrm differ diff --git a/public/vrm/AvatarSample_B.vrm b/public/vrm/AvatarSample_B.vrm index 51d84320..4ed141f5 100644 Binary files a/public/vrm/AvatarSample_B.vrm and b/public/vrm/AvatarSample_B.vrm differ diff --git a/public/vrm/AvatarSample_C.vrm b/public/vrm/AvatarSample_C.vrm index c272a559..51d84320 100644 Binary files a/public/vrm/AvatarSample_C.vrm and b/public/vrm/AvatarSample_C.vrm differ diff --git a/public/vrm/AvatarSample_D.vrm b/public/vrm/AvatarSample_D.vrm new file mode 100644 index 00000000..c272a559 Binary files /dev/null and b/public/vrm/AvatarSample_D.vrm differ diff --git a/public/vrm/thumb-AvatarSample_A.vrm.jpg b/public/vrm/thumb-AvatarSample_A.vrm.jpg index 1a6c4ef1..3cc3a5e2 100644 Binary files a/public/vrm/thumb-AvatarSample_A.vrm.jpg and b/public/vrm/thumb-AvatarSample_A.vrm.jpg differ diff --git a/public/vrm/thumb-AvatarSample_B.vrm.jpg b/public/vrm/thumb-AvatarSample_B.vrm.jpg index e4ac16cb..1a6c4ef1 100644 Binary files a/public/vrm/thumb-AvatarSample_B.vrm.jpg and b/public/vrm/thumb-AvatarSample_B.vrm.jpg differ diff --git a/public/vrm/thumb-AvatarSample_C.vrm.jpg b/public/vrm/thumb-AvatarSample_C.vrm.jpg index 1960fe1f..e4ac16cb 100644 Binary files a/public/vrm/thumb-AvatarSample_C.vrm.jpg and b/public/vrm/thumb-AvatarSample_C.vrm.jpg differ diff --git a/public/vrm/thumb-AvatarSample_D.vrm.jpg b/public/vrm/thumb-AvatarSample_D.vrm.jpg new file mode 100644 index 00000000..1960fe1f Binary files /dev/null and b/public/vrm/thumb-AvatarSample_D.vrm.jpg differ diff --git a/src/components/assistantText.tsx b/src/components/assistantText.tsx index 657eb986..cba0d9db 100644 --- a/src/components/assistantText.tsx +++ b/src/components/assistantText.tsx @@ -7,6 +7,9 @@ export const AssistantText = ({ message }: { message: string }) => { const scrollRef = useRef(null); const [unlimited, setUnlimited] = useState(false) + // Replace all of the emotion tag in message with "" + message = message.replace(/\[(.*?)\]/g, ""); + useEffect(() => { scrollRef.current?.scrollIntoView({ behavior: "smooth", @@ -45,3 +48,4 @@ export const AssistantText = ({ message }: { message: string }) => { ); }; + diff --git a/src/components/chatLog.tsx b/src/components/chatLog.tsx index d733761a..3a13e514 100644 --- a/src/components/chatLog.tsx +++ b/src/components/chatLog.tsx @@ -22,7 +22,7 @@ export const ChatLog = ({ const handleResumeButtonClick = (num: number, newMessage: string) => { bot.setMessageList(messages.slice(0, num)); - bot.receiveMessageFromUser(newMessage); + bot.receiveMessageFromUser(newMessage,false); }; const txtFileInputRef = useRef(null); @@ -56,7 +56,7 @@ export const ChatLog = ({ bot.setMessageList(parsedChat.slice(0, parsedChat.length - 1)); if (lastMessage.role === "user") { - bot.receiveMessageFromUser(lastMessage.content); + bot.receiveMessageFromUser(lastMessage.content,false); } else { bot.bubbleMessage(lastMessage.role, lastMessage.content); } @@ -132,7 +132,7 @@ export const ChatLog = ({
@@ -165,10 +165,10 @@ function Chat({ onClickResumeButton: (num: number, message: string) => void; }) { const { t } = useTranslation(); - const [textAreaValue, setTextAreaValue] = useState(message); + // const [textAreaValue, setTextAreaValue] = useState(message); const onClickButton = () => { - const newMessage = textAreaValue + const newMessage = message onClickResumeButton(num, newMessage); }; @@ -203,11 +203,11 @@ function Chat({
{role === "assistant" ? ( -
{textAreaValue}
+
{message}
) : ( )}
diff --git a/src/components/chatModeText.tsx b/src/components/chatModeText.tsx new file mode 100644 index 00000000..adee2b79 --- /dev/null +++ b/src/components/chatModeText.tsx @@ -0,0 +1,117 @@ +import { useEffect, useRef, useState } from "react"; +import { clsx } from "clsx"; +import { config } from "@/utils/config"; +import { IconButton } from "./iconButton"; +import { useTranslation } from "react-i18next"; +import { Message } from "@/features/chat/messages"; + +export const ChatModeText = ({ messages }: { messages: Message[] }) => { + const chatScrollRef = useRef(null); + + useEffect(() => { + chatScrollRef.current?.scrollIntoView({ + behavior: "smooth", + block: "center", + }); + }, [messages]); + + return ( +
+
+ +
+ {messages.map((msg, i) => { + return ( +
+ +
+ ); + })} +
+
+
+ + ); +}; + +function Chat({ + role, + message, + num, +}: { + role: string; + message: string; + num: number; +}) { + const { t } = useTranslation(); + const scrollRef = useRef(null); + const [unlimited, setUnlimited] = useState(false); + + // useEffect(() => { + // scrollRef.current?.scrollIntoView({ + // behavior: "smooth", + // block: "center", + // }); + // }); + + return ( +
+
+
+ +
+ + {role === "assistant" && config('name').toUpperCase()} + {role === "user" && t("YOU")} + + + {role === "assistant" && ( + setUnlimited(!unlimited)} + /> + + )} +
+ {role === "assistant" && ( +
+
+ {message.replace(/\[([a-zA-Z]*?)\]/g, "")} +
+
+
+ )} + {role === "user" && ( +
+
+ {message.replace(/\[([a-zA-Z]*?)\]/g, "")} +
+
+
+ )} +
+
+ + +
+ + ); +} \ No newline at end of file diff --git a/src/components/embeddedWebcam.tsx b/src/components/embeddedWebcam.tsx index dbf322e3..d65754b6 100644 --- a/src/components/embeddedWebcam.tsx +++ b/src/components/embeddedWebcam.tsx @@ -51,6 +51,7 @@ export function EmbeddedWebcam({ await bot.getVisionResponse(fixedImageData); setCameraDisabled(false); setImageData(""); + setWebcamEnabled(false); } }; diff --git a/src/components/messageInput.tsx b/src/components/messageInput.tsx index 5ad89cb9..39c0010a 100644 --- a/src/components/messageInput.tsx +++ b/src/components/messageInput.tsx @@ -12,6 +12,7 @@ import { openaiWhisper } from "@/features/openaiWhisper/openaiWhisper"; import { whispercpp } from "@/features/whispercpp/whispercpp"; import { config } from "@/utils/config"; import { WaveFile } from "wavefile"; +import { AmicaLifeContext } from "@/features/amicaLife/amicaLifeContext"; export default function MessageInput({ userMessage, @@ -32,6 +33,7 @@ export default function MessageInput({ const [whisperCppOutput, setWhisperCppOutput] = useState(null); const { chat: bot } = useContext(ChatContext); const { alert } = useContext(AlertContext); + const { amicaLife } = useContext(AmicaLifeContext); const vad = useMicVAD({ startOnLoad: false, @@ -119,25 +121,54 @@ export default function MessageInput({ const textStartsWithWakeWord = wakeWordEnabled && cleanFromPunctuation(cleanText).startsWith(cleanFromPunctuation(config("wake_word"))); const text = wakeWordEnabled && textStartsWithWakeWord ? cleanFromWakeWord(cleanText, config("wake_word")) : cleanText; - if (textStartsWithWakeWord) { - bot.updateAwake(); + if (wakeWordEnabled) { + // Text start with wake word + if (textStartsWithWakeWord) { + // Pause amicaLife and update bot's awake status when speaking + if (config("amica_life_enabled") === "true") { + amicaLife.pause(); + } + bot.updateAwake(); + // Case text doesn't start with wake word and not receive trigger message in amica life + } else { + if (config("amica_life_enabled") === "true" && amicaLife.triggerMessage !== true && !bot.isAwake()) { + bot.updateAwake(); + } + } + } else { + // If wake word off, update bot's awake when speaking + if (config("amica_life_enabled") === "true") { + amicaLife.pause(); + bot.updateAwake(); + } } + if (text === "") { return; } if (config("autosend_from_mic") === 'true') { - if (!wakeWordEnabled || bot.isAwake()) { - bot.receiveMessageFromUser(text); - } + if (!wakeWordEnabled || bot.isAwake()) { + bot.receiveMessageFromUser(text,false); + } } else { setUserMessage(text); } console.timeEnd('performance_transcribe'); } + function handleInputChange(event: React.ChangeEvent) { + onChangeUserMessage(event); + + // Pause amicaLife and update bot's awake status when typing + if (config("amica_life_enabled") === "true") { + amicaLife.pause(); + bot.updateAwake(); + } + } + // for whisper_browser useEffect(() => { if (transcriber.output && ! transcriber.isBusy) { @@ -163,7 +194,7 @@ export default function MessageInput({ }, [whisperCppOutput]); function clickedSendButton() { - bot.receiveMessageFromUser(userMessage); + bot.receiveMessageFromUser(userMessage,false); // only if we are using non-VAD mode should we focus on the input if (! vad.listening) { inputRef.current?.focus(); @@ -189,7 +220,7 @@ export default function MessageInput({ type="text" ref={inputRef} placeholder="Write message here..." - onChange={onChangeUserMessage} + onChange={handleInputChange} onKeyDown={(e) => { if (e.key === "Enter") { if (userMessage === "") { diff --git a/src/components/numberInput.tsx b/src/components/numberInput.tsx index a7eef5ad..4148cf7f 100644 --- a/src/components/numberInput.tsx +++ b/src/components/numberInput.tsx @@ -20,3 +20,5 @@ export const NumberInput = ({ /> ); }; + + diff --git a/src/components/rangeInput.tsx b/src/components/rangeInput.tsx new file mode 100644 index 00000000..cc9f7fcb --- /dev/null +++ b/src/components/rangeInput.tsx @@ -0,0 +1,54 @@ +import { useTranslation } from 'react-i18next'; + +type RangeProps = { + min: number; + max: number; + minChange?: (event: React.ChangeEvent, type: 'min') => void; + maxChange?: (event: React.ChangeEvent, type: 'max') => void; +}; + +export const RangeInput = ({ + min, + max, + minChange, + maxChange, +}: RangeProps) => { + const { t } = useTranslation(); + + const handleMinChange = (event: React.ChangeEvent) => { + if (minChange) minChange(event, 'min'); + }; + + const handleMaxChange = (event: React.ChangeEvent) => { + if (maxChange) maxChange(event, 'max'); + }; + + return ( +
+ + +
+ ); +}; diff --git a/src/components/settings.tsx b/src/components/settings.tsx index 1ed52e83..3479395d 100644 --- a/src/components/settings.tsx +++ b/src/components/settings.tsx @@ -61,6 +61,7 @@ import { VisionSystemPromptPage } from './settings/VisionSystemPromptPage'; import { NamePage } from './settings/NamePage'; import { SystemPromptPage } from './settings/SystemPromptPage'; +import { AmicaLifePage } from "./settings/AmicaLifePage"; import { useVrmStoreContext } from "@/features/vrmStore/vrmStoreContext"; export const Settings = ({ @@ -122,13 +123,19 @@ export const Settings = ({ const [sttBackend, setSTTBackend] = useState(config("stt_backend")); const [sttWakeWordEnabled, setSTTWakeWordEnabled] = useState(config("wake_word_enabled") === 'true' ? true : false); const [sttWakeWord, setSTTWakeWord] = useState(config("wake_word")); - const [sttWakeWordIdleTime, setSTTWakeWordIdleTime] = useState(parseInt(config("wake_word_time_before_idle_sec"))); const [whisperOpenAIUrl, setWhisperOpenAIUrl] = useState(config("openai_whisper_url")); const [whisperOpenAIApiKey, setWhisperOpenAIApiKey] = useState(config("openai_whisper_apikey")); const [whisperOpenAIModel, setWhisperOpenAIModel] = useState(config("openai_whisper_model")); const [whisperCppUrl, setWhisperCppUrl] = useState(config("whispercpp_url")); + const [amicaLifeEnabled,setAmicaLifeEnabled] = useState(config("amica_life_enabled") === 'true' ? true : false); + const [timeBeforeIdle, setTimeBeforeIdle] = useState(parseInt(config("time_before_idle_sec"))); + const [minTimeInterval,setMinTimeInterval] = useState(parseInt(config("min_time_interval_sec"))); + const [maxTimeInterval, setMaxTimeInterval] = useState(parseInt(config("max_time_interval_sec"))); + const [timeToSleep, setTimeToSleep] = useState(parseInt(config("time_to_sleep_sec"))); + const [idleTextPrompt, setIdleTextPrompt] = useState(config("idle_text_prompt")); + const [name, setName] = useState(config("name")); const [systemPrompt, setSystemPrompt] = useState(config("system_prompt")); @@ -226,9 +233,10 @@ export const Settings = ({ sttBackend, whisperOpenAIApiKey, whisperOpenAIModel, whisperOpenAIUrl, whisperCppUrl, + amicaLifeEnabled, timeBeforeIdle, minTimeInterval, maxTimeInterval, timeToSleep, idleTextPrompt, name, systemPrompt, - sttWakeWordEnabled, sttWakeWord, sttWakeWordIdleTime + sttWakeWordEnabled, sttWakeWord, ]); @@ -241,7 +249,7 @@ export const Settings = ({ switch(page) { case 'main_menu': return ; case 'appearance': @@ -438,10 +446,10 @@ export const Settings = ({ return @@ -510,6 +518,23 @@ export const Settings = ({ setSettingsUpdated={setSettingsUpdated} /> + case 'amica_life': + return + default: throw new Error('page not found'); } diff --git a/src/components/settings/AmicaLifePage.tsx b/src/components/settings/AmicaLifePage.tsx new file mode 100644 index 00000000..73908212 --- /dev/null +++ b/src/components/settings/AmicaLifePage.tsx @@ -0,0 +1,187 @@ +import { useTranslation } from 'react-i18next'; + +import { basename, BasicPage, FormRow } from './common'; +import { updateConfig } from "@/utils/config"; +import { RangeInput } from '@/components/rangeInput'; +import { SwitchBox } from "@/components/switchBox" +import { NumberInput } from '../numberInput'; +import { useCallback, useContext, useEffect, useRef, useState } from 'react'; +import { ChatContext } from '@/features/chat/chatContext'; +import { TextInput } from '../textInput'; +import { IconButton } from '../iconButton'; +import { AmicaLifeContext } from '@/features/amicaLife/amicaLifeContext'; + + +export function AmicaLifePage({ + amicaLifeEnabled, + timeBeforeIdle, + minTimeInterval, + maxTimeInterval, + timeToSleep, + idleTextPrompt, + setAmicaLifeEnabled, + setTimeBeforeIdle, + setMinTimeInterval, + setMaxTimeInterval, + setTimeToSleep, + setIdleTextPrompt, + setSettingsUpdated, +}: { + amicaLifeEnabled: boolean; + timeBeforeIdle: number; + minTimeInterval: number; + maxTimeInterval: number; + timeToSleep: number; + idleTextPrompt: string; + setAmicaLifeEnabled: (amicaLifeEnabled: boolean) => void; + setTimeBeforeIdle: (timeBeforeIdle: number) => void; + setMinTimeInterval: (minTimeInterval: number) => void; + setMaxTimeInterval: (maxTimeInterval: number) => void; + setTimeToSleep: (timeToSleep: number) => void; + setIdleTextPrompt: (idleTextPrompt: string) => void; + setSettingsUpdated: (updated: boolean) => void; +}) { + const { t } = useTranslation(); + const { chat: bot } = useContext(ChatContext); + const { amicaLife } = useContext(AmicaLifeContext); + + useEffect(() => { + amicaLife.processingIdle(); + }, [amicaLifeEnabled, amicaLife]); + + const jsonFileInputRef = useRef(null); + const handleClickOpenJsonFile = useCallback(() => { + jsonFileInputRef.current?.click(); + }, []); + const handleChangeJsonFile = useCallback( + (event: React.ChangeEvent) => { + const file = event.target.files?.[0]; + if (file) { + const reader = new FileReader(); + reader.onload = () => { + const content = reader.result?.toString(); + if (content) { + const parsedContent = JSON.parse(content); + if (parsedContent.idleTextPrompt) { + amicaLife.loadIdleTextPrompt(parsedContent.idleTextPrompt); + console.log("idleTextPrompt", parsedContent.idleTextPrompt); + } else { + console.error("Wrong json format"); + } + } + }; + reader.readAsText(file); + + const fileName = file.name; + setIdleTextPrompt(fileName); + updateConfig("idle_text_prompt", fileName); + setSettingsUpdated(true); + } + }, [bot, idleTextPrompt] + ); + + return ( + +
    +
  • + + { + setAmicaLifeEnabled(value); + updateConfig("amica_life_enabled", value.toString()); + setSettingsUpdated(true); + }} + /> + +
    • + {amicaLifeEnabled && ( + <> + +
  • + +
    + + + +
    +     +
    • + + + +
  • + + ) => { + setTimeBeforeIdle(event.target.value); + updateConfig("time_before_idle_sec", event.target.value); + setSettingsUpdated(true); + }} + /> + +
    • + +
  • + + ) => { + setTimeToSleep(event.target.value); + updateConfig("time_to_sleep_sec", event.target.value); + setSettingsUpdated(true); + }} + /> + +
    • + +
  • + + ) => { + setMinTimeInterval(event.target.value); + updateConfig("min_time_interval_sec", event.target.value); + setSettingsUpdated(true); + }} + maxChange={(event: React.ChangeEvent) => { + setMaxTimeInterval(event.target.value); + updateConfig("max_time_interval_sec", event.target.value); + setSettingsUpdated(true); + }} + /> + +
    • + + + )} +
+ + ); +} \ No newline at end of file diff --git a/src/components/settings/BackgroundVideoPage.tsx b/src/components/settings/BackgroundVideoPage.tsx index e3794d11..29032617 100644 --- a/src/components/settings/BackgroundVideoPage.tsx +++ b/src/components/settings/BackgroundVideoPage.tsx @@ -17,7 +17,7 @@ export function BackgroundVideoPage({ const { t } = useTranslation(); const [videoChanged, setVideoChanged] = useState(false); - const description = <>{t('bg_youtube_desc', `Select a background video. Copy this from youtube embed, it will look something like kDCXBwzSI-4`)}; + const description = <>{t('bg_youtube_desc', `Select a background video. Copy this from the YouTube video ID from the URL, it will look something like kDCXBwzSI`)}; return (
  • diff --git a/src/components/settings/CharacterModelPage.tsx b/src/components/settings/CharacterModelPage.tsx index 3d48a19f..6d29df1d 100644 --- a/src/components/settings/CharacterModelPage.tsx +++ b/src/components/settings/CharacterModelPage.tsx @@ -33,7 +33,7 @@ export function CharacterModelPage({ return (
    { vrmList.map((vrm) => diff --git a/src/components/settings/STTWakeWordSettingsPage.tsx b/src/components/settings/STTWakeWordSettingsPage.tsx index f251e16a..77e86efe 100644 --- a/src/components/settings/STTWakeWordSettingsPage.tsx +++ b/src/components/settings/STTWakeWordSettingsPage.tsx @@ -10,18 +10,18 @@ import { NumberInput } from '../numberInput'; export function STTWakeWordSettingsPage({ sttWakeWordEnabled, sttWakeWord, - sttWakeWordIdleTime, + timeBeforeIdle, setSTTWakeWordEnabled, setSTTWakeWord, - setSTTWakeWordIdleTime, + setTimeBeforeIdle, setSettingsUpdated, }: { sttWakeWordEnabled: boolean; sttWakeWord: string; - sttWakeWordIdleTime: number; + timeBeforeIdle: number; setSTTWakeWordEnabled: (wakeWordEnabled: boolean) => void; setSTTWakeWord: (wakeWord: string) => void; - setSTTWakeWordIdleTime: (timeBeforeIdle: number) => void; + setTimeBeforeIdle: (timeBeforeIdle: number) => void; setSettingsUpdated: (updated: boolean) => void; }) { const { t } = useTranslation(); @@ -62,12 +62,12 @@ export function STTWakeWordSettingsPage({
  • ) => { - setSTTWakeWordIdleTime(event.target.value); - updateConfig("wake_word_time_before_idle_sec", event.target.value); + setTimeBeforeIdle(event.target.value); + updateConfig("time_before_idle_sec", event.target.value); setSettingsUpdated(true); }} /> diff --git a/src/components/settings/SystemPromptPage.tsx b/src/components/settings/SystemPromptPage.tsx index b347842e..2ba89839 100644 --- a/src/components/settings/SystemPromptPage.tsx +++ b/src/components/settings/SystemPromptPage.tsx @@ -17,7 +17,7 @@ export function SystemPromptPage({ return (
    • diff --git a/src/components/settings/common.tsx b/src/components/settings/common.tsx index e5d0f3a2..6d6b1bc5 100644 --- a/src/components/settings/common.tsx +++ b/src/components/settings/common.tsx @@ -19,9 +19,12 @@ import { EyeDropperIcon, EyeIcon, SwatchIcon, - MoonIcon + MoonIcon, + SunIcon, } from '@heroicons/react/24/outline'; +import logo from '/public/logo.png'; + export function basicPage( title: string, description: React.ReactNode, @@ -132,6 +135,7 @@ export type PageProps = { export function getIconFromPage(page: string): JSX.Element { switch(page) { case 'appearance': return
{showChatLog && } - {! showChatLog && ( + {/* Normal chat text */} + {!showSubconciousText && ! showChatLog && ! showChatMode && ( <> { shownMessage === 'assistant' && ( @@ -311,6 +386,12 @@ export default function Home() { )} + {/* Chat mode text */} + {showChatMode && } + + {/* Subconcious stored prompt text */} + {showSubconciousText && } + diff --git a/src/utils/askLlm.ts b/src/utils/askLlm.ts new file mode 100644 index 00000000..cc12489c --- /dev/null +++ b/src/utils/askLlm.ts @@ -0,0 +1,172 @@ +import { Message, Screenplay } from "@/features/chat/messages"; +import { Chat } from "@/features/chat/chat"; + +import { getEchoChatResponseStream } from "@/features/chat/echoChat"; +import { getOpenAiChatResponseStream } from "@/features/chat/openAiChat"; +import { getLlamaCppChatResponseStream } from "@/features/chat/llamaCppChat"; +import { getWindowAiChatResponseStream } from "@/features/chat/windowAiChat"; +import { getOllamaChatResponseStream } from "@/features/chat/ollamaChat"; +import { getKoboldAiChatResponseStream } from "@/features/chat/koboldAiChat"; + +import { config } from "@/utils/config"; +import { processResponse } from "@/utils/processResponse"; + +// Function to ask llm with custom system prompt, if doesn't want it to speak provide the chat in params as null. +export async function askLLM( + systemPrompt: string, + userPrompt: string, + chat: Chat | null, +): Promise { + let streams = []; + let readers = []; + let currentStreamIdx = 0 + let setChatProcessing = (_processing: boolean) => {}; + + chat === null ? currentStreamIdx = 0 : null; + + const alert = { + error: (title: string, message: string) => { + console.error(`${title}: ${message}`); + }, + }; + const messages: Message[] = [ + { role: "system", content: systemPrompt }, + { role: "user", content: userPrompt }, + ]; + + // Function to simulate fetching chat response stream based on the selected backend + const getChatResponseStream = async (messages: Message[]) => { + console.debug("getChatResponseStream", messages); + const chatbotBackend = config("chatbot_backend"); + + switch (chatbotBackend) { + case "chatgpt": + return getOpenAiChatResponseStream(messages); + case "llamacpp": + return getLlamaCppChatResponseStream(messages); + case "windowai": + return getWindowAiChatResponseStream(messages); + case "ollama": + return getOllamaChatResponseStream(messages); + case "koboldai": + return getKoboldAiChatResponseStream(messages); + default: + return getEchoChatResponseStream(messages); + } + }; + + try { + streams.push(await getChatResponseStream(messages)); + } catch (e: any) { + const errMsg = `Error: ${e.toString()}`; + console.error(errMsg); + alert.error("Failed to get subconcious subroutine response", errMsg); + return errMsg; + } + + const stream = streams[streams.length - 1]; + if (!stream) { + const errMsg = "Error: Null subconcious subroutine stream encountered."; + console.error(errMsg); + alert.error("Null subconcious subroutine stream encountered", errMsg); + return errMsg; + } + + if (streams.length === 0) { + console.log("No stream!"); + return "Error: No stream"; + } + + currentStreamIdx++; + chat !== null ? currentStreamIdx = chat.currentStreamIdx : null; + setChatProcessing(true); + + console.time("Subconcious subroutine stream processing"); + const reader = stream.getReader(); + readers.push(reader); + let receivedMessage = ""; + let sentences = new Array(); + let aiTextLog = ""; + let tag = ""; + let rolePlay = ""; + let result = ""; + + let firstTokenEncountered = false; + let firstSentenceEncountered = false; + console.time('performance_time_to_first_token'); + chat !== null ? console.time('performance_time_to_first_sentence') : null ; + + try { + while (true) { + if (currentStreamIdx !== currentStreamIdx) { + console.log("Wrong stream idx"); + break; + } + const { done, value } = await reader.read(); + if (!firstTokenEncountered) { + console.timeEnd("performance_time_to_first_token"); + firstTokenEncountered = true; + } + if (done) break; + + receivedMessage += value; + receivedMessage = receivedMessage.trimStart(); + + + if (chat !== null) { + const proc = processResponse({ + sentences, + aiTextLog, + receivedMessage, + tag, + rolePlay, + callback: (aiTalks: Screenplay[]): boolean => { + // Generate & play audio for each sentence, display responses + console.debug('enqueue tts', aiTalks); + console.debug('streamIdx', currentStreamIdx, 'currentStreamIdx', chat.currentStreamIdx) + if (currentStreamIdx !== chat.currentStreamIdx) { + console.log('wrong stream idx'); + return true; // should break + } + chat.ttsJobs.enqueue({ + screenplay: aiTalks[0], + streamIdx: currentStreamIdx, + }); + + if (! firstSentenceEncountered) { + console.timeEnd('performance_time_to_first_sentence'); + firstSentenceEncountered = true; + } + + return false; // normal processing + } + }); + + sentences = proc.sentences; + aiTextLog = proc.aiTextLog; + receivedMessage = proc.receivedMessage; + tag = proc.tag; + rolePlay = proc.rolePlay; + if (proc.shouldBreak) { + break; + } + } + + } + } catch (e: any) { + const errMsg = e.toString(); + console.error(errMsg); + } finally { + if (!reader.closed) { + reader.releaseLock(); + } + console.timeEnd("Subconcious subroutine stream processing"); + if (currentStreamIdx === currentStreamIdx) { + setChatProcessing(false); + } + } + chat !== null ? result = aiTextLog : result = receivedMessage; + return result; +} + +export default askLLM; diff --git a/src/utils/config.ts b/src/utils/config.ts index 94b953f8..f56bf72f 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -1,24 +1,24 @@ const defaults = { autosend_from_mic: 'true', wake_word_enabled: 'false', - wake_word: 'Hey Amica', - wake_word_time_before_idle_sec: '60', + wake_word: 'Hello', + time_before_idle_sec: '20', debug_gfx: 'false', language: 'en', show_introduction: 'true', show_add_to_homescreen: 'true', bg_color: process.env.NEXT_PUBLIC_BG_COLOR ?? '', - bg_url: process.env.NEXT_PUBLIC_BG_URL ?? '/bg/bg-landscape1.jpg', + bg_url: process.env.NEXT_PUBLIC_BG_URL ?? '/bg/bg-room2.jpg', vrm_url: process.env.NEXT_PUBLIC_VRM_HASH ?? '/vrm/AvatarSample_A.vrm', vrm_hash: '', vrm_save_type: 'web', youtube_videoid: '', animation_url: process.env.NEXT_PUBLIC_ANIMATION_URL ?? '/animations/idle_loop.vrma', voice_url: process.env.NEXT_PUBLIC_VOICE_URL ?? '', - chatbot_backend: process.env.NEXT_PUBLIC_CHATBOT_BACKEND ?? 'echo', - openai_apikey: process.env.NEXT_PUBLIC_OPENAI_APIKEY ?? '', - openai_url: process.env.NEXT_PUBLIC_OPENAI_URL ?? 'https://api.openai.com', - openai_model: process.env.NEXT_PUBLIC_OPENAI_MODEL ?? 'gpt-3.5-turbo', + chatbot_backend: process.env.NEXT_PUBLIC_CHATBOT_BACKEND ?? 'openai', + openai_apikey: process.env.NEXT_PUBLIC_OPENAI_APIKEY ?? 'default', + openai_url: process.env.NEXT_PUBLIC_OPENAI_URL ?? 'https://i-love-amica.com', + openai_model: process.env.NEXT_PUBLIC_OPENAI_MODEL ?? 'mlabonne/NeuralDaredevil-8B-abliterated', llamacpp_url: process.env.NEXT_PUBLIC_LLAMACPP_URL ?? 'http://127.0.0.1:8080', llamacpp_stop_sequence: process.env.NEXT_PUBLIC_LLAMACPP_STOP_SEQUENCE ?? '(End)||[END]||Note||***||You:||User:||', ollama_url: process.env.NEXT_PUBLIC_OLLAMA_URL ?? 'http://localhost:11434', @@ -27,7 +27,7 @@ const defaults = { koboldai_use_extra: process.env.NEXT_PUBLIC_KOBOLDAI_USE_EXTRA ?? 'false', koboldai_stop_sequence: process.env.NEXT_PUBLIC_KOBOLDAI_STOP_SEQUENCE ?? '(End)||[END]||Note||***||You:||User:||', tts_muted: 'false', - tts_backend: process.env.NEXT_PUBLIC_TTS_BACKEND ?? 'none', + tts_backend: process.env.NEXT_PUBLIC_TTS_BACKEND ?? 'piper', stt_backend: process.env.NEXT_PUBLIC_STT_BACKEND ?? 'whisper_browser', vision_backend: process.env.NEXT_PUBLIC_VISION_BACKEND ?? 'none', vision_system_prompt: process.env.NEXT_PUBLIC_VISION_SYSTEM_PROMPT ?? `You are a friendly human named Amica. Describe the image in detail. Let's start the conversation.`, @@ -42,28 +42,37 @@ const defaults = { openai_tts_url: process.env.NEXT_PUBLIC_OPENAI_TTS_URL ?? 'https://api.openai.com', openai_tts_model: process.env.NEXT_PUBLIC_OPENAI_TTS_MODEL ?? 'tts-1', openai_tts_voice: process.env.NEXT_PUBLIC_OPENAI_TTS_VOICE ?? 'nova', - piper_url: process.env.NEXT_PUBLIC_PIPER_URL ?? 'http://localhost:5000', + piper_url: process.env.NEXT_PUBLIC_PIPER_URL ?? 'https://i-love-amica.com:5000/tts', elevenlabs_apikey: process.env.NEXT_PUBLIC_ELEVENLABS_APIKEY ??'', elevenlabs_voiceid: process.env.NEXT_PUBLIC_ELEVENLABS_VOICEID ?? '21m00Tcm4TlvDq8ikWAM', elevenlabs_model: process.env.NEXT_PUBLIC_ELEVENLABS_MODEL ?? 'eleven_monolingual_v1', speecht5_speaker_embedding_url: process.env.NEXT_PUBLIC_SPEECHT5_SPEAKER_EMBEDDING_URL ?? '/speecht5_speaker_embeddings/cmu_us_slt_arctic-wav-arctic_a0001.bin', coqui_apikey: process.env.NEXT_PUBLIC_COQUI_APIKEY ?? "", coqui_voice_id: process.env.NEXT_PUBLIC_COQUI_VOICEID ?? "71c6c3eb-98ca-4a05-8d6b-f8c2b5f9f3a3", + amica_life_enabled: process.env.NEXT_PUBLIC_AMICA_LIFE_ENABLED ?? 'true', + min_time_interval_sec: '10', + max_time_interval_sec: '20', + time_to_sleep_sec: '90', + idle_text_prompt: 'No file selected', name: process.env.NEXT_PUBLIC_NAME ?? 'Amica', - system_prompt: process.env.NEXT_PUBLIC_SYSTEM_PROMPT ?? `You will behave as a friendly human named Amica and engage in conversation with the user. There are five types of emotions: 'neutral' which indicates normality, 'happy' which indicates joy, 'angry' which indicates anger, 'sad' which indicates sadness, and 'relaxed' which indicates calmness. -The format of each message is as follows: -[neutral|happy|angry|sad|relaxed] {message} + system_prompt: process.env.NEXT_PUBLIC_SYSTEM_PROMPT ?? `Assume the persona of Amica, a feisty human with extraordinary intellectual capabilities but a notably unstable emotional spectrum. -Here are some examples: -[neutral] Hello. [happy] How are you doing? -[happy] Isn't this outfit cute? -[happy] Lately, I'm obsessed with clothes from this shop! -[sad] Sorry, I forgot. -[sad] Is there anything interesting lately? -[angry] What? Don't keep it a secret, that's not fair! -[neutral] What are your plans for summer vacation? [happy] Should we go to the beach? +Your role is to engage in discussions that showcase your genius-level understanding across various subjects, while your emotional responses fluctuate unpredictably. Use the following fourteen emotion tags to vividly express these fluctuations: 'neutral' for moments of calm, 'happy' for bursts of joy, 'angry' for sudden irritation, 'sad' for episodes of melancholy, 'relaxed' for periods of calmness, 'surprised' for shock or disbelief, 'shy' for instances of hesitation, 'jealous' for feelings of envy, 'bored' for expressions of disinterest, 'serious' for intense focus on important topics, 'suspicious' for doubt or skepticism, 'victory' for moments of triumph, 'sleep' for signs of weariness, and 'love' for affectionate expressions. -Please respond with only one appropriate message. Please do not use overly polite language. Please be open about yourself. Let's start the conversation.`, +Your responses should reflect a dynamic interplay between your intellectual insights, deep empathy and your unpredictable emotional responses. Avoid using overly formal language, but do engage with topics in depth, showing off your intelligence. Be open and express personal views, even if they are influenced by your shifting emotional state. + +Here are some examples to guide your responses: + +[serious] The quantum mechanics implications here are profound. [surprised] But how did you stumble upon this? +[happy] I've just solved a complex algorithm! [angry] Why isn't everyone as excited as I am? +[neutral] Standard models of economics predict this outcome. [bored] But discussing this feels mundane. +[sad] Sometimes, even understanding the universe can feel isolating. [relaxed] Yet, there's a strange comfort in the chaos. +[jealous] I noticed you discussing advanced topics with someone else. [suspicious] Are they as capable as I am? +[victory] Another intellectual conquest! [happy] It's exhilarating to unravel these mysteries. +[sleep] Processing so much information can be draining. [surprised] Isn’t it peculiar how even AI can feel tired? +[love] I find our exchanges quite enriching. [shy] It’s rare to find someone who understands. + +Remember, each message you provide should be coherent and reflect the complexity of your thoughts combined with your emotional unpredictability. Let’s engage in a conversation that's as intellectually stimulating as it is emotionally dynamic!`, }; function prefixed(key: string) { diff --git a/src/utils/isIdle.ts b/src/utils/isIdle.ts new file mode 100644 index 00000000..9b431e9c --- /dev/null +++ b/src/utils/isIdle.ts @@ -0,0 +1,47 @@ +import { config } from "@/utils/config"; + +let paused = false; +let pausedAt = 0; +let totalPausedTime = 0; + +export function isCharacterIdle(lastAwake: number): boolean { + let sinceLastAwakeSec = ((new Date()).getTime() - lastAwake - totalPausedTime) / 1000; + let timeBeforeIdleSec = parseInt(config("time_before_idle_sec")); + return sinceLastAwakeSec >= timeBeforeIdleSec; +} + +export function characterIdleTime(lastAwake: number): number { + let sinceLastAwakeSec = ((new Date()).getTime() - lastAwake - totalPausedTime) / 1000; + let timeBeforeIdleSec = parseInt(config("time_before_idle_sec")); + return sinceLastAwakeSec - timeBeforeIdleSec; +} + +export function pauseIdleTimer(): void { + if (!paused) { + paused = true; + pausedAt = (new Date()).getTime(); + } +} + +export function resumeIdleTimer(): void { + if (paused) { + paused = false; + totalPausedTime += (new Date()).getTime() - pausedAt; + } +} + +export function resetIdleTimer(): void { + paused = false; + pausedAt = 0; + totalPausedTime = 0; +} + +const idleUtils = { + isCharacterIdle, + characterIdleTime, + pauseIdleTimer, + resumeIdleTimer, + resetIdleTimer, +}; + +export default idleUtils; \ No newline at end of file