From f35184a7e5edd033532ca6f2d991830ca6031b58 Mon Sep 17 00:00:00 2001 From: DJDavid98 Date: Sat, 16 Dec 2023 16:23:53 +0100 Subject: [PATCH] Add pronoun-based TTS female voice selection --- package-lock.json | 6 ++-- src/js/chat/Chat.tsx | 44 ++++++++++++++++++------- src/js/hooks/use-tts.ts | 60 +++++++++++++++++++++++++++++------ src/js/utils/chat-messages.ts | 14 ++++++++ 4 files changed, 100 insertions(+), 24 deletions(-) diff --git a/package-lock.json b/package-lock.json index 33bb854..c93092a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3707,9 +3707,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001498", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001498.tgz", - "integrity": "sha512-LFInN2zAwx3ANrGCDZ5AKKJroHqNKyjXitdV5zRIVIaQlXKj3GmxUKagoKsjqUfckpAObPCEWnk5EeMlyMWcgw==", + "version": "1.0.30001570", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001570.tgz", + "integrity": "sha512-+3e0ASu4sw1SWaoCtvPeyXp+5PsjigkSt8OXZbF9StH5pQWbxEjLAZE3n8Aup5udop1uRiKA7a4utUk/uoSpUw==", "dev": true, "funding": [ { diff --git a/src/js/chat/Chat.tsx b/src/js/chat/Chat.tsx index 588adc7..ac52824 100644 --- a/src/js/chat/Chat.tsx +++ b/src/js/chat/Chat.tsx @@ -5,8 +5,11 @@ import { ChatSystemMessage, ChatUserMessage, DisplayableMessage, - getChatWebsocketMessageTimestamp, isSongRequest, removeEmotes, - SystemMessageType, tokenizeMessage, ttsNameSubstitutions + getChatWebsocketMessageTimestamp, + isSongRequest, + removeEmotes, + SystemMessageType, + tokenizeMessage, } from '../utils/chat-messages'; import { ChatMessage } from './ChatMessage'; import DurationUnitFormat from 'intl-unofficial-duration-unit-format'; @@ -56,7 +59,7 @@ export const Chat: FC = () => { message: `Joined room #${room}`, }; addMessage(data); - tts.readText(data.message); + tts.readText({ message: data.message }); }, follow(message) { const data: ChatSystemMessage = { @@ -69,7 +72,7 @@ export const Chat: FC = () => { data.message = data.message.replace('!', `, that's ${message.total} in total!`); } addMessage(data); - tts.readText(data.message); + tts.readText({ message: data.message }); }, donation(message) { const data: ChatSystemMessage = { @@ -79,7 +82,7 @@ export const Chat: FC = () => { message: `${message.from} just donated!`, }; addMessage(data); - tts.readText(data.message); + tts.readText({ message: data.message }); }, chat(message) { if (!chatSongPreviews && isSongRequest(message.message)) { @@ -90,8 +93,9 @@ export const Chat: FC = () => { tokens, emoteOnly } = tokenizeMessage(message.message, message.tags.emotes); + const messageId = message.tags.id || window.crypto.randomUUID(); const data: ChatUserMessage = { - id: message.tags.id || window.crypto.randomUUID(), + id: messageId, name: message.name, username: message.username, nameColor: message.tags.color, @@ -104,7 +108,12 @@ export const Chat: FC = () => { }; addMessage(data); if (!emoteOnly) { - tts.readText(`${ttsNameSubstitutions(message.username)}. ${removeEmotes(tokens)}`); + tts.readText({ + id: messageId, + name: message.username, + message: removeEmotes(tokens), + pronouns: message.pronouns + }); } }, clearChat() { @@ -125,7 +134,7 @@ export const Chat: FC = () => { message: `Chat connected`, }; addMessage(data); - tts.readText(data.message); + tts.readText({ message: data.message }); }, disconnect() { const data: ChatSystemMessage = { @@ -135,7 +144,7 @@ export const Chat: FC = () => { message: `Chat disconnected`, }; addMessage(data); - tts.readText(data.message); + tts.readText({ message: data.message }); }, ban(message) { let data: ChatSystemMessage | undefined; @@ -152,17 +161,28 @@ export const Chat: FC = () => { message: `${message.username} has been ${action}${bannedBy}${reason}`, }; } - const filterMessagesFromUser = (oldState: Array) => ( - oldState.filter(m => 'username' in m ? m.username !== message.username : true) - ); + const filteredMessageIds: string[] = []; + const filterMessagesFromUser = (oldState: Array) => { + return ( + oldState.filter(m => { + const keepMessage = 'username' in m ? m.username !== message.username : true; + if (!keepMessage && m.id) { + filteredMessageIds.push(m.id); + } + return keepMessage; + }) + ); + }; if (data) { addMessage(data, filterMessagesFromUser); } else { setMessages((oldState) => filterMessagesFromUser(oldState)); } + tts.clearIds(filteredMessageIds); }, messageDeleted(data) { setMessages((oldState) => oldState.filter(message => message.id !== data.id)); + tts.clearIds([data.id]); } }; for (const eventKey in listeners) { diff --git a/src/js/hooks/use-tts.ts b/src/js/hooks/use-tts.ts index 53c3367..8efd5bb 100644 --- a/src/js/hooks/use-tts.ts +++ b/src/js/hooks/use-tts.ts @@ -1,18 +1,32 @@ import { useCallback, useEffect, useRef } from 'react'; import { VoiceData } from '../model/eleven-labs'; -import { ttsMessageSubstitutions } from '../utils/chat-messages'; +import { + mapPronounsToGender, + ttsMessageSubstitutions, + ttsNameSubstitutions, + VoiceGender +} from '../utils/chat-messages'; + +interface TtsInput { + id?: string; + name?: string; + message: string; + pronouns?: string[]; +} export interface TtsApi { - readText: (input: string) => Promise; + readText: (input: TtsInput) => Promise; clearQueue: VoidFunction; + clearIds: (ids: string[]) => void; } export const useTts = (token: string | null, enabled: boolean | null): TtsApi => { const voicesRef = useRef([]); const mountedRef = useRef(true); - const textQueueRef = useRef([]); + const textQueueRef = useRef([]); + const currentlyReadingRef = useRef(null); const audioPlayerRef = useRef(null); - const getVoice = useCallback((targetGender = 'male') => { + const getVoice = useCallback((targetGender: VoiceGender) => { return voicesRef.current.find(voice => { const { age, gender, 'use case': useCase } = voice.labels; return age === 'young' && gender === targetGender && useCase === 'narration'; @@ -28,6 +42,9 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi => } audioPlayerRef.current = null; } + if (currentlyReadingRef.current) { + currentlyReadingRef.current = null; + } }, []); const processQueue = useCallback(async (debugSource: string) => { @@ -42,8 +59,8 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi => return; } - // TODO Base voice on user pronouns - const voice = getVoice(); + const firstQueueItem = textQueueRef.current[0] as TtsInput; + const voice = getVoice(mapPronounsToGender(firstQueueItem?.pronouns)); if (!voice) { console.error('No voice found (%s)', debugSource); return; @@ -57,7 +74,9 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi => } audioPlayerRef.current = new Audio(); - const text = textQueueRef.current.shift() as string; + const ttsInput = textQueueRef.current.shift() as TtsInput; + currentlyReadingRef.current = ttsInput; + const textToRead = (ttsInput.name ? `${ttsNameSubstitutions(ttsInput.name)}. ` : '') + ttsMessageSubstitutions(ttsInput.message); const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voice_id}/stream`, { method: 'POST', headers: { @@ -65,7 +84,7 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi => 'xi-api-key': token, 'accept': 'audio/mpeg' }, - body: JSON.stringify({ text: ttsMessageSubstitutions(text) }) + body: JSON.stringify({ text: textToRead }) }); const audioBlob = await response.blob(); @@ -83,7 +102,7 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi => }); }, [enabled, token, getVoice, clearPlayingAudio]); - const readText = useCallback(async (text: string) => { + const readText = useCallback(async (text: TtsInput) => { if (!enabled) return; textQueueRef.current.push(text); @@ -95,6 +114,28 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi => clearPlayingAudio(); }, [clearPlayingAudio]); + const clearIds = useCallback((clearedIds: string[]) => { + if (clearedIds.length === 0) return; + + const clearedIdsSet = new Set(clearedIds); + if (currentlyReadingRef.current) { + const currentId = currentlyReadingRef.current?.id; + if (currentId && clearedIdsSet.has(currentId)) { + clearPlayingAudio(); + } + } + + if (textQueueRef.current.length > 0) { + textQueueRef.current = textQueueRef.current.filter(queueItem => { + return !queueItem.id || !clearedIdsSet.has(queueItem.id); + }); + } + + if (!currentlyReadingRef.current) { + void processQueue('clearIds'); + } + }, [clearPlayingAudio, processQueue]); + useEffect(() => { if (!enabled || !token || voicesRef.current.length) return; @@ -120,5 +161,6 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi => return { readText, clearQueue, + clearIds, }; }; diff --git a/src/js/utils/chat-messages.ts b/src/js/utils/chat-messages.ts index 0235c32..7d602db 100644 --- a/src/js/utils/chat-messages.ts +++ b/src/js/utils/chat-messages.ts @@ -253,3 +253,17 @@ export const getAccentColor = (message: ChatUserMessage | ChatSystemMessage) => return message.nameColor; }; + + +export type VoiceGender = 'male' | 'female'; + +export const mapPronounsToGender = (pronouns?: string[]): VoiceGender => { + const firstPronoun = pronouns?.[0]; + switch (firstPronoun) { + case 'She/Her': + case 'She/They': + return 'female'; + default: + return 'male'; + } +};