Skip to content

Commit

Permalink
Add pronoun-based TTS female voice selection
Browse files Browse the repository at this point in the history
  • Loading branch information
@DJDavid98
DJDavid98 committed Dec 16, 2023
1 parent cf2dc3d commit f35184a
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 24 deletions.
6 changes: 3 additions & 3 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

44 changes: 32 additions & 12 deletions src/js/chat/Chat.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ import {
ChatSystemMessage,
ChatUserMessage,
DisplayableMessage,
getChatWebsocketMessageTimestamp, isSongRequest, removeEmotes,
SystemMessageType, tokenizeMessage, ttsNameSubstitutions
getChatWebsocketMessageTimestamp,
isSongRequest,
removeEmotes,
SystemMessageType,
tokenizeMessage,
} from '../utils/chat-messages';
import { ChatMessage } from './ChatMessage';
import DurationUnitFormat from 'intl-unofficial-duration-unit-format';
Expand Down Expand Up @@ -56,7 +59,7 @@ export const Chat: FC = () => {
message: `Joined room #${room}`,
};
addMessage(data);
tts.readText(data.message);
tts.readText({ message: data.message });
},
follow(message) {
const data: ChatSystemMessage = {
Expand All @@ -69,7 +72,7 @@ export const Chat: FC = () => {
data.message = data.message.replace('!', `, that's ${message.total} in total!`);
}
addMessage(data);
tts.readText(data.message);
tts.readText({ message: data.message });
},
donation(message) {
const data: ChatSystemMessage = {
Expand All @@ -79,7 +82,7 @@ export const Chat: FC = () => {
message: `${message.from} just donated!`,
};
addMessage(data);
tts.readText(data.message);
tts.readText({ message: data.message });
},
chat(message) {
if (!chatSongPreviews && isSongRequest(message.message)) {
Expand All @@ -90,8 +93,9 @@ export const Chat: FC = () => {
tokens,
emoteOnly
} = tokenizeMessage(message.message, message.tags.emotes);
const messageId = message.tags.id || window.crypto.randomUUID();
const data: ChatUserMessage = {
id: message.tags.id || window.crypto.randomUUID(),
id: messageId,
name: message.name,
username: message.username,
nameColor: message.tags.color,
Expand All @@ -104,7 +108,12 @@ export const Chat: FC = () => {
};
addMessage(data);
if (!emoteOnly) {
tts.readText(`${ttsNameSubstitutions(message.username)}. ${removeEmotes(tokens)}`);
tts.readText({
id: messageId,
name: message.username,
message: removeEmotes(tokens),
pronouns: message.pronouns
});
}
},
clearChat() {
Expand All @@ -125,7 +134,7 @@ export const Chat: FC = () => {
message: `Chat connected`,
};
addMessage(data);
tts.readText(data.message);
tts.readText({ message: data.message });
},
disconnect() {
const data: ChatSystemMessage = {
Expand All @@ -135,7 +144,7 @@ export const Chat: FC = () => {
message: `Chat disconnected`,
};
addMessage(data);
tts.readText(data.message);
tts.readText({ message: data.message });
},
ban(message) {
let data: ChatSystemMessage | undefined;
Expand All @@ -152,17 +161,28 @@ export const Chat: FC = () => {
message: `${message.username} has been ${action}${bannedBy}${reason}`,
};
}
const filterMessagesFromUser = (oldState: Array<DisplayableMessage>) => (
oldState.filter(m => 'username' in m ? m.username !== message.username : true)
);
const filteredMessageIds: string[] = [];
const filterMessagesFromUser = (oldState: Array<DisplayableMessage>) => {
return (
oldState.filter(m => {
const keepMessage = 'username' in m ? m.username !== message.username : true;
if (!keepMessage && m.id) {
filteredMessageIds.push(m.id);
}
return keepMessage;
})
);
};
if (data) {
addMessage(data, filterMessagesFromUser);
} else {
setMessages((oldState) => filterMessagesFromUser(oldState));
}
tts.clearIds(filteredMessageIds);
},
messageDeleted(data) {
setMessages((oldState) => oldState.filter(message => message.id !== data.id));
tts.clearIds([data.id]);
}
};
for (const eventKey in listeners) {
Expand Down
60 changes: 51 additions & 9 deletions src/js/hooks/use-tts.ts
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,32 @@
import { useCallback, useEffect, useRef } from 'react';
import { VoiceData } from '../model/eleven-labs';
import { ttsMessageSubstitutions } from '../utils/chat-messages';
import {
mapPronounsToGender,
ttsMessageSubstitutions,
ttsNameSubstitutions,
VoiceGender
} from '../utils/chat-messages';

interface TtsInput {
id?: string;
name?: string;
message: string;
pronouns?: string[];
}

export interface TtsApi {
readText: (input: string) => Promise<void>;
readText: (input: TtsInput) => Promise<void>;
clearQueue: VoidFunction;
clearIds: (ids: string[]) => void;
}

export const useTts = (token: string | null, enabled: boolean | null): TtsApi => {
const voicesRef = useRef<VoiceData['voices']>([]);
const mountedRef = useRef(true);
const textQueueRef = useRef<string[]>([]);
const textQueueRef = useRef<TtsInput[]>([]);
const currentlyReadingRef = useRef<TtsInput | null>(null);
const audioPlayerRef = useRef<HTMLAudioElement | null>(null);
const getVoice = useCallback((targetGender = 'male') => {
const getVoice = useCallback((targetGender: VoiceGender) => {
return voicesRef.current.find(voice => {
const { age, gender, 'use case': useCase } = voice.labels;
return age === 'young' && gender === targetGender && useCase === 'narration';
Expand All @@ -28,6 +42,9 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi =>
}
audioPlayerRef.current = null;
}
if (currentlyReadingRef.current) {
currentlyReadingRef.current = null;
}
}, []);

const processQueue = useCallback(async (debugSource: string) => {
Expand All @@ -42,8 +59,8 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi =>
return;
}

// TODO Base voice on user pronouns
const voice = getVoice();
const firstQueueItem = textQueueRef.current[0] as TtsInput;
const voice = getVoice(mapPronounsToGender(firstQueueItem?.pronouns));
if (!voice) {
console.error('No voice found (%s)', debugSource);
return;
Expand All @@ -57,15 +74,17 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi =>
}
audioPlayerRef.current = new Audio();

const text = textQueueRef.current.shift() as string;
const ttsInput = textQueueRef.current.shift() as TtsInput;
currentlyReadingRef.current = ttsInput;
const textToRead = (ttsInput.name ? `${ttsNameSubstitutions(ttsInput.name)}. ` : '') + ttsMessageSubstitutions(ttsInput.message);
const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voice_id}/stream`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'xi-api-key': token,
'accept': 'audio/mpeg'
},
body: JSON.stringify({ text: ttsMessageSubstitutions(text) })
body: JSON.stringify({ text: textToRead })
});
const audioBlob = await response.blob();

Expand All @@ -83,7 +102,7 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi =>
});
}, [enabled, token, getVoice, clearPlayingAudio]);

const readText = useCallback(async (text: string) => {
const readText = useCallback(async (text: TtsInput) => {
if (!enabled) return;

textQueueRef.current.push(text);
Expand All @@ -95,6 +114,28 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi =>
clearPlayingAudio();
}, [clearPlayingAudio]);

const clearIds = useCallback((clearedIds: string[]) => {
if (clearedIds.length === 0) return;

const clearedIdsSet = new Set(clearedIds);
if (currentlyReadingRef.current) {
const currentId = currentlyReadingRef.current?.id;
if (currentId && clearedIdsSet.has(currentId)) {
clearPlayingAudio();
}
}

if (textQueueRef.current.length > 0) {
textQueueRef.current = textQueueRef.current.filter(queueItem => {
return !queueItem.id || !clearedIdsSet.has(queueItem.id);
});
}

if (!currentlyReadingRef.current) {
void processQueue('clearIds');
}
}, [clearPlayingAudio, processQueue]);

useEffect(() => {
if (!enabled || !token || voicesRef.current.length) return;

Expand All @@ -120,5 +161,6 @@ export const useTts = (token: string | null, enabled: boolean | null): TtsApi =>
return {
readText,
clearQueue,
clearIds,
};
};
14 changes: 14 additions & 0 deletions src/js/utils/chat-messages.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,3 +253,17 @@ export const getAccentColor = (message: ChatUserMessage | ChatSystemMessage) =>

return message.nameColor;
};


export type VoiceGender = 'male' | 'female';

export const mapPronounsToGender = (pronouns?: string[]): VoiceGender => {
const firstPronoun = pronouns?.[0];
switch (firstPronoun) {
case 'She/Her':
case 'She/They':
return 'female';
default:
return 'male';
}
};

0 comments on commit f35184a

Please sign in to comment.