Skip to content

Commit

Permalink
Merge pull request #1797 from elizaOS/fix/lint_errors
Browse files Browse the repository at this point in the history 
chore: remove unused import and var
  • Loading branch information
@shakkernerd
shakkernerd authored Jan 4, 2025
2 parents 3fec0bd + 9a6e380 commit 6e30c41
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 72 deletions.
1 change: 0 additions & 1 deletion packages/client-discord/src/actions/chat_with_attachments.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { composeContext } from "@elizaos/core";
import { generateText, trimTokens } from "@elizaos/core";
import type { TiktokenModel } from "js-tiktoken";
import { models } from "@elizaos/core";
import { parseJSONObjectFromText } from "@elizaos/core";
import {
Expand Down
149 changes: 78 additions & 71 deletions packages/client-twitter/src/plugins/SttTtsSpacesPlugin.ts
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
// src/plugins/SttTtsPlugin.ts

import { spawn } from 'child_process';
import { ITranscriptionService } from '@elizaos/core';
import { Space, JanusClient, AudioDataWithUser } from 'agent-twitter-client';
import { spawn } from "child_process";
import { ITranscriptionService } from "@elizaos/core";
import { Space, JanusClient, AudioDataWithUser } from "agent-twitter-client";

interface PluginConfig {
openAiApiKey?: string; // for STT & ChatGPT
Expand All @@ -14,7 +14,7 @@ interface PluginConfig {
elevenLabsModel?: string; // e.g. "eleven_monolingual_v1"
systemPrompt?: string; // ex. "You are a helpful AI assistant"
chatContext?: Array<{
role: 'system' | 'user' | 'assistant';
role: "system" | "user" | "assistant";
content: string;
}>;
transcriptionService: ITranscriptionService;
Expand All @@ -33,12 +33,12 @@ export class SttTtsPlugin implements Plugin {
private openAiApiKey?: string;
private elevenLabsApiKey?: string;

private gptModel = 'gpt-3.5-turbo';
private voiceId = '21m00Tcm4TlvDq8ikWAM';
private elevenLabsModel = 'eleven_monolingual_v1';
private systemPrompt = 'You are a helpful AI assistant.';
private gptModel = "gpt-3.5-turbo";
private voiceId = "21m00Tcm4TlvDq8ikWAM";
private elevenLabsModel = "eleven_monolingual_v1";
private systemPrompt = "You are a helpful AI assistant.";
private chatContext: Array<{
role: 'system' | 'user' | 'assistant';
role: "system" | "user" | "assistant";
content: string;
}> = [];

Expand All @@ -63,24 +63,26 @@ export class SttTtsPlugin implements Plugin {
private ttsQueue: string[] = [];
private isSpeaking = false;

onAttach(space: Space) {
console.log('[SttTtsPlugin] onAttach => space was attached');
onAttach(_space: Space) {
console.log("[SttTtsPlugin] onAttach => space was attached");
}

init(params: { space: Space; pluginConfig?: Record<string, any> }): void {
console.log(
'[SttTtsPlugin] init => Space fully ready. Subscribing to events.',
"[SttTtsPlugin] init => Space fully ready. Subscribing to events."
);

this.space = params.space;
this.janus = (this.space as any)?.janusClient as JanusClient | undefined;
this.janus = (this.space as any)?.janusClient as
| JanusClient
| undefined;

const config = params.pluginConfig as PluginConfig;
this.openAiApiKey = config?.openAiApiKey;
this.elevenLabsApiKey = config?.elevenLabsApiKey;
this.transcriptionService = config.transcriptionService;
if (config?.gptModel) this.gptModel = config.gptModel;
if (typeof config?.silenceThreshold === 'number') {
if (typeof config?.silenceThreshold === "number") {
this.silenceThreshold = config.silenceThreshold;
}
if (config?.voiceId) {
Expand All @@ -95,24 +97,24 @@ export class SttTtsPlugin implements Plugin {
if (config?.chatContext) {
this.chatContext = config.chatContext;
}
console.log('[SttTtsPlugin] Plugin config =>', config);
console.log("[SttTtsPlugin] Plugin config =>", config);

// Listen for mute events
this.space.on(
'muteStateChanged',
"muteStateChanged",
(evt: { userId: string; muted: boolean }) => {
console.log('[SttTtsPlugin] Speaker muteStateChanged =>', evt);
console.log("[SttTtsPlugin] Speaker muteStateChanged =>", evt);
if (evt.muted) {
this.handleMute(evt.userId).catch((err) =>
console.error('[SttTtsPlugin] handleMute error =>', err),
console.error("[SttTtsPlugin] handleMute error =>", err)
);
} else {
this.speakerUnmuted.set(evt.userId, true);
if (!this.pcmBuffers.has(evt.userId)) {
this.pcmBuffers.set(evt.userId, []);
}
}
},
}
);
}

Expand Down Expand Up @@ -157,22 +159,22 @@ export class SttTtsPlugin implements Plugin {
const view = new DataView(buffer);

// RIFF chunk descriptor
this.writeString(view, 0, 'RIFF');
this.writeString(view, 0, "RIFF");
view.setUint32(4, 36 + dataSize, true); // file size - 8
this.writeString(view, 8, 'WAVE');
this.writeString(view, 8, "WAVE");

// fmt sub-chunk
this.writeString(view, 12, 'fmt ');
view.setUint32(16, 16, true); // Subchunk1Size (16 for PCM)
view.setUint16(20, 1, true); // AudioFormat (1 = PCM)
this.writeString(view, 12, "fmt ");
view.setUint32(16, 16, true); // Subchunk1Size (16 for PCM)
view.setUint16(20, 1, true); // AudioFormat (1 = PCM)
view.setUint16(22, numChannels, true); // NumChannels
view.setUint32(24, sampleRate, true); // SampleRate
view.setUint32(28, byteRate, true); // ByteRate
view.setUint16(32, blockAlign, true); // BlockAlign
view.setUint16(34, 16, true); // BitsPerSample (16)
view.setUint32(24, sampleRate, true); // SampleRate
view.setUint32(28, byteRate, true); // ByteRate
view.setUint16(32, blockAlign, true); // BlockAlign
view.setUint16(34, 16, true); // BitsPerSample (16)

// data sub-chunk
this.writeString(view, 36, 'data');
this.writeString(view, 36, "data");
view.setUint32(40, dataSize, true);

// Write PCM samples
Expand All @@ -199,11 +201,11 @@ export class SttTtsPlugin implements Plugin {
this.pcmBuffers.set(userId, []);

if (!chunks.length) {
console.log('[SttTtsPlugin] No audio chunks for user =>', userId);
console.log("[SttTtsPlugin] No audio chunks for user =>", userId);
return;
}
console.log(
`[SttTtsPlugin] Flushing STT buffer for user=${userId}, chunks=${chunks.length}`,
`[SttTtsPlugin] Flushing STT buffer for user=${userId}, chunks=${chunks.length}`
);

const totalLen = chunks.reduce((acc, c) => acc + c.length, 0);
Expand All @@ -221,14 +223,19 @@ export class SttTtsPlugin implements Plugin {
const sttText = await this.transcriptionService.transcribe(wavBuffer);

if (!sttText || !sttText.trim()) {
console.log('[SttTtsPlugin] No speech recognized for user =>', userId);
console.log(
"[SttTtsPlugin] No speech recognized for user =>",
userId
);
return;
}
console.log(`[SttTtsPlugin] STT => user=${userId}, text="${sttText}"`);

// GPT answer
const replyText = await this.askChatGPT(sttText);
console.log(`[SttTtsPlugin] GPT => user=${userId}, reply="${replyText}"`);
console.log(
`[SttTtsPlugin] GPT => user=${userId}, reply="${replyText}"`
);

// Use the standard speak method with queue
await this.speakText(replyText);
Expand All @@ -242,7 +249,7 @@ export class SttTtsPlugin implements Plugin {
if (!this.isSpeaking) {
this.isSpeaking = true;
this.processTtsQueue().catch((err) => {
console.error('[SttTtsPlugin] processTtsQueue error =>', err);
console.error("[SttTtsPlugin] processTtsQueue error =>", err);
});
}
}
Expand All @@ -260,7 +267,7 @@ export class SttTtsPlugin implements Plugin {
const pcm = await this.convertMp3ToPcm(ttsAudio, 48000);
await this.streamToJanus(pcm, 48000);
} catch (err) {
console.error('[SttTtsPlugin] TTS streaming error =>', err);
console.error("[SttTtsPlugin] TTS streaming error =>", err);
}
}
this.isSpeaking = false;
Expand All @@ -271,20 +278,20 @@ export class SttTtsPlugin implements Plugin {
*/
private async askChatGPT(userText: string): Promise<string> {
if (!this.openAiApiKey) {
throw new Error('[SttTtsPlugin] No OpenAI API key for ChatGPT');
throw new Error("[SttTtsPlugin] No OpenAI API key for ChatGPT");
}
const url = 'https://api.openai.com/v1/chat/completions';
const url = "https://api.openai.com/v1/chat/completions";
const messages = [
{ role: 'system', content: this.systemPrompt },
{ role: "system", content: this.systemPrompt },
...this.chatContext,
{ role: 'user', content: userText },
{ role: "user", content: userText },
];

const resp = await fetch(url, {
method: 'POST',
method: "POST",
headers: {
Authorization: `Bearer ${this.openAiApiKey}`,
'Content-Type': 'application/json',
"Content-Type": "application/json",
},
body: JSON.stringify({
model: this.gptModel,
Expand All @@ -295,14 +302,14 @@ export class SttTtsPlugin implements Plugin {
if (!resp.ok) {
const errText = await resp.text();
throw new Error(
`[SttTtsPlugin] ChatGPT error => ${resp.status} ${errText}`,
`[SttTtsPlugin] ChatGPT error => ${resp.status} ${errText}`
);
}

const json = await resp.json();
const reply = json.choices?.[0]?.message?.content || '';
this.chatContext.push({ role: 'user', content: userText });
this.chatContext.push({ role: 'assistant', content: reply });
const reply = json.choices?.[0]?.message?.content || "";
this.chatContext.push({ role: "user", content: userText });
this.chatContext.push({ role: "assistant", content: reply });
return reply.trim();
}

Expand All @@ -311,14 +318,14 @@ export class SttTtsPlugin implements Plugin {
*/
private async elevenLabsTts(text: string): Promise<Buffer> {
if (!this.elevenLabsApiKey) {
throw new Error('[SttTtsPlugin] No ElevenLabs API key');
throw new Error("[SttTtsPlugin] No ElevenLabs API key");
}
const url = `https://api.elevenlabs.io/v1/text-to-speech/${this.voiceId}`;
const resp = await fetch(url, {
method: 'POST',
method: "POST",
headers: {
'Content-Type': 'application/json',
'xi-api-key': this.elevenLabsApiKey,
"Content-Type": "application/json",
"xi-api-key": this.elevenLabsApiKey,
},
body: JSON.stringify({
text,
Expand All @@ -329,7 +336,7 @@ export class SttTtsPlugin implements Plugin {
if (!resp.ok) {
const errText = await resp.text();
throw new Error(
`[SttTtsPlugin] ElevenLabs TTS error => ${resp.status} ${errText}`,
`[SttTtsPlugin] ElevenLabs TTS error => ${resp.status} ${errText}`
);
}
const arrayBuf = await resp.arrayBuffer();
Expand All @@ -341,37 +348,37 @@ export class SttTtsPlugin implements Plugin {
*/
private convertMp3ToPcm(
mp3Buf: Buffer,
outRate: number,
outRate: number
): Promise<Int16Array> {
return new Promise((resolve, reject) => {
const ff = spawn('ffmpeg', [
'-i',
'pipe:0',
'-f',
's16le',
'-ar',
const ff = spawn("ffmpeg", [
"-i",
"pipe:0",
"-f",
"s16le",
"-ar",
outRate.toString(),
'-ac',
'1',
'pipe:1',
"-ac",
"1",
"pipe:1",
]);
let raw = Buffer.alloc(0);

ff.stdout.on('data', (chunk: Buffer) => {
ff.stdout.on("data", (chunk: Buffer) => {
raw = Buffer.concat([raw, chunk]);
});
ff.stderr.on('data', () => {
ff.stderr.on("data", () => {
// ignoring ffmpeg logs
});
ff.on('close', (code) => {
ff.on("close", (code) => {
if (code !== 0) {
reject(new Error(`ffmpeg error code=${code}`));
return;
}
const samples = new Int16Array(
raw.buffer,
raw.byteOffset,
raw.byteLength / 2,
raw.byteLength / 2
);
resolve(samples);
});
Expand All @@ -387,7 +394,7 @@ export class SttTtsPlugin implements Plugin {
*/
private async streamToJanus(
samples: Int16Array,
sampleRate: number,
sampleRate: number
): Promise<void> {
// TODO: Check if better than 480 fixed
const FRAME_SIZE = Math.floor(sampleRate * 0.01); // 10ms frames => 480 @48kHz
Expand All @@ -408,25 +415,25 @@ export class SttTtsPlugin implements Plugin {

public setSystemPrompt(prompt: string) {
this.systemPrompt = prompt;
console.log('[SttTtsPlugin] setSystemPrompt =>', prompt);
console.log("[SttTtsPlugin] setSystemPrompt =>", prompt);
}

/**
* Change the GPT model at runtime (e.g. "gpt-4", "gpt-3.5-turbo", etc.).
*/
public setGptModel(model: string) {
this.gptModel = model;
console.log('[SttTtsPlugin] setGptModel =>', model);
console.log("[SttTtsPlugin] setGptModel =>", model);
}

/**
* Add a message (system, user or assistant) to the chat context.
* E.g. to store conversation history or inject a persona.
*/
public addMessage(role: 'system' | 'user' | 'assistant', content: string) {
public addMessage(role: "system" | "user" | "assistant", content: string) {
this.chatContext.push({ role, content });
console.log(
`[SttTtsPlugin] addMessage => role=${role}, content=${content}`,
`[SttTtsPlugin] addMessage => role=${role}, content=${content}`
);
}

Expand All @@ -435,11 +442,11 @@ export class SttTtsPlugin implements Plugin {
*/
public clearChatContext() {
this.chatContext = [];
console.log('[SttTtsPlugin] clearChatContext => done');
console.log("[SttTtsPlugin] clearChatContext => done");
}

cleanup(): void {
console.log('[SttTtsPlugin] cleanup => releasing resources');
console.log("[SttTtsPlugin] cleanup => releasing resources");
this.pcmBuffers.clear();
this.speakerUnmuted.clear();
this.ttsQueue = [];
Expand Down

0 comments on commit 6e30c41

Please sign in to comment.