From b81785d8e269a893cc14b3cf419354fb5d6d3715 Mon Sep 17 00:00:00 2001 From: Shannon Code Date: Mon, 18 Nov 2024 20:55:49 -0500 Subject: [PATCH 1/6] ignore local lock --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index ad4176f7b2..bdd4993bcd 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,5 @@ characters/ packages/core/src/providers/cache packages/core/src/providers/cache/* cache/* + +pnpm-lock.yaml From 8afd007c61a4e2849c6bba103a203edd5758ed75 Mon Sep 17 00:00:00 2001 From: Shannon Code Date: Mon, 18 Nov 2024 20:56:39 -0500 Subject: [PATCH 2/6] now with env scoping --- docs/docs/guides/secrets-management.md | 5 ++++- docs/docs/packages/agent.md | 14 ++++++++++++ packages/agent/src/index.ts | 26 ++++++++++++++++----- packages/core/src/settings.ts | 31 ++++++++++++++++++++++++++ pnpm-lock.yaml | 2 +- 5 files changed, 70 insertions(+), 8 deletions(-) diff --git a/docs/docs/guides/secrets-management.md b/docs/docs/guides/secrets-management.md index c35c65345e..636f783c0c 100644 --- a/docs/docs/guides/secrets-management.md +++ b/docs/docs/guides/secrets-management.md @@ -12,7 +12,8 @@ A comprehensive guide for managing secrets, API keys, and sensitive configuratio Eliza uses a hierarchical environment variable system: -1. Character-specific secrets (highest priority) +1. Character-specific namespaced environment variables (highest priority) +1. Character-specific secrets 2. Environment variables 3. Default values (lowest priority) @@ -89,6 +90,8 @@ Define secrets in character files: } ``` +Alternatively, you can use the `CHARACTER.YOUR_CHARACTER_NAME.SECRET_NAME` format inside your `.env` file. + Access secrets in code: ```typescript diff --git a/docs/docs/packages/agent.md b/docs/docs/packages/agent.md index 87108048b7..2f16b85871 100644 --- a/docs/docs/packages/agent.md +++ b/docs/docs/packages/agent.md @@ -159,6 +159,15 @@ export async function initializeClients( ### Token Management +Tokens can be configured in two ways: + +1. Using namespaced environment variables: +```env +CHARACTER.YOUR_CHARACTER_NAME.OPENAI_API_KEY=sk-... +CHARACTER.YOUR_CHARACTER_NAME.ANTHROPIC_API_KEY=sk-... +``` + +2. Using character settings: ```typescript export function getTokenForProvider( provider: ModelProviderName, @@ -179,6 +188,11 @@ export function getTokenForProvider( } ``` +The system will check for tokens in the following order: +1. Character-specific namespaced env variables +2. Character settings from JSON +3. Global environment variables + ### Database Selection ```typescript diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 725a65cfca..34ac2bc92c 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -76,8 +76,25 @@ export async function loadCharacters( for (const path of characterPaths) { try { const character = JSON.parse(fs.readFileSync(path, "utf8")); + + const characterId = character.id || character.name; + const characterPrefix = `CHARACTER.${characterId.toUpperCase().replace(/ /g, '_')}.`; + + const characterSettings = Object.entries(process.env) + .filter(([key]) => key.startsWith(characterPrefix)) + .reduce((settings, [key, value]) => { + const settingKey = key.slice(characterPrefix.length); + return { ...settings, [settingKey]: value }; + }, {}); + + if (Object.keys(characterSettings).length > 0) { + character.settings = character.settings || {}; + character.settings.secrets = { + ...characterSettings, + ...character.settings.secrets + }; + } - // is there a "plugins" field? if (character.plugins) { console.log("Plugins are: ", character.plugins); @@ -99,13 +116,10 @@ export async function loadCharacters( } } } - - if (loadedCharacters.length === 0) { + loadedCharacters.length === 0 && console.log("No characters found, using default character"); - loadedCharacters.push(defaultCharacter); - } - return loadedCharacters; + return loadedCharacters.length > 0 ? loadedCharacters : [defaultCharacter]; } export function getTokenForProvider( diff --git a/packages/core/src/settings.ts b/packages/core/src/settings.ts index 9de8564cba..2d51b0c764 100644 --- a/packages/core/src/settings.ts +++ b/packages/core/src/settings.ts @@ -6,6 +6,10 @@ interface Settings { [key: string]: string | undefined; } +interface NamespacedSettings { + [namespace: string]: Settings; +} + let environmentSettings: Settings = {}; /** @@ -81,6 +85,15 @@ export function loadEnvConfig(): Settings { } console.log(`Loaded .env file from: ${envPath}`); + + // Parse namespaced settings + const namespacedSettings = parseNamespacedSettings(process.env as Settings); + + // Attach to process.env for backward compatibility + Object.entries(namespacedSettings).forEach(([namespace, settings]) => { + process.env[`__namespaced_${namespace}`] = JSON.stringify(settings); + }); + return process.env as Settings; } @@ -115,3 +128,21 @@ export function hasEnvVariable(key: string): boolean { // Initialize settings based on environment export const settings = isBrowser() ? environmentSettings : loadEnvConfig(); export default settings; + +// Add this function to parse namespaced settings +function parseNamespacedSettings(env: Settings): NamespacedSettings { + const namespaced: NamespacedSettings = {}; + + for (const [key, value] of Object.entries(env)) { + if (!value) continue; + + const [namespace, ...rest] = key.split('.'); + if (!namespace || rest.length === 0) continue; + + const settingKey = rest.join('.'); + namespaced[namespace] = namespaced[namespace] || {}; + namespaced[namespace][settingKey] = value; + } + + return namespaced; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a97848a2ed..7fd37dabce 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22169,7 +22169,7 @@ snapshots: log-symbols@4.1.0: dependencies: - chalk: 4.1.0 + chalk: 4.1.2 is-unicode-supported: 0.1.0 log-symbols@6.0.0: From 93f3a54983c0c65742a486dbbae74197fa7dbfb6 Mon Sep 17 00:00:00 2001 From: Shannon Code Date: Tue, 19 Nov 2024 02:35:39 -0500 Subject: [PATCH 3/6] re number list --- docs/docs/guides/secrets-management.md | 6 +- packages/client-ui/static/index.html | 701 +++++++++++++++++++++++++ scripts/devall.sh | 12 + 3 files changed, 716 insertions(+), 3 deletions(-) create mode 100644 packages/client-ui/static/index.html create mode 100644 scripts/devall.sh diff --git a/docs/docs/guides/secrets-management.md b/docs/docs/guides/secrets-management.md index 636f783c0c..9dc86f8952 100644 --- a/docs/docs/guides/secrets-management.md +++ b/docs/docs/guides/secrets-management.md @@ -13,9 +13,9 @@ A comprehensive guide for managing secrets, API keys, and sensitive configuratio Eliza uses a hierarchical environment variable system: 1. Character-specific namespaced environment variables (highest priority) -1. Character-specific secrets -2. Environment variables -3. Default values (lowest priority) +2. Character-specific secrets +3. Environment variables +4. Default values (lowest priority) ### Secret Types diff --git a/packages/client-ui/static/index.html b/packages/client-ui/static/index.html new file mode 100644 index 0000000000..bbff5d786c --- /dev/null +++ b/packages/client-ui/static/index.html @@ -0,0 +1,701 @@ + + + + Eliza Admin Panel + + + + + +
+ + +
+ +
+
+ + +
+
+ + +
+ +
+
+
+ + +
+
+
+
+
+
+ + + + \ No newline at end of file diff --git a/scripts/devall.sh b/scripts/devall.sh new file mode 100644 index 0000000000..7fb48f7d1d --- /dev/null +++ b/scripts/devall.sh @@ -0,0 +1,12 @@ +echo "Passing arguments: $* --character=\"characters/fusion.character.json\"" +npx concurrently --raw \ + "pnpm --dir packages/core dev -- $* --character=\"characters/fusion.character.json\"" \ + "pnpm --dir packages/client-telegram dev -- $* --character=\"characters/fusion.character.json\"" \ + "pnpm --dir packages/client-discord dev -- $* --character=\"characters/fusion.character.json\"" \ + "pnpm --dir packages/client-twitter dev -- $* --character=\"characters/fusion.character.json\"" \ + "pnpm --dir packages/client-ui dev -- $* --character=\"characters/fusion.character.json\"" \ + "pnpm --dir packages/plugin-bootstrap dev -- $* --character=\"characters/fusion.character.json\"" \ + "pnpm --dir packages/plugin-node dev -- $* --character=\"characters/fusion.character.json\"" \ + "pnpm --dir packages/adapter-sqlite dev -- $* --character=\"characters/fusion.character.json\"" \ + "pnpm --dir packages/adapter-postgres dev -- $* --character=\"characters/fusion.character.json\"" \ + "node -e \"setTimeout(() => process.exit(0), 5000)\" && pnpm --dir packages/agent dev -- $* --characters=\"characters/fusion.character.json,characters/shannonai.character.json\"" \ No newline at end of file From 40bdb8fabb7ac7582b17b9956cf701156427eba5 Mon Sep 17 00:00:00 2001 From: Shannon Code Date: Tue, 19 Nov 2024 07:35:28 -0500 Subject: [PATCH 4/6] those don't belong here --- packages/client-ui/static/index.html | 701 --------------------------- scripts/devall.sh | 12 - 2 files changed, 713 deletions(-) delete mode 100644 packages/client-ui/static/index.html delete mode 100644 scripts/devall.sh diff --git a/packages/client-ui/static/index.html b/packages/client-ui/static/index.html deleted file mode 100644 index bbff5d786c..0000000000 --- a/packages/client-ui/static/index.html +++ /dev/null @@ -1,701 +0,0 @@ - - - - Eliza Admin Panel - - - - - -
- - -
- -
-
- - -
-
- - -
- -
-
-
- - -
-
-
-
-
-
- - - - \ No newline at end of file diff --git a/scripts/devall.sh b/scripts/devall.sh deleted file mode 100644 index 7fb48f7d1d..0000000000 --- a/scripts/devall.sh +++ /dev/null @@ -1,12 +0,0 @@ -echo "Passing arguments: $* --character=\"characters/fusion.character.json\"" -npx concurrently --raw \ - "pnpm --dir packages/core dev -- $* --character=\"characters/fusion.character.json\"" \ - "pnpm --dir packages/client-telegram dev -- $* --character=\"characters/fusion.character.json\"" \ - "pnpm --dir packages/client-discord dev -- $* --character=\"characters/fusion.character.json\"" \ - "pnpm --dir packages/client-twitter dev -- $* --character=\"characters/fusion.character.json\"" \ - "pnpm --dir packages/client-ui dev -- $* --character=\"characters/fusion.character.json\"" \ - "pnpm --dir packages/plugin-bootstrap dev -- $* --character=\"characters/fusion.character.json\"" \ - "pnpm --dir packages/plugin-node dev -- $* --character=\"characters/fusion.character.json\"" \ - "pnpm --dir packages/adapter-sqlite dev -- $* --character=\"characters/fusion.character.json\"" \ - "pnpm --dir packages/adapter-postgres dev -- $* --character=\"characters/fusion.character.json\"" \ - "node -e \"setTimeout(() => process.exit(0), 5000)\" && pnpm --dir packages/agent dev -- $* --characters=\"characters/fusion.character.json,characters/shannonai.character.json\"" \ No newline at end of file From dc060addc2a9e26cfe09726fb3f06bc3ac6dbca5 Mon Sep 17 00:00:00 2001 From: odilitime Date: Thu, 26 Dec 2024 01:01:29 +0000 Subject: [PATCH 5/6] add being able to load character secrets from env --- agent/src/index.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/agent/src/index.ts b/agent/src/index.ts index b0ac9dbe48..57f8c262b5 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -178,6 +178,25 @@ export async function loadCharacters( const character = JSON.parse(content); validateCharacterConfig(character); + // .id isn't really valid + const characterId = character.id || character.name; + const characterPrefix = `CHARACTER.${characterId.toUpperCase().replace(/ /g, '_')}.`; + + const characterSettings = Object.entries(process.env) + .filter(([key]) => key.startsWith(characterPrefix)) + .reduce((settings, [key, value]) => { + const settingKey = key.slice(characterPrefix.length); + return { ...settings, [settingKey]: value }; + }, {}); + + if (Object.keys(characterSettings).length > 0) { + character.settings = character.settings || {}; + character.settings.secrets = { + ...characterSettings, + ...character.settings.secrets + }; + } + // Handle plugins if (isAllStrings(character.plugins)) { elizaLogger.info("Plugins are: ", character.plugins); From 85128ceeb7503e045e3b969cfd56c671401dbac7 Mon Sep 17 00:00:00 2001 From: odilitime Date: Thu, 26 Dec 2024 01:01:47 +0000 Subject: [PATCH 6/6] error --- packages/agent/src/index.ts | 350 ------------------------------------ 1 file changed, 350 deletions(-) delete mode 100644 packages/agent/src/index.ts diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts deleted file mode 100644 index 34ac2bc92c..0000000000 --- a/packages/agent/src/index.ts +++ /dev/null @@ -1,350 +0,0 @@ -import { PostgresDatabaseAdapter } from "@ai16z/adapter-postgres"; -import { SqliteDatabaseAdapter } from "@ai16z/adapter-sqlite"; -import { DirectClientInterface } from "@ai16z/client-direct"; -import { DiscordClientInterface } from "@ai16z/client-discord"; -import { AutoClientInterface } from "@ai16z/client-auto"; -import { TelegramClientInterface } from "@ai16z/client-telegram"; -import { TwitterClientInterface } from "@ai16z/client-twitter"; -import { defaultCharacter } from "@ai16z/eliza"; -import { AgentRuntime } from "@ai16z/eliza"; -import { settings } from "@ai16z/eliza"; -import { - Character, - IAgentRuntime, - IDatabaseAdapter, - ModelProviderName, -} from "@ai16z/eliza"; -import { bootstrapPlugin } from "@ai16z/plugin-bootstrap"; -import { solanaPlugin } from "@ai16z/plugin-solana"; -import { nodePlugin } from "@ai16z/plugin-node"; -import Database from "better-sqlite3"; -import fs from "fs"; -import readline from "readline"; -import yargs from "yargs"; -import { character } from "./character.ts"; - -export const wait = (minTime: number = 1000, maxTime: number = 3000) => { - const waitTime = - Math.floor(Math.random() * (maxTime - minTime + 1)) + minTime; - return new Promise((resolve) => setTimeout(resolve, waitTime)); -}; - -export function parseArguments(): { - character?: string; - characters?: string; -} { - try { - return yargs(process.argv.slice(2)) - .option("character", { - type: "string", - description: "Path to the character JSON file", - }) - .option("characters", { - type: "string", - description: - "Comma separated list of paths to character JSON files", - }) - .parseSync(); - } catch (error) { - console.error("Error parsing arguments:", error); - return {}; - } -} - -export async function loadCharacters( - charactersArg: string -): Promise { - let characterPaths = charactersArg - ?.split(",") - .map((path) => path.trim()) - .map((path) => { - if (path.startsWith("../characters")) { - return `../${path}`; - } - if (path.startsWith("characters")) { - return `../../${path}`; - } - if (path.startsWith("./characters")) { - return `../.${path}`; - } - return path; - }); - - const loadedCharacters = []; - - if (characterPaths?.length > 0) { - for (const path of characterPaths) { - try { - const character = JSON.parse(fs.readFileSync(path, "utf8")); - - const characterId = character.id || character.name; - const characterPrefix = `CHARACTER.${characterId.toUpperCase().replace(/ /g, '_')}.`; - - const characterSettings = Object.entries(process.env) - .filter(([key]) => key.startsWith(characterPrefix)) - .reduce((settings, [key, value]) => { - const settingKey = key.slice(characterPrefix.length); - return { ...settings, [settingKey]: value }; - }, {}); - - if (Object.keys(characterSettings).length > 0) { - character.settings = character.settings || {}; - character.settings.secrets = { - ...characterSettings, - ...character.settings.secrets - }; - } - - if (character.plugins) { - console.log("Plugins are: ", character.plugins); - - const importedPlugins = await Promise.all( - character.plugins.map(async (plugin) => { - // if the plugin name doesnt start with @eliza, - - const importedPlugin = await import(plugin); - return importedPlugin; - }) - ); - - character.plugins = importedPlugins; - } - - loadedCharacters.push(character); - } catch (e) { - console.error(`Error loading character from ${path}: ${e}`); - } - } - } - loadedCharacters.length === 0 && - console.log("No characters found, using default character"); - - return loadedCharacters.length > 0 ? loadedCharacters : [defaultCharacter]; -} - -export function getTokenForProvider( - provider: ModelProviderName, - character: Character -) { - switch (provider) { - case ModelProviderName.OPENAI: - return ( - character.settings?.secrets?.OPENAI_API_KEY || - settings.OPENAI_API_KEY - ); - case ModelProviderName.LLAMACLOUD: - return ( - character.settings?.secrets?.LLAMACLOUD_API_KEY || - settings.LLAMACLOUD_API_KEY || - character.settings?.secrets?.TOGETHER_API_KEY || - settings.TOGETHER_API_KEY || - character.settings?.secrets?.XAI_API_KEY || - settings.XAI_API_KEY || - character.settings?.secrets?.OPENAI_API_KEY || - settings.OPENAI_API_KEY - ); - case ModelProviderName.ANTHROPIC: - return ( - character.settings?.secrets?.ANTHROPIC_API_KEY || - character.settings?.secrets?.CLAUDE_API_KEY || - settings.ANTHROPIC_API_KEY || - settings.CLAUDE_API_KEY - ); - case ModelProviderName.REDPILL: - return ( - character.settings?.secrets?.REDPILL_API_KEY || - settings.REDPILL_API_KEY - ); - case ModelProviderName.OPENROUTER: - return ( - character.settings?.secrets?.OPENROUTER || - settings.OPENROUTER_API_KEY - ); - case ModelProviderName.GROK: - return ( - character.settings?.secrets?.GROK_API_KEY || - settings.GROK_API_KEY - ); - case ModelProviderName.HEURIST: - return ( - character.settings?.secrets?.HEURIST_API_KEY || - settings.HEURIST_API_KEY - ); - case ModelProviderName.GROQ: - return ( - character.settings?.secrets?.GROQ_API_KEY || - settings.GROQ_API_KEY - ); - } -} - -function initializeDatabase() { - if (process.env.POSTGRES_URL) { - return new PostgresDatabaseAdapter({ - connectionString: process.env.POSTGRES_URL, - }); - } else { - return new SqliteDatabaseAdapter(new Database("./db.sqlite")); - } -} - -export async function initializeClients( - character: Character, - runtime: IAgentRuntime -) { - const clients = []; - const clientTypes = - character.clients?.map((str) => str.toLowerCase()) || []; - - if (clientTypes.includes("auto")) { - const autoClient = await AutoClientInterface.start(runtime); - if (autoClient) clients.push(autoClient); - } - - if (clientTypes.includes("discord")) { - clients.push(await DiscordClientInterface.start(runtime)); - } - - if (clientTypes.includes("telegram")) { - const telegramClient = await TelegramClientInterface.start(runtime); - if (telegramClient) clients.push(telegramClient); - } - - if (clientTypes.includes("twitter")) { - const twitterClients = await TwitterClientInterface.start(runtime); - clients.push(twitterClients); - } - - if (character.plugins?.length > 0) { - for (const plugin of character.plugins) { - if (plugin.clients) { - for (const client of plugin.clients) { - clients.push(await client.start(runtime)); - } - } - } - } - - return clients; -} - -export async function createAgent( - character: Character, - db: any, - token: string -) { - console.log("Creating runtime for character", character.name); - return new AgentRuntime({ - databaseAdapter: db, - token, - modelProvider: character.modelProvider, - evaluators: [], - character, - plugins: [ - bootstrapPlugin, - nodePlugin, - character.settings.secrets?.WALLET_PUBLIC_KEY ? solanaPlugin : null, - ].filter(Boolean), - providers: [], - actions: [], - services: [], - managers: [], - }); -} - -async function startAgent(character: Character, directClient: any) { - try { - const token = getTokenForProvider(character.modelProvider, character); - const db = initializeDatabase(); - - const runtime = await createAgent(character, db, token); - - const clients = await initializeClients( - character, - runtime as IAgentRuntime - ); - - directClient.registerAgent(await runtime); - - return clients; - } catch (error) { - console.error( - `Error starting agent for character ${character.name}:`, - error - ); - throw error; - } -} - -const startAgents = async () => { - const directClient = await DirectClientInterface.start(); - const args = parseArguments(); - - let charactersArg = args.characters || args.character; - - let characters = [character]; - - if (charactersArg) { - characters = await loadCharacters(charactersArg); - } - - try { - for (const character of characters) { - await startAgent(character, directClient); - } - } catch (error) { - console.error("Error starting agents:", error); - } - - function chat() { - const agentId = characters[0].name ?? "Agent"; - rl.question("You: ", async (input) => { - await handleUserInput(input, agentId); - if (input.toLowerCase() !== "exit") { - chat(); // Loop back to ask another question - } - }); - } - - console.log("Chat started. Type 'exit' to quit."); - chat(); -}; - -startAgents().catch((error) => { - console.error("Unhandled error in startAgents:", error); - process.exit(1); // Exit the process after logging -}); - -const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, -}); - -async function handleUserInput(input, agentId) { - if (input.toLowerCase() === "exit") { - rl.close(); - return; - } - - try { - const serverPort = parseInt(settings.SERVER_PORT || "3000"); - - const response = await fetch( - `http://localhost:${serverPort}/${agentId}/message`, - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - text: input, - userId: "user", - userName: "User", - }), - } - ); - - const data = await response.json(); - data.forEach((message) => console.log(`${"Agent"}: ${message.text}`)); - } catch (error) { - console.error("Error fetching response:", error); - } -}