diff --git a/src/appmixer/ai/agenttools/ToolOutput/ToolOutput.js b/src/appmixer/ai/agenttools/ToolOutput/ToolOutput.js new file mode 100644 index 000000000..571e7a702 --- /dev/null +++ b/src/appmixer/ai/agenttools/ToolOutput/ToolOutput.js @@ -0,0 +1,31 @@ +'use strict'; + +module.exports = { + + receive: async function(context) { + + // Find the toolCallId in the message scope, looking into the first + // component with type 'appmixer.utils.ai.ToolStart' and taking + // toolCallId from the output message that is stored in the flow state. + const flowDescriptor = context.flowDescriptor; + const scope = context.messages.in.scope; + let toolCallId; + for (const componentId of Object.keys(scope)) { + const component = flowDescriptor[componentId]; + if (component && component.type === 'appmixer.ai.agenttools.ToolStart') { + const key = componentId + ':' + context.messages.in.correlationId; + toolCallId = await context.flow.stateGet(key); + await context.flow.stateUnset(key); + } + } + + if (!toolCallId) { + await context.log({ step: 'no-tool-call-id', scope, flowDescriptor }); + throw new context.CancelError('No toolCallId found in the scope. Are you sure you used ai.ToolStart to start your tool chain?'); + } + + await context.log({ step: 'tool-output', toolCallId, output: context.messages.in.content.output }); + // The AI agent expects to see the output in the flow state under the toolCallId key. + return context.flow.stateSet(toolCallId, { output: context.messages.in.content.output }); + } +}; diff --git a/src/appmixer/ai/agenttools/ToolOutput/component.json b/src/appmixer/ai/agenttools/ToolOutput/component.json new file mode 100644 index 000000000..3e7a06a2a --- /dev/null +++ b/src/appmixer/ai/agenttools/ToolOutput/component.json @@ -0,0 +1,24 @@ +{ + "name": "appmixer.ai.agenttools.ToolOutput", + "author": "Appmixer ", + "description": "Determines the end of tool chain for AI Agent. Set the output of the tool chain. This output will be sent back to the AI Agent to serve as context.", + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "output": { "type": "string" } + } + }, + "inspector": { + "inputs": { + "output": { + "type": "textarea", + "label": "Output", + "index": 1 + } + } + } + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzUyMjFfMzI1MzcpIj4KPHBhdGggZD0iTTEyLjY2NjcgNi42NjY2N1YySDgiIHN0cm9rZT0iIzE4MTgxOCIgc3Ryb2tlLXdpZHRoPSIyLjEiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMTguNjY4MyA2LjY2NjVINS42NjgyOUM0LjM3OTYzIDYuNjY2NSAzLjMzNDk2IDcuNzExMTcgMy4zMzQ5NiA4Ljk5OTg0VjE4LjMzMzJDMy4zMzQ5NiAxOS42MjE4IDQuMzc5NjMgMjAuNjY2NSA1LjY2ODI5IDIwLjY2NjVIMTguNjY4M0MxOS45NTcgMjAuNjY2NSAyMS4wMDE2IDE5LjYyMTggMjEuMDAxNiAxOC4zMzMyVjguOTk5ODRDMjEuMDAxNiA3LjcxMTE3IDE5Ljk1NyA2LjY2NjUgMTguNjY4MyA2LjY2NjVaIiBzdHJva2U9IiMxODE4MTgiIHN0cm9rZS13aWR0aD0iMi4xIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KPHBhdGggZD0iTTEuMDAwOTggMTMuNjY2M0gzLjMzNDMxIiBzdHJva2U9IiMxODE4MTgiIHN0cm9rZS13aWR0aD0iMi4xIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KPHBhdGggZD0iTTIxLjAwMSAxMy42NjYzSDIzLjAzNDUiIHN0cm9rZT0iIzE4MTgxOCIgc3Ryb2tlLXdpZHRoPSIyLjEiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8ZyBmaWx0ZXI9InVybCgjZmlsdGVyMF9kXzUyMjFfMzI1MzcpIj4KPHBhdGggZD0iTTcuMDAwOTggMTMuNDU5NUw5LjYwNzE4IDE2LjA2NTdDMTAuMzA2OCAxNi43NjUzIDExLjQ0OTUgMTYuNzM2NyAxMi4xMTMyIDE2LjAwM0wxNy4wMDEgMTAuNTk5OSIgc3Ryb2tlPSIjMjBCNzVBIiBzdHJva2Utd2lkdGg9IjIuODU1MTIiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIvPgo8L2c+CjwvZz4KPGRlZnM+CjxmaWx0ZXIgaWQ9ImZpbHRlcjBfZF81MjIxXzMyNTM3IiB4PSIzLjQ3MjY5IiB5PSI3LjA3MTgxIiB3aWR0aD0iMTcuMDU2NiIgaGVpZ2h0PSIxMy4wMjg3IiBmaWx0ZXJVbml0cz0idXNlclNwYWNlT25Vc2UiIGNvbG9yLWludGVycG9sYXRpb24tZmlsdGVycz0ic1JHQiI+CjxmZUZsb29kIGZsb29kLW9wYWNpdHk9IjAiIHJlc3VsdD0iQmFja2dyb3VuZEltYWdlRml4Ii8+CjxmZUNvbG9yTWF0cml4IGluPSJTb3VyY2VBbHBoYSIgdHlwZT0ibWF0cml4IiB2YWx1ZXM9IjAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDEyNyAwIiByZXN1bHQ9ImhhcmRBbHBoYSIvPgo8ZmVPZmZzZXQvPgo8ZmVHYXVzc2lhbkJsdXIgc3RkRGV2aWF0aW9uPSIxLjA1MDI3Ii8+CjxmZUNvbXBvc2l0ZSBpbjI9ImhhcmRBbHBoYSIgb3BlcmF0b3I9Im91dCIvPgo8ZmVDb2xvck1hdHJpeCB0eXBlPSJtYXRyaXgiIHZhbHVlcz0iMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMC4yNSAwIi8+CjxmZUJsZW5kIG1vZGU9Im5vcm1hbCIgaW4yPSJCYWNrZ3JvdW5kSW1hZ2VGaXgiIHJlc3VsdD0iZWZmZWN0MV9kcm9wU2hhZG93XzUyMjFfMzI1MzciLz4KPGZlQmxlbmQgbW9kZT0ibm9ybWFsIiBpbj0iU291cmNlR3JhcGhpYyIgaW4yPSJlZmZlY3QxX2Ryb3BTaGFkb3dfNTIyMV8zMjUzNyIgcmVzdWx0PSJzaGFwZSIvPgo8L2ZpbHRlcj4KPGNsaXBQYXRoIGlkPSJjbGlwMF81MjIxXzMyNTM3Ij4KPHJlY3Qgd2lkdGg9IjI0IiBoZWlnaHQ9IjI0IiBmaWxsPSJ3aGl0ZSIvPgo8L2NsaXBQYXRoPgo8L2RlZnM+Cjwvc3ZnPgo=" +} diff --git a/src/appmixer/ai/agenttools/ToolOutput/icon.svg b/src/appmixer/ai/agenttools/ToolOutput/icon.svg new file mode 100644 index 000000000..fa873f377 --- /dev/null +++ b/src/appmixer/ai/agenttools/ToolOutput/icon.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/appmixer/ai/agenttools/ToolStart/ToolStart.js b/src/appmixer/ai/agenttools/ToolStart/ToolStart.js new file mode 100644 index 000000000..45fc7668a --- /dev/null +++ b/src/appmixer/ai/agenttools/ToolStart/ToolStart.js @@ -0,0 +1,32 @@ +'use strict'; + +module.exports = { + + receive: async function(context) { + + if (context.properties.generateOutputPortOptions) { + return this.getOutputPortOptions(context); + } + const { toolCalls } = context.messages.in.originalContent; + for (const toolCall of toolCalls) { + // Process only those tool calls that are for this component. + // This is because the AI Agent fans out all tool calls by using sendJson(..., 'tools'). + if (toolCall.componentId === context.componentId) { + const out = { args: toolCall.args, toolCallId: toolCall.id }; + await context.flow.stateSet(context.componentId + ':' + context.messages.in.correlationId, toolCall.id); + await context.sendJson(out, 'out'); + } + } + }, + + getOutputPortOptions(context) { + + const options = []; + const parameters = context.properties.parameters?.ADD || []; + parameters.forEach(parameter => { + options.push({ label: parameter.name, value: 'args.' + parameter.name, schema: { type: parameter.type } }); + }); + options.push({ label: 'Tool Call ID', value: 'toolCallId', schema: { type: 'string' } }); + return context.sendJson(options, 'out'); + } +}; diff --git a/src/appmixer/ai/agenttools/ToolStart/component.json b/src/appmixer/ai/agenttools/ToolStart/component.json new file mode 100644 index 000000000..7d681203b --- /dev/null +++ b/src/appmixer/ai/agenttools/ToolStart/component.json @@ -0,0 +1,65 @@ +{ + "name": "appmixer.ai.agenttools.ToolStart", + "author": "Appmixer ", + "description": "Define a tool chain (i.e. a set of actions that might be selected by the AI model to run based on the user prompt). The tool can provide additional context to the AI agent to be able to reply to the user or call actions. This has to be the first component connected to the AI Agent tools output port. The tool chain must be ended with 'Tool Output'.", + "properties": { + "schema": { + "type": "object", + "properties": { + "generateOutputPortOptions": { "type": "boolean" }, + "description": { "type": "string" }, + "parameters": { "type": "object" } + } + }, + "inspector": { + "inputs": { + "description": { + "type": "textarea", + "label": "Description", + "index": 1 + }, + "parameters": { + "type": "expression", + "levels": ["ADD"], + "label": "Parameters", + "index": 2, + "fields": { + "name": { + "type": "text", + "label": "Name" + }, + "description": { + "type": "textarea", + "label": "Description" + }, + "type": { + "type": "select", + "label": "Type", + "options": [ + { "value": "string", "label": "String" }, + { "value": "number", "label": "Number" }, + { "value": "boolean", "label": "Boolean" } + ] + } + } + } + } + } + }, + "inPorts": [{ + "name": "in" + }], + "outPorts": [{ + "name": "out", + "source": { + "url": "/component/appmixer/ai/agenttools/ToolStart?outPort=out", + "data": { + "properties": { + "generateOutputPortOptions": true, + "parameters": "properties/parameters" + } + } + } + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzUyMjFfMzI0OTcpIj4KPHBhdGggZD0iTTExLjk5OTggNS4yVjFINy43OTk4IiBzdHJva2U9IiMxODE4MTgiIHN0cm9rZS13aWR0aD0iMi4xIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KPHBhdGggZD0iTTcuNzk5NjEgMTcuOEwzLjU5OTYxIDIyVjcuMjk5OTVDMy41OTk2MSA2Ljc0MyAzLjgyMDg2IDYuMjA4ODUgNC4yMTQ2OSA1LjgxNTAzQzQuNjA4NTEgNS40MjEyIDUuMTQyNjUgNS4xOTk5NSA1LjY5OTYxIDUuMTk5OTVIMTguMjk5NkMxOC44NTY2IDUuMTk5OTUgMTkuMzkwNyA1LjQyMTIgMTkuNzg0NSA1LjgxNTAzQzIwLjE3ODQgNi4yMDg4NSAyMC4zOTk2IDYuNzQzIDIwLjM5OTYgNy4yOTk5NVYxNS43QzIwLjM5OTYgMTYuMjU2OSAyMC4xNzg0IDE2Ljc5MSAxOS43ODQ1IDE3LjE4NDlDMTkuMzkwNyAxNy41Nzg3IDE4Ljg1NjYgMTcuOCAxOC4yOTk2IDE3LjhINy43OTk2MVoiIHN0cm9rZT0iIzE4MTgxOCIgc3Ryb2tlLXdpZHRoPSIyLjEiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMS41IDExLjQ5OThIMy42IiBzdHJva2U9IiMxODE4MTgiIHN0cm9rZS13aWR0aD0iMi4xIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KPHBhdGggZD0iTTguODQ5NjEgMTAuNDVWMTIuNTUiIHN0cm9rZT0iIzE4MTgxOCIgc3Ryb2tlLXdpZHRoPSIyLjEiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMTUuMTUwNCAxMC40NVYxMi41NSIgc3Ryb2tlPSIjMTgxODE4IiBzdHJva2Utd2lkdGg9IjIuMSIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIi8+CjxwYXRoIGQ9Ik0yMC40MDA0IDExLjQ5OThIMjIuNTAwNCIgc3Ryb2tlPSIjMTgxODE4IiBzdHJva2Utd2lkdGg9IjIuMSIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIi8+CjwvZz4KPGRlZnM+CjxjbGlwUGF0aCBpZD0iY2xpcDBfNTIyMV8zMjQ5NyI+CjxyZWN0IHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgZmlsbD0id2hpdGUiLz4KPC9jbGlwUGF0aD4KPC9kZWZzPgo8L3N2Zz4K" +} diff --git a/src/appmixer/ai/agenttools/ToolStart/icon.svg b/src/appmixer/ai/agenttools/ToolStart/icon.svg new file mode 100644 index 000000000..b8e110fa5 --- /dev/null +++ b/src/appmixer/ai/agenttools/ToolStart/icon.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/src/appmixer/ai/agenttools/bundle.json b/src/appmixer/ai/agenttools/bundle.json new file mode 100644 index 000000000..20da803db --- /dev/null +++ b/src/appmixer/ai/agenttools/bundle.json @@ -0,0 +1,9 @@ +{ + "name": "appmixer.ai.agenttools", + "version": "1.0.0", + "changelog": { + "1.0.0": [ + "First version." + ] + } +} diff --git a/src/appmixer/ai/agenttools/module.json b/src/appmixer/ai/agenttools/module.json new file mode 100644 index 000000000..da1f12d3f --- /dev/null +++ b/src/appmixer/ai/agenttools/module.json @@ -0,0 +1,10 @@ +{ + "name": "appmixer.ai.agenttools", + "label": "Agent Tools", + "category": "ai", + "categoryIndex": 0, + "index": 2, + "categoryLabel": "AI", + "description": "Generic tool components for building AI agents.", + "icon": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzUyMjFfMzI0OTApIj4KPHBhdGggZD0iTTEyLjE1OTcgNi44Nzk5VjIuMzk5OUg3LjY3OTY5IiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMTcuOTIxMiA2Ljg3OTY0SDUuNDQxMTdDNC4yMDQwNSA2Ljg3OTY0IDMuMjAxMTcgNy44ODI1MiAzLjIwMTE3IDkuMTE5NjRWMTguMDc5NkMzLjIwMTE3IDE5LjMxNjggNC4yMDQwNSAyMC4zMTk2IDUuNDQxMTcgMjAuMzE5NkgxNy45MjEyQzE5LjE1ODMgMjAuMzE5NiAyMC4xNjEyIDE5LjMxNjggMjAuMTYxMiAxOC4wNzk2VjkuMTE5NjRDMjAuMTYxMiA3Ljg4MjUyIDE5LjE1ODMgNi44Nzk2NCAxNy45MjEyIDYuODc5NjRaIiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMC45NTk5NjEgMTMuNTk5NkgzLjE5OTk2IiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMjEuMTIwMSAxMy41OTk2SDIyLjQwMDEiIHN0cm9rZT0iIzMwMzIzNiIgc3Ryb2tlLXdpZHRoPSIyLjAxNiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIi8+CjxwYXRoIGQ9Ik0xNC43ODQyIDEyLjQ3OTdWMTQuNzE5NyIgc3Ryb2tlPSIjMzAzMjM2IiBzdHJva2Utd2lkdGg9IjIuMDE2IiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KPHBhdGggZD0iTTguNjM5NjUgMTIuNDc5N1YxNC43MTk3IiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8L2c+CjxkZWZzPgo8Y2xpcFBhdGggaWQ9ImNsaXAwXzUyMjFfMzI0OTAiPgo8cmVjdCB3aWR0aD0iMjQiIGhlaWdodD0iMjMuMDQiIGZpbGw9IndoaXRlIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgwIDAuNDc5OTgpIi8+CjwvY2xpcFBhdGg+CjwvZGVmcz4KPC9zdmc+Cg==" +} diff --git a/src/appmixer/ai/openai/AIAgent/AIAgent.js b/src/appmixer/ai/openai/AIAgent/AIAgent.js new file mode 100644 index 000000000..48a747f12 --- /dev/null +++ b/src/appmixer/ai/openai/AIAgent/AIAgent.js @@ -0,0 +1,232 @@ +'use strict'; + +const OpenAI = require('openai'); + +const COLLECT_TOOL_OUTPUTS_POLL_TIMEOUT = 60 * 1000; // 60 seconds +const COLLECT_TOOL_OUTPUTS_POLL_INTERVAL = 1 * 1000; // 1 second +const MAX_RUN_DURATION = 5 * 60 * 1000; // 5 minutes + +module.exports = { + + start: async function(context) { + + const assistant = await this.createAssistant(context); + return context.stateSet('assistant', assistant); + }, + + createAssistant: async function(context) { + + const flowDescriptor = context.flowDescriptor; + const agentComponentId = context.componentId; + const toolsPort = 'tools'; + + // Create a new assistant with tools defined in the branches connected to my 'tools' output port. + const tools = {}; + let error; + + // Find all components connected to my 'tools' output port. + Object.keys(flowDescriptor).forEach((componentId) => { + const component = flowDescriptor[componentId]; + const sources = component.source; + Object.keys(sources || {}).forEach((inPort) => { + const source = sources[inPort]; + if (source[agentComponentId] && source[agentComponentId].includes(toolsPort)) { + tools[componentId] = component; + if (component.type !== 'appmixer.ai.agenttools.ToolStart') { + error = `Component ${componentId} is not of type 'ToolStart' but ${comopnent.type}. + Every tool chain connected to the '${toolsPort}' port of the AI Agent + must start with 'ToolStart' and end with 'ToolOutput'. + This is where you describe what the tool does and what parameters should the AI model provide to it.`; + } + } + }); + }); + + // Teach the user via logs that they need to use the 'ToolStart' component. + if (error) { + throw new context.CancelError(error); + } + + const toolsDefinition = this.getToolsDefinition(tools); + + const instructions = context.properties.instructions || null; + await context.log({ step: 'create-assistant', tools: toolsDefinition, instructions }); + + const apiKey = context.auth.apiKey; + const client = new OpenAI({ apiKey }); + const assistant = await client.beta.assistants.create({ + model: context.properties.model || 'gpt-4o', + instructions, + tools: toolsDefinition + }); + + await context.log({ step: 'created-assistant', assistant }); + return assistant; + }, + + getToolsDefinition: function(tools) { + + // https://platform.openai.com/docs/assistants/tools/function-calling + const toolsDefinition = []; + + Object.keys(tools).forEach((componentId) => { + const component = tools[componentId]; + const parameters = component.config.properties.parameters?.ADD || []; + const toolParameters = { + type: 'object', + properties: {} + }; + parameters.forEach((parameter) => { + toolParameters.properties[parameter.name] = { + type: parameter.type, + description: parameter.description + }; + }); + const toolDefinition = { + type: 'function', + function: { + name: componentId, + description: component.config.properties.description, + } + }; + if (parameters.length) { + toolDefinition.function.parameters = toolParameters; + } + toolsDefinition.push(toolDefinition); + }); + return toolsDefinition; + }, + + handleRunStatus: async function(context, client, thread, run) { + + if (Date.now() - (run.created_at * 1000) > MAX_RUN_DURATION) { + await context.log({ step: 'run-timeout', run }); + await client.beta.threads.runs.cancel(thread.id, run.id); + throw new context.CancelError('The run took too long to complete.'); + } + + await context.log({ step: 'run-status', run }); + // Check if the run is completed + if (run.status === 'completed') { + let messages = await client.beta.threads.messages.list(thread.id); + await context.log({ step: 'completed-run', run, messages }); + await context.sendJson({ + answer: messages.data[0].content[0].text.value, + prompt: context.messages.in.content.prompt + }, 'out'); + } else if (run.status === 'requires_action') { + await this.handleRequiresAction(context, client, thread, run); + } else { + // incomplete, cancelled, failed, expired + await context.log({ step: 'unexpected-run-state', run }); + } + }, + + handleRequiresAction: async function(context, client, thread, run) { + + await context.log({ step: 'requires-action', run }); + + // Check if there are tools that require outputs. + if ( + run.required_action && + run.required_action.submit_tool_outputs && + run.required_action.submit_tool_outputs.tool_calls + ) { + const toolCalls = []; + for (const toolCall of run.required_action.submit_tool_outputs.tool_calls) { + const componentId = toolCall.function.name; + const args = JSON.parse(toolCall.function.arguments); + toolCalls.push({ componentId, args, id: toolCall.id }); + } + + // Send to all tools. Each ai.ToolStart ignores tool calls that are not intended for it. + await context.sendJson({ toolCalls, prompt: context.messages.in.content.prompt }, 'tools'); + + // Output of each tool is expected to be stored in the service state + // under the ID of the tool call. This is done in the ToolStartOutput component. + // Collect outputs of all the required tool calls. + await context.log({ step: 'collect-tools-output', threadId: thread.id, runId: run.id }); + const outputs = []; + const pollStart = Date.now(); + const runExpiresAt = run.expires_at; + while ( + (outputs.length < toolCalls.length) && + (runExpiresAt ? + Date.now() / 1000 < runExpiresAt : + Date.now() - pollStart < COLLECT_TOOL_OUTPUTS_POLL_TIMEOUT) + ) { + for (const toolCall of toolCalls) { + const result = await context.flow.stateGet(toolCall.id); + if (result) { + outputs.push({ tool_call_id: toolCall.id, output: result.output }); + await context.flow.stateUnset(toolCall.id); + } + } + // Sleep. + await new Promise((resolve) => setTimeout(resolve, COLLECT_TOOL_OUTPUTS_POLL_INTERVAL)); + } + await context.log({ step: 'collected-tools-output', threadId: thread.id, runId: run.id, outputs }); + + // Submit tool outputs to the assistant. + if (outputs && outputs.length) { + await context.log({ step: 'tool-outputs', tools: toolCalls, outputs }); + run = await client.beta.threads.runs.submitToolOutputsAndPoll( + thread.id, + run.id, + { tool_outputs: outputs } + ); + // Check status after submitting tool outputs. + await this.handleRunStatus(context, client, thread, run); + + } else { + await context.log({ step: 'no-tool-outputs', tools: toolCalls }); + } + } + }, + + receive: async function(context) { + + const { prompt } = context.messages.in.content; + let threadId = context.messages.in.content.threadId || context.messages.in.correlationId; + const apiKey = context.auth.apiKey; + const client = new OpenAI({ apiKey }); + const assistant = await context.stateGet('assistant'); + + // Check if a thread with a given ID exists. + let thread; + if (threadId) { + thread = await context.stateGet(threadId); + } + if (!thread) { + await context.log({ step: 'create-thread', assistantId: assistant.id, internalThreadId: threadId }); + thread = await client.beta.threads.create(); + await context.stateSet(threadId, thread); + } else { + await context.log({ step: 'use-thread', assistantId: assistant.id, thread }); + } + + await context.log({ step: 'create-thread-message', assistantId: assistant.id, threadId: thread.id }); + await client.beta.threads.messages.create(thread.id, { + role: 'user', + content: prompt + }); + + await context.log({ step: 'create-thread-run', assistantId: assistant.id, threadId: thread.id }); + let run = await client.beta.threads.runs.create(thread.id, { + assistant_id: assistant.id + }); + await context.log({ step: 'created-thread-run', assistantId: assistant.id, threadId: thread.id, run }); + + // Poll the run status until it reaches a terminal state. + run = await client.beta.threads.runs.poll(thread.id, run.id); + await this.handleRunStatus(context, client, thread, run); + }, + + stop: async function(context) { + + const apiKey = context.auth.apiKey; + const client = new OpenAI({ apiKey }); + const assistant = await context.stateGet('assistant'); + await client.beta.assistants.del(assistant.id); + } +}; diff --git a/src/appmixer/ai/openai/AIAgent/component.json b/src/appmixer/ai/openai/AIAgent/component.json new file mode 100644 index 000000000..63797b0da --- /dev/null +++ b/src/appmixer/ai/openai/AIAgent/component.json @@ -0,0 +1,86 @@ +{ + "name": "appmixer.ai.openai.AIAgent", + "author": "Appmixer ", + "description": "Build an AI agent responding with contextual answers or performing contextual actions.", + "auth": { + "service": "appmixer:ai:openai" + }, + "properties": { + "schema": { + "type": "object", + "properties": { + "model": { "type": "string" }, + "instructions": { "type": "string", "maxLength": 256000 } + } + }, + "inspector": { + "inputs": { + "model": { + "type": "text", + "index": 1, + "label": "Model", + "tooltip": "ID of the model to use.", + "defaultValue": "gpt-4o", + "source": { + "url": "/component/appmixer/ai/openai/ListModels?outPort=out", + "data": { + "transform": "./ListModels#toSelectOptions" + } + } + }, + "instructions": { + "type": "textarea", + "label": "Instructions", + "index": 2, + "tooltip": "The system instructions that the assistant uses. The maximum length is 256,000 characters. For example 'You are a personal math tutor.'." + } + } + } + }, + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "prompt": { "type": "string" }, + "threadId": { "type": "string" } + }, + "required": ["prompt"] + }, + "inspector": { + "inputs": { + "prompt": { + "label": "Prompt", + "type": "textarea", + "index": 1 + }, + "threadId": { + "label": "Thread ID", + "type": "text", + "index": 2, + "tooltip": "By setting a thread ID you can keep the context of the conversation." + } + } + } + }], + "outPorts": [{ + "name": "out", + "options": [{ + "label": "Answer", + "value": "answer", + "schema": { "type": "string" } + }, { + "label": "Prompt", + "value": "prompt", + "schema": { "type": "string" } + }] + }, { + "name": "tools", + "options": [{ + "label": "Prompt", + "value": "prompt", + "schema": { "type": "string" } + }] + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzUyMjFfMzI0OTApIj4KPHBhdGggZD0iTTEyLjE1OTcgNi44Nzk5VjIuMzk5OUg3LjY3OTY5IiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMTcuOTIxMiA2Ljg3OTY0SDUuNDQxMTdDNC4yMDQwNSA2Ljg3OTY0IDMuMjAxMTcgNy44ODI1MiAzLjIwMTE3IDkuMTE5NjRWMTguMDc5NkMzLjIwMTE3IDE5LjMxNjggNC4yMDQwNSAyMC4zMTk2IDUuNDQxMTcgMjAuMzE5NkgxNy45MjEyQzE5LjE1ODMgMjAuMzE5NiAyMC4xNjEyIDE5LjMxNjggMjAuMTYxMiAxOC4wNzk2VjkuMTE5NjRDMjAuMTYxMiA3Ljg4MjUyIDE5LjE1ODMgNi44Nzk2NCAxNy45MjEyIDYuODc5NjRaIiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMC45NTk5NjEgMTMuNTk5NkgzLjE5OTk2IiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMjEuMTIwMSAxMy41OTk2SDIyLjQwMDEiIHN0cm9rZT0iIzMwMzIzNiIgc3Ryb2tlLXdpZHRoPSIyLjAxNiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIi8+CjxwYXRoIGQ9Ik0xNC43ODQyIDEyLjQ3OTdWMTQuNzE5NyIgc3Ryb2tlPSIjMzAzMjM2IiBzdHJva2Utd2lkdGg9IjIuMDE2IiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KPHBhdGggZD0iTTguNjM5NjUgMTIuNDc5N1YxNC43MTk3IiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8L2c+CjxkZWZzPgo8Y2xpcFBhdGggaWQ9ImNsaXAwXzUyMjFfMzI0OTAiPgo8cmVjdCB3aWR0aD0iMjQiIGhlaWdodD0iMjMuMDQiIGZpbGw9IndoaXRlIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgwIDAuNDc5OTgpIi8+CjwvY2xpcFBhdGg+CjwvZGVmcz4KPC9zdmc+Cg==" +} diff --git a/src/appmixer/ai/openai/AIAgent/icon.svg b/src/appmixer/ai/openai/AIAgent/icon.svg new file mode 100644 index 000000000..640f610a1 --- /dev/null +++ b/src/appmixer/ai/openai/AIAgent/icon.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/src/appmixer/ai/openai/CreateSpeech/CreateSpeech.js b/src/appmixer/ai/openai/CreateSpeech/CreateSpeech.js new file mode 100644 index 000000000..ff26ed93e --- /dev/null +++ b/src/appmixer/ai/openai/CreateSpeech/CreateSpeech.js @@ -0,0 +1,29 @@ +'use strict'; + +module.exports = { + + receive: async function(context) { + + const { input, voice, responseFormat, speed, model } = context.messages.in.content; + const apiKey = context.auth.apiKey; + + const url = 'https://api.openai.com/v1/audio/speech'; + const { data: readStream } = await context.httpRequest.post(url, { + model: model || 'tts-1', + input, + voice, + response_format: responseFormat, + speed + }, { + responseType: 'stream', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json' + } + }); + + const filename = `generated-audio-${(new Date).toISOString()}.${responseFormat}`; + const file = await context.saveFileStream(filename, readStream); + return context.sendJson({ fileId: file.fileId, input, fileSize: file.length }, 'out'); + } +}; diff --git a/src/appmixer/ai/openai/CreateSpeech/component.json b/src/appmixer/ai/openai/CreateSpeech/component.json new file mode 100644 index 000000000..ccce45051 --- /dev/null +++ b/src/appmixer/ai/openai/CreateSpeech/component.json @@ -0,0 +1,95 @@ +{ + "name": "appmixer.ai.openai.CreateSpeech", + "author": "Appmixer ", + "description": "Generates audio from the input text.", + "auth": { + "service": "appmixer:ai:openai" + }, + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "input": { "type": "string" }, + "voice": { "type": "string" }, + "responseFormat": { "type": "string" }, + "speed": { "type": "number" }, + "model": { "type": "string" } + }, + "required": ["input"] + }, + "inspector": { + "inputs": { + "input": { + "label": "Text", + "type": "textarea", + "index": 1 + }, + "voice": { + "label": "Voice", + "type": "select", + "defaultValue": "alloy", + "options": [ + { "label": "Alloy (woman)", "value": "alloy" }, + { "label": "Echo (man)", "value": "echo" }, + { "label": "Fable (woman)", "value": "fable" }, + { "label": "Onyx (man)", "value": "onyx" }, + { "label": "Nova (woman)", "value": "nova" }, + { "label": "Shimmer (woman)", "value": "shimmer" } + ], + "index": 2 + }, + "responseFormat": { + "label": "Response Format", + "type": "select", + "defaultValue": "mp3", + "options": [ + { "label": "mp3", "value": "mp3" }, + { "label": "wav", "value": "wav" }, + { "label": "aac", "value": "aac" }, + { "label": "flac", "value": "flac" }, + { "label": "opus", "value": "opus" }, + { "label": "pcm", "value": "pcm" } + ], + "index": 3 + }, + "speed": { + "label": "Speed", + "type": "number", + "index": 4, + "defaultValue": 1 + }, + "model": { + "type": "text", + "index": 5, + "label": "Model", + "tooltip": "ID of the model to use.", + "defaultValue": "tts-1", + "source": { + "url": "/component/appmixer/ai/openai/ListModels?outPort=out", + "data": { + "transform": "./ListModels#toSelectOptions" + } + } + } + } + } + }], + "outPorts": [{ + "name": "out", + "options": [{ + "label": "File ID", + "value": "fileId", + "schema": { "type": "string", "format": "appmixer-file-id" } + },{ + "label": "File Size", + "value": "fileSize", + "schema": { "type": "number" } + },{ + "label": "Text", + "value": "input", + "schema": { "type": "string" } + }] + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNHB4IiBoZWlnaHQ9IjI0cHgiIGZpbGw9ImN1cnJlbnRDb2xvciIgdmlld0JveD0iMCAwIDI0IDI0IiBjb2xvcj0iYmxhY2siPjxwYXRoIGQ9Ik0yMi40MTggOS44MjJhNS45MDMgNS45MDMgMCAwIDAtLjUyLTQuOTEgNi4xIDYuMSAwIDAgMC0yLjgyMi0yLjUxMyA2LjIwNCA2LjIwNCAwIDAgMC0zLjc4LS4zODlBNi4wNTUgNi4wNTUgMCAwIDAgMTMuMjMyLjUxOCA2LjEyOSA2LjEyOSAwIDAgMCAxMC43MjYgMGE2LjE4NSA2LjE4NSAwIDAgMC0zLjYxNSAxLjE1M0E2LjA1MiA2LjA1MiAwIDAgMCA0Ljg4IDQuMTg3YTYuMTAyIDYuMTAyIDAgMCAwLTIuMzQ0IDEuMDE4QTYuMDA4IDYuMDA4IDAgMCAwIC44MjggNy4wODdhNS45ODEgNS45ODEgMCAwIDAgLjc1NCA3LjA5IDUuOTA0IDUuOTA0IDAgMCAwIC41MiA0LjkxMSA2LjEwMSA2LjEwMSAwIDAgMCAyLjgyMSAyLjUxMyA2LjIwNSA2LjIwNSAwIDAgMCAzLjc4LjM4OSA2LjA1NyA2LjA1NyAwIDAgMCAyLjA2NSAxLjQ5MiA2LjEzIDYuMTMgMCAwIDAgMi41MDUuNTE4IDYuMTg1IDYuMTg1IDAgMCAwIDMuNjE3LTEuMTU0IDYuMDUyIDYuMDUyIDAgMCAwIDIuMjMyLTMuMDM1IDYuMTAxIDYuMTAxIDAgMCAwIDIuMzQzLTEuMDE4IDYuMDA5IDYuMDA5IDAgMCAwIDEuNzA5LTEuODgzIDUuOTgxIDUuOTgxIDAgMCAwLS43NTYtNy4wODhabS05LjE0MyAxMi42MDlhNC41ODMgNC41ODMgMCAwIDEtMi45MTgtMS4wNGMuMDM3LS4wMi4xMDItLjA1Ni4xNDQtLjA4MWw0Ljg0NC0yLjc2YS43ODMuNzgzIDAgMCAwIC4zOTctLjY4di02LjczOEwxNy43OSAxMi4zYS4wNzIuMDcyIDAgMCAxIC4wNC4wNTV2NS41OGE0LjQ3MyA0LjQ3MyAwIDAgMS0xLjMzNSAzLjE3NiA0LjU5NiA0LjU5NiAwIDAgMS0zLjIxOSAxLjMyMVptLTkuNzkzLTQuMTI3YTQuNDMyIDQuNDMyIDAgMCAxLS41NDQtMy4wMTRjLjAzNi4wMjEuMDk5LjA2LjE0NC4wODVsNC44NDMgMi43NmEuNzk2Ljc5NiAwIDAgMCAuNzk1IDBsNS45MTMtMy4zNjlWMTcuMWEuMDcxLjA3MSAwIDAgMS0uMDI5LjA2Mkw5LjcwOCAxOS45NWE0LjYxNyA0LjYxNyAwIDAgMS0zLjQ1OC40NDcgNC41NTYgNC41NTYgMCAwIDEtMi43NjgtMi4wOTNaTTIuMjA4IDcuODcyQTQuNTI3IDQuNTI3IDAgMCAxIDQuNTggNS45bC0uMDAyLjE2NHY1LjUyYS43NjguNzY4IDAgMCAwIC4zOTcuNjhsNS45MTMgMy4zNjktMi4wNDcgMS4xNjZhLjA3NS4wNzUgMCAwIDEtLjA2OS4wMDZsLTQuODk2LTIuNzkyYTQuNTEgNC41MSAwIDAgMS0yLjEyLTIuNzMgNC40NSA0LjQ1IDAgMCAxIC40NTItMy40MTFabTE2LjgxOCAzLjg2MS01LjkxMy0zLjM2OCAyLjA0Ny0xLjE2NmEuMDc0LjA3NCAwIDAgMSAuMDctLjAwNmw0Ljg5NiAyLjc4OWE0LjUyNiA0LjUyNiAwIDAgMSAxLjc2MiAxLjgxNSA0LjQ0OCA0LjQ0OCAwIDAgMS0uNDE4IDQuODA4IDQuNTU2IDQuNTU2IDAgMCAxLTIuMDQ5IDEuNDk0di01LjY4NmEuNzY3Ljc2NyAwIDAgMC0uMzk1LS42OFptMi4wMzgtMy4wMjVhNi44NzQgNi44NzQgMCAwIDAtLjE0NC0uMDg1bC00Ljg0My0yLjc2YS43OTcuNzk3IDAgMCAwLS43OTYgMEw5LjM2OCA5LjIzVjYuOWEuMDcyLjA3MiAwIDAgMSAuMDMtLjA2Mmw0Ljg5NS0yLjc4N2E0LjYwOCA0LjYwOCAwIDAgMSA0Ljg4NS4yMDcgNC41MSA0LjUxIDAgMCAxIDEuNTk5IDEuOTU1Yy4zMzMuNzg4LjQzMyAxLjY1NC4yODcgMi40OTZaTTguMjU1IDEyLjg2NSA2LjIwOCAxMS43YS4wNzEuMDcxIDAgMCAxLS4wNC0uMDU2di01LjU4YzAtLjg1NC4yNDgtMS42OS43MTMtMi40MTJhNC41NCA0LjU0IDAgMCAxIDEuOTEzLTEuNjU4IDQuNjE0IDQuNjE0IDAgMCAxIDQuODUuNjE2Yy0uMDM3LjAyLS4xMDIuMDU1LS4xNDQuMDhMOC42NTcgNS40NTJhLjc4Mi43ODIgMCAwIDAtLjM5OC42OGwtLjAwNCA2LjczNFpNOS4zNjcgMTAuNSAxMi4wMDEgOWwyLjYzMyAxLjV2M0wxMi4wMDEgMTVsLTIuNjM0LTEuNXYtM1oiLz48L3N2Zz4=" +} diff --git a/src/appmixer/ai/openai/CreateTranscription/CreateTranscription.js b/src/appmixer/ai/openai/CreateTranscription/CreateTranscription.js new file mode 100644 index 000000000..138cd94c2 --- /dev/null +++ b/src/appmixer/ai/openai/CreateTranscription/CreateTranscription.js @@ -0,0 +1,34 @@ +'use strict'; + +const FormData = require('form-data'); + +module.exports = { + + receive: async function(context) { + + const { fileId, responseFormat, model } = context.messages.in.content; + const apiKey = context.auth.apiKey; + + const readStream = await context.getFileReadStream(fileId); + const fileInfo = await context.getFileInfo(fileId); + const formData = new FormData(); + + formData.append('file', readStream, { + filename: fileInfo.filename, + contentType: fileInfo.contentType, + knownLength: fileInfo.length + }); + formData.append('model', model || 'whisper-1'); + formData.append('response_format', responseFormat || 'text'); + + const url = 'https://api.openai.com/v1/audio/transcriptions'; + const { data } = await context.httpRequest.post(url, formData, { + headers: Object.assign({ + 'Authorization': `Bearer ${apiKey}` + }, formData.getHeaders()) + }); + + await context.log({ step: 'response', data: (data || '').substring(0, 100) }); + return context.sendJson({ text: data }, 'out'); + } +}; diff --git a/src/appmixer/ai/openai/CreateTranscription/component.json b/src/appmixer/ai/openai/CreateTranscription/component.json new file mode 100644 index 000000000..445522b23 --- /dev/null +++ b/src/appmixer/ai/openai/CreateTranscription/component.json @@ -0,0 +1,63 @@ +{ + "name": "appmixer.ai.openai.CreateTranscription", + "author": "Appmixer ", + "description": "Transcribes audio into the input language.", + "auth": { + "service": "appmixer:ai:openai" + }, + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "fileId": { "type": "string" }, + "responseFormat": { "type": "string" }, + "model": { "type": "string" } + }, + "required": ["fileId"] + }, + "inspector": { + "inputs": { + "fileId": { + "label": "File ID", + "type": "filepicker", + "index": 1, + "tooltip": "The audio file to transcribe, in one of these formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm." + }, + "responseFormat": { + "label": "Response Format", + "type": "select", + "index": 2, + "defaultValue": "text", + "options": [ + { "label": "Text", "value": "text" }, + { "label": "SRT", "value": "srt" }, + { "label": "VTT", "value": "vtt" } + ] + }, + "model": { + "type": "text", + "index": 3, + "label": "Model", + "tooltip": "ID of the model to use.", + "defaultValue": "whisper-1", + "source": { + "url": "/component/appmixer/ai/openai/ListModels?outPort=out", + "data": { + "transform": "./ListModels#toSelectOptions" + } + } + } + } + } + }], + "outPorts": [{ + "name": "out", + "options": [{ + "label": "Text", + "value": "text", + "schema": { "type": "string" } + }] + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNHB4IiBoZWlnaHQ9IjI0cHgiIGZpbGw9ImN1cnJlbnRDb2xvciIgdmlld0JveD0iMCAwIDI0IDI0IiBjb2xvcj0iYmxhY2siPjxwYXRoIGQ9Ik0yMi40MTggOS44MjJhNS45MDMgNS45MDMgMCAwIDAtLjUyLTQuOTEgNi4xIDYuMSAwIDAgMC0yLjgyMi0yLjUxMyA2LjIwNCA2LjIwNCAwIDAgMC0zLjc4LS4zODlBNi4wNTUgNi4wNTUgMCAwIDAgMTMuMjMyLjUxOCA2LjEyOSA2LjEyOSAwIDAgMCAxMC43MjYgMGE2LjE4NSA2LjE4NSAwIDAgMC0zLjYxNSAxLjE1M0E2LjA1MiA2LjA1MiAwIDAgMCA0Ljg4IDQuMTg3YTYuMTAyIDYuMTAyIDAgMCAwLTIuMzQ0IDEuMDE4QTYuMDA4IDYuMDA4IDAgMCAwIC44MjggNy4wODdhNS45ODEgNS45ODEgMCAwIDAgLjc1NCA3LjA5IDUuOTA0IDUuOTA0IDAgMCAwIC41MiA0LjkxMSA2LjEwMSA2LjEwMSAwIDAgMCAyLjgyMSAyLjUxMyA2LjIwNSA2LjIwNSAwIDAgMCAzLjc4LjM4OSA2LjA1NyA2LjA1NyAwIDAgMCAyLjA2NSAxLjQ5MiA2LjEzIDYuMTMgMCAwIDAgMi41MDUuNTE4IDYuMTg1IDYuMTg1IDAgMCAwIDMuNjE3LTEuMTU0IDYuMDUyIDYuMDUyIDAgMCAwIDIuMjMyLTMuMDM1IDYuMTAxIDYuMTAxIDAgMCAwIDIuMzQzLTEuMDE4IDYuMDA5IDYuMDA5IDAgMCAwIDEuNzA5LTEuODgzIDUuOTgxIDUuOTgxIDAgMCAwLS43NTYtNy4wODhabS05LjE0MyAxMi42MDlhNC41ODMgNC41ODMgMCAwIDEtMi45MTgtMS4wNGMuMDM3LS4wMi4xMDItLjA1Ni4xNDQtLjA4MWw0Ljg0NC0yLjc2YS43ODMuNzgzIDAgMCAwIC4zOTctLjY4di02LjczOEwxNy43OSAxMi4zYS4wNzIuMDcyIDAgMCAxIC4wNC4wNTV2NS41OGE0LjQ3MyA0LjQ3MyAwIDAgMS0xLjMzNSAzLjE3NiA0LjU5NiA0LjU5NiAwIDAgMS0zLjIxOSAxLjMyMVptLTkuNzkzLTQuMTI3YTQuNDMyIDQuNDMyIDAgMCAxLS41NDQtMy4wMTRjLjAzNi4wMjEuMDk5LjA2LjE0NC4wODVsNC44NDMgMi43NmEuNzk2Ljc5NiAwIDAgMCAuNzk1IDBsNS45MTMtMy4zNjlWMTcuMWEuMDcxLjA3MSAwIDAgMS0uMDI5LjA2Mkw5LjcwOCAxOS45NWE0LjYxNyA0LjYxNyAwIDAgMS0zLjQ1OC40NDcgNC41NTYgNC41NTYgMCAwIDEtMi43NjgtMi4wOTNaTTIuMjA4IDcuODcyQTQuNTI3IDQuNTI3IDAgMCAxIDQuNTggNS45bC0uMDAyLjE2NHY1LjUyYS43NjguNzY4IDAgMCAwIC4zOTcuNjhsNS45MTMgMy4zNjktMi4wNDcgMS4xNjZhLjA3NS4wNzUgMCAwIDEtLjA2OS4wMDZsLTQuODk2LTIuNzkyYTQuNTEgNC41MSAwIDAgMS0yLjEyLTIuNzMgNC40NSA0LjQ1IDAgMCAxIC40NTItMy40MTFabTE2LjgxOCAzLjg2MS01LjkxMy0zLjM2OCAyLjA0Ny0xLjE2NmEuMDc0LjA3NCAwIDAgMSAuMDctLjAwNmw0Ljg5NiAyLjc4OWE0LjUyNiA0LjUyNiAwIDAgMSAxLjc2MiAxLjgxNSA0LjQ0OCA0LjQ0OCAwIDAgMS0uNDE4IDQuODA4IDQuNTU2IDQuNTU2IDAgMCAxLTIuMDQ5IDEuNDk0di01LjY4NmEuNzY3Ljc2NyAwIDAgMC0uMzk1LS42OFptMi4wMzgtMy4wMjVhNi44NzQgNi44NzQgMCAwIDAtLjE0NC0uMDg1bC00Ljg0My0yLjc2YS43OTcuNzk3IDAgMCAwLS43OTYgMEw5LjM2OCA5LjIzVjYuOWEuMDcyLjA3MiAwIDAgMSAuMDMtLjA2Mmw0Ljg5NS0yLjc4N2E0LjYwOCA0LjYwOCAwIDAgMSA0Ljg4NS4yMDcgNC41MSA0LjUxIDAgMCAxIDEuNTk5IDEuOTU1Yy4zMzMuNzg4LjQzMyAxLjY1NC4yODcgMi40OTZaTTguMjU1IDEyLjg2NSA2LjIwOCAxMS43YS4wNzEuMDcxIDAgMCAxLS4wNC0uMDU2di01LjU4YzAtLjg1NC4yNDgtMS42OS43MTMtMi40MTJhNC41NCA0LjU0IDAgMCAxIDEuOTEzLTEuNjU4IDQuNjE0IDQuNjE0IDAgMCAxIDQuODUuNjE2Yy0uMDM3LjAyLS4xMDIuMDU1LS4xNDQuMDhMOC42NTcgNS40NTJhLjc4Mi43ODIgMCAwIDAtLjM5OC42OGwtLjAwNCA2LjczNFpNOS4zNjcgMTAuNSAxMi4wMDEgOWwyLjYzMyAxLjV2M0wxMi4wMDEgMTVsLTIuNjM0LTEuNXYtM1oiLz48L3N2Zz4=" +} diff --git a/src/appmixer/ai/openai/DescribeImages/DescribeImages.js b/src/appmixer/ai/openai/DescribeImages/DescribeImages.js new file mode 100644 index 000000000..d3a7e51b1 --- /dev/null +++ b/src/appmixer/ai/openai/DescribeImages/DescribeImages.js @@ -0,0 +1,71 @@ +'use strict'; + +const path = require('path'); + +module.exports = { + + receive: async function(context) { + + const { prompt, images, model } = context.messages.in.content; + const apiKey = context.auth.apiKey; + const imageFileIds = (images.ADD || []).map(image => (image.fileId || null)).filter(fileId => fileId !== null); + + const imageContent = await Promise.all(imageFileIds.map(async (fileId) => { + const fileInfo = await context.getFileInfo(fileId); + const fileContent = await context.loadFile(fileId); + const base64 = fileContent.toString('base64'); + + let contentType = fileInfo.contentType; + if (!contentType) { + const ext = path.extname(fileInfo.filename).toLowerCase(); + if (ext === '.png') { + contentType = 'image/png'; + } else if (ext === '.jpg' || ext === '.jpeg') { + contentType = 'image/jpeg'; + } else if (ext === '.gif') { + contentType = 'image/gif'; + } else if (ext === '.webp') { + contentType = 'image/webp'; + } + } + + if (!['image/png', 'image/jpeg', 'image/gif', 'image/webp'].includes(contentType)) { + throw new Error(`Unsupported image type: ${contentType}`); + } + return { + type: 'image_url', + image_url: { + url: `data:${contentType};base64,${base64}` + } + }; + })); + + const url = 'https://api.openai.com/v1/chat/completions'; + const payload = { + model: model || 'gpt-4o', + messages: [ + { + role: 'user', + content: [{ + type: 'text', + text: prompt + }].concat(imageContent) + } + ] + }; + const { data } = await context.httpRequest.post(url, payload, { + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json' + } + }); + + let answer = ''; + + if (data && data.choices) { + answer = data.choices[0].message.content; + } + + return context.sendJson({ answer, prompt }, 'out'); + } +}; diff --git a/src/appmixer/ai/openai/DescribeImages/component.json b/src/appmixer/ai/openai/DescribeImages/component.json new file mode 100644 index 000000000..e6bd179d5 --- /dev/null +++ b/src/appmixer/ai/openai/DescribeImages/component.json @@ -0,0 +1,70 @@ +{ + "name": "appmixer.ai.openai.DescribeImages", + "author": "Appmixer ", + "description": "Answer questions about the content of one or more images.", + "auth": { + "service": "appmixer:ai:openai" + }, + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "prompt": { "type": "string" }, + "images": {}, + "model": { "type": "string" } + }, + "required": ["prompt", "images"] + }, + "inspector": { + "inputs": { + "prompt": { + "label": "Prompt", + "type": "textarea", + "index": 1, + "tooltip": "The question to answer about the content of one or more images." + }, + "images": { + "label": "Images", + "type": "expression", + "index": 2, + "levels": ["ADD"], + "fields": { + "fileId": { + "type": "filepicker", + "label": "File ID", + "tooltip": "The image file to attach. Only PNG, JPEG, non-animated GIF and WEBP image types are supported.", + "index": 1 + } + }, + "model": { + "type": "text", + "index": 3, + "label": "Model", + "tooltip": "ID of the model to use.", + "defaultValue": "gpt-4o", + "source": { + "url": "/component/appmixer/ai/openai/ListModels?outPort=out", + "data": { + "transform": "./ListModels#toSelectOptions" + } + } + } + } + } + } + }], + "outPorts": [{ + "name": "out", + "options": [{ + "label": "Answer", + "value": "answer", + "schema": { "type": "string" } + }, { + "label": "Prompt", + "value": "prompt", + "schema": { "type": "string" } + }] + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNHB4IiBoZWlnaHQ9IjI0cHgiIGZpbGw9ImN1cnJlbnRDb2xvciIgdmlld0JveD0iMCAwIDI0IDI0IiBjb2xvcj0iYmxhY2siPjxwYXRoIGQ9Ik0yMi40MTggOS44MjJhNS45MDMgNS45MDMgMCAwIDAtLjUyLTQuOTEgNi4xIDYuMSAwIDAgMC0yLjgyMi0yLjUxMyA2LjIwNCA2LjIwNCAwIDAgMC0zLjc4LS4zODlBNi4wNTUgNi4wNTUgMCAwIDAgMTMuMjMyLjUxOCA2LjEyOSA2LjEyOSAwIDAgMCAxMC43MjYgMGE2LjE4NSA2LjE4NSAwIDAgMC0zLjYxNSAxLjE1M0E2LjA1MiA2LjA1MiAwIDAgMCA0Ljg4IDQuMTg3YTYuMTAyIDYuMTAyIDAgMCAwLTIuMzQ0IDEuMDE4QTYuMDA4IDYuMDA4IDAgMCAwIC44MjggNy4wODdhNS45ODEgNS45ODEgMCAwIDAgLjc1NCA3LjA5IDUuOTA0IDUuOTA0IDAgMCAwIC41MiA0LjkxMSA2LjEwMSA2LjEwMSAwIDAgMCAyLjgyMSAyLjUxMyA2LjIwNSA2LjIwNSAwIDAgMCAzLjc4LjM4OSA2LjA1NyA2LjA1NyAwIDAgMCAyLjA2NSAxLjQ5MiA2LjEzIDYuMTMgMCAwIDAgMi41MDUuNTE4IDYuMTg1IDYuMTg1IDAgMCAwIDMuNjE3LTEuMTU0IDYuMDUyIDYuMDUyIDAgMCAwIDIuMjMyLTMuMDM1IDYuMTAxIDYuMTAxIDAgMCAwIDIuMzQzLTEuMDE4IDYuMDA5IDYuMDA5IDAgMCAwIDEuNzA5LTEuODgzIDUuOTgxIDUuOTgxIDAgMCAwLS43NTYtNy4wODhabS05LjE0MyAxMi42MDlhNC41ODMgNC41ODMgMCAwIDEtMi45MTgtMS4wNGMuMDM3LS4wMi4xMDItLjA1Ni4xNDQtLjA4MWw0Ljg0NC0yLjc2YS43ODMuNzgzIDAgMCAwIC4zOTctLjY4di02LjczOEwxNy43OSAxMi4zYS4wNzIuMDcyIDAgMCAxIC4wNC4wNTV2NS41OGE0LjQ3MyA0LjQ3MyAwIDAgMS0xLjMzNSAzLjE3NiA0LjU5NiA0LjU5NiAwIDAgMS0zLjIxOSAxLjMyMVptLTkuNzkzLTQuMTI3YTQuNDMyIDQuNDMyIDAgMCAxLS41NDQtMy4wMTRjLjAzNi4wMjEuMDk5LjA2LjE0NC4wODVsNC44NDMgMi43NmEuNzk2Ljc5NiAwIDAgMCAuNzk1IDBsNS45MTMtMy4zNjlWMTcuMWEuMDcxLjA3MSAwIDAgMS0uMDI5LjA2Mkw5LjcwOCAxOS45NWE0LjYxNyA0LjYxNyAwIDAgMS0zLjQ1OC40NDcgNC41NTYgNC41NTYgMCAwIDEtMi43NjgtMi4wOTNaTTIuMjA4IDcuODcyQTQuNTI3IDQuNTI3IDAgMCAxIDQuNTggNS45bC0uMDAyLjE2NHY1LjUyYS43NjguNzY4IDAgMCAwIC4zOTcuNjhsNS45MTMgMy4zNjktMi4wNDcgMS4xNjZhLjA3NS4wNzUgMCAwIDEtLjA2OS4wMDZsLTQuODk2LTIuNzkyYTQuNTEgNC41MSAwIDAgMS0yLjEyLTIuNzMgNC40NSA0LjQ1IDAgMCAxIC40NTItMy40MTFabTE2LjgxOCAzLjg2MS01LjkxMy0zLjM2OCAyLjA0Ny0xLjE2NmEuMDc0LjA3NCAwIDAgMSAuMDctLjAwNmw0Ljg5NiAyLjc4OWE0LjUyNiA0LjUyNiAwIDAgMSAxLjc2MiAxLjgxNSA0LjQ0OCA0LjQ0OCAwIDAgMS0uNDE4IDQuODA4IDQuNTU2IDQuNTU2IDAgMCAxLTIuMDQ5IDEuNDk0di01LjY4NmEuNzY3Ljc2NyAwIDAgMC0uMzk1LS42OFptMi4wMzgtMy4wMjVhNi44NzQgNi44NzQgMCAwIDAtLjE0NC0uMDg1bC00Ljg0My0yLjc2YS43OTcuNzk3IDAgMCAwLS43OTYgMEw5LjM2OCA5LjIzVjYuOWEuMDcyLjA3MiAwIDAgMSAuMDMtLjA2Mmw0Ljg5NS0yLjc4N2E0LjYwOCA0LjYwOCAwIDAgMSA0Ljg4NS4yMDcgNC41MSA0LjUxIDAgMCAxIDEuNTk5IDEuOTU1Yy4zMzMuNzg4LjQzMyAxLjY1NC4yODcgMi40OTZaTTguMjU1IDEyLjg2NSA2LjIwOCAxMS43YS4wNzEuMDcxIDAgMCAxLS4wNC0uMDU2di01LjU4YzAtLjg1NC4yNDgtMS42OS43MTMtMi40MTJhNC41NCA0LjU0IDAgMCAxIDEuOTEzLTEuNjU4IDQuNjE0IDQuNjE0IDAgMCAxIDQuODUuNjE2Yy0uMDM3LjAyLS4xMDIuMDU1LS4xNDQuMDhMOC42NTcgNS40NTJhLjc4Mi43ODIgMCAwIDAtLjM5OC42OGwtLjAwNCA2LjczNFpNOS4zNjcgMTAuNSAxMi4wMDEgOWwyLjYzMyAxLjV2M0wxMi4wMDEgMTVsLTIuNjM0LTEuNXYtM1oiLz48L3N2Zz4=" +} diff --git a/src/appmixer/ai/openai/GenerateEmbeddings/GenerateEmbeddings.js b/src/appmixer/ai/openai/GenerateEmbeddings/GenerateEmbeddings.js new file mode 100644 index 000000000..6883313b9 --- /dev/null +++ b/src/appmixer/ai/openai/GenerateEmbeddings/GenerateEmbeddings.js @@ -0,0 +1,81 @@ +'use strict'; + +const { RecursiveCharacterTextSplitter } = require('langchain/text_splitter'); +const OpenAI = require('openai'); + +// See https://platform.openai.com/docs/api-reference/embeddings/create#embeddings-create-input. +const MAX_INPUT_LENGTH = 8192 * 4; // max 8192 tokens, 1 token ~ 4 characters. +const MAX_BATCH_SIZE = 2048; + +module.exports = { + + receive: async function(context) { + + const messageId = context.messages.in.messageId; + const { + text, + model = 'text-embedding-ada-002', + chunkSize = 500, + chunkOverlap = 50 + } = context.messages.in.content; + + const chunks = await this.splitText(text, chunkSize, chunkOverlap); + await context.log({ step: 'split-text', message: 'Text succesfully split into chunks.', chunksLength: chunks.length }); + + const apiKey = context.auth.apiKey; + const client = new OpenAI({ apiKey }); + + // Process chunks in batches. + // the batch size is calculated based on the chunk size and the maximum input length in + // order not to exceed the maximum input length defined in + // https://platform.openai.com/docs/api-reference/embeddings/create#embeddings-create-input + // We devide the maximum input length by 2 to stay on the safe side + // because the token to character ratio might not be accurate. + const batchSize = Math.min(Math.floor((MAX_INPUT_LENGTH / 2) / chunkSize), MAX_BATCH_SIZE); + const embeddings = []; + // For convenience, the GenerateEmbeddings component returns the first vector. + // This makes it easy to genereate embedding for a prompt and send it e.g. to the pinecone.QueryVectors component + // without having to apply modifiers to the embedding array returned. + let firstVector = null; + for (let i = 0; i < chunks.length; i += batchSize) { + const batch = chunks.slice(i, i + batchSize); + + const response = await client.embeddings.create({ + model, + input: batch, + encoding_format: 'float' + }); + + // Collect embeddings for the current batch. + response.data.forEach((item, index) => { + if (!firstVector) { + firstVector = item.embedding; + } + try { + const embedding = { + text: batch[index], + vector: item.embedding, + index: i + index + }; + embeddings.push(embedding); + } catch (err) { + // It does not make sense to retry the component call. + // Things "won't" improve. + throw new context.CancelError(err); + } + }); + } + return context.sendJson({ embeddings, firstVector }, 'out'); + }, + + splitText(text, chunkSize, chunkOverlap) { + + const splitter = new RecursiveCharacterTextSplitter({ + chunkSize, + chunkOverlap + }); + + return splitter.splitText(text); + } +}; + diff --git a/src/appmixer/ai/openai/GenerateEmbeddings/component.json b/src/appmixer/ai/openai/GenerateEmbeddings/component.json new file mode 100644 index 000000000..124ebc124 --- /dev/null +++ b/src/appmixer/ai/openai/GenerateEmbeddings/component.json @@ -0,0 +1,90 @@ +{ + "name": "appmixer.ai.openai.GenerateEmbeddings", + "author": "Appmixer ", + "description": "Generate embeddings for text data. The text is split into chunks and embedding is returned for each chunk.
The returned embeddings is an array of the form: [{ \"index\": 0, \"text\": \"chunk1\", \"vector\": [1.1, 1.2, 1.3] }].
TIP: use the JSONata modifier to convert the embeddings array into custom formats. For convenience, the component also returns the first vector in the embeddings array which is useful when querying vector databases to find relevant chunks.", + "auth": { + "service": "appmixer:ai:openai" + }, + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "text": { "type": "string", "maxLength": 512000 }, + "model": { "type": "string" }, + "chunkSize": { "type": "integer" }, + "chunkOverlap": { "type": "integer" }, + "embeddingTemplate": { "type": "string" } + } + }, + "inspector": { + "inputs": { + "text": { + "type": "textarea", + "label": "Text", + "tooltip": "Enter the text to generate embeddings for. The text will be split into chunks and embeddings will be generated for each chunk. The maximum length is 512,000 characters. If you need more than 512,000 characters, use the 'Generate Embeddings From File' component.", + "index": 1 + }, + "model": { + "type": "text", + "index": 2, + "label": "Model", + "tooltip": "ID of the model to use.", + "defaultValue": "text-embedding-ada-002", + "source": { + "url": "/component/appmixer/ai/openai/ListModels?outPort=out", + "data": { + "transform": "./ListModels#toSelectOptions" + } + } + }, + "chunkSize": { + "type": "number", + "label": "Chunk Size", + "defaultValue": 500, + "tooltip": "Maximum size of each chunk for text splitting. The default is 500.", + "index": 3 + }, + "chunkOverlap": { + "type": "number", + "label": "Chunk Overlap", + "defaultValue": 50, + "tooltip": "Overlap between chunks for text splitting to maintain context. The default is 50.", + "index": 4 + } + } + } + }], + "outPorts": [{ + "name": "out", + "options": [{ + "label": "Embeddings", + "value": "embeddings", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "index": { "type": "string" }, + "vector": { "type": "array", "items": { "type": "number" } }, + "text": { "type": "string" } + } + }, + "examples": [ + [{ "index": 0, "text": "chunk1", "vector": [1.1, 1.2, 1.3] }, { "index": 1, "text": "chunk2", "vector": [2.1, 2.2, 2.3] }] + ] + } + }, { + "label": "First Vector", + "value": "firstVector", + "schema": { + "type": "array", + "items": { "type": "number" }, + "examples": [ + [-0.0120379254, -0.0376950279, -0.0133513855, -0.0365983546, -0.0247007012, 0.0158507861, -0.0143460445, 0.00486809108] + ] + } + }] + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNHB4IiBoZWlnaHQ9IjI0cHgiIGZpbGw9ImN1cnJlbnRDb2xvciIgdmlld0JveD0iMCAwIDI0IDI0IiBjb2xvcj0iYmxhY2siPjxwYXRoIGQ9Ik0yMi40MTggOS44MjJhNS45MDMgNS45MDMgMCAwIDAtLjUyLTQuOTEgNi4xIDYuMSAwIDAgMC0yLjgyMi0yLjUxMyA2LjIwNCA2LjIwNCAwIDAgMC0zLjc4LS4zODlBNi4wNTUgNi4wNTUgMCAwIDAgMTMuMjMyLjUxOCA2LjEyOSA2LjEyOSAwIDAgMCAxMC43MjYgMGE2LjE4NSA2LjE4NSAwIDAgMC0zLjYxNSAxLjE1M0E2LjA1MiA2LjA1MiAwIDAgMCA0Ljg4IDQuMTg3YTYuMTAyIDYuMTAyIDAgMCAwLTIuMzQ0IDEuMDE4QTYuMDA4IDYuMDA4IDAgMCAwIC44MjggNy4wODdhNS45ODEgNS45ODEgMCAwIDAgLjc1NCA3LjA5IDUuOTA0IDUuOTA0IDAgMCAwIC41MiA0LjkxMSA2LjEwMSA2LjEwMSAwIDAgMCAyLjgyMSAyLjUxMyA2LjIwNSA2LjIwNSAwIDAgMCAzLjc4LjM4OSA2LjA1NyA2LjA1NyAwIDAgMCAyLjA2NSAxLjQ5MiA2LjEzIDYuMTMgMCAwIDAgMi41MDUuNTE4IDYuMTg1IDYuMTg1IDAgMCAwIDMuNjE3LTEuMTU0IDYuMDUyIDYuMDUyIDAgMCAwIDIuMjMyLTMuMDM1IDYuMTAxIDYuMTAxIDAgMCAwIDIuMzQzLTEuMDE4IDYuMDA5IDYuMDA5IDAgMCAwIDEuNzA5LTEuODgzIDUuOTgxIDUuOTgxIDAgMCAwLS43NTYtNy4wODhabS05LjE0MyAxMi42MDlhNC41ODMgNC41ODMgMCAwIDEtMi45MTgtMS4wNGMuMDM3LS4wMi4xMDItLjA1Ni4xNDQtLjA4MWw0Ljg0NC0yLjc2YS43ODMuNzgzIDAgMCAwIC4zOTctLjY4di02LjczOEwxNy43OSAxMi4zYS4wNzIuMDcyIDAgMCAxIC4wNC4wNTV2NS41OGE0LjQ3MyA0LjQ3MyAwIDAgMS0xLjMzNSAzLjE3NiA0LjU5NiA0LjU5NiAwIDAgMS0zLjIxOSAxLjMyMVptLTkuNzkzLTQuMTI3YTQuNDMyIDQuNDMyIDAgMCAxLS41NDQtMy4wMTRjLjAzNi4wMjEuMDk5LjA2LjE0NC4wODVsNC44NDMgMi43NmEuNzk2Ljc5NiAwIDAgMCAuNzk1IDBsNS45MTMtMy4zNjlWMTcuMWEuMDcxLjA3MSAwIDAgMS0uMDI5LjA2Mkw5LjcwOCAxOS45NWE0LjYxNyA0LjYxNyAwIDAgMS0zLjQ1OC40NDcgNC41NTYgNC41NTYgMCAwIDEtMi43NjgtMi4wOTNaTTIuMjA4IDcuODcyQTQuNTI3IDQuNTI3IDAgMCAxIDQuNTggNS45bC0uMDAyLjE2NHY1LjUyYS43NjguNzY4IDAgMCAwIC4zOTcuNjhsNS45MTMgMy4zNjktMi4wNDcgMS4xNjZhLjA3NS4wNzUgMCAwIDEtLjA2OS4wMDZsLTQuODk2LTIuNzkyYTQuNTEgNC41MSAwIDAgMS0yLjEyLTIuNzMgNC40NSA0LjQ1IDAgMCAxIC40NTItMy40MTFabTE2LjgxOCAzLjg2MS01LjkxMy0zLjM2OCAyLjA0Ny0xLjE2NmEuMDc0LjA3NCAwIDAgMSAuMDctLjAwNmw0Ljg5NiAyLjc4OWE0LjUyNiA0LjUyNiAwIDAgMSAxLjc2MiAxLjgxNSA0LjQ0OCA0LjQ0OCAwIDAgMS0uNDE4IDQuODA4IDQuNTU2IDQuNTU2IDAgMCAxLTIuMDQ5IDEuNDk0di01LjY4NmEuNzY3Ljc2NyAwIDAgMC0uMzk1LS42OFptMi4wMzgtMy4wMjVhNi44NzQgNi44NzQgMCAwIDAtLjE0NC0uMDg1bC00Ljg0My0yLjc2YS43OTcuNzk3IDAgMCAwLS43OTYgMEw5LjM2OCA5LjIzVjYuOWEuMDcyLjA3MiAwIDAgMSAuMDMtLjA2Mmw0Ljg5NS0yLjc4N2E0LjYwOCA0LjYwOCAwIDAgMSA0Ljg4NS4yMDcgNC41MSA0LjUxIDAgMCAxIDEuNTk5IDEuOTU1Yy4zMzMuNzg4LjQzMyAxLjY1NC4yODcgMi40OTZaTTguMjU1IDEyLjg2NSA2LjIwOCAxMS43YS4wNzEuMDcxIDAgMCAxLS4wNC0uMDU2di01LjU4YzAtLjg1NC4yNDgtMS42OS43MTMtMi40MTJhNC41NCA0LjU0IDAgMCAxIDEuOTEzLTEuNjU4IDQuNjE0IDQuNjE0IDAgMCAxIDQuODUuNjE2Yy0uMDM3LjAyLS4xMDIuMDU1LS4xNDQuMDhMOC42NTcgNS40NTJhLjc4Mi43ODIgMCAwIDAtLjM5OC42OGwtLjAwNCA2LjczNFpNOS4zNjcgMTAuNSAxMi4wMDEgOWwyLjYzMyAxLjV2M0wxMi4wMDEgMTVsLTIuNjM0LTEuNXYtM1oiLz48L3N2Zz4=" +} diff --git a/src/appmixer/ai/openai/GenerateEmbeddingsFromFile/GenerateEmbeddingsFromFile.js b/src/appmixer/ai/openai/GenerateEmbeddingsFromFile/GenerateEmbeddingsFromFile.js new file mode 100644 index 000000000..5a34440c4 --- /dev/null +++ b/src/appmixer/ai/openai/GenerateEmbeddingsFromFile/GenerateEmbeddingsFromFile.js @@ -0,0 +1,147 @@ +'use strict'; + +const { Transform } = require('stream'); +const { RecursiveCharacterTextSplitter } = require('langchain/text_splitter'); +const OpenAI = require('openai'); + +// See https://platform.openai.com/docs/api-reference/embeddings/create#embeddings-create-input. +const MAX_INPUT_LENGTH = 8192 * 4; // max 8192 tokens, 1 token ~ 4 characters. +const MAX_BATCH_SIZE = 2048; +const FILE_PART_SIZE = 1024 * 1024; // 1MB + +module.exports = { + + receive: async function(context) { + + const messageId = context.messages.in.messageId; + const { + fileId + } = context.messages.in.content; + + const apiKey = context.auth.apiKey; + const client = new OpenAI({ apiKey }); + + const readStream = await context.getFileReadStream(fileId); + const fileInfo = await context.getFileInfo(fileId); + await context.log({ step: 'split-file', message: 'Splitting file into parts.', partSize: FILE_PART_SIZE, fileInfo }); + let firstVector; + const partsStream = splitStream(readStream, FILE_PART_SIZE); + for await (const part of partsStream) { + const embeddings = await this.generateEmbeddings(context, client, part.toString()); + if (!firstVector) { + firstVector = embeddings[0].vector; + } + await context.sendJson({ embeddings, firstVector }, 'out'); + } + }, + + generateEmbeddings: async function(context, client, text) { + + const { + model = 'text-embedding-ada-002', + chunkSize = 500, + chunkOverlap = 50 + } = context.messages.in.content; + + const chunks = await this.splitText(text, chunkSize, chunkOverlap); + await context.log({ + step: 'split-text', + message: 'Text succesfully split into chunks.', + chunksLength: chunks.length, + textLength: text.length + }); + + + // Process chunks in batches. + // the batch size is calculated based on the chunk size and the maximum input length in + // order not to exceed the maximum input length defined in + // https://platform.openai.com/docs/api-reference/embeddings/create#embeddings-create-input + // We devide the maximum input length by 2 to stay on the safe side + // because the token to character ratio might not be accurate. + const batchSize = Math.min(Math.floor((MAX_INPUT_LENGTH / 2) / chunkSize), MAX_BATCH_SIZE); + const embeddings = []; + // For convenience, the GenerateEmbeddings component returns the first vector. + // This makes it easy to genereate embedding for a prompt and send it e.g. to the pinecone.QueryVectors component + // without having to apply modifiers to the embedding array returned. + let firstVector = null; + for (let i = 0; i < chunks.length; i += batchSize) { + const batch = chunks.slice(i, i + batchSize); + + const response = await client.embeddings.create({ + model, + input: batch, + encoding_format: 'float' + }); + + // Collect embeddings for the current batch. + response.data.forEach((item, index) => { + if (!firstVector) { + firstVector = item.embedding; + } + try { + const embedding = { + text: batch[index], + vector: item.embedding, + index: i + index + }; + embeddings.push(embedding); + } catch (err) { + // It does not make sense to retry the component call. + // Things "won't" improve. + throw new context.CancelError(err); + } + }); + } + return embeddings; + }, + + splitText(text, chunkSize, chunkOverlap) { + + const splitter = new RecursiveCharacterTextSplitter({ + chunkSize, + chunkOverlap + }); + + return splitter.splitText(text); + } +}; + +/** + * Splits a readable stream into chunks of n bytes. + * @param {Readable} inputStream - The readable stream to split. + * @param {number} chunkSize - Size of each chunk in bytes. + * @returns {Readable} - A readable stream emitting chunks. + */ +function splitStream(inputStream, chunkSize) { + + let leftover = Buffer.alloc(0); + + const transformStream = new Transform({ + transform(chunk, encoding, callback) { + // Combine leftover buffer with the new chunk + const combined = Buffer.concat([leftover, chunk]); + const combinedLength = combined.length; + + // Emit chunks of the desired size + let offset = 0; + while (offset + chunkSize <= combinedLength) { + this.push(combined.slice(offset, offset + chunkSize)); + offset += chunkSize; + } + + // Store leftover data + leftover = combined.slice(offset); + + callback(); + }, + flush(callback) { + // Push any remaining data as the final chunk + if (leftover.length > 0) { + this.push(leftover); + } + callback(); + }, + }); + + return inputStream.pipe(transformStream); + } \ No newline at end of file diff --git a/src/appmixer/ai/openai/GenerateEmbeddingsFromFile/component.json b/src/appmixer/ai/openai/GenerateEmbeddingsFromFile/component.json new file mode 100644 index 000000000..cb7f665f4 --- /dev/null +++ b/src/appmixer/ai/openai/GenerateEmbeddingsFromFile/component.json @@ -0,0 +1,90 @@ +{ + "name": "appmixer.ai.openai.GenerateEmbeddingsFromFile", + "author": "Appmixer ", + "description": "Generate embeddings for a text file. The text is split into parts, each part is split into chunks and embedding is returned for each chunk. The component emits embeddings array for each file part (1MB).
The returned embeddings is an array of the form: [{ \"index\": 0, \"text\": \"chunk1\", \"vector\": [1.1, 1.2, 1.3] }].
TIP: use the JSONata modifier to convert the embeddings array into custom formats.", + "auth": { + "service": "appmixer:ai:openai" + }, + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "fileId": { "type": "string" }, + "model": { "type": "string" }, + "chunkSize": { "type": "integer" }, + "chunkOverlap": { "type": "integer" }, + "embeddingTemplate": { "type": "string" } + } + }, + "inspector": { + "inputs": { + "fileId": { + "label": "File ID", + "type": "filepicker", + "index": 1, + "tooltip": "The text file to generate embeddings for. Use plain text or CSV files only." + }, + "model": { + "type": "text", + "index": 2, + "label": "Model", + "tooltip": "ID of the model to use.", + "defaultValue": "text-embedding-ada-002", + "source": { + "url": "/component/appmixer/ai/openai/ListModels?outPort=out", + "data": { + "transform": "./ListModels#toSelectOptions" + } + } + }, + "chunkSize": { + "type": "number", + "label": "Chunk Size", + "defaultValue": 500, + "tooltip": "Maximum size of each chunk for text splitting. The default is 500.", + "index": 3 + }, + "chunkOverlap": { + "type": "number", + "label": "Chunk Overlap", + "defaultValue": 50, + "tooltip": "Overlap between chunks for text splitting to maintain context. The default is 50.", + "index": 4 + } + } + } + }], + "outPorts": [{ + "name": "out", + "options": [{ + "label": "Embeddings", + "value": "embeddings", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "index": { "type": "string" }, + "vector": { "type": "array", "items": { "type": "number" } }, + "text": { "type": "string" } + } + }, + "examples": [ + [{ "index": 0, "text": "chunk1", "vector": [1.1, 1.2, 1.3] }, { "index": 1, "text": "chunk2", "vector": [2.1, 2.2, 2.3] }] + ] + } + }, { + "label": "First Vector", + "value": "firstVector", + "schema": { + "type": "array", + "items": { "type": "number" }, + "examples": [ + [-0.0120379254, -0.0376950279, -0.0133513855, -0.0365983546, -0.0247007012, 0.0158507861, -0.0143460445, 0.00486809108] + ] + } + }] + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNHB4IiBoZWlnaHQ9IjI0cHgiIGZpbGw9ImN1cnJlbnRDb2xvciIgdmlld0JveD0iMCAwIDI0IDI0IiBjb2xvcj0iYmxhY2siPjxwYXRoIGQ9Ik0yMi40MTggOS44MjJhNS45MDMgNS45MDMgMCAwIDAtLjUyLTQuOTEgNi4xIDYuMSAwIDAgMC0yLjgyMi0yLjUxMyA2LjIwNCA2LjIwNCAwIDAgMC0zLjc4LS4zODlBNi4wNTUgNi4wNTUgMCAwIDAgMTMuMjMyLjUxOCA2LjEyOSA2LjEyOSAwIDAgMCAxMC43MjYgMGE2LjE4NSA2LjE4NSAwIDAgMC0zLjYxNSAxLjE1M0E2LjA1MiA2LjA1MiAwIDAgMCA0Ljg4IDQuMTg3YTYuMTAyIDYuMTAyIDAgMCAwLTIuMzQ0IDEuMDE4QTYuMDA4IDYuMDA4IDAgMCAwIC44MjggNy4wODdhNS45ODEgNS45ODEgMCAwIDAgLjc1NCA3LjA5IDUuOTA0IDUuOTA0IDAgMCAwIC41MiA0LjkxMSA2LjEwMSA2LjEwMSAwIDAgMCAyLjgyMSAyLjUxMyA2LjIwNSA2LjIwNSAwIDAgMCAzLjc4LjM4OSA2LjA1NyA2LjA1NyAwIDAgMCAyLjA2NSAxLjQ5MiA2LjEzIDYuMTMgMCAwIDAgMi41MDUuNTE4IDYuMTg1IDYuMTg1IDAgMCAwIDMuNjE3LTEuMTU0IDYuMDUyIDYuMDUyIDAgMCAwIDIuMjMyLTMuMDM1IDYuMTAxIDYuMTAxIDAgMCAwIDIuMzQzLTEuMDE4IDYuMDA5IDYuMDA5IDAgMCAwIDEuNzA5LTEuODgzIDUuOTgxIDUuOTgxIDAgMCAwLS43NTYtNy4wODhabS05LjE0MyAxMi42MDlhNC41ODMgNC41ODMgMCAwIDEtMi45MTgtMS4wNGMuMDM3LS4wMi4xMDItLjA1Ni4xNDQtLjA4MWw0Ljg0NC0yLjc2YS43ODMuNzgzIDAgMCAwIC4zOTctLjY4di02LjczOEwxNy43OSAxMi4zYS4wNzIuMDcyIDAgMCAxIC4wNC4wNTV2NS41OGE0LjQ3MyA0LjQ3MyAwIDAgMS0xLjMzNSAzLjE3NiA0LjU5NiA0LjU5NiAwIDAgMS0zLjIxOSAxLjMyMVptLTkuNzkzLTQuMTI3YTQuNDMyIDQuNDMyIDAgMCAxLS41NDQtMy4wMTRjLjAzNi4wMjEuMDk5LjA2LjE0NC4wODVsNC44NDMgMi43NmEuNzk2Ljc5NiAwIDAgMCAuNzk1IDBsNS45MTMtMy4zNjlWMTcuMWEuMDcxLjA3MSAwIDAgMS0uMDI5LjA2Mkw5LjcwOCAxOS45NWE0LjYxNyA0LjYxNyAwIDAgMS0zLjQ1OC40NDcgNC41NTYgNC41NTYgMCAwIDEtMi43NjgtMi4wOTNaTTIuMjA4IDcuODcyQTQuNTI3IDQuNTI3IDAgMCAxIDQuNTggNS45bC0uMDAyLjE2NHY1LjUyYS43NjguNzY4IDAgMCAwIC4zOTcuNjhsNS45MTMgMy4zNjktMi4wNDcgMS4xNjZhLjA3NS4wNzUgMCAwIDEtLjA2OS4wMDZsLTQuODk2LTIuNzkyYTQuNTEgNC41MSAwIDAgMS0yLjEyLTIuNzMgNC40NSA0LjQ1IDAgMCAxIC40NTItMy40MTFabTE2LjgxOCAzLjg2MS01LjkxMy0zLjM2OCAyLjA0Ny0xLjE2NmEuMDc0LjA3NCAwIDAgMSAuMDctLjAwNmw0Ljg5NiAyLjc4OWE0LjUyNiA0LjUyNiAwIDAgMSAxLjc2MiAxLjgxNSA0LjQ0OCA0LjQ0OCAwIDAgMS0uNDE4IDQuODA4IDQuNTU2IDQuNTU2IDAgMCAxLTIuMDQ5IDEuNDk0di01LjY4NmEuNzY3Ljc2NyAwIDAgMC0uMzk1LS42OFptMi4wMzgtMy4wMjVhNi44NzQgNi44NzQgMCAwIDAtLjE0NC0uMDg1bC00Ljg0My0yLjc2YS43OTcuNzk3IDAgMCAwLS43OTYgMEw5LjM2OCA5LjIzVjYuOWEuMDcyLjA3MiAwIDAgMSAuMDMtLjA2Mmw0Ljg5NS0yLjc4N2E0LjYwOCA0LjYwOCAwIDAgMSA0Ljg4NS4yMDcgNC41MSA0LjUxIDAgMCAxIDEuNTk5IDEuOTU1Yy4zMzMuNzg4LjQzMyAxLjY1NC4yODcgMi40OTZaTTguMjU1IDEyLjg2NSA2LjIwOCAxMS43YS4wNzEuMDcxIDAgMCAxLS4wNC0uMDU2di01LjU4YzAtLjg1NC4yNDgtMS42OS43MTMtMi40MTJhNC41NCA0LjU0IDAgMCAxIDEuOTEzLTEuNjU4IDQuNjE0IDQuNjE0IDAgMCAxIDQuODUuNjE2Yy0uMDM3LjAyLS4xMDIuMDU1LS4xNDQuMDhMOC42NTcgNS40NTJhLjc4Mi43ODIgMCAwIDAtLjM5OC42OGwtLjAwNCA2LjczNFpNOS4zNjcgMTAuNSAxMi4wMDEgOWwyLjYzMyAxLjV2M0wxMi4wMDEgMTVsLTIuNjM0LTEuNXYtM1oiLz48L3N2Zz4=" +} diff --git a/src/appmixer/ai/openai/GenerateImage/GenerateImage.js b/src/appmixer/ai/openai/GenerateImage/GenerateImage.js new file mode 100644 index 000000000..e3b3c028b --- /dev/null +++ b/src/appmixer/ai/openai/GenerateImage/GenerateImage.js @@ -0,0 +1,36 @@ +'use strict'; + +module.exports = { + + receive: async function(context) { + + const { prompt, size, model } = context.messages.in.content; + const apiKey = context.auth.apiKey; + + const url = 'https://api.openai.com/v1/images/generations'; + const { data } = await context.httpRequest.post(url, { + model: model || 'dall-e-3', + prompt, + size, + n: 1 + }, { + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json' + } + }); + + let imageUrl; + if (data && data.data && data.data.length > 0) { + imageUrl = data.data[0].url; + } + + if (imageUrl) { + const response = await context.httpRequest.get(imageUrl, { responseType: 'stream' }); + const readStream = response.data; + const filename = `generated-image-${(new Date).toISOString()}.png`; + const file = await context.saveFileStream(filename, readStream); + return context.sendJson({ fileId: file.fileId, prompt, size }, 'out'); + } + } +}; diff --git a/src/appmixer/ai/openai/GenerateImage/component.json b/src/appmixer/ai/openai/GenerateImage/component.json new file mode 100644 index 000000000..a047e1ee0 --- /dev/null +++ b/src/appmixer/ai/openai/GenerateImage/component.json @@ -0,0 +1,70 @@ +{ + "name": "appmixer.ai.openai.GenerateImage", + "author": "Appmixer ", + "description": "Generates an image from a text prompt.", + "auth": { + "service": "appmixer:ai:openai" + }, + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "prompt": { "type": "string" }, + "size": { "type": "string" }, + "model": { "type": "string" } + }, + "required": ["prompt"] + }, + "inspector": { + "inputs": { + "prompt": { + "label": "Prompt", + "type": "textarea", + "index": 1 + }, + "size": { + "label": "Size", + "type": "select", + "defaultValue": "1024x1024", + "options": [ + { "label": "1024x1024", "value": "1024x1024" }, + { "label": "1792x1024", "value": "1792x1024" }, + { "label": "1024x1792", "value": "1024x1792" } + ], + "index": 2 + }, + "model": { + "type": "text", + "index": 3, + "label": "Model", + "tooltip": "ID of the model to use.", + "defaultValue": "dall-e-3", + "source": { + "url": "/component/appmixer/ai/openai/ListModels?outPort=out", + "data": { + "transform": "./ListModels#toSelectOptions" + } + } + } + } + } + }], + "outPorts": [{ + "name": "out", + "options": [{ + "label": "File ID", + "value": "fileId", + "schema": { "type": "string", "format": "appmixer-file-id" } + },{ + "label": "Prompt", + "value": "prompt", + "schema": { "type": "string" } + },{ + "label": "Size", + "value": "size", + "schema": { "type": "string" } + }] + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNHB4IiBoZWlnaHQ9IjI0cHgiIGZpbGw9ImN1cnJlbnRDb2xvciIgdmlld0JveD0iMCAwIDI0IDI0IiBjb2xvcj0iYmxhY2siPjxwYXRoIGQ9Ik0yMi40MTggOS44MjJhNS45MDMgNS45MDMgMCAwIDAtLjUyLTQuOTEgNi4xIDYuMSAwIDAgMC0yLjgyMi0yLjUxMyA2LjIwNCA2LjIwNCAwIDAgMC0zLjc4LS4zODlBNi4wNTUgNi4wNTUgMCAwIDAgMTMuMjMyLjUxOCA2LjEyOSA2LjEyOSAwIDAgMCAxMC43MjYgMGE2LjE4NSA2LjE4NSAwIDAgMC0zLjYxNSAxLjE1M0E2LjA1MiA2LjA1MiAwIDAgMCA0Ljg4IDQuMTg3YTYuMTAyIDYuMTAyIDAgMCAwLTIuMzQ0IDEuMDE4QTYuMDA4IDYuMDA4IDAgMCAwIC44MjggNy4wODdhNS45ODEgNS45ODEgMCAwIDAgLjc1NCA3LjA5IDUuOTA0IDUuOTA0IDAgMCAwIC41MiA0LjkxMSA2LjEwMSA2LjEwMSAwIDAgMCAyLjgyMSAyLjUxMyA2LjIwNSA2LjIwNSAwIDAgMCAzLjc4LjM4OSA2LjA1NyA2LjA1NyAwIDAgMCAyLjA2NSAxLjQ5MiA2LjEzIDYuMTMgMCAwIDAgMi41MDUuNTE4IDYuMTg1IDYuMTg1IDAgMCAwIDMuNjE3LTEuMTU0IDYuMDUyIDYuMDUyIDAgMCAwIDIuMjMyLTMuMDM1IDYuMTAxIDYuMTAxIDAgMCAwIDIuMzQzLTEuMDE4IDYuMDA5IDYuMDA5IDAgMCAwIDEuNzA5LTEuODgzIDUuOTgxIDUuOTgxIDAgMCAwLS43NTYtNy4wODhabS05LjE0MyAxMi42MDlhNC41ODMgNC41ODMgMCAwIDEtMi45MTgtMS4wNGMuMDM3LS4wMi4xMDItLjA1Ni4xNDQtLjA4MWw0Ljg0NC0yLjc2YS43ODMuNzgzIDAgMCAwIC4zOTctLjY4di02LjczOEwxNy43OSAxMi4zYS4wNzIuMDcyIDAgMCAxIC4wNC4wNTV2NS41OGE0LjQ3MyA0LjQ3MyAwIDAgMS0xLjMzNSAzLjE3NiA0LjU5NiA0LjU5NiAwIDAgMS0zLjIxOSAxLjMyMVptLTkuNzkzLTQuMTI3YTQuNDMyIDQuNDMyIDAgMCAxLS41NDQtMy4wMTRjLjAzNi4wMjEuMDk5LjA2LjE0NC4wODVsNC44NDMgMi43NmEuNzk2Ljc5NiAwIDAgMCAuNzk1IDBsNS45MTMtMy4zNjlWMTcuMWEuMDcxLjA3MSAwIDAgMS0uMDI5LjA2Mkw5LjcwOCAxOS45NWE0LjYxNyA0LjYxNyAwIDAgMS0zLjQ1OC40NDcgNC41NTYgNC41NTYgMCAwIDEtMi43NjgtMi4wOTNaTTIuMjA4IDcuODcyQTQuNTI3IDQuNTI3IDAgMCAxIDQuNTggNS45bC0uMDAyLjE2NHY1LjUyYS43NjguNzY4IDAgMCAwIC4zOTcuNjhsNS45MTMgMy4zNjktMi4wNDcgMS4xNjZhLjA3NS4wNzUgMCAwIDEtLjA2OS4wMDZsLTQuODk2LTIuNzkyYTQuNTEgNC41MSAwIDAgMS0yLjEyLTIuNzMgNC40NSA0LjQ1IDAgMCAxIC40NTItMy40MTFabTE2LjgxOCAzLjg2MS01LjkxMy0zLjM2OCAyLjA0Ny0xLjE2NmEuMDc0LjA3NCAwIDAgMSAuMDctLjAwNmw0Ljg5NiAyLjc4OWE0LjUyNiA0LjUyNiAwIDAgMSAxLjc2MiAxLjgxNSA0LjQ0OCA0LjQ0OCAwIDAgMS0uNDE4IDQuODA4IDQuNTU2IDQuNTU2IDAgMCAxLTIuMDQ5IDEuNDk0di01LjY4NmEuNzY3Ljc2NyAwIDAgMC0uMzk1LS42OFptMi4wMzgtMy4wMjVhNi44NzQgNi44NzQgMCAwIDAtLjE0NC0uMDg1bC00Ljg0My0yLjc2YS43OTcuNzk3IDAgMCAwLS43OTYgMEw5LjM2OCA5LjIzVjYuOWEuMDcyLjA3MiAwIDAgMSAuMDMtLjA2Mmw0Ljg5NS0yLjc4N2E0LjYwOCA0LjYwOCAwIDAgMSA0Ljg4NS4yMDcgNC41MSA0LjUxIDAgMCAxIDEuNTk5IDEuOTU1Yy4zMzMuNzg4LjQzMyAxLjY1NC4yODcgMi40OTZaTTguMjU1IDEyLjg2NSA2LjIwOCAxMS43YS4wNzEuMDcxIDAgMCAxLS4wNC0uMDU2di01LjU4YzAtLjg1NC4yNDgtMS42OS43MTMtMi40MTJhNC41NCA0LjU0IDAgMCAxIDEuOTEzLTEuNjU4IDQuNjE0IDQuNjE0IDAgMCAxIDQuODUuNjE2Yy0uMDM3LjAyLS4xMDIuMDU1LS4xNDQuMDhMOC42NTcgNS40NTJhLjc4Mi43ODIgMCAwIDAtLjM5OC42OGwtLjAwNCA2LjczNFpNOS4zNjcgMTAuNSAxMi4wMDEgOWwyLjYzMyAxLjV2M0wxMi4wMDEgMTVsLTIuNjM0LTEuNXYtM1oiLz48L3N2Zz4=" +} diff --git a/src/appmixer/ai/openai/ListModels/ListModels.js b/src/appmixer/ai/openai/ListModels/ListModels.js new file mode 100644 index 000000000..5bd312176 --- /dev/null +++ b/src/appmixer/ai/openai/ListModels/ListModels.js @@ -0,0 +1,24 @@ +'use strict'; + +module.exports = { + + receive: async function(context) { + const { prompt } = context.messages.in.content; + const apiKey = context.auth.apiKey; + + const url = 'https://api.openai.com/v1/models'; + const { data } = await context.httpRequest.get(url, { + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json' + } + }); + return context.sendJson({ models: data.data }, 'out'); + }, + + toSelectOptions(out) { + return out.models.map(model => { + return { label: model.id, value: model.id }; + }); + } +}; diff --git a/src/appmixer/ai/openai/ListModels/component.json b/src/appmixer/ai/openai/ListModels/component.json new file mode 100644 index 000000000..63244711c --- /dev/null +++ b/src/appmixer/ai/openai/ListModels/component.json @@ -0,0 +1,30 @@ +{ + "name": "appmixer.ai.openai.ListModels", + "author": "Appmixer ", + "description": "List models.", + "auth": { + "service": "appmixer:ai:openai" + }, + "inPorts": [{ + "name": "in" + }], + "outPorts": [{ + "name": "out", + "options": [{ + "label": "Models", + "value": "models", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": "string" }, + "created": { "type": "integer" }, + "owned_by": { "type": "string" } + } + } + } + }] + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNHB4IiBoZWlnaHQ9IjI0cHgiIGZpbGw9ImN1cnJlbnRDb2xvciIgdmlld0JveD0iMCAwIDI0IDI0IiBjb2xvcj0iYmxhY2siPjxwYXRoIGQ9Ik0yMi40MTggOS44MjJhNS45MDMgNS45MDMgMCAwIDAtLjUyLTQuOTEgNi4xIDYuMSAwIDAgMC0yLjgyMi0yLjUxMyA2LjIwNCA2LjIwNCAwIDAgMC0zLjc4LS4zODlBNi4wNTUgNi4wNTUgMCAwIDAgMTMuMjMyLjUxOCA2LjEyOSA2LjEyOSAwIDAgMCAxMC43MjYgMGE2LjE4NSA2LjE4NSAwIDAgMC0zLjYxNSAxLjE1M0E2LjA1MiA2LjA1MiAwIDAgMCA0Ljg4IDQuMTg3YTYuMTAyIDYuMTAyIDAgMCAwLTIuMzQ0IDEuMDE4QTYuMDA4IDYuMDA4IDAgMCAwIC44MjggNy4wODdhNS45ODEgNS45ODEgMCAwIDAgLjc1NCA3LjA5IDUuOTA0IDUuOTA0IDAgMCAwIC41MiA0LjkxMSA2LjEwMSA2LjEwMSAwIDAgMCAyLjgyMSAyLjUxMyA2LjIwNSA2LjIwNSAwIDAgMCAzLjc4LjM4OSA2LjA1NyA2LjA1NyAwIDAgMCAyLjA2NSAxLjQ5MiA2LjEzIDYuMTMgMCAwIDAgMi41MDUuNTE4IDYuMTg1IDYuMTg1IDAgMCAwIDMuNjE3LTEuMTU0IDYuMDUyIDYuMDUyIDAgMCAwIDIuMjMyLTMuMDM1IDYuMTAxIDYuMTAxIDAgMCAwIDIuMzQzLTEuMDE4IDYuMDA5IDYuMDA5IDAgMCAwIDEuNzA5LTEuODgzIDUuOTgxIDUuOTgxIDAgMCAwLS43NTYtNy4wODhabS05LjE0MyAxMi42MDlhNC41ODMgNC41ODMgMCAwIDEtMi45MTgtMS4wNGMuMDM3LS4wMi4xMDItLjA1Ni4xNDQtLjA4MWw0Ljg0NC0yLjc2YS43ODMuNzgzIDAgMCAwIC4zOTctLjY4di02LjczOEwxNy43OSAxMi4zYS4wNzIuMDcyIDAgMCAxIC4wNC4wNTV2NS41OGE0LjQ3MyA0LjQ3MyAwIDAgMS0xLjMzNSAzLjE3NiA0LjU5NiA0LjU5NiAwIDAgMS0zLjIxOSAxLjMyMVptLTkuNzkzLTQuMTI3YTQuNDMyIDQuNDMyIDAgMCAxLS41NDQtMy4wMTRjLjAzNi4wMjEuMDk5LjA2LjE0NC4wODVsNC44NDMgMi43NmEuNzk2Ljc5NiAwIDAgMCAuNzk1IDBsNS45MTMtMy4zNjlWMTcuMWEuMDcxLjA3MSAwIDAgMS0uMDI5LjA2Mkw5LjcwOCAxOS45NWE0LjYxNyA0LjYxNyAwIDAgMS0zLjQ1OC40NDcgNC41NTYgNC41NTYgMCAwIDEtMi43NjgtMi4wOTNaTTIuMjA4IDcuODcyQTQuNTI3IDQuNTI3IDAgMCAxIDQuNTggNS45bC0uMDAyLjE2NHY1LjUyYS43NjguNzY4IDAgMCAwIC4zOTcuNjhsNS45MTMgMy4zNjktMi4wNDcgMS4xNjZhLjA3NS4wNzUgMCAwIDEtLjA2OS4wMDZsLTQuODk2LTIuNzkyYTQuNTEgNC41MSAwIDAgMS0yLjEyLTIuNzMgNC40NSA0LjQ1IDAgMCAxIC40NTItMy40MTFabTE2LjgxOCAzLjg2MS01LjkxMy0zLjM2OCAyLjA0Ny0xLjE2NmEuMDc0LjA3NCAwIDAgMSAuMDctLjAwNmw0Ljg5NiAyLjc4OWE0LjUyNiA0LjUyNiAwIDAgMSAxLjc2MiAxLjgxNSA0LjQ0OCA0LjQ0OCAwIDAgMS0uNDE4IDQuODA4IDQuNTU2IDQuNTU2IDAgMCAxLTIuMDQ5IDEuNDk0di01LjY4NmEuNzY3Ljc2NyAwIDAgMC0uMzk1LS42OFptMi4wMzgtMy4wMjVhNi44NzQgNi44NzQgMCAwIDAtLjE0NC0uMDg1bC00Ljg0My0yLjc2YS43OTcuNzk3IDAgMCAwLS43OTYgMEw5LjM2OCA5LjIzVjYuOWEuMDcyLjA3MiAwIDAgMSAuMDMtLjA2Mmw0Ljg5NS0yLjc4N2E0LjYwOCA0LjYwOCAwIDAgMSA0Ljg4NS4yMDcgNC41MSA0LjUxIDAgMCAxIDEuNTk5IDEuOTU1Yy4zMzMuNzg4LjQzMyAxLjY1NC4yODcgMi40OTZaTTguMjU1IDEyLjg2NSA2LjIwOCAxMS43YS4wNzEuMDcxIDAgMCAxLS4wNC0uMDU2di01LjU4YzAtLjg1NC4yNDgtMS42OS43MTMtMi40MTJhNC41NCA0LjU0IDAgMCAxIDEuOTEzLTEuNjU4IDQuNjE0IDQuNjE0IDAgMCAxIDQuODUuNjE2Yy0uMDM3LjAyLS4xMDIuMDU1LS4xNDQuMDhMOC42NTcgNS40NTJhLjc4Mi43ODIgMCAwIDAtLjM5OC42OGwtLjAwNCA2LjczNFpNOS4zNjcgMTAuNSAxMi4wMDEgOWwyLjYzMyAxLjV2M0wxMi4wMDEgMTVsLTIuNjM0LTEuNXYtM1oiLz48L3N2Zz4=" +} diff --git a/src/appmixer/ai/openai/Moderate/Moderate.js b/src/appmixer/ai/openai/Moderate/Moderate.js new file mode 100644 index 000000000..59e69265f --- /dev/null +++ b/src/appmixer/ai/openai/Moderate/Moderate.js @@ -0,0 +1,31 @@ +'use strict'; + +module.exports = { + + receive: async function(context) { + + const { input } = context.messages.in.content; + const apiKey = context.auth.apiKey; + + const url = 'https://api.openai.com/v1/moderations'; + const { data } = await context.httpRequest.post(url, { + model: context.config.ModerateModel || 'text-moderation-latest', + input + }, { + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json' + } + }); + + if (data.results) { + const moderation = data.results[0]; + + if (moderation.flagged) { + return context.sendJson({ moderation, input }, 'IsFlagged'); + } else { + return context.sendJson({ moderation, input }, 'NotFlagged'); + } + } + } +}; diff --git a/src/appmixer/ai/openai/Moderate/component.json b/src/appmixer/ai/openai/Moderate/component.json new file mode 100644 index 000000000..028537800 --- /dev/null +++ b/src/appmixer/ai/openai/Moderate/component.json @@ -0,0 +1,296 @@ +{ + "name": "appmixer.ai.openai.Moderate", + "author": "Appmixer ", + "description": "Classify if text is potentially harmful.", + "auth": { + "service": "appmixer:ai:openai" + }, + "inPorts": [ + { + "name": "in", + "schema": { + "type": "object", + "properties": { + "input": { "type": "string" }, + "model": { "type": "string" } + }, + "required": ["input"] + }, + "inspector": { + "inputs": { + "input": { + "label": "Text", + "type": "textarea", + "index": 1 + }, + "model": { + "type": "text", + "index": 2, + "label": "Model", + "tooltip": "ID of the model to use.", + "defaultValue": "text-moderation-latest", + "source": { + "url": "/component/appmixer/ai/openai/ListModels?outPort=out", + "data": { + "transform": "./ListModels#toSelectOptions" + } + } + } + } + } + } + ], + "outPorts": [ + { + "name": "IsFlagged", + "options": [ + { + "label": "Moderation", + "value": "moderation", + "schema": { + "type": "object", + "properties": { + "flagged": { + "type": "boolean", + "title": "Is Flagged" + }, + "categories": { + "type": "object", + "title": "Categories", + "properties": { + "sexual": { + "type": "boolean", + "title": "Is Sexual" + }, + "hate": { + "type": "boolean", + "title": "Is Hate" + }, + "harassment": { + "type": "boolean", + "title": "Is Harassment" + }, + "self-harm": { + "type": "boolean", + "title": "Is Self-Harm" + }, + "sexual/minors": { + "type": "boolean", + "title": "Is Sexual/Minors" + }, + "hate/threatening": { + "type": "boolean", + "title": "Is Hate/Threatening" + }, + "violence/graphic": { + "type": "boolean", + "title": "Is Violence/Graphic" + }, + "self-harm/intent": { + "type": "boolean", + "title": "Is Self-Harm/Intent" + }, + "self-harm/instructions": { + "type": "boolean", + "title": "Is Self-Harm/Instructions" + }, + "harassment/threatening": { + "type": "boolean", + "title": "Is Harassment/Threatening" + }, + "violence": { + "type": "boolean", + "title": "Is Violence" + } + } + }, + "category_scores": { + "type": "object", + "title": "Category Scores", + "properties": { + "sexual": { + "type": "number", + "title": "Sexual Score" + }, + "hate": { + "type": "number", + "title": "Hate Score" + }, + "harassment": { + "type": "number", + "title": "Harassment Score" + }, + "self-harm": { + "type": "number", + "title": "Self-Harm Score" + }, + "sexual/minors": { + "type": "number", + "title": "Sexual/Minors Score" + }, + "hate/threatening": { + "type": "number", + "title": "Hate/Threatening Score" + }, + "violence/graphic": { + "type": "number", + "title": "Violence/Graphic Score" + }, + "self-harm/intent": { + "type": "number", + "title": "Self-Harm/Intent Score" + }, + "self-harm/instructions": { + "type": "number", + "title": "Self-Harm/Instructions Score" + }, + "harassment/threatening": { + "type": "number", + "title": "Harassment/Threatening Score" + }, + "violence": { + "type": "number", + "title": "Violence Score" + } + } + } + } + } + }, + { + "label": "Text", + "value": "input", + "schema": { + "type": "string" + } + } + ] + }, + { + "name": "NotFlagged", + "options": [ + { + "label": "Moderation", + "value": "moderation", + "schema": { + "type": "object", + "properties": { + "flagged": { + "type": "boolean", + "title": "Is Flagged" + }, + "categories": { + "type": "object", + "title": "Categories", + "properties": { + "sexual": { + "type": "boolean", + "title": "Is Sexual" + }, + "hate": { + "type": "boolean", + "title": "Is Hate" + }, + "harassment": { + "type": "boolean", + "title": "Is Harassment" + }, + "self-harm": { + "type": "boolean", + "title": "Is Self-Harm" + }, + "sexual/minors": { + "type": "boolean", + "title": "Is Sexual/Minors" + }, + "hate/threatening": { + "type": "boolean", + "title": "Is Hate/Threatening" + }, + "violence/graphic": { + "type": "boolean", + "title": "Is Violence/Graphic" + }, + "self-harm/intent": { + "type": "boolean", + "title": "Is Self-Harm/Intent" + }, + "self-harm/instructions": { + "type": "boolean", + "title": "Is Self-Harm/Instructions" + }, + "harassment/threatening": { + "type": "boolean", + "title": "Is Harassment/Threatening" + }, + "violence": { + "type": "boolean", + "title": "Is Violence" + } + } + }, + "category_scores": { + "type": "object", + "title": "Category Scores", + "properties": { + "sexual": { + "type": "number", + "title": "Sexual Score" + }, + "hate": { + "type": "number", + "title": "Hate Score" + }, + "harassment": { + "type": "number", + "title": "Harassment Score" + }, + "self-harm": { + "type": "number", + "title": "Self-Harm Score" + }, + "sexual/minors": { + "type": "number", + "title": "Sexual/Minors Score" + }, + "hate/threatening": { + "type": "number", + "title": "Hate/Threatening Score" + }, + "violence/graphic": { + "type": "number", + "title": "Violence/Graphic Score" + }, + "self-harm/intent": { + "type": "number", + "title": "Self-Harm/Intent Score" + }, + "self-harm/instructions": { + "type": "number", + "title": "Self-Harm/Instructions Score" + }, + "harassment/threatening": { + "type": "number", + "title": "Harassment/Threatening Score" + }, + "violence": { + "type": "number", + "title": "Violence Score" + } + } + } + } + } + }, + { + "label": "Text", + "value": "input", + "schema": { + "type": "string" + } + } + ] + } + ], + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNHB4IiBoZWlnaHQ9IjI0cHgiIGZpbGw9ImN1cnJlbnRDb2xvciIgdmlld0JveD0iMCAwIDI0IDI0IiBjb2xvcj0iYmxhY2siPjxwYXRoIGQ9Ik0yMi40MTggOS44MjJhNS45MDMgNS45MDMgMCAwIDAtLjUyLTQuOTEgNi4xIDYuMSAwIDAgMC0yLjgyMi0yLjUxMyA2LjIwNCA2LjIwNCAwIDAgMC0zLjc4LS4zODlBNi4wNTUgNi4wNTUgMCAwIDAgMTMuMjMyLjUxOCA2LjEyOSA2LjEyOSAwIDAgMCAxMC43MjYgMGE2LjE4NSA2LjE4NSAwIDAgMC0zLjYxNSAxLjE1M0E2LjA1MiA2LjA1MiAwIDAgMCA0Ljg4IDQuMTg3YTYuMTAyIDYuMTAyIDAgMCAwLTIuMzQ0IDEuMDE4QTYuMDA4IDYuMDA4IDAgMCAwIC44MjggNy4wODdhNS45ODEgNS45ODEgMCAwIDAgLjc1NCA3LjA5IDUuOTA0IDUuOTA0IDAgMCAwIC41MiA0LjkxMSA2LjEwMSA2LjEwMSAwIDAgMCAyLjgyMSAyLjUxMyA2LjIwNSA2LjIwNSAwIDAgMCAzLjc4LjM4OSA2LjA1NyA2LjA1NyAwIDAgMCAyLjA2NSAxLjQ5MiA2LjEzIDYuMTMgMCAwIDAgMi41MDUuNTE4IDYuMTg1IDYuMTg1IDAgMCAwIDMuNjE3LTEuMTU0IDYuMDUyIDYuMDUyIDAgMCAwIDIuMjMyLTMuMDM1IDYuMTAxIDYuMTAxIDAgMCAwIDIuMzQzLTEuMDE4IDYuMDA5IDYuMDA5IDAgMCAwIDEuNzA5LTEuODgzIDUuOTgxIDUuOTgxIDAgMCAwLS43NTYtNy4wODhabS05LjE0MyAxMi42MDlhNC41ODMgNC41ODMgMCAwIDEtMi45MTgtMS4wNGMuMDM3LS4wMi4xMDItLjA1Ni4xNDQtLjA4MWw0Ljg0NC0yLjc2YS43ODMuNzgzIDAgMCAwIC4zOTctLjY4di02LjczOEwxNy43OSAxMi4zYS4wNzIuMDcyIDAgMCAxIC4wNC4wNTV2NS41OGE0LjQ3MyA0LjQ3MyAwIDAgMS0xLjMzNSAzLjE3NiA0LjU5NiA0LjU5NiAwIDAgMS0zLjIxOSAxLjMyMVptLTkuNzkzLTQuMTI3YTQuNDMyIDQuNDMyIDAgMCAxLS41NDQtMy4wMTRjLjAzNi4wMjEuMDk5LjA2LjE0NC4wODVsNC44NDMgMi43NmEuNzk2Ljc5NiAwIDAgMCAuNzk1IDBsNS45MTMtMy4zNjlWMTcuMWEuMDcxLjA3MSAwIDAgMS0uMDI5LjA2Mkw5LjcwOCAxOS45NWE0LjYxNyA0LjYxNyAwIDAgMS0zLjQ1OC40NDcgNC41NTYgNC41NTYgMCAwIDEtMi43NjgtMi4wOTNaTTIuMjA4IDcuODcyQTQuNTI3IDQuNTI3IDAgMCAxIDQuNTggNS45bC0uMDAyLjE2NHY1LjUyYS43NjguNzY4IDAgMCAwIC4zOTcuNjhsNS45MTMgMy4zNjktMi4wNDcgMS4xNjZhLjA3NS4wNzUgMCAwIDEtLjA2OS4wMDZsLTQuODk2LTIuNzkyYTQuNTEgNC41MSAwIDAgMS0yLjEyLTIuNzMgNC40NSA0LjQ1IDAgMCAxIC40NTItMy40MTFabTE2LjgxOCAzLjg2MS01LjkxMy0zLjM2OCAyLjA0Ny0xLjE2NmEuMDc0LjA3NCAwIDAgMSAuMDctLjAwNmw0Ljg5NiAyLjc4OWE0LjUyNiA0LjUyNiAwIDAgMSAxLjc2MiAxLjgxNSA0LjQ0OCA0LjQ0OCAwIDAgMS0uNDE4IDQuODA4IDQuNTU2IDQuNTU2IDAgMCAxLTIuMDQ5IDEuNDk0di01LjY4NmEuNzY3Ljc2NyAwIDAgMC0uMzk1LS42OFptMi4wMzgtMy4wMjVhNi44NzQgNi44NzQgMCAwIDAtLjE0NC0uMDg1bC00Ljg0My0yLjc2YS43OTcuNzk3IDAgMCAwLS43OTYgMEw5LjM2OCA5LjIzVjYuOWEuMDcyLjA3MiAwIDAgMSAuMDMtLjA2Mmw0Ljg5NS0yLjc4N2E0LjYwOCA0LjYwOCAwIDAgMSA0Ljg4NS4yMDcgNC41MSA0LjUxIDAgMCAxIDEuNTk5IDEuOTU1Yy4zMzMuNzg4LjQzMyAxLjY1NC4yODcgMi40OTZaTTguMjU1IDEyLjg2NSA2LjIwOCAxMS43YS4wNzEuMDcxIDAgMCAxLS4wNC0uMDU2di01LjU4YzAtLjg1NC4yNDgtMS42OS43MTMtMi40MTJhNC41NCA0LjU0IDAgMCAxIDEuOTEzLTEuNjU4IDQuNjE0IDQuNjE0IDAgMCAxIDQuODUuNjE2Yy0uMDM3LjAyLS4xMDIuMDU1LS4xNDQuMDhMOC42NTcgNS40NTJhLjc4Mi43ODIgMCAwIDAtLjM5OC42OGwtLjAwNCA2LjczNFpNOS4zNjcgMTAuNSAxMi4wMDEgOWwyLjYzMyAxLjV2M0wxMi4wMDEgMTVsLTIuNjM0LTEuNXYtM1oiLz48L3N2Zz4=" +} \ No newline at end of file diff --git a/src/appmixer/ai/openai/SendPrompt/SendPrompt.js b/src/appmixer/ai/openai/SendPrompt/SendPrompt.js new file mode 100644 index 000000000..5723013fc --- /dev/null +++ b/src/appmixer/ai/openai/SendPrompt/SendPrompt.js @@ -0,0 +1,40 @@ +'use strict'; + +module.exports = { + + receive: async function(context) { + + const { prompt, model } = context.messages.in.content; + const apiKey = context.auth.apiKey; + + const url = 'https://api.openai.com/v1/chat/completions'; + const { data } = await context.httpRequest.post(url, { + model: model || 'gpt-4o', + messages: [ + { + role: 'system', + content: 'You are a helpful assistant.', + name: 'system' + }, + { + role: 'user', + content: prompt, + name: 'user' + } + ] + }, { + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json' + } + }); + + let answer = ''; + + if (data && data.choices) { + answer = data.choices[0].message.content; + } + + return context.sendJson({ answer }, 'out'); + } +}; diff --git a/src/appmixer/ai/openai/SendPrompt/component.json b/src/appmixer/ai/openai/SendPrompt/component.json new file mode 100644 index 000000000..7439dc09b --- /dev/null +++ b/src/appmixer/ai/openai/SendPrompt/component.json @@ -0,0 +1,54 @@ +{ + "name": "appmixer.ai.openai.SendPrompt", + "author": "Appmixer ", + "description": "Send a prompt to a Large Language Model and receive a response.", + "auth": { + "service": "appmixer:ai:openai" + }, + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "prompt": { "type": "string" }, + "model": { "type": "string" } + }, + "required": ["prompt"] + }, + "inspector": { + "inputs": { + "prompt": { + "label": "Prompt", + "type": "textarea", + "index": 1 + }, + "model": { + "type": "text", + "index": 2, + "label": "Model", + "tooltip": "ID of the model to use.", + "defaultValue": "gpt-4o", + "source": { + "url": "/component/appmixer/ai/openai/ListModels?outPort=out", + "data": { + "transform": "./ListModels#toSelectOptions" + } + } + } + } + } + }], + "outPorts": [{ + "name": "out", + "options": [{ + "label": "Answer", + "value": "answer", + "schema": { "type": "string" } + }, { + "label": "Prompt", + "value": "prompt", + "schema": { "type": "string" } + }] + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNHB4IiBoZWlnaHQ9IjI0cHgiIGZpbGw9ImN1cnJlbnRDb2xvciIgdmlld0JveD0iMCAwIDI0IDI0IiBjb2xvcj0iYmxhY2siPjxwYXRoIGQ9Ik0yMi40MTggOS44MjJhNS45MDMgNS45MDMgMCAwIDAtLjUyLTQuOTEgNi4xIDYuMSAwIDAgMC0yLjgyMi0yLjUxMyA2LjIwNCA2LjIwNCAwIDAgMC0zLjc4LS4zODlBNi4wNTUgNi4wNTUgMCAwIDAgMTMuMjMyLjUxOCA2LjEyOSA2LjEyOSAwIDAgMCAxMC43MjYgMGE2LjE4NSA2LjE4NSAwIDAgMC0zLjYxNSAxLjE1M0E2LjA1MiA2LjA1MiAwIDAgMCA0Ljg4IDQuMTg3YTYuMTAyIDYuMTAyIDAgMCAwLTIuMzQ0IDEuMDE4QTYuMDA4IDYuMDA4IDAgMCAwIC44MjggNy4wODdhNS45ODEgNS45ODEgMCAwIDAgLjc1NCA3LjA5IDUuOTA0IDUuOTA0IDAgMCAwIC41MiA0LjkxMSA2LjEwMSA2LjEwMSAwIDAgMCAyLjgyMSAyLjUxMyA2LjIwNSA2LjIwNSAwIDAgMCAzLjc4LjM4OSA2LjA1NyA2LjA1NyAwIDAgMCAyLjA2NSAxLjQ5MiA2LjEzIDYuMTMgMCAwIDAgMi41MDUuNTE4IDYuMTg1IDYuMTg1IDAgMCAwIDMuNjE3LTEuMTU0IDYuMDUyIDYuMDUyIDAgMCAwIDIuMjMyLTMuMDM1IDYuMTAxIDYuMTAxIDAgMCAwIDIuMzQzLTEuMDE4IDYuMDA5IDYuMDA5IDAgMCAwIDEuNzA5LTEuODgzIDUuOTgxIDUuOTgxIDAgMCAwLS43NTYtNy4wODhabS05LjE0MyAxMi42MDlhNC41ODMgNC41ODMgMCAwIDEtMi45MTgtMS4wNGMuMDM3LS4wMi4xMDItLjA1Ni4xNDQtLjA4MWw0Ljg0NC0yLjc2YS43ODMuNzgzIDAgMCAwIC4zOTctLjY4di02LjczOEwxNy43OSAxMi4zYS4wNzIuMDcyIDAgMCAxIC4wNC4wNTV2NS41OGE0LjQ3MyA0LjQ3MyAwIDAgMS0xLjMzNSAzLjE3NiA0LjU5NiA0LjU5NiAwIDAgMS0zLjIxOSAxLjMyMVptLTkuNzkzLTQuMTI3YTQuNDMyIDQuNDMyIDAgMCAxLS41NDQtMy4wMTRjLjAzNi4wMjEuMDk5LjA2LjE0NC4wODVsNC44NDMgMi43NmEuNzk2Ljc5NiAwIDAgMCAuNzk1IDBsNS45MTMtMy4zNjlWMTcuMWEuMDcxLjA3MSAwIDAgMS0uMDI5LjA2Mkw5LjcwOCAxOS45NWE0LjYxNyA0LjYxNyAwIDAgMS0zLjQ1OC40NDcgNC41NTYgNC41NTYgMCAwIDEtMi43NjgtMi4wOTNaTTIuMjA4IDcuODcyQTQuNTI3IDQuNTI3IDAgMCAxIDQuNTggNS45bC0uMDAyLjE2NHY1LjUyYS43NjguNzY4IDAgMCAwIC4zOTcuNjhsNS45MTMgMy4zNjktMi4wNDcgMS4xNjZhLjA3NS4wNzUgMCAwIDEtLjA2OS4wMDZsLTQuODk2LTIuNzkyYTQuNTEgNC41MSAwIDAgMS0yLjEyLTIuNzMgNC40NSA0LjQ1IDAgMCAxIC40NTItMy40MTFabTE2LjgxOCAzLjg2MS01LjkxMy0zLjM2OCAyLjA0Ny0xLjE2NmEuMDc0LjA3NCAwIDAgMSAuMDctLjAwNmw0Ljg5NiAyLjc4OWE0LjUyNiA0LjUyNiAwIDAgMSAxLjc2MiAxLjgxNSA0LjQ0OCA0LjQ0OCAwIDAgMS0uNDE4IDQuODA4IDQuNTU2IDQuNTU2IDAgMCAxLTIuMDQ5IDEuNDk0di01LjY4NmEuNzY3Ljc2NyAwIDAgMC0uMzk1LS42OFptMi4wMzgtMy4wMjVhNi44NzQgNi44NzQgMCAwIDAtLjE0NC0uMDg1bC00Ljg0My0yLjc2YS43OTcuNzk3IDAgMCAwLS43OTYgMEw5LjM2OCA5LjIzVjYuOWEuMDcyLjA3MiAwIDAgMSAuMDMtLjA2Mmw0Ljg5NS0yLjc4N2E0LjYwOCA0LjYwOCAwIDAgMSA0Ljg4NS4yMDcgNC41MSA0LjUxIDAgMCAxIDEuNTk5IDEuOTU1Yy4zMzMuNzg4LjQzMyAxLjY1NC4yODcgMi40OTZaTTguMjU1IDEyLjg2NSA2LjIwOCAxMS43YS4wNzEuMDcxIDAgMCAxLS4wNC0uMDU2di01LjU4YzAtLjg1NC4yNDgtMS42OS43MTMtMi40MTJhNC41NCA0LjU0IDAgMCAxIDEuOTEzLTEuNjU4IDQuNjE0IDQuNjE0IDAgMCAxIDQuODUuNjE2Yy0uMDM3LjAyLS4xMDIuMDU1LS4xNDQuMDhMOC42NTcgNS40NTJhLjc4Mi43ODIgMCAwIDAtLjM5OC42OGwtLjAwNCA2LjczNFpNOS4zNjcgMTAuNSAxMi4wMDEgOWwyLjYzMyAxLjV2M0wxMi4wMDEgMTVsLTIuNjM0LTEuNXYtM1oiLz48L3N2Zz4=" +} diff --git a/src/appmixer/ai/openai/TransformTextToJSON/TransformTextToJSON.js b/src/appmixer/ai/openai/TransformTextToJSON/TransformTextToJSON.js new file mode 100644 index 000000000..10ce751d6 --- /dev/null +++ b/src/appmixer/ai/openai/TransformTextToJSON/TransformTextToJSON.js @@ -0,0 +1,65 @@ +'use strict'; + +module.exports = { + + receive: async function(context) { + + const { text, jsonSchema: jsonSchemaString, model } = context.messages.in.content; + + const jsonSchema = JSON.parse(jsonSchemaString); + + if (context.properties.generateOutputPortOptions) { + return this.getOutputPortOptions(context, jsonSchema); + } + + const apiKey = context.auth.apiKey; + + const url = 'https://api.openai.com/v1/chat/completions'; + const { data } = await context.httpRequest.post(url, { + model: model || 'gpt-4o-2024-08-06', + messages: [ + { + role: 'system', + content: 'You are an expert at structured data extraction. You will be given unstructured text and should convert it into the given structure.', + name: 'system' + }, + { + role: 'user', + content: text, + name: 'user' + } + ], + response_format: { + type: 'json_schema', + json_schema: { + name: 'json_extraction', + schema: jsonSchema + } + } + }, { + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json' + } + }); + + const json = JSON.parse(data.choices[0].message.content); + return context.sendJson({ json }, 'out'); + }, + + getOutputPortOptions: function(context, jsonSchema) { + + return context.sendJson([ + { + value: 'json', + label: 'JSON', + schema: jsonSchema + }, + { + value: 'text', + label: 'Text', + schema: { type: 'string' } + } + ], 'out'); + } +}; diff --git a/src/appmixer/ai/openai/TransformTextToJSON/component.json b/src/appmixer/ai/openai/TransformTextToJSON/component.json new file mode 100644 index 000000000..96e5f186c --- /dev/null +++ b/src/appmixer/ai/openai/TransformTextToJSON/component.json @@ -0,0 +1,65 @@ +{ + "name": "appmixer.ai.openai.TransformTextToJSON", + "author": "Appmixer ", + "description": "Extract structured JSON data from text using AI.", + "auth": { + "service": "appmixer:ai:openai" + }, + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "text": { "type": "string" }, + "jsonSchema": { "type": "string" }, + "model": { "type": "string" } + }, + "required": ["text", "jsonSchema"] + }, + "inspector": { + "inputs": { + "text": { + "label": "Text", + "type": "textarea", + "index": 1, + "tooltip": "The text from which to extract structured JSON data. Example: John is 25 years old.." + }, + "jsonSchema": { + "label": "Output JSON Schema", + "type": "textarea", + "index": 2, + "tooltip": "The schema that defines the structure of the output JSON. Use JSON Schema format. Example: {\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"string\"}, \"age\":{\"type\":\"number\"}}}. It must be a valid JSON schema and must be of \"type\": \"object\". If you want to produce an array, you can nest the array under an object property of type array. Example: {\"type\":\"object\",\"properties\":{\"contacts\":{\"type\":\"array\",\"items\":{\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"string\"},\"age\":{\"type\":\"number\"}}}}}}." + }, + "model": { + "type": "text", + "index": 3, + "label": "Model", + "tooltip": "ID of the model to use.", + "defaultValue": "gpt-4o-2024-08-06", + "source": { + "url": "/component/appmixer/ai/openai/ListModels?outPort=out", + "data": { + "transform": "./ListModels#toSelectOptions" + } + } + } + } + } + }], + "outPorts": [{ + "name": "out", + "source": { + "url": "/component/appmixer/utils/ai/TransformTextToJSON?outPort=out", + "data": { + "properties": { + "generateOutputPortOptions": true + }, + "messages": { + "in/text": "dummy", + "in/jsonSchema": "inputs/in/jsonSchema" + } + } + } + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNHB4IiBoZWlnaHQ9IjI0cHgiIGZpbGw9ImN1cnJlbnRDb2xvciIgdmlld0JveD0iMCAwIDI0IDI0IiBjb2xvcj0iYmxhY2siPjxwYXRoIGQ9Ik0yMi40MTggOS44MjJhNS45MDMgNS45MDMgMCAwIDAtLjUyLTQuOTEgNi4xIDYuMSAwIDAgMC0yLjgyMi0yLjUxMyA2LjIwNCA2LjIwNCAwIDAgMC0zLjc4LS4zODlBNi4wNTUgNi4wNTUgMCAwIDAgMTMuMjMyLjUxOCA2LjEyOSA2LjEyOSAwIDAgMCAxMC43MjYgMGE2LjE4NSA2LjE4NSAwIDAgMC0zLjYxNSAxLjE1M0E2LjA1MiA2LjA1MiAwIDAgMCA0Ljg4IDQuMTg3YTYuMTAyIDYuMTAyIDAgMCAwLTIuMzQ0IDEuMDE4QTYuMDA4IDYuMDA4IDAgMCAwIC44MjggNy4wODdhNS45ODEgNS45ODEgMCAwIDAgLjc1NCA3LjA5IDUuOTA0IDUuOTA0IDAgMCAwIC41MiA0LjkxMSA2LjEwMSA2LjEwMSAwIDAgMCAyLjgyMSAyLjUxMyA2LjIwNSA2LjIwNSAwIDAgMCAzLjc4LjM4OSA2LjA1NyA2LjA1NyAwIDAgMCAyLjA2NSAxLjQ5MiA2LjEzIDYuMTMgMCAwIDAgMi41MDUuNTE4IDYuMTg1IDYuMTg1IDAgMCAwIDMuNjE3LTEuMTU0IDYuMDUyIDYuMDUyIDAgMCAwIDIuMjMyLTMuMDM1IDYuMTAxIDYuMTAxIDAgMCAwIDIuMzQzLTEuMDE4IDYuMDA5IDYuMDA5IDAgMCAwIDEuNzA5LTEuODgzIDUuOTgxIDUuOTgxIDAgMCAwLS43NTYtNy4wODhabS05LjE0MyAxMi42MDlhNC41ODMgNC41ODMgMCAwIDEtMi45MTgtMS4wNGMuMDM3LS4wMi4xMDItLjA1Ni4xNDQtLjA4MWw0Ljg0NC0yLjc2YS43ODMuNzgzIDAgMCAwIC4zOTctLjY4di02LjczOEwxNy43OSAxMi4zYS4wNzIuMDcyIDAgMCAxIC4wNC4wNTV2NS41OGE0LjQ3MyA0LjQ3MyAwIDAgMS0xLjMzNSAzLjE3NiA0LjU5NiA0LjU5NiAwIDAgMS0zLjIxOSAxLjMyMVptLTkuNzkzLTQuMTI3YTQuNDMyIDQuNDMyIDAgMCAxLS41NDQtMy4wMTRjLjAzNi4wMjEuMDk5LjA2LjE0NC4wODVsNC44NDMgMi43NmEuNzk2Ljc5NiAwIDAgMCAuNzk1IDBsNS45MTMtMy4zNjlWMTcuMWEuMDcxLjA3MSAwIDAgMS0uMDI5LjA2Mkw5LjcwOCAxOS45NWE0LjYxNyA0LjYxNyAwIDAgMS0zLjQ1OC40NDcgNC41NTYgNC41NTYgMCAwIDEtMi43NjgtMi4wOTNaTTIuMjA4IDcuODcyQTQuNTI3IDQuNTI3IDAgMCAxIDQuNTggNS45bC0uMDAyLjE2NHY1LjUyYS43NjguNzY4IDAgMCAwIC4zOTcuNjhsNS45MTMgMy4zNjktMi4wNDcgMS4xNjZhLjA3NS4wNzUgMCAwIDEtLjA2OS4wMDZsLTQuODk2LTIuNzkyYTQuNTEgNC41MSAwIDAgMS0yLjEyLTIuNzMgNC40NSA0LjQ1IDAgMCAxIC40NTItMy40MTFabTE2LjgxOCAzLjg2MS01LjkxMy0zLjM2OCAyLjA0Ny0xLjE2NmEuMDc0LjA3NCAwIDAgMSAuMDctLjAwNmw0Ljg5NiAyLjc4OWE0LjUyNiA0LjUyNiAwIDAgMSAxLjc2MiAxLjgxNSA0LjQ0OCA0LjQ0OCAwIDAgMS0uNDE4IDQuODA4IDQuNTU2IDQuNTU2IDAgMCAxLTIuMDQ5IDEuNDk0di01LjY4NmEuNzY3Ljc2NyAwIDAgMC0uMzk1LS42OFptMi4wMzgtMy4wMjVhNi44NzQgNi44NzQgMCAwIDAtLjE0NC0uMDg1bC00Ljg0My0yLjc2YS43OTcuNzk3IDAgMCAwLS43OTYgMEw5LjM2OCA5LjIzVjYuOWEuMDcyLjA3MiAwIDAgMSAuMDMtLjA2Mmw0Ljg5NS0yLjc4N2E0LjYwOCA0LjYwOCAwIDAgMSA0Ljg4NS4yMDcgNC41MSA0LjUxIDAgMCAxIDEuNTk5IDEuOTU1Yy4zMzMuNzg4LjQzMyAxLjY1NC4yODcgMi40OTZaTTguMjU1IDEyLjg2NSA2LjIwOCAxMS43YS4wNzEuMDcxIDAgMCAxLS4wNC0uMDU2di01LjU4YzAtLjg1NC4yNDgtMS42OS43MTMtMi40MTJhNC41NCA0LjU0IDAgMCAxIDEuOTEzLTEuNjU4IDQuNjE0IDQuNjE0IDAgMCAxIDQuODUuNjE2Yy0uMDM3LjAyLS4xMDIuMDU1LS4xNDQuMDhMOC42NTcgNS40NTJhLjc4Mi43ODIgMCAwIDAtLjM5OC42OGwtLjAwNCA2LjczNFpNOS4zNjcgMTAuNSAxMi4wMDEgOWwyLjYzMyAxLjV2M0wxMi4wMDEgMTVsLTIuNjM0LTEuNXYtM1oiLz48L3N2Zz4=" +} diff --git a/src/appmixer/ai/openai/auth.js b/src/appmixer/ai/openai/auth.js new file mode 100644 index 000000000..468025fec --- /dev/null +++ b/src/appmixer/ai/openai/auth.js @@ -0,0 +1,35 @@ +'use strict'; + +module.exports = { + + type: 'apiKey', + + definition: () => { + + return { + auth: { + apiKey: { + type: 'text', + name: 'API Key', + tooltip: 'Log into your OpenAI account and find API Keys page in the Settings.' + } + }, + + validate: async (context) => { + + const url = 'https://api.openai.com/v1/models'; + return context.httpRequest.get(url, { + headers: { + 'Authorization': `Bearer ${context.apiKey}`, + 'Content-Type': 'application/json' + } + }); + }, + + accountNameFromProfileInfo: (context) => { + const apiKey = context.apiKey; + return apiKey.substr(0, 6) + '...' + apiKey.substr(-6); + } + }; + } +}; \ No newline at end of file diff --git a/src/appmixer/ai/openai/bundle.json b/src/appmixer/ai/openai/bundle.json new file mode 100644 index 000000000..3c801aaf4 --- /dev/null +++ b/src/appmixer/ai/openai/bundle.json @@ -0,0 +1,9 @@ +{ + "name": "appmixer.ai.openai", + "version": "1.0.0", + "changelog": { + "1.0.0": [ + "First version." + ] + } +} diff --git a/src/appmixer/ai/openai/module.json b/src/appmixer/ai/openai/module.json new file mode 100644 index 000000000..fd8970413 --- /dev/null +++ b/src/appmixer/ai/openai/module.json @@ -0,0 +1,10 @@ +{ + "name": "appmixer.ai.openai", + "label": "OpenAI", + "category": "ai", + "categoryIndex": 0, + "index": 1, + "categoryLabel": "AI", + "description": "OpenAI components for building AI agents.", + "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNHB4IiBoZWlnaHQ9IjI0cHgiIGZpbGw9ImN1cnJlbnRDb2xvciIgdmlld0JveD0iMCAwIDI0IDI0IiBjb2xvcj0iYmxhY2siPjxwYXRoIGQ9Ik0yMi40MTggOS44MjJhNS45MDMgNS45MDMgMCAwIDAtLjUyLTQuOTEgNi4xIDYuMSAwIDAgMC0yLjgyMi0yLjUxMyA2LjIwNCA2LjIwNCAwIDAgMC0zLjc4LS4zODlBNi4wNTUgNi4wNTUgMCAwIDAgMTMuMjMyLjUxOCA2LjEyOSA2LjEyOSAwIDAgMCAxMC43MjYgMGE2LjE4NSA2LjE4NSAwIDAgMC0zLjYxNSAxLjE1M0E2LjA1MiA2LjA1MiAwIDAgMCA0Ljg4IDQuMTg3YTYuMTAyIDYuMTAyIDAgMCAwLTIuMzQ0IDEuMDE4QTYuMDA4IDYuMDA4IDAgMCAwIC44MjggNy4wODdhNS45ODEgNS45ODEgMCAwIDAgLjc1NCA3LjA5IDUuOTA0IDUuOTA0IDAgMCAwIC41MiA0LjkxMSA2LjEwMSA2LjEwMSAwIDAgMCAyLjgyMSAyLjUxMyA2LjIwNSA2LjIwNSAwIDAgMCAzLjc4LjM4OSA2LjA1NyA2LjA1NyAwIDAgMCAyLjA2NSAxLjQ5MiA2LjEzIDYuMTMgMCAwIDAgMi41MDUuNTE4IDYuMTg1IDYuMTg1IDAgMCAwIDMuNjE3LTEuMTU0IDYuMDUyIDYuMDUyIDAgMCAwIDIuMjMyLTMuMDM1IDYuMTAxIDYuMTAxIDAgMCAwIDIuMzQzLTEuMDE4IDYuMDA5IDYuMDA5IDAgMCAwIDEuNzA5LTEuODgzIDUuOTgxIDUuOTgxIDAgMCAwLS43NTYtNy4wODhabS05LjE0MyAxMi42MDlhNC41ODMgNC41ODMgMCAwIDEtMi45MTgtMS4wNGMuMDM3LS4wMi4xMDItLjA1Ni4xNDQtLjA4MWw0Ljg0NC0yLjc2YS43ODMuNzgzIDAgMCAwIC4zOTctLjY4di02LjczOEwxNy43OSAxMi4zYS4wNzIuMDcyIDAgMCAxIC4wNC4wNTV2NS41OGE0LjQ3MyA0LjQ3MyAwIDAgMS0xLjMzNSAzLjE3NiA0LjU5NiA0LjU5NiAwIDAgMS0zLjIxOSAxLjMyMVptLTkuNzkzLTQuMTI3YTQuNDMyIDQuNDMyIDAgMCAxLS41NDQtMy4wMTRjLjAzNi4wMjEuMDk5LjA2LjE0NC4wODVsNC44NDMgMi43NmEuNzk2Ljc5NiAwIDAgMCAuNzk1IDBsNS45MTMtMy4zNjlWMTcuMWEuMDcxLjA3MSAwIDAgMS0uMDI5LjA2Mkw5LjcwOCAxOS45NWE0LjYxNyA0LjYxNyAwIDAgMS0zLjQ1OC40NDcgNC41NTYgNC41NTYgMCAwIDEtMi43NjgtMi4wOTNaTTIuMjA4IDcuODcyQTQuNTI3IDQuNTI3IDAgMCAxIDQuNTggNS45bC0uMDAyLjE2NHY1LjUyYS43NjguNzY4IDAgMCAwIC4zOTcuNjhsNS45MTMgMy4zNjktMi4wNDcgMS4xNjZhLjA3NS4wNzUgMCAwIDEtLjA2OS4wMDZsLTQuODk2LTIuNzkyYTQuNTEgNC41MSAwIDAgMS0yLjEyLTIuNzMgNC40NSA0LjQ1IDAgMCAxIC40NTItMy40MTFabTE2LjgxOCAzLjg2MS01LjkxMy0zLjM2OCAyLjA0Ny0xLjE2NmEuMDc0LjA3NCAwIDAgMSAuMDctLjAwNmw0Ljg5NiAyLjc4OWE0LjUyNiA0LjUyNiAwIDAgMSAxLjc2MiAxLjgxNSA0LjQ0OCA0LjQ0OCAwIDAgMS0uNDE4IDQuODA4IDQuNTU2IDQuNTU2IDAgMCAxLTIuMDQ5IDEuNDk0di01LjY4NmEuNzY3Ljc2NyAwIDAgMC0uMzk1LS42OFptMi4wMzgtMy4wMjVhNi44NzQgNi44NzQgMCAwIDAtLjE0NC0uMDg1bC00Ljg0My0yLjc2YS43OTcuNzk3IDAgMCAwLS43OTYgMEw5LjM2OCA5LjIzVjYuOWEuMDcyLjA3MiAwIDAgMSAuMDMtLjA2Mmw0Ljg5NS0yLjc4N2E0LjYwOCA0LjYwOCAwIDAgMSA0Ljg4NS4yMDcgNC41MSA0LjUxIDAgMCAxIDEuNTk5IDEuOTU1Yy4zMzMuNzg4LjQzMyAxLjY1NC4yODcgMi40OTZaTTguMjU1IDEyLjg2NSA2LjIwOCAxMS43YS4wNzEuMDcxIDAgMCAxLS4wNC0uMDU2di01LjU4YzAtLjg1NC4yNDgtMS42OS43MTMtMi40MTJhNC41NCA0LjU0IDAgMCAxIDEuOTEzLTEuNjU4IDQuNjE0IDQuNjE0IDAgMCAxIDQuODUuNjE2Yy0uMDM3LjAyLS4xMDIuMDU1LS4xNDQuMDhMOC42NTcgNS40NTJhLjc4Mi43ODIgMCAwIDAtLjM5OC42OGwtLjAwNCA2LjczNFpNOS4zNjcgMTAuNSAxMi4wMDEgOWwyLjYzMyAxLjV2M0wxMi4wMDEgMTVsLTIuNjM0LTEuNXYtM1oiLz48L3N2Zz4=" +} diff --git a/src/appmixer/ai/openai/package.json b/src/appmixer/ai/openai/package.json new file mode 100644 index 000000000..a8132549f --- /dev/null +++ b/src/appmixer/ai/openai/package.json @@ -0,0 +1,10 @@ +{ + "name": "appmixer.ai.openai", + "version": "1.0.0", + "dependencies": { + "form-data": "4.0.0", + "langchain": "0.3.6", + "mustache": "4.2.0", + "openai": "4.80.1" + } +} diff --git a/src/appmixer/utils/ai/AIAgent/AIAgent.js b/src/appmixer/utils/ai/AIAgent/AIAgent.js new file mode 100644 index 000000000..2ff7962e3 --- /dev/null +++ b/src/appmixer/utils/ai/AIAgent/AIAgent.js @@ -0,0 +1,230 @@ +'use strict'; + +const { max } = require('lodash'); +const OpenAI = require('openai'); + +const COLLECT_TOOL_OUTPUTS_POLL_TIMEOUT = 60 * 1000; // 60 seconds +const COLLECT_TOOL_OUTPUTS_POLL_INTERVAL = 1 * 1000; // 1 second +const MAX_RUN_DURATION = 5 * 60 * 1000; // 5 minutes + +module.exports = { + + start: async function(context) { + + const assistant = await this.createAssistant(context); + return context.stateSet('assistant', assistant); + }, + + createAssistant: async function(context) { + + const flowDescriptor = context.flowDescriptor; + const agentComponentId = context.componentId; + const toolsPort = 'tools'; + + // Create a new assistant with tools defined in the branches connected to my 'tools' output port. + const tools = {}; + let error; + + // Find all components connected to my 'tools' output port. + Object.keys(flowDescriptor).forEach((componentId) => { + const component = flowDescriptor[componentId]; + const sources = component.source; + Object.keys(sources || {}).forEach((inPort) => { + const source = sources[inPort]; + if (source[agentComponentId] && source[agentComponentId].includes(toolsPort)) { + tools[componentId] = component; + // assert(flowDescriptor[componentId].type === 'appmixer.utils.ai.ToolStart') + if (component.type !== 'appmixer.utils.ai.ToolStart') { + error = `Component ${componentId} is not of type 'ai.ToolStart' but ${comopnent.type}. + Every tool chain connected to the '${toolsPort}' port of the AI Agent + must start with 'ai.ToolStart' and end with 'ai.ToolOutput'. + This is where you describe what the tool does and what parameters should the AI model provide to it.`; + } + } + }); + }); + + // Teach the user via logs that they need to use the 'ai.ToolStart' component. + if (error) { + throw new context.CancelError(error); + } + + const toolsDefinition = this.getToolsDefinition(tools); + + const instructions = context.properties.instructions || null; + await context.log({ step: 'create-assistant', tools: toolsDefinition, instructions }); + + const apiKey = context.config.apiKey; + + if (!apiKey) { + throw new context.CancelError('Missing \'apiKey\' system setting of the appmixer.utils.ai module pointing to OpenAI. Please provide it in the Connector Configuration section of the Appmixer Backoffice.'); + } + + const client = new OpenAI({ apiKey }); + const assistant = await client.beta.assistants.create({ + model: context.config.AIAgentModel || 'gpt-4o', + instructions, + tools: toolsDefinition + }); + + await context.log({ step: 'created-assistant', assistant }); + return assistant; + }, + + getToolsDefinition: function(tools) { + + // https://platform.openai.com/docs/assistants/tools/function-calling + const toolsDefinition = []; + + Object.keys(tools).forEach((componentId) => { + const component = tools[componentId]; + const parameters = component.config.properties.parameters?.ADD || []; + const toolParameters = { + type: 'object', + properties: {} + }; + parameters.forEach((parameter) => { + toolParameters.properties[parameter.name] = { + type: parameter.type, + description: parameter.description + }; + }); + const toolDefinition = { + type: 'function', + function: { + name: componentId, + description: component.config.properties.description, + } + }; + if (parameters.length) { + toolDefinition.function.parameters = toolParameters; + } + toolsDefinition.push(toolDefinition); + }); + return toolsDefinition; + }, + + handleRunStatus: async function(context, client, thread, run) { + + if (Date.now() - (run.created_at * 1000) > MAX_RUN_DURATION) { + await context.log({ step: 'run-timeout', run }); + await client.beta.threads.runs.cancel(thread.id, run.id); + throw new context.CancelError('The run took too long to complete.'); + } + + await context.log({ step: 'run-status', run }); + // Check if the run is completed + if (run.status === 'completed') { + let messages = await client.beta.threads.messages.list(thread.id); + await context.log({ step: 'completed-run', run, messages }); + await context.sendJson({ + answer: messages.data[0].content[0].text.value, + prompt: context.messages.in.content.prompt + }, 'out'); + } else if (run.status === 'requires_action') { + await this.handleRequiresAction(context, client, thread, run); + } else { + await context.log({ step: 'unexpected-run-state', run }); + } + }, + + handleRequiresAction: async function(context, client, thread, run) { + + await context.log({ step: 'requires-action', run }); + + // Check if there are tools that require outputs. + if ( + run.required_action && + run.required_action.submit_tool_outputs && + run.required_action.submit_tool_outputs.tool_calls + ) { + const toolCalls = []; + for (const toolCall of run.required_action.submit_tool_outputs.tool_calls) { + const componentId = toolCall.function.name; + const args = JSON.parse(toolCall.function.arguments); + toolCalls.push({ componentId, args, id: toolCall.id }); + } + + // Send to all tools. Each ai.ToolStart ignores tool calls that are not intended for it. + await context.sendJson({ toolCalls, prompt: context.messages.in.content.prompt }, 'tools'); + + // Output of each tool is expected to be stored in the service state + // under the ID of the tool call. This is done in the ToolStartOutput component. + // Collect outputs of all the required tool calls. + await context.log({ step: 'collect-tools-output', threadId: thread.id, runId: run.id }); + const outputs = []; + const pollStart = Date.now(); + while ((outputs.length < toolCalls.length) && (Date.now() - pollStart < COLLECT_TOOL_OUTPUTS_POLL_TIMEOUT)) { + for (const toolCall of toolCalls) { + const result = await context.flow.stateGet(toolCall.id); + if (result) { + outputs.push({ tool_call_id: toolCall.id, output: result.output }); + await context.flow.stateUnset(toolCall.id); + } + } + // Sleep. + await new Promise((resolve) => setTimeout(resolve, COLLECT_TOOL_OUTPUTS_POLL_INTERVAL)); + } + await context.log({ step: 'collected-tools-output', threadId: thread.id, runId: run.id, outputs }); + + // Submit tool outputs to the assistant. + if (outputs && outputs.length) { + await context.log({ step: 'tool-outputs', tools: toolCalls, outputs }); + run = await client.beta.threads.runs.submitToolOutputsAndPoll( + thread.id, + run.id, + { tool_outputs: outputs } + ); + // Check status after submitting tool outputs. + await this.handleRunStatus(context, client, thread, run); + + } else { + await context.log({ step: 'no-tool-outputs', tools: toolCalls }); + } + } + }, + + receive: async function(context) { + + const { prompt } = context.messages.in.content; + let threadId = context.messages.in.content.threadId || context.messages.in.correlationId; + const apiKey = context.config.apiKey; + const client = new OpenAI({ apiKey }); + const assistant = await context.stateGet('assistant'); + + // Check if a thread with a given ID exists. + let thread; + if (threadId) { + thread = await context.stateGet(threadId); + } + if (!thread) { + await context.log({ step: 'create-thread', assistantId: assistant.id, internalThreadId: threadId }); + thread = await client.beta.threads.create(); + await context.stateSet(threadId, thread); + } else { + await context.log({ step: 'use-thread', assistantId: assistant.id, thread }); + } + + await context.log({ step: 'create-thread-message', assistantId: assistant.id, threadId: thread.id }); + await client.beta.threads.messages.create(thread.id, { + role: 'user', + content: prompt + }); + + await context.log({ step: 'create-thread-run', assistantId: assistant.id, threadId: thread.id }); + const run = await client.beta.threads.runs.createAndPoll(thread.id, { + assistant_id: assistant.id + }); + + await context.log({ step: 'created-thread-run', assistantId: assistant.id, threadId: thread.id, runId: run.id }); + await this.handleRunStatus(context, client, thread, run); + }, + + stop: async function(context) { + + const apiKey = context.config.apiKey; + const client = new OpenAI({ apiKey }); + const assistant = await context.stateGet('assistant'); + await client.beta.assistants.del(assistant.id); + } +}; diff --git a/src/appmixer/utils/ai/AIAgent/component.json b/src/appmixer/utils/ai/AIAgent/component.json new file mode 100644 index 000000000..6e8f8ea71 --- /dev/null +++ b/src/appmixer/utils/ai/AIAgent/component.json @@ -0,0 +1,69 @@ +{ + "name": "appmixer.utils.ai.AIAgent", + "author": "Appmixer ", + "description": "Build an AI agent responding with contextual answers or performing contextual actions.", + "properties": { + "schema": { + "type": "object", + "properties": { + "instructions": { "type": "string", "maxLength": 256000 } + } + }, + "inspector": { + "inputs": { + "instructions": { + "type": "textarea", + "label": "Instructions", + "index": 1, + "tooltip": "The system instructions that the assistant uses. The maximum length is 256,000 characters. For example 'You are a personal math tutor.'." + } + } + } + }, + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "prompt": { "type": "string" }, + "threadId": { "type": "string" } + }, + "required": ["prompt"] + }, + "inspector": { + "inputs": { + "prompt": { + "label": "Prompt", + "type": "textarea", + "index": 1 + }, + "threadId": { + "label": "Thread ID", + "type": "text", + "index": 2, + "tooltip": "By setting a thread ID you can keep the context of the conversation." + } + } + } + }], + "outPorts": [{ + "name": "out", + "options": [{ + "label": "Answer", + "value": "answer", + "schema": { "type": "string" } + }, { + "label": "Prompt", + "value": "prompt", + "schema": { "type": "string" } + }] + }, { + "name": "tools", + "options": [{ + "label": "Prompt", + "value": "prompt", + "schema": { "type": "string" } + }] + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzUyMjFfMzI0OTApIj4KPHBhdGggZD0iTTEyLjE1OTcgNi44Nzk5VjIuMzk5OUg3LjY3OTY5IiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMTcuOTIxMiA2Ljg3OTY0SDUuNDQxMTdDNC4yMDQwNSA2Ljg3OTY0IDMuMjAxMTcgNy44ODI1MiAzLjIwMTE3IDkuMTE5NjRWMTguMDc5NkMzLjIwMTE3IDE5LjMxNjggNC4yMDQwNSAyMC4zMTk2IDUuNDQxMTcgMjAuMzE5NkgxNy45MjEyQzE5LjE1ODMgMjAuMzE5NiAyMC4xNjEyIDE5LjMxNjggMjAuMTYxMiAxOC4wNzk2VjkuMTE5NjRDMjAuMTYxMiA3Ljg4MjUyIDE5LjE1ODMgNi44Nzk2NCAxNy45MjEyIDYuODc5NjRaIiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMC45NTk5NjEgMTMuNTk5NkgzLjE5OTk2IiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMjEuMTIwMSAxMy41OTk2SDIyLjQwMDEiIHN0cm9rZT0iIzMwMzIzNiIgc3Ryb2tlLXdpZHRoPSIyLjAxNiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIi8+CjxwYXRoIGQ9Ik0xNC43ODQyIDEyLjQ3OTdWMTQuNzE5NyIgc3Ryb2tlPSIjMzAzMjM2IiBzdHJva2Utd2lkdGg9IjIuMDE2IiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KPHBhdGggZD0iTTguNjM5NjUgMTIuNDc5N1YxNC43MTk3IiBzdHJva2U9IiMzMDMyMzYiIHN0cm9rZS13aWR0aD0iMi4wMTYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8L2c+CjxkZWZzPgo8Y2xpcFBhdGggaWQ9ImNsaXAwXzUyMjFfMzI0OTAiPgo8cmVjdCB3aWR0aD0iMjQiIGhlaWdodD0iMjMuMDQiIGZpbGw9IndoaXRlIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgwIDAuNDc5OTgpIi8+CjwvY2xpcFBhdGg+CjwvZGVmcz4KPC9zdmc+Cg==" +} diff --git a/src/appmixer/utils/ai/AIAgent/icon.svg b/src/appmixer/utils/ai/AIAgent/icon.svg new file mode 100644 index 000000000..640f610a1 --- /dev/null +++ b/src/appmixer/utils/ai/AIAgent/icon.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/src/appmixer/utils/ai/GenerateEmbeddings/GenerateEmbeddings.js b/src/appmixer/utils/ai/GenerateEmbeddings/GenerateEmbeddings.js new file mode 100644 index 000000000..f71ce2402 --- /dev/null +++ b/src/appmixer/utils/ai/GenerateEmbeddings/GenerateEmbeddings.js @@ -0,0 +1,113 @@ +'use strict'; + +const { RecursiveCharacterTextSplitter } = require('langchain/text_splitter'); +const OpenAI = require('openai'); + +// See https://platform.openai.com/docs/api-reference/embeddings/create#embeddings-create-input. +const MAX_INPUT_LENGTH = 8192 * 4; // max 8192 tokens, 1 token ~ 4 characters. +const MAX_BATCH_SIZE = 2048; + +module.exports = { + + receive: async function(context) { + + const messageId = context.messages.in.messageId; + const { + text, + model = 'text-embedding-ada-002', + chunkSize = 500, + chunkOverlap = 50, + embeddingTemplate = '{ "id": "{messageId}:{chunkIndex}", "values": {embedding}, "metadata": { "text": "{chunkText}" } }' + } = context.messages.in.content; + + const chunks = await this.splitText(text, chunkSize, chunkOverlap); + await context.log({ step: 'split-text', message: 'Text succesfully split into chunks.', chunksLength: chunks.length }); + + const apiKey = context.config.apiKey; + if (!apiKey) { + throw new context.CancelError('Missing \'apiKey\' system setting of the appmixer.utils.ai module pointing to OpenAI. Please provide it in the Connector Configuration section of the Appmixer Backoffice.'); + } + const client = new OpenAI({ apiKey }); + + // Process chunks in batches. + // the batch size is calculated based on the chunk size and the maximum input length in + // order not to exceed the maximum input length defined in + // https://platform.openai.com/docs/api-reference/embeddings/create#embeddings-create-input + // We devide the maximum input length by 2 to stay on the safe side + // because the token to character ratio might not be accurate. + const batchSize = Math.min(Math.floor((MAX_INPUT_LENGTH / 2) / chunkSize), MAX_BATCH_SIZE); + const embeddings = []; + // For convenience, the GenerateEmbeddings component returns the first vector. + // This makes it easy to genereate embedding for a prompt and send it e.g. to the pinecone.QueryVectors component + // without having to apply modifiers to the embedding array returned. + let firstVector = null; + for (let i = 0; i < chunks.length; i += batchSize) { + const batch = chunks.slice(i, i + batchSize); + + const response = await client.embeddings.create({ + model, + input: batch, + encoding_format: 'float' + }); + + // Collect embeddings for the current batch. + response.data.forEach((item, index) => { + if (!firstVector) { + firstVector = item.embedding; + } + try { + const embedding = this.createEmbedding(embeddingTemplate, { + embedding: JSON.stringify(item.embedding), + chunkIndex: i + index, + chunkText: JSON.stringify(batch[index]).slice(1, -1), + messageId, + now: Date.now() + }); + embeddings.push(embedding); + } catch (err) { + // It does not make sense to retry the component call. + // Things "won't" improve. + throw new context.CancelError(err); + } + }); + } + return context.sendJson({ embeddings, firstVector }, 'out'); + }, + + createEmbedding(embeddingTemplate, args) { + + const unknownPlaceholders = []; + const embeddingString = embeddingTemplate.replace(/{([^{}]+)}/g, (match, key) => { + if (key in args) { + return args[key]; + } else { + unknownPlaceholders.push(key); + return match; + } + }); + + if (unknownPlaceholders.length) { + throw new Error(`Unknown placeholders in the embedding template: ${unknownPlaceholders.join(', ')}. Template: ${embeddingTemplate}.`); + } + + let embedding; + try { + embedding = JSON.parse(embeddingString); + } catch (err) { + throw new Error(`Failed to parse the embedding string. Error: ${err.message}. ${embeddingString}.`); + } + + return embedding; + }, + + splitText(text, chunkSize, chunkOverlap) { + + const splitter = new RecursiveCharacterTextSplitter({ + chunkSize, + chunkOverlap + }); + + return splitter.splitText(text); + } +}; + diff --git a/src/appmixer/utils/ai/GenerateEmbeddings/component.json b/src/appmixer/utils/ai/GenerateEmbeddings/component.json new file mode 100644 index 000000000..cb1188acf --- /dev/null +++ b/src/appmixer/utils/ai/GenerateEmbeddings/component.json @@ -0,0 +1,91 @@ +{ + "name": "appmixer.utils.ai.GenerateEmbeddings", + "author": "Appmixer ", + "description": "Generate embeddings for text data. It automatically splits the text into chunks and get embedding for each chunk.", + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "text": { "type": "string", "maxLength": 512000 }, + "model": { "type": "string" }, + "chunkSize": { "type": "integer" }, + "chunkOverlap": { "type": "integer" }, + "embeddingTemplate": { "type": "string" } + } + }, + "inspector": { + "inputs": { + "text": { + "type": "textarea", + "label": "Text", + "tooltip": "Enter the text to generate embeddings for. The text will be split into chunks and embeddings will be generated for each chunk. The maximum length is 512,000 characters. If you need more than 512,000 characters, use the 'Generate Embeddings From File' component.", + "index": 1 + }, + "model": { + "type": "select", + "label": "Model", + "index": 2, + "defaultValue": "text-embedding-ada-002", + "options": [ + { "label": "text-embedding-3-large (3072 dimension)", "value": "text-embedding-3-large" }, + { "label": "text-embedding-3-small (1536 dimension)", "value": "text-embedding-3-small" }, + { "label": "text-embedding-ada-002 (1536 dimension)", "value": "text-embedding-ada-002" } + ] + }, + "chunkSize": { + "type": "number", + "label": "Chunk Size", + "defaultValue": 500, + "tooltip": "Maximum size of each chunk for text splitting. The default is 500.", + "index": 3 + }, + "chunkOverlap": { + "type": "number", + "label": "Chunk Overlap", + "defaultValue": 50, + "tooltip": "Overlap between chunks for text splitting to maintain context. The default is 50.", + "index": 4 + }, + "embeddingTemplate": { + "type": "textarea", + "label": "Embedding Template", + "defaultValue": "{ \"id\": \"{messageId}:{chunkIndex}\", \"values\": {embedding}, \"metadata\": { \"text\": \"{chunkText}\" } }", + "tooltip": "The template for the embeddings. Use the placeholders {embedding} (a vector, e.g. [1.1, 2.2, 3.3]), {now} (a UNIX timestamp in milliseconds), {messageId} (a unique ID for the run of the 'Generate Embeddings' component), {chunkIndex} (the index of the chunk) and {chunkText} (the text of the chunk) to define the structure of the embeddings. Default template is { \"id\": \"{messageId}:{chunkIndex}\", \"values\": {embedding}, \"metadata\": { \"text\": \"{chunkText}\" } }. This template defines embeddings that can be directly used with the Pinecone vector database.", + "index": 5 + } + } + } + }], + "outPorts": [{ + "name": "out", + "options": [{ + "label": "Embeddings", + "value": "embeddings", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": "string" }, + "values": { "type": "array", "items": { "type": "number" } }, + "metadata": { + "type": "object", + "properties": { + "text": { "type": "string" } + } + } + } + } + } + }, { + "label": "First Vector", + "value": "firstVector", + "schema": { + "type": "array", + "items": { "type": "number" } + } + }] + }], + "icon": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGUAAABiCAYAAABJeR13AAABDWlDQ1BJQ0MgUHJvZmlsZQAAKJFjYGCSSCwoyGESYGDIzSspCnJ3UoiIjFJgf8HAwcDJwMegyaCWmFxc4BgQ4MMABDAaFXy7xsAIoi/rgszClMcLuFJSi5OB9B8gTkkuKCphYGBMALKVy0sKQOwWIFskKRvMngFiFwEdCGSvAbHTIewDYDUQ9hWwmpAgZyD7BZDNlwRh/wCx08FsJg4QG2ov2A2OQHenKgB9T6LjCYGS1IoSEO2cX1BZlJmeUaIAsckzL1lPR8HIwMiEgQEU3hDVnwPB4cgodgYhhgAIscq9wJgIYmBg2YkQCwP6b40+A4MsM0JMTYmBQaiegWFjQXJpURnUGEbGswwMhPgAhBpLo/XijBMAAABWZVhJZk1NACoAAAAIAAGHaQAEAAAAAQAAABoAAAAAAAOShgAHAAAAEgAAAESgAgAEAAAAAQAAAGWgAwAEAAAAAQAAAGIAAAAAQVNDSUkAAABTY3JlZW5zaG90r1CPNgAAAdVpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDYuMC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iPgogICAgICAgICA8ZXhpZjpQaXhlbFlEaW1lbnNpb24+OTg8L2V4aWY6UGl4ZWxZRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFhEaW1lbnNpb24+MTAxPC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC94OnhtcG1ldGE+CsiCPvwAAAz7SURBVHgB7V1XqBU9EI69996wi72+2MVeUewNRB/UJx9UFDsootgVRLFgQURF7KJYQFGx96547Vix97r/fPnJkt3NtnP37NkrO3DvbpJJMpnZSZlMcjJpBCyGSHEgc6SoiYnhHIiFEsEPIXSh/Pr1i129epV9//7dNztOnTrF0tLSfOdDHuT1C6ARtILmUAFjSljw9+9frUWLFhjDtDp16mjUaM9VT5o0iefLli2bdvz4cc/5jh07piEP6pw8ebLnfKANNCIfaAbtYQELqyLU8/TpU95INBR/Fy5c8Fx9/fr19bxTp071nA+4oj6U4RVAm8iH55MnT7xmTTdeqN2XuRswh6nxtkBfqp4mv+uRNi8yrvxug65Hm2kzh3XEJLyEKpQk0P9PFhkLJYJijYUSCyWCHIggSbGmxEKJIAciSFKsKbFQIsiBCJKUNSyaHj58yA4ePGiobtmyZWzHjh2GOLvAixcv9KQjR46wCRMm6GGnFzLJ6Mkow2u+58+f6/nwcvjwYdahQwdWvnx5Q3xSAum2CTgU8PnzZ23BggVajRo1DCYLakiGDdeqVUtbuHCh9uXLF4eWpy8pabYv0gCtVKlSGZb5bh9OmTJltF27dqWP+za5MyE+aBWcMmUKmzVrFgRuKJqstaxcuXIsV65chvioB759+8bIIGkx4WfKlImhrTNmzAi2CTbCSjiaiLRoR+fOnbW9e/dqX79+TbjcVGcE7WhDp06dLO3zY7X20o5Au6+dO3dq9PXoRBcuXDhpKu6lccnCQTsLFSqktxNtDrIrC0woGNRLly6tEwqB0K5dsviS8nKvXLliEAzGmKAG/8CEglkWdaz6X5BfTsolYEMANEZu86JFi2ww/UUHNtDTVJHdvHmTD3g0hrB9+/Y5Dn4fPnxgWLuQhjnipSoxb968rEKFCqxAgQKOJNAYww4cOMBxwIPr16874ntK9CdDNTYx1/DFYEC0A2yzYrDMmjWrIQ8RG7kwaMQk5eLFi3bN0Xbv3m2gG7xIL9hqChXMaE/dMg1USRqr3ZEjR/IkTHuhBapp76ZNm9jQoUPZz58/VcVENi5Hjhxs7dq1bODAgRYaMV2GNont4hUrVrB27dpZ8MwR2bNnZzQOMUyrLaCSKjFNa968ueELoIyewpUqVVIVqZ0/f14jQjyV4bWuMPFIMLYaU7FixYTaBS8Z8NoMStsXxoYTJ05YBOglIk+ePEo0LLKEhmTJkoXboPDlqTRKWUDIkbQuYdDsOXPmsD9//rAfP34wclFSjpUYfxIB2OVu3brF6tata8xulhLCpJK6zxNh+/oKqAJLke/fvzeMIX78ryyFhRwxceJEvf3wH6Ou2UIB2uyXT8CHXxl4bQalpuTMmZOdO3eO3b59m/3+/dsoRUUI1t41a9YoUv6PwixLLkfVN9tmTnHCoEGD2OzZszkVGDfQFsuXLdE4ZMgQNmLECClG/YpxioTC8DSDUihAAnK9evXM+MowGR6V8SISg6EMdl2cjBOVd3PXhG7NCWDba9asmROKa1q88+jKovARYqGEz3PXGmOhuLIofIRYKOHz3LXGWCiuLAofIRZK+Dx3rdF2SuyaM0UIsDQULFiQ1a5d2xMFWInDkoB1EtYZYnVOCzaWP39+Rvs+avuTp9KTg5ShNAWLMrIX8fXTypUrHTkyZswYVrRoUYaFsGB+iRIlGG3EMbJVMbLR8fRixYopTSeOhSc5McMIBfszq1at4uzA4Z/Ro0dzZwYVf+7du8dow4m9efNGlWyIA87cuXMNcakO2HZfMMRt2bJFNyI6Efrq1Sun5EDSzB4jWFkvWbKEzZ8/31J+kSJFuDkdWwheoG3btl7QPOFAg2lH0hUXpvsBAwaw8ePHW3HNxjCEyeaVkIGNStdUBkk6mWso78GDB6pqbeMuX75syI968EfdkUbjhDIfeThqpF3a0aNHNRqHtKVLlxrKwGFWxF+6dEmZX0SCVlEfnmiLGRI1SMLh4s6dO+biNKWmYPCDfYocAYiO1MPq1auVRLx8+ZJvxXbt2tWSXrJkSYZtaQHmI+INGjTgbRTpqXjmzp2bkVeMpWqlUDD40VFnRs4PfB/BkssUgX2BkydPmmKDCWLWhG5UABh96NAh3eq8efNmphKKwBdPzMJkUFln5fRE35s2bconI275UX+PHj0YeG0GpVCA1LBhQ/5nzqAKYwMrWUKBM/fr16/1akeNGsWntcJZfM+ePfzDcWOyrCnYZKP9d73MIF9at27NZs6cma4iIz/72rp1q95ADODY/8YAKQCDOXwE3EDePsA0OcoQaaFgoSfPZKDucMzo2bOnYXNo27ZtrjyOheLKIm8IGNfkrqt///48I1b0OCsiAGOf8CYRceanLBS3rs6cN+xwpDVl+/btOj8wILZp00YP9+nTR39/+/Ytw9jjBLJQ4u7LiVMOaTR5N3RdvXr1MgzO3bt399WFyUKJNcWB8U5JZ86c4c6AAqdfv37ilT/Rhcmag7EHY5AdyEKJqluToD0580JRejqecteFYjDNFF4lotj79++LVwZTD+xdY8eOVVp9ZYeHWCg62/y9mIXiNmag9HHjxnGLsMrFR9aUeEzxJwuOTbauhG7AQ2bzqV5RvawpMG9EGWy7L/KO52YW4Wrq1Aj5WLQTntc0WUuwYHQyo+AYtljdo/zBgwcrq5E1JZl+Z16Pk2OygclKo0aNLPQqhYK1QatWrVJmkJQXg3379mXLly+3EC4iMJbA+IjZGoAcyVmVKlVEsv6UhZJMTYG5yavJCdsOD8nj0mz/Uq5T3r17x2R111sWwgtcZcXhI1SHr8kJihcvznBYR4DQGggJAkNb8Pfx40eB4noQSEdM8gs+FNBmBqWmVKtWjXubw+vcS/eFxssrb3MlfsJy15UvXz4GA58bAEecoIIdDAJo0qSJQbhyGZilQftwNgRnRNavX89g3Q0C8NXjQ3EDbHLBpxq8toBlhyWBCHjRU8H8L72bXC1bttTL6t27tydqSJB6HtBBH5MhLGizezrV43eTK4gTBcrui4hPGdSsWZPXjcEYzg9eAJZjOEMAoCHdunVTDqB2ZakGWzvcMOKV3VcYFdvVgWMVOE6AQ6Bu3vyiDHRzN27cYPRV8+4AeyVnz57lXRoMlcK1CF0xVv14wvkCG1/weMGkJkoQOaGgn8fX7hewShdahryZM2d2PEfit/ww8SPXfYXZ+KjWFQslgpKJhRILJYIciCBJsabEQokgByJIku2UGLava9eu6U5vTrTj1jgnMG8qRfWSHFUbzLS6GTOxdQALuxtg6l+9enWmLE9lxyAjmUaLN1+mCiKC46vMLLgQQL4gBxcGZBSg21l1PgR9uQGuD8HFD2ZQagokDZNyIkAVWLLhfEj79u3Z/v37eZo4eoCLA5RfiqWE8CPQU2zcuJHNmzdPrxxtQFvMoGqzGUcVhgUCvJZ9DTieWUoI45Y7uthA/0II0fO73YU5uJ6JNnY8l+OnzjBwQbudh36iF+bgl49Ut+nZXi0F2xCulpKv7+BSVPzzerUUHLVxtZTs16soLnJR2CVct26dwV1WEAmNwtVSgk842GT58gWy9IQ/M26ngDnIAipN8RtnNm87XcIGjenSpYv+42VEUGS1B2MIbUXbagj4ZL6E7dGjR37ZZ8G31RSL9Fwi5OsKcYWfGD/ssn369IlbdfGMIsDyjO0APJ1Avq4Qh2MxY003WMSUYASuGidi9D9cevmvg3lzbfHixYE0ObDbVjFg4RpYIRjc24trYv9VwJE/8tLU21u2bFnloJ1I+wMTCirHtbc4xycL5l/UGGiILBC0GWNLUBCoUECU/GOXQjgdO3bkRKumf0E1JNnlgHYwno5g6B+daN+0adMCrT6wgZ4I1IGI5L6/RKkehxds02IaGNUFo4FYKYBpL0xJYtorkmAqoY+QTZ8+XUQF8wxUxFJh6MrkMYaotXxhGTkObQuyy5JYpwXefcmFQ+Vx1ThNFf8ZgdDUn7cpmV1xUrovlQ4/fvyY+/wOHz5cT4bXilePFVwcKhz+vB6LRkU4oid+9hxOcsOGDdPrd3p59uwZ27Bhg46ClTpsXxn+559krcE7nScxaMzp06fNKLZh8TPkxCVfP2EufjId+VQWbLsK03tLhl25XuIVhhf944hfUsSBWCgpYrxTtbFQnLiTorRYKClivFO1sVCcuJOitFgoKWK8U7WxUJy4k6K0WCgpYrxTtaEKBTcQyYAjZl5B3svG/rZXSDSfmVZz2Gv9ieCFKhScL6Tjc5xO/AyIfIDUjXhxbBvMkW8wcssHXMFQUYZbHqRjaxe/bwIAzaA9LAjN9iUaBPM3TgBXrVrVcOGNSHd6woYF+1XlypWd0CxpaWlp3G7WuHFjS5pTBLxu7t69yw8jJesmPVX9oQtFRUQcZ+RAqN2Xseo4ZMeBWCh2nElh/H/liwvAjmbZQwAAAABJRU5ErkJggg==" +} diff --git a/src/appmixer/utils/ai/ToolOutput/ToolOutput.js b/src/appmixer/utils/ai/ToolOutput/ToolOutput.js new file mode 100644 index 000000000..f960c0d7b --- /dev/null +++ b/src/appmixer/utils/ai/ToolOutput/ToolOutput.js @@ -0,0 +1,31 @@ +'use strict'; + +module.exports = { + + receive: async function(context) { + + // Find the toolCallId in the message scope, looking into the first + // component with type 'appmixer.utils.ai.ToolStart' and taking + // toolCallId from the output message that is stored in the flow state. + const flowDescriptor = context.flowDescriptor; + const scope = context.messages.in.scope; + let toolCallId; + for (const componentId of Object.keys(scope)) { + const component = flowDescriptor[componentId]; + if (component && component.type === 'appmixer.utils.ai.ToolStart') { + const key = componentId + ':' + context.messages.in.correlationId; + toolCallId = await context.flow.stateGet(key); + await context.flow.stateUnset(key); + } + } + + if (!toolCallId) { + await context.log({ step: 'no-tool-call-id', scope, flowDescriptor }); + throw new context.CancelError('No toolCallId found in the scope. Are you sure you used ai.ToolStart to start your tool chain?'); + } + + await context.log({ step: 'tool-output', toolCallId, output: context.messages.in.content.output }); + // The AI agent expects to see the output in the flow state under the toolCallId key. + return context.flow.stateSet(toolCallId, { output: context.messages.in.content.output }); + } +}; diff --git a/src/appmixer/utils/ai/ToolOutput/component.json b/src/appmixer/utils/ai/ToolOutput/component.json new file mode 100644 index 000000000..ccf99e686 --- /dev/null +++ b/src/appmixer/utils/ai/ToolOutput/component.json @@ -0,0 +1,24 @@ +{ + "name": "appmixer.utils.ai.ToolOutput", + "author": "Appmixer ", + "description": "Determines the end of tool chain for AI Agent. Set the output of the tool chain.", + "inPorts": [{ + "name": "in", + "schema": { + "type": "object", + "properties": { + "output": { "type": "string" } + } + }, + "inspector": { + "inputs": { + "output": { + "type": "textarea", + "label": "Output", + "index": 1 + } + } + } + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzUyMjFfMzI1MzcpIj4KPHBhdGggZD0iTTEyLjY2NjcgNi42NjY2N1YySDgiIHN0cm9rZT0iIzE4MTgxOCIgc3Ryb2tlLXdpZHRoPSIyLjEiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMTguNjY4MyA2LjY2NjVINS42NjgyOUM0LjM3OTYzIDYuNjY2NSAzLjMzNDk2IDcuNzExMTcgMy4zMzQ5NiA4Ljk5OTg0VjE4LjMzMzJDMy4zMzQ5NiAxOS42MjE4IDQuMzc5NjMgMjAuNjY2NSA1LjY2ODI5IDIwLjY2NjVIMTguNjY4M0MxOS45NTcgMjAuNjY2NSAyMS4wMDE2IDE5LjYyMTggMjEuMDAxNiAxOC4zMzMyVjguOTk5ODRDMjEuMDAxNiA3LjcxMTE3IDE5Ljk1NyA2LjY2NjUgMTguNjY4MyA2LjY2NjVaIiBzdHJva2U9IiMxODE4MTgiIHN0cm9rZS13aWR0aD0iMi4xIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KPHBhdGggZD0iTTEuMDAwOTggMTMuNjY2M0gzLjMzNDMxIiBzdHJva2U9IiMxODE4MTgiIHN0cm9rZS13aWR0aD0iMi4xIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KPHBhdGggZD0iTTIxLjAwMSAxMy42NjYzSDIzLjAzNDUiIHN0cm9rZT0iIzE4MTgxOCIgc3Ryb2tlLXdpZHRoPSIyLjEiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8ZyBmaWx0ZXI9InVybCgjZmlsdGVyMF9kXzUyMjFfMzI1MzcpIj4KPHBhdGggZD0iTTcuMDAwOTggMTMuNDU5NUw5LjYwNzE4IDE2LjA2NTdDMTAuMzA2OCAxNi43NjUzIDExLjQ0OTUgMTYuNzM2NyAxMi4xMTMyIDE2LjAwM0wxNy4wMDEgMTAuNTk5OSIgc3Ryb2tlPSIjMjBCNzVBIiBzdHJva2Utd2lkdGg9IjIuODU1MTIiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIvPgo8L2c+CjwvZz4KPGRlZnM+CjxmaWx0ZXIgaWQ9ImZpbHRlcjBfZF81MjIxXzMyNTM3IiB4PSIzLjQ3MjY5IiB5PSI3LjA3MTgxIiB3aWR0aD0iMTcuMDU2NiIgaGVpZ2h0PSIxMy4wMjg3IiBmaWx0ZXJVbml0cz0idXNlclNwYWNlT25Vc2UiIGNvbG9yLWludGVycG9sYXRpb24tZmlsdGVycz0ic1JHQiI+CjxmZUZsb29kIGZsb29kLW9wYWNpdHk9IjAiIHJlc3VsdD0iQmFja2dyb3VuZEltYWdlRml4Ii8+CjxmZUNvbG9yTWF0cml4IGluPSJTb3VyY2VBbHBoYSIgdHlwZT0ibWF0cml4IiB2YWx1ZXM9IjAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDEyNyAwIiByZXN1bHQ9ImhhcmRBbHBoYSIvPgo8ZmVPZmZzZXQvPgo8ZmVHYXVzc2lhbkJsdXIgc3RkRGV2aWF0aW9uPSIxLjA1MDI3Ii8+CjxmZUNvbXBvc2l0ZSBpbjI9ImhhcmRBbHBoYSIgb3BlcmF0b3I9Im91dCIvPgo8ZmVDb2xvck1hdHJpeCB0eXBlPSJtYXRyaXgiIHZhbHVlcz0iMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMC4yNSAwIi8+CjxmZUJsZW5kIG1vZGU9Im5vcm1hbCIgaW4yPSJCYWNrZ3JvdW5kSW1hZ2VGaXgiIHJlc3VsdD0iZWZmZWN0MV9kcm9wU2hhZG93XzUyMjFfMzI1MzciLz4KPGZlQmxlbmQgbW9kZT0ibm9ybWFsIiBpbj0iU291cmNlR3JhcGhpYyIgaW4yPSJlZmZlY3QxX2Ryb3BTaGFkb3dfNTIyMV8zMjUzNyIgcmVzdWx0PSJzaGFwZSIvPgo8L2ZpbHRlcj4KPGNsaXBQYXRoIGlkPSJjbGlwMF81MjIxXzMyNTM3Ij4KPHJlY3Qgd2lkdGg9IjI0IiBoZWlnaHQ9IjI0IiBmaWxsPSJ3aGl0ZSIvPgo8L2NsaXBQYXRoPgo8L2RlZnM+Cjwvc3ZnPgo=" +} diff --git a/src/appmixer/utils/ai/ToolOutput/icon.svg b/src/appmixer/utils/ai/ToolOutput/icon.svg new file mode 100644 index 000000000..fa873f377 --- /dev/null +++ b/src/appmixer/utils/ai/ToolOutput/icon.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/appmixer/utils/ai/ToolStart/ToolStart.js b/src/appmixer/utils/ai/ToolStart/ToolStart.js new file mode 100644 index 000000000..45fc7668a --- /dev/null +++ b/src/appmixer/utils/ai/ToolStart/ToolStart.js @@ -0,0 +1,32 @@ +'use strict'; + +module.exports = { + + receive: async function(context) { + + if (context.properties.generateOutputPortOptions) { + return this.getOutputPortOptions(context); + } + const { toolCalls } = context.messages.in.originalContent; + for (const toolCall of toolCalls) { + // Process only those tool calls that are for this component. + // This is because the AI Agent fans out all tool calls by using sendJson(..., 'tools'). + if (toolCall.componentId === context.componentId) { + const out = { args: toolCall.args, toolCallId: toolCall.id }; + await context.flow.stateSet(context.componentId + ':' + context.messages.in.correlationId, toolCall.id); + await context.sendJson(out, 'out'); + } + } + }, + + getOutputPortOptions(context) { + + const options = []; + const parameters = context.properties.parameters?.ADD || []; + parameters.forEach(parameter => { + options.push({ label: parameter.name, value: 'args.' + parameter.name, schema: { type: parameter.type } }); + }); + options.push({ label: 'Tool Call ID', value: 'toolCallId', schema: { type: 'string' } }); + return context.sendJson(options, 'out'); + } +}; diff --git a/src/appmixer/utils/ai/ToolStart/component.json b/src/appmixer/utils/ai/ToolStart/component.json new file mode 100644 index 000000000..7b22bfc7f --- /dev/null +++ b/src/appmixer/utils/ai/ToolStart/component.json @@ -0,0 +1,65 @@ +{ + "name": "appmixer.utils.ai.ToolStart", + "author": "Appmixer ", + "description": "Define a tool chain (i.e. a set of actions that might be selected by the AI model to run based on the user prompt). The tool can provide additional context to the AI agent to be able to reply to the user or call actions. This has to be the first component connected to the AI Agent tools output port. The tool chain must be ended with 'Tool Output'.", + "properties": { + "schema": { + "type": "object", + "properties": { + "generateOutputPortOptions": { "type": "boolean" }, + "description": { "type": "string" }, + "parameters": { "type": "object" } + } + }, + "inspector": { + "inputs": { + "description": { + "type": "textarea", + "label": "Description", + "index": 1 + }, + "parameters": { + "type": "expression", + "levels": ["ADD"], + "label": "Parameters", + "index": 2, + "fields": { + "name": { + "type": "text", + "label": "Name" + }, + "description": { + "type": "textarea", + "label": "Description" + }, + "type": { + "type": "select", + "label": "Type", + "options": [ + { "value": "string", "label": "String" }, + { "value": "number", "label": "Number" }, + { "value": "boolean", "label": "Boolean" } + ] + } + } + } + } + } + }, + "inPorts": [{ + "name": "in" + }], + "outPorts": [{ + "name": "out", + "source": { + "url": "/component/appmixer/utils/ai/ToolStart?outPort=out", + "data": { + "properties": { + "generateOutputPortOptions": true, + "parameters": "properties/parameters" + } + } + } + }], + "icon": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzUyMjFfMzI0OTcpIj4KPHBhdGggZD0iTTExLjk5OTggNS4yVjFINy43OTk4IiBzdHJva2U9IiMxODE4MTgiIHN0cm9rZS13aWR0aD0iMi4xIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KPHBhdGggZD0iTTcuNzk5NjEgMTcuOEwzLjU5OTYxIDIyVjcuMjk5OTVDMy41OTk2MSA2Ljc0MyAzLjgyMDg2IDYuMjA4ODUgNC4yMTQ2OSA1LjgxNTAzQzQuNjA4NTEgNS40MjEyIDUuMTQyNjUgNS4xOTk5NSA1LjY5OTYxIDUuMTk5OTVIMTguMjk5NkMxOC44NTY2IDUuMTk5OTUgMTkuMzkwNyA1LjQyMTIgMTkuNzg0NSA1LjgxNTAzQzIwLjE3ODQgNi4yMDg4NSAyMC4zOTk2IDYuNzQzIDIwLjM5OTYgNy4yOTk5NVYxNS43QzIwLjM5OTYgMTYuMjU2OSAyMC4xNzg0IDE2Ljc5MSAxOS43ODQ1IDE3LjE4NDlDMTkuMzkwNyAxNy41Nzg3IDE4Ljg1NjYgMTcuOCAxOC4yOTk2IDE3LjhINy43OTk2MVoiIHN0cm9rZT0iIzE4MTgxOCIgc3Ryb2tlLXdpZHRoPSIyLjEiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMS41IDExLjQ5OThIMy42IiBzdHJva2U9IiMxODE4MTgiIHN0cm9rZS13aWR0aD0iMi4xIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KPHBhdGggZD0iTTguODQ5NjEgMTAuNDVWMTIuNTUiIHN0cm9rZT0iIzE4MTgxOCIgc3Ryb2tlLXdpZHRoPSIyLjEiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgo8cGF0aCBkPSJNMTUuMTUwNCAxMC40NVYxMi41NSIgc3Ryb2tlPSIjMTgxODE4IiBzdHJva2Utd2lkdGg9IjIuMSIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIi8+CjxwYXRoIGQ9Ik0yMC40MDA0IDExLjQ5OThIMjIuNTAwNCIgc3Ryb2tlPSIjMTgxODE4IiBzdHJva2Utd2lkdGg9IjIuMSIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIi8+CjwvZz4KPGRlZnM+CjxjbGlwUGF0aCBpZD0iY2xpcDBfNTIyMV8zMjQ5NyI+CjxyZWN0IHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgZmlsbD0id2hpdGUiLz4KPC9jbGlwUGF0aD4KPC9kZWZzPgo8L3N2Zz4K" +} diff --git a/src/appmixer/utils/ai/ToolStart/icon.svg b/src/appmixer/utils/ai/ToolStart/icon.svg new file mode 100644 index 000000000..b8e110fa5 --- /dev/null +++ b/src/appmixer/utils/ai/ToolStart/icon.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/src/appmixer/utils/ai/bundle.json b/src/appmixer/utils/ai/bundle.json index 107242f8b..f72cac75a 100644 --- a/src/appmixer/utils/ai/bundle.json +++ b/src/appmixer/utils/ai/bundle.json @@ -1,6 +1,6 @@ { "name": "appmixer.utils.ai", - "version": "1.1.1", + "version": "1.2.0", "changelog": { "1.0.0": [ "Initial version." @@ -10,6 +10,10 @@ ], "1.1.1": [ "DescribeImages: images are now required" + ], + "1.2.0": [ + "New components: AIAgent, ToolStart and ToolOutput for defining AI agents.", + "New component: GenerateEmbeddings to generate embeddings from text." ] } } diff --git a/src/appmixer/utils/ai/package-lock.json b/src/appmixer/utils/ai/package-lock.json new file mode 100644 index 000000000..ad69e404d --- /dev/null +++ b/src/appmixer/utils/ai/package-lock.json @@ -0,0 +1,629 @@ +{ + "name": "appmixer.utils.ai", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "appmixer.utils.ai", + "version": "1.0.0", + "dependencies": { + "form-data": "4.0.0", + "langchain": "0.3.6", + "mustache": "4.2.0", + "openai": "4.76.0" + } + }, + "node_modules/@cfworker/json-schema": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.0.3.tgz", + "integrity": "sha512-ZykIcDTVv5UNmKWSTLAs3VukO6NDJkkSKxrgUTDPBkAlORVT3H9n5DbRjRl8xIotklscHdbLIa0b9+y3mQq73g==", + "peer": true + }, + "node_modules/@langchain/core": { + "version": "0.3.23", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.23.tgz", + "integrity": "sha512-Aut43dEJYH/ibccSErFOLQzymkBG4emlN16P0OHWwx02bDosOR9ilZly4JJiCSYcprn2X2H8nee6P/4VMg1oQA==", + "peer": true, + "dependencies": { + "@cfworker/json-schema": "^4.0.2", + "ansi-styles": "^5.0.0", + "camelcase": "6", + "decamelize": "1.2.0", + "js-tiktoken": "^1.0.12", + "langsmith": "^0.2.8", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^10.0.0", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/openai": { + "version": "0.3.14", + "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.3.14.tgz", + "integrity": "sha512-lNWjUo1tbvsss45IF7UQtMu1NJ6oUKvhgPYWXnX9f/d6OmuLu7D99HQ3Y88vLcUo9XjjOy417olYHignMduMjA==", + "dependencies": { + "js-tiktoken": "^1.0.12", + "openai": "^4.71.0", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": ">=0.2.26 <0.4.0" + } + }, + "node_modules/@langchain/textsplitters": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@langchain/textsplitters/-/textsplitters-0.1.0.tgz", + "integrity": "sha512-djI4uw9rlkAb5iMhtLED+xJebDdAG935AdP4eRTB02R7OB/act55Bj9wsskhZsvuyQRpO4O1wQOp85s6T6GWmw==", + "dependencies": { + "js-tiktoken": "^1.0.12" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": ">=0.2.21 <0.4.0" + } + }, + "node_modules/@types/node": { + "version": "18.19.68", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.68.tgz", + "integrity": "sha512-QGtpFH1vB99ZmTa63K4/FU8twThj4fuVSBkGddTp7uIL/cuoLWIUSL2RcOaigBhfR+hg5pgGkBnkoOxrTVBMKw==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==" + }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==" + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/agentkeepalive": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "engines": { + "node": ">=14" + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.15.tgz", + "integrity": "sha512-65ruOWWXDEZHHbAo7EjOcNxOGasQKbL4Fq3jEr2xsCqSsoOo6VVSqzWQb6PRIqypFSDcma4jO90YP0w5X8qVXQ==", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/langchain": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.3.6.tgz", + "integrity": "sha512-erZOIKXzwCOrQHqY9AyjkQmaX62zUap1Sigw1KrwMUOnVoLKkVNRmAyxFlNZDZ9jLs/58MaQcaT9ReJtbj3x6w==", + "dependencies": { + "@langchain/openai": ">=0.1.0 <0.4.0", + "@langchain/textsplitters": ">=0.0.0 <0.2.0", + "js-tiktoken": "^1.0.12", + "js-yaml": "^4.1.0", + "jsonpointer": "^5.0.1", + "langsmith": "^0.2.0", + "openapi-types": "^12.1.3", + "p-retry": "4", + "uuid": "^10.0.0", + "yaml": "^2.2.1", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/anthropic": "*", + "@langchain/aws": "*", + "@langchain/cohere": "*", + "@langchain/core": ">=0.2.21 <0.4.0", + "@langchain/google-genai": "*", + "@langchain/google-vertexai": "*", + "@langchain/groq": "*", + "@langchain/mistralai": "*", + "@langchain/ollama": "*", + "axios": "*", + "cheerio": "*", + "handlebars": "^4.7.8", + "peggy": "^3.0.2", + "typeorm": "*" + }, + "peerDependenciesMeta": { + "@langchain/anthropic": { + "optional": true + }, + "@langchain/aws": { + "optional": true + }, + "@langchain/cohere": { + "optional": true + }, + "@langchain/google-genai": { + "optional": true + }, + "@langchain/google-vertexai": { + "optional": true + }, + "@langchain/groq": { + "optional": true + }, + "@langchain/mistralai": { + "optional": true + }, + "@langchain/ollama": { + "optional": true + }, + "axios": { + "optional": true + }, + "cheerio": { + "optional": true + }, + "handlebars": { + "optional": true + }, + "peggy": { + "optional": true + }, + "typeorm": { + "optional": true + } + } + }, + "node_modules/langsmith": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.2.13.tgz", + "integrity": "sha512-16EOM5nhU6GlMCKGm5sgBIAKOKzS2d30qcDZmF21kSLZJiUhUNTROwvYdqgZLrGfIIzmSMJHCKA7RFd5qf50uw==", + "dependencies": { + "@types/uuid": "^10.0.0", + "commander": "^10.0.1", + "p-queue": "^6.6.2", + "p-retry": "4", + "semver": "^7.6.3", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "openai": "*" + }, + "peerDependenciesMeta": { + "openai": { + "optional": true + } + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "bin": { + "mustache": "bin/mustache" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/openai": { + "version": "4.76.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.76.0.tgz", + "integrity": "sha512-QBGIetjX1C9xDp5XGa/3mPnfKI9BgAe2xHQX6PmO98wuW9qQaurBaumcYptQWc9LHZZq7cH/Y1Rjnsr6uUDdVw==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/openapi-types": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==" + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, + "node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/yaml": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.6.1.tgz", + "integrity": "sha512-7r0XPzioN/Q9kXBro/XPnA6kznR73DHq+GXh5ON7ZozRO6aMjbmiBuKste2wslTFkC5d1dw0GooOCepZXJ2SAg==", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/zod": { + "version": "3.24.1", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.1.tgz", + "integrity": "sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.24.1", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.1.tgz", + "integrity": "sha512-3h08nf3Vw3Wl3PK+q3ow/lIil81IT2Oa7YpQyUUDsEWbXveMesdfK1xBd2RhCkynwZndAxixji/7SYJJowr62w==", + "peerDependencies": { + "zod": "^3.24.1" + } + } + } +} diff --git a/src/appmixer/utils/ai/package.json b/src/appmixer/utils/ai/package.json index 7916d331a..1eadc8f24 100644 --- a/src/appmixer/utils/ai/package.json +++ b/src/appmixer/utils/ai/package.json @@ -2,6 +2,9 @@ "name": "appmixer.utils.ai", "version": "1.0.0", "dependencies": { - "form-data": "4.0.0" + "form-data": "4.0.0", + "langchain": "0.3.6", + "mustache": "4.2.0", + "openai": "4.76.0" } }