Merge pull request #427 from Ironclad/niels/update-assemblyai

Update LeMUR nodes + AssemblyAI SDK + configure integration in user agent

README.md

@@ -45,7 +45,7 @@ Rivet currently has LLM support for:
 - [OpenAI GPT-3.5 and GPT-4](https://openai.com/gpt-4)
 - [Anthropic Claude Instant and Claude 2](https://www.anthropic.com/index/claude-2)
 - [Anthropic Claude 3 Haiku, Sonnet, and Opus] (https://www.anthropic.com/news/claude-3-family)
-- [AssemblyAI LeMUR framework for voice data](https://www.assemblyai.com/models/lemur)
+- [AssemblyAI LeMUR framework for voice data](https://www.assemblyai.com/products/speech-understanding?utm_source=rivet)
 
 Rivet has embedding/vector database support for:
 
@@ -54,7 +54,7 @@ Rivet has embedding/vector database support for:
 
 Rivet also supports these additional integrations:
 
-- [Audio Transcription from AssemblyAI](https://www.assemblyai.com/models/core-transcription)
+- [Speech-to-Text from AssemblyAI](https://www.assemblyai.com/discover/products/speech-to-text?utm_source=rivet)
 
 For more information on how to use the application and all of its capabilities, see [the documentation](https://rivet.ironcladapp.com/docs)!
 

packages/app/src/plugins.ts

@@ -90,15 +90,15 @@ export const pluginInfos: PluginInfo[] = [
     id: 'assemblyAi',
     name: 'AssemblyAI',
     description: dedent`
-      Use [AssemblyAI](https://www.assemblyai.com/) to build AI applications with voice data.
+      Use [AssemblyAI](https://www.assemblyai.com/?utm_source=rivet) to build AI applications with voice data.
 
       - [Transcribe audio files](https://rivet.ironcladapp.com/docs/user-guide/plugins/built-in/assemblyai#transcribe-audio-node)
       - [Summarize audio files using LeMUR](https://rivet.ironcladapp.com/docs/user-guide/plugins/built-in/assemblyai#lemur-summary-node)
       - [Answer questions about audio files using LeMUR Q&A](https://rivet.ironcladapp.com/docs/user-guide/plugins/built-in/assemblyai#lemur-qa)
       - [Extract action items from audio files using LeMUR](https://rivet.ironcladapp.com/docs/user-guide/plugins/built-in/assemblyai#lemur-action-items)
       - [Run custom LLM tasks against audio files using LeMUR](https://rivet.ironcladapp.com/docs/user-guide/plugins/built-in/assemblyai#lemur-custom-task)
 
-      Follow this [step-by-step tutorial to see these capabilities in action](https://www.assemblyai.com/blog/podcast-qa-application-rivet/).
+      Follow this [step-by-step tutorial to see these capabilities in action](https://www.assemblyai.com/blog/podcast-qa-application-rivet?utm_source=rivet).
     `,
     author: 'AssemblyAI',
     logoImage: AssemblyAiLogo,
@@ -278,8 +278,8 @@ export const pluginInfos: PluginInfo[] = [
     description: dedent`
       Adds several utilities nodes that help with concurrency, pipelines and iterations.
 
-      - Iterator node:  The iterator Node will map an array of input objects through a provided graph.  It allows for parallel execution with concurrency controls and caching.  
-      - Pipeline node: The PipelineNode will take and input and run it through a pipeline of graphs (stages).  Each stage's output will be the next graph's input.  In addition, the pipeline node can have optional pre/post graph.  
+      - Iterator node:  The iterator Node will map an array of input objects through a provided graph.  It allows for parallel execution with concurrency controls and caching.
+      - Pipeline node: The PipelineNode will take and input and run it through a pipeline of graphs (stages).  Each stage's output will be the next graph's input.  In addition, the pipeline node can have optional pre/post graph.
       - Pinecone Search/Upsert node: These nodes will allow you to search or upsert documents in a pinecone collection.  Allows for metadata, sparse vectors as optional inputs
     `,
     tag: 'latest',
@@ -300,5 +300,5 @@ export const pluginInfos: PluginInfo[] = [
     package: 'rivet-plugin-qdrant',
     tag: 'latest',
     logoImage: QdrantLogo,
-  }
+  },
 ];

packages/cli/README.md

@@ -45,7 +45,7 @@ Rivet currently has LLM support for:
 - [OpenAI GPT-3.5 and GPT-4](https://openai.com/gpt-4)
 - [Anthropic Claude Instant and Claude 2](https://www.anthropic.com/index/claude-2)
 - [Anthropic Claude 3 Haiku, Sonnet, and Opus] (https://www.anthropic.com/news/claude-3-family)
-- [AssemblyAI LeMUR framework for voice data](https://www.assemblyai.com/models/lemur)
+- [AssemblyAI LeMUR framework for voice data](https://www.assemblyai.com/discover/products/speech-understanding?utm_source=rivet)
 
 Rivet has embedding/vector database support for:
 
@@ -54,7 +54,7 @@ Rivet has embedding/vector database support for:
 
 Rivet also supports these additional integrations:
 
-- [Audio Transcription from AssemblyAI](https://www.assemblyai.com/models/core-transcription)
+- [Speech-to-Text from AssemblyAI](https://www.assemblyai.com/products/speech-to-text)
 
 For more information on how to use the application and all of its capabilities, see [the documentation](https://rivet.ironcladapp.com/docs)!
 

packages/core/README.md

@@ -45,7 +45,7 @@ Rivet currently has LLM support for:
 - [OpenAI GPT-3.5 and GPT-4](https://openai.com/gpt-4)
 - [Anthropic Claude Instant and Claude 2](https://www.anthropic.com/index/claude-2)
 - [Anthropic Claude 3 Haiku, Sonnet, and Opus] (https://www.anthropic.com/news/claude-3-family)
-- [AssemblyAI LeMUR framework for voice data](https://www.assemblyai.com/models/lemur)
+- [AssemblyAI LeMUR framework for voice data](https://www.assemblyai.com/discover/products/speech-understanding?utm_source=rivet)
 
 Rivet has embedding/vector database support for:
 
@@ -54,7 +54,7 @@ Rivet has embedding/vector database support for:
 
 Rivet also supports these additional integrations:
 
-- [Audio Transcription from AssemblyAI](https://www.assemblyai.com/models/core-transcription)
+- [Speech-to-Text from AssemblyAI](https://www.assemblyai.com/products/speech-to-text?utm_source=rivet)
 
 For more information on how to use the application and all of its capabilities, see [the documentation](https://rivet.ironcladapp.com/docs)!
 

packages/core/package.json

@@ -47,7 +47,7 @@
     "@gentrace/core": "^2.2.5",
     "@google-cloud/vertexai": "^0.1.3",
     "@huggingface/inference": "^2.6.4",
-    "assemblyai": "^4.4.2",
+    "assemblyai": "^4.6.0",
     "autoevals": "^0.0.26",
     "crypto-js": "^4.2.0",
     "emittery": "^1.0.1",

packages/core/src/plugins/assemblyAi/LemurActionItemsNode.ts

@@ -1,6 +1,6 @@
 import { nanoid } from 'nanoid/non-secure';
 import { dedent } from 'ts-dedent';
-import { AssemblyAI, type LemurActionItemsParams } from 'assemblyai';
+import type { LemurActionItemsParams } from 'assemblyai';
 import {
   type ChartNode,
   type EditorDefinition,
@@ -17,10 +17,10 @@ import {
 import { pluginNodeDefinition } from '../../model/NodeDefinition.js';
 import {
   type LemurNodeData,
-  getApiKey,
+  getClient,
   getLemurParams,
   lemurEditorDefinitions,
-  lemurTranscriptIdsInputDefinition,
+  lemurInputDefinitions,
 } from './lemurHelpers.js';
 
 export type LemurActionItemsNode = ChartNode<'assemblyAiLemurActionItems', LemurActionItemsNodeData>;
@@ -48,7 +48,7 @@ export const LemurActionItemsNodeImpl: PluginNodeImpl<LemurActionItemsNode> = {
 
   getInputDefinitions(): NodeInputDefinition[] {
     return [
-      lemurTranscriptIdsInputDefinition,
+      ...lemurInputDefinitions,
       {
         id: 'context' as PortId,
         dataType: 'string',
@@ -92,8 +92,7 @@ export const LemurActionItemsNodeImpl: PluginNodeImpl<LemurActionItemsNode> = {
   },
 
   async process(data, inputs: Inputs, context: InternalProcessContext): Promise<Outputs> {
-    const apiKey = getApiKey(context);
-    const client = new AssemblyAI({ apiKey });
+    const client = getClient(context);
 
     const params: LemurActionItemsParams = getLemurParams(inputs, data);
 

packages/core/src/plugins/assemblyAi/LemurQaNode.ts

@@ -20,10 +20,10 @@ import {
 } from '../../index.js';
 import {
   type LemurNodeData,
-  getApiKey,
+  getClient,
   getLemurParams,
   lemurEditorDefinitions,
-  lemurTranscriptIdsInputDefinition,
+  lemurInputDefinitions,
 } from './lemurHelpers.js';
 import { coerceType } from '../../utils/coerceType.js';
 import { pluginNodeDefinition } from '../../model/NodeDefinition.js';
@@ -58,7 +58,7 @@ export const LemurQaNodeImpl = {
 
   getInputDefinitions(): NodeInputDefinition[] {
     return [
-      lemurTranscriptIdsInputDefinition,
+      ...lemurInputDefinitions,
       {
         id: 'questions' as PortId,
         dataType: ['string', 'string[]', 'object', 'object[]', 'any', 'any[]'],
@@ -122,8 +122,7 @@ export const LemurQaNodeImpl = {
   },
 
   async process(data, inputs: Inputs, context: InternalProcessContext): Promise<Outputs> {
-    const apiKey = getApiKey(context);
-    const client = new AssemblyAI({ apiKey });
+    const client = getClient(context);
 
     const questions = getQuestions(inputs).map((question) => applyQuestionEditors(data, question));
 

packages/core/src/plugins/assemblyAi/LemurSummaryNode.ts

@@ -16,10 +16,10 @@ import {
 } from '../../index.js';
 import {
   type LemurNodeData,
-  getApiKey,
+  getClient,
   getLemurParams,
   lemurEditorDefinitions,
-  lemurTranscriptIdsInputDefinition,
+  lemurInputDefinitions,
 } from './lemurHelpers.js';
 import { pluginNodeDefinition } from '../../model/NodeDefinition.js';
 
@@ -50,7 +50,7 @@ export const LemurSummaryNodeImpl: PluginNodeImpl<LemurSummaryNode> = {
 
   getInputDefinitions(): NodeInputDefinition[] {
     return [
-      lemurTranscriptIdsInputDefinition,
+      ...lemurInputDefinitions,
       {
         id: 'context' as PortId,
         dataType: 'string',
@@ -94,8 +94,7 @@ export const LemurSummaryNodeImpl: PluginNodeImpl<LemurSummaryNode> = {
   },
 
   async process(data, inputs: Inputs, context: InternalProcessContext): Promise<Outputs> {
-    const apiKey = getApiKey(context);
-    const client = new AssemblyAI({ apiKey });
+    const client = getClient(context);
     const params: LemurSummaryParams = getLemurParams(inputs, data);
 
     if (data.answer_format) {

packages/core/src/plugins/assemblyAi/LemurTaskNode.ts

@@ -1,6 +1,6 @@
 import { nanoid } from 'nanoid/non-secure';
 import { dedent } from 'ts-dedent';
-import { AssemblyAI, type LemurTaskParams } from 'assemblyai';
+import { type LemurTaskParams } from 'assemblyai';
 import {
   type ChartNode,
   type EditorDefinition,
@@ -16,10 +16,10 @@ import {
 } from '../../index.js';
 import {
   type LemurNodeData,
-  getApiKey,
+  getClient,
   getLemurParams,
   lemurEditorDefinitions,
-  lemurTranscriptIdsInputDefinition,
+  lemurInputDefinitions,
 } from './lemurHelpers.js';
 import { pluginNodeDefinition } from '../../model/NodeDefinition.js';
 import { coerceTypeOptional } from '../../utils/coerceType.js';
@@ -51,7 +51,7 @@ export const LemurTaskNodeImpl: PluginNodeImpl<LemurTaskNode> = {
 
   getInputDefinitions(): NodeInputDefinition[] {
     return [
-      lemurTranscriptIdsInputDefinition,
+      ...lemurInputDefinitions,
       {
         id: 'prompt' as PortId,
         dataType: 'string',
@@ -95,8 +95,7 @@ export const LemurTaskNodeImpl: PluginNodeImpl<LemurTaskNode> = {
   },
 
   async process(data, inputs: Inputs, context: InternalProcessContext): Promise<Outputs> {
-    const apiKey = getApiKey(context);
-    const client = new AssemblyAI({ apiKey });
+    const client = getClient(context);
     const params: LemurTaskParams = {
       prompt: coerceTypeOptional(inputs['prompt' as PortId], 'string') || data.prompt || '',
       ...getLemurParams(inputs, data),

packages/core/src/plugins/assemblyAi/TranscribeAudioNode.ts

@@ -1,6 +1,5 @@
 import { nanoid } from 'nanoid/non-secure';
 import { dedent } from 'ts-dedent';
-import { AssemblyAI } from 'assemblyai';
 import type { TranscribeParams, TranscriptParams } from 'assemblyai';
 import {
   type AnyDataValue,
@@ -18,7 +17,7 @@ import {
   type PortId,
   type StringDataValue,
 } from '../../index.js';
-import { getApiKey } from './lemurHelpers.js';
+import { getClient } from './lemurHelpers.js';
 import { pluginNodeDefinition } from '../../model/NodeDefinition.js';
 import { coerceType } from '../../utils/coerceType.js';
 
@@ -83,7 +82,7 @@ export const TranscribeAudioNodeImpl: PluginNodeImpl<TranscribeAudioNode> = {
         language: 'json',
         dataKey: 'transcriptParameters',
         helperMessage: `Configure additional parameters using a JSON object. This will override any other fields you have set.
-        For a detailed list of parameters, see [the AssemblyAI API documentation](https://www.assemblyai.com/docs/api-reference/transcript#create-a-transcript).`,
+        For a detailed list of parameters, see [the AssemblyAI API documentation](https://www.assemblyai.com/docs/api-reference/transcripts/submit?utm_source=rivet).`,
       },
     ];
   },
@@ -105,8 +104,7 @@ export const TranscribeAudioNodeImpl: PluginNodeImpl<TranscribeAudioNode> = {
     const input = inputs['audio' as PortId] as AudioDataValue | StringDataValue | AnyDataValue;
     if (!input) throw new Error('Audio input is required.');
 
-    const apiKey = getApiKey(context);
-    const client = new AssemblyAI({ apiKey });
+    const client = getClient(context);
 
     let audioUrl: string;
     if (input.type === 'audio') {