diff --git a/Conductor/Client/Ai/Configuration.cs b/Conductor/Client/Ai/Configuration.cs new file mode 100644 index 00000000..51a4e844 --- /dev/null +++ b/Conductor/Client/Ai/Configuration.cs @@ -0,0 +1,59 @@ +using Newtonsoft.Json.Converters; +using System.Runtime.Serialization; +using System.Text.Json.Serialization; + +namespace Conductor.Client.Ai +{ + public class Configuration + { + /// + /// Defines LLMProvider + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum LLMProviderEnum + { + /// + /// Enum AZURE_OPEN_AI for value: azure_openai + /// + [EnumMember(Value = "azure_openai")] + AZURE_OPEN_AI = 1, + + /// + /// Enum OPEN_AI for value: openai + /// + [EnumMember(Value = "openai")] + OPEN_AI = 2, + + /// + /// Enum GCP_VERTEX_AI for value: vertex_ai + /// + [EnumMember(Value = "vertex_ai")] + GCP_VERTEX_AI = 3, + + /// + /// Enum HUGGING_FACE for value: huggingface + /// + [EnumMember(Value = "huggingface")] + HUGGING_FACE = 4, + } + + /// + /// Defines VectorDB + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum VectorDBEnum + { + /// + /// Enum PINECONE_DB for value: pineconedb + /// + [EnumMember(Value = "pineconedb")] + PINECONE_DB = 1, + + /// + /// Enum WEAVIATE_DB for value: weaviatedb + /// + [EnumMember(Value = "weaviatedb")] + WEAVIATE_DB = 2, + } + } +} \ No newline at end of file diff --git a/Conductor/Client/Ai/Integrations.cs b/Conductor/Client/Ai/Integrations.cs new file mode 100644 index 00000000..cfc13d82 --- /dev/null +++ b/Conductor/Client/Ai/Integrations.cs @@ -0,0 +1,192 @@ +using System.Collections.Generic; +using EnvironmentInstance = System.Environment; + +namespace Conductor.Client.Ai +{ + /// + /// Integration configuration abstract base class. + /// + public abstract class IntegrationConfig + { + /// + /// Converts the configuration to a dictionary. + /// + /// A dictionary representation of the configuration. + public abstract Dictionary ToDictionary(); + } + + /// + /// Configuration class for Weaviate integration. + /// + public class WeaviateConfig : IntegrationConfig + { + /// + /// Gets or Sets ApiKey + /// + public string ApiKey { get; set; } + + /// + /// Gets or Sets Endpoint + /// + public string Endpoint { get; set; } + + /// + /// Gets or Sets Class + /// + public string ClassName { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + /// + public WeaviateConfig(string apiKey, string endpoint, string className) + { + ApiKey = apiKey; + Endpoint = endpoint; + ClassName = className; + } + + /// + /// Inherited method + /// + /// + public override Dictionary ToDictionary() + { + return new Dictionary + { + { Constants.APIKEY, ApiKey }, + { Constants.ENDPOINT, Endpoint } + }; + } + } + + /// + /// Configuration class for OpenAIConfig integration. + /// + public class OpenAIConfig : IntegrationConfig + { + /// + /// Gets or Sets ApiKey + /// + public string ApiKey { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + public OpenAIConfig(string apiKey = null) + { + ApiKey = apiKey ?? EnvironmentInstance.GetEnvironmentVariable(Constants.OPENAIAPIKEY); + } + + /// + /// Inherited method + /// + /// + public override Dictionary ToDictionary() + { + return new Dictionary + { + { Constants.APIKEY, ApiKey } + }; + } + } + + /// + /// Configuration class for AzureOpenAIConfig integration. + /// + public class AzureOpenAIConfig : IntegrationConfig + { + /// + /// Gets or Sets ApiKey + /// + public string ApiKey { get; set; } + + /// + /// Gets or Sets Endpoint + /// + public string Endpoint { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + public AzureOpenAIConfig(string apiKey, string endpoint) + { + ApiKey = apiKey; + Endpoint = endpoint; + } + + /// + /// Inherited method + /// + /// + public override Dictionary ToDictionary() + { + return new Dictionary + { + { Constants.APIKEY, ApiKey }, + { Constants.ENDPOINT, Endpoint } + }; + } + } + + /// + /// Configuration class for PineconeConfig integration. + /// + public class PineconeConfig : IntegrationConfig + { + /// + /// Gets or Sets ApiKey + /// + public string ApiKey { get; set; } + + /// + /// Gets or Sets Endpoint + /// + public string Endpoint { get; set; } + + /// + /// Gets or Sets Environment + /// + public string Environment { get; set; } + + /// + /// Gets or Sets ProjectName + /// + public string ProjectName { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + /// + /// + public PineconeConfig(string apiKey = null, string endpoint = null, string environment = null, string projectName = null) + { + ApiKey = apiKey ?? EnvironmentInstance.GetEnvironmentVariable(Constants.PINECONEAPIKEY); + Endpoint = endpoint ?? EnvironmentInstance.GetEnvironmentVariable(Constants.PINECONEENDPOINT); + Environment = environment ?? EnvironmentInstance.GetEnvironmentVariable(Constants.PINECONEENV); + ProjectName = projectName ?? EnvironmentInstance.GetEnvironmentVariable(Constants.PINECONEPROJECT); + } + + /// + /// Inherited method + /// + /// + public override Dictionary ToDictionary() + { + return new Dictionary + { + { Constants.APIKEY, ApiKey }, + { Constants.ENDPOINT, Endpoint }, + { Constants.PROJECTNAME, ProjectName }, + { Constants.ENVIRONMENT, Environment } + }; + } + } +} \ No newline at end of file diff --git a/Conductor/Client/Ai/Orchestrator.cs b/Conductor/Client/Ai/Orchestrator.cs new file mode 100644 index 00000000..56e2d78e --- /dev/null +++ b/Conductor/Client/Ai/Orchestrator.cs @@ -0,0 +1,237 @@ +using Conductor.Api; +using Conductor.Client.Authentication; +using Conductor.Client.Models; +using System; +using System.Collections.Generic; +using static Conductor.Client.Ai.Configuration; +using Config = Conductor.Client.Configuration; + +namespace Conductor.Client.Ai +{ + /// + /// Orchestrator + /// + public class Orchestrator + { + private readonly IntegrationResourceApi _integrationResourceApi; + private readonly WorkflowResourceApi _workflowResourceApi; + private readonly PromptResourceApi _promptResourceApi; + private readonly string _promptTestWorkflowName; + + /// + /// Initializes a new instance of the class + /// + /// + /// + public Orchestrator(Config apiConfiguration, string promptTestWorkflowName = "") + { + var orkesApiClients = new OrkesApiClient(apiConfiguration, new OrkesAuthenticationSettings(Constants.KEY_ID, Constants.KEY_SECRET)); + + _integrationResourceApi = orkesApiClients.GetClient(); + _workflowResourceApi = orkesApiClients.GetClient(); + _promptResourceApi = orkesApiClients.GetClient(); + _promptTestWorkflowName = string.IsNullOrEmpty(promptTestWorkflowName) ? Constants.PROMPTTESTWORKFLOWDEFAULTNAME + Guid.NewGuid().ToString() : promptTestWorkflowName; + } + + /// + /// Method to add prompt template + /// + /// + /// + /// + /// + /// + /// + public Orchestrator AddPromptTemplate(string promptTemplate, string description, string name, List models = null) + { + try + { + _promptResourceApi.SaveMessageTemplate(promptTemplate, description, name, models); + return this; + } + catch (Exception ex) + { + string errorMessage = string.Format(Constants.ADD_PROMPT_TEMPLATE_ERROR_MESSAGE, ex.Message); + throw new Exception(errorMessage, ex); + } + } + + /// + /// Method to get prompt template + /// + /// + /// + /// + public MessageTemplate GetPromptTemplate(string templateName) + { + try + { + return _promptResourceApi.GetMessageTemplate(templateName); + } + catch (Exception ex) + { + string errorMessage = string.Format(Constants.GET_PROMPT_TEMPLATE_ERROR_MESSAGE, ex.Message); + throw new Exception(errorMessage, ex); + } + } + + /// + /// Method to associate prompt template with integration + /// + /// + /// + /// + /// + public void AssociatePromptTemplate(string integrationProvider, List aiModels, string PromptName) + { + try + { + foreach (var aiModel in aiModels) + _integrationResourceApi.AssociatePromptWithIntegration(integrationProvider, aiModel, PromptName); + } + catch (Exception ex) + { + string errorMessage = string.Format(Constants.ASSOCIATE_PROMPT_TEMPLATE_ERROR_MESSAGE, ex.Message); + throw new Exception(errorMessage, ex); + } + } + + /// + /// Method to test prompt template + /// + /// + /// + /// + public string TestPromptTemplate(PromptTemplateTestRequest promptTemplateTestRequest) + { + try + { + return _promptResourceApi.TestMessageTemplate(promptTemplateTestRequest); + } + catch (Exception ex) + { + string errorMessage = string.Format(Constants.TEST_MESSAGE_TEMPLATE_ERROR_MESSAGE, ex.Message); + throw new Exception(errorMessage, ex); + } + } + + /// + /// Method to add AI Integration using LLMProvider + /// + /// + /// + /// + /// + /// + /// + /// + public void AddAIIntegration(string aiIntegrationName, LLMProviderEnum provider, List models, string description, IntegrationConfig config, bool overwrite = false) + { + try + { + var details = new IntegrationUpdate(); + details.Configuration = config.ToDictionary(); + details.Type = provider.ToString(); + details.Category = IntegrationUpdate.CategoryEnum.AIMODEL; + details.Enabled = true; + details.Description = description; + var existingIntegration = _integrationResourceApi.GetIntegrationProvider(aiIntegrationName); + if (existingIntegration == null || overwrite) + _integrationResourceApi.SaveIntegrationProvider(details, aiIntegrationName); + foreach (var model in models) + { + var apiDetails = new IntegrationApiUpdate(); + apiDetails.Enabled = true; + apiDetails.Description = description; + var existingIntegrationApi = _integrationResourceApi.GetIntegrationApi(aiIntegrationName, model); + if (existingIntegrationApi == null || overwrite) + _integrationResourceApi.SaveIntegrationApi(apiDetails, model, aiIntegrationName); + } + } + catch (Exception ex) + { + string errorMessage = string.Format(Constants.ADD_AI_INTEGRATION_ERROR_MESSAGE, ex.Message); + throw new Exception(errorMessage, ex); + } + } + + /// + /// Method to add AI Integration using VectorDB + /// + /// + /// + /// + /// + /// + /// + /// + public void AddVectorStore(string dbIntegrationName, VectorDBEnum provider, List indices, IntegrationConfig config, string description = null, bool overwrite = false) + { + try + { + var vectorDb = new IntegrationUpdate(); + vectorDb.Configuration = config.ToDictionary(); + vectorDb.Type = provider.ToString(); + vectorDb.Category = IntegrationUpdate.CategoryEnum.VECTORDB; + vectorDb.Enabled = true; + vectorDb.Description = description; + var existingIntegration = _integrationResourceApi.GetIntegrationProvider(dbIntegrationName); + if (existingIntegration == null || overwrite) + _integrationResourceApi.SaveIntegrationProvider(vectorDb, dbIntegrationName); + foreach (var index in indices) + { + var apiDetails = new IntegrationApiUpdate(); + apiDetails.Enabled = true; + apiDetails.Description = description; + var existingIntegrationApi = _integrationResourceApi.GetIntegrationApi(dbIntegrationName, index); + if (existingIntegrationApi == null || overwrite) + _integrationResourceApi.SaveIntegrationApi(apiDetails, index, dbIntegrationName); + } + } + catch (Exception ex) + { + string errorMessage = string.Format(Constants.ADD_AI_INTEGRATION_ERROR_MESSAGE, ex.Message); + throw new Exception(errorMessage, ex); + } + } + + /// + /// Method to get token used for integration provider + /// + /// + /// + /// + public Dictionary GetTokenUsed(string aiIntegration) + { + try + { + return _integrationResourceApi.GetTokenUsageForIntegrationProvider(aiIntegration); + } + catch (Exception ex) + { + string errorMessage = string.Format(Constants.GET_TOKEN_USED_BY_INTEGRATION_ERROR_MESSAGE, ex.Message); + throw new Exception(errorMessage, ex); + } + } + + /// + /// Method to get token used by model + /// + /// + /// + /// + /// + public int? GetTokenUsedByModel(string aiIntegration, string model) + { + try + { + return _integrationResourceApi.GetTokenUsageForIntegration(aiIntegration, model); + } + catch (Exception ex) + { + string errorMessage = string.Format(Constants.GET_TOKEN_USED_BY_MODEL_ERROR_MESSAGE, ex.Message); + throw new Exception(errorMessage, ex); + } + } + } +} \ No newline at end of file diff --git a/Conductor/Client/Constants.cs b/Conductor/Client/Constants.cs index dd2b3454..a2fa20b6 100644 --- a/Conductor/Client/Constants.cs +++ b/Conductor/Client/Constants.cs @@ -5,11 +5,67 @@ /// public static class Constants { + //Authentication Keys public const string KEY_ID = ""; public const string KEY_SECRET = ""; public const string OWNER_EMAIL = ""; public const int REST_CLIENT_REQUEST_TIME_OUT = 20000; public const int MAX_TOKEN_REFRESH_RETRY_COUNT = 3; + //Error Messages + public const string ADD_AI_INTEGRATION_ERROR_MESSAGE = "Failed to Add AI Integration: {0}"; + public const string ADD_PROMPT_TEMPLATE_ERROR_MESSAGE = "Failed to Add Prompt Template : {0}"; + public const string ASSOCIATE_PROMPT_TEMPLATE_ERROR_MESSAGE = "Failed to Associate a Prompt Template : {0}"; + public const string GET_PROMPT_TEMPLATE_ERROR_MESSAGE = "Failed to get Prompt Template: {0}"; + public const string GET_TOKEN_USED_BY_INTEGRATION_ERROR_MESSAGE = "Failed to get Token Used By Integration Provider: {0}"; + public const string GET_TOKEN_USED_BY_MODEL_ERROR_MESSAGE = "Failed to get Token Used By Model: {0}"; + public const string TEST_MESSAGE_TEMPLATE_ERROR_MESSAGE = "Failed to Test a Prompt Template : {0}"; + + //Llm Task keys + public const string LLM_GENERATE_EMBEDDINGS = "llm_generate_embeddings"; + public const string LLM_INDEX_DOCUMENT_TASKNAME = "llm_index_document"; + public const string LLM_INDEX_TEXT_TASKNAME = "llm_index_text"; + public const string LLM_TEXT_COMPLETE = "llm_text_complete"; + + //String Keys + public const string APIKEY = "api_key"; + public const string CHUNKOVERLAP = "chunkOverlap"; + public const string CHUNKSIZE = "chunkSize"; + public const string DOCID = "docId"; + public const string EMBEDDING_MODEL = "embeddingModel"; + public const string EMBEDDING_MODEL_PROVIDER = "embeddingModelProvider"; + public const string EMBEDDINGS = "embeddings"; + public const string ENDPOINT = "endpoint"; + public const string ENVIRONMENT = "environment"; + public const string INDEX = "index"; + public const string INSTRUCTIONTEMPLATE = "instructionTemplate"; + public const string LLM_QUERY_EMBEDDING_TASKNAME = "llm_get_embeddings"; + public const string LLMPROVIDER = "llmProvider"; + public const string MAXRESULTS = "maxResults"; + public const string MAXTOKENS = "maxTokens"; + public const string MEDIATYPE = "mediaType"; + public const string MESSAGES = "messages"; + public const string METADATA = "docId"; + public const string MODEL = "model"; + public const string NAMESPACE = "nameSpace"; + public const string PROJECTNAME = "projectName"; + public const string PROMPTNAME = "promptName"; + public const string PROMPTVARIABLES = "promptVariables"; + public const string QUERY = "query"; + public const string STOPWORDS = "stopWords"; + public const string TASK_NAME = "taskName"; + public const string TEMPERATURE = "temperature"; + public const string TEXT = "text"; + public const string TOPP = "topP"; + public const string URL = "url"; + public const string VECTORDB = "vectorDB"; + public const string PROMPTTESTWORKFLOWDEFAULTNAME = "prompt_test_"; + + //LLM Environment Variables + public const string PINECONEPROJECT = "PINECONE_PROJECT"; + public const string PINECONEENV = "PINECONE_ENV"; + public const string PINECONEENDPOINT = "PINECONE_ENDPOINT"; + public const string PINECONEAPIKEY = "PINECONE_API_KEY"; + public const string OPENAIAPIKEY = "OPENAI_API_KEY"; } -} +} \ No newline at end of file diff --git a/Conductor/Client/Models/IntegrationDefFormField.cs b/Conductor/Client/Models/IntegrationDefFormField.cs index 715739fa..92ca2633 100644 --- a/Conductor/Client/Models/IntegrationDefFormField.cs +++ b/Conductor/Client/Models/IntegrationDefFormField.cs @@ -155,7 +155,27 @@ public enum FieldNameEnum /// Enum ValueSubjectNameStrategy for value: valueSubjectNameStrategy /// [EnumMember(Value = "valueSubjectNameStrategy")] - ValueSubjectNameStrategy = 27 + ValueSubjectNameStrategy = 27, + /// + /// Enum File for value: file + /// + [EnumMember(Value = "file")] + File = 28, + /// + /// Enum JdbcDriver for value: jdbcDriver + /// + [EnumMember(Value = "jdbcDriver")] + JdbcDriver = 29, + /// + /// Enum DataSourceURL for value: datasourceURL + /// + [EnumMember(Value = "datasourceURL")] + DataSourceURL = 30, + /// + /// Enum Subscription for value: subscription + /// + [EnumMember(Value = "subscription")] + Subscription = 31 } /// @@ -184,7 +204,12 @@ public enum FieldTypeEnum /// Enum PASSWORD for value: PASSWORD /// [EnumMember(Value = "PASSWORD")] - PASSWORD = 3 + PASSWORD = 3, + /// + /// Enum FILE value: FILE + /// + [EnumMember(Value = "FILE")] + FILE = 4 } /// diff --git a/Conductor/Client/Models/WorkflowTask.cs b/Conductor/Client/Models/WorkflowTask.cs index 4c4558fb..1487e76e 100644 --- a/Conductor/Client/Models/WorkflowTask.cs +++ b/Conductor/Client/Models/WorkflowTask.cs @@ -131,7 +131,42 @@ public enum WorkflowTaskTypeEnum /// Enum SETVARIABLE for value: SET_VARIABLE /// [EnumMember(Value = "SET_VARIABLE")] - SETVARIABLE = 22 + SETVARIABLE = 22, + /// + /// Enum LLMCHATCOMPLETE for value: LLM_CHAT_COMPLETE + /// + [EnumMember(Value = "LLM_CHAT_COMPLETE")] + LLMCHATCOMPLETE = 23, + /// + /// Enum LLMGENERATEEMBEDDINGS for value: LLM_GENERATE_EMBEDDINGS + /// + [EnumMember(Value = "LLM_GENERATE_EMBEDDINGS")] + LLMGENERATEEMBEDDINGS = 24, + /// + /// Enum LLMINDEXDOCUMENT for value: LLM_INDEX_DOCUMENT + /// + [EnumMember(Value = "LLM_INDEX_DOCUMENT")] + LLMINDEXDOCUMENT = 25, + /// + /// Enum LLMINDEXTEXT for value: LLM_INDEX_TEXT + /// + [EnumMember(Value = "LLM_INDEX_TEXT")] + LLMINDEXTEXT = 26, + /// + /// Enum LLMGETEMBEDDINGS for value: LLM_GET_EMBEDDINGS + /// + [EnumMember(Value = "LLM_GET_EMBEDDINGS")] + LLMGETEMBEDDINGS = 27, + /// + /// Enum LLMSEARCHINDEX for value: LLM_SEARCH_INDEX + /// + [EnumMember(Value = "LLM_SEARCH_INDEX")] + LLMSEARCHINDEX = 28, + /// + /// Enum LLMINDEXTEXT for value: LLM_INDEX_TEXT + /// + [EnumMember(Value = "LLM_TEXT_COMPLETE")] + LLMTEXTCOMPLETE = 29 } /// /// Gets or Sets WorkflowTaskType diff --git a/Conductor/Definition/TaskType/LlmTasks/LlmChatComplete.cs b/Conductor/Definition/TaskType/LlmTasks/LlmChatComplete.cs new file mode 100644 index 00000000..8993697a --- /dev/null +++ b/Conductor/Definition/TaskType/LlmTasks/LlmChatComplete.cs @@ -0,0 +1,165 @@ +using Conductor.Client; +using Conductor.Client.Models; +using System.Collections.Generic; + +namespace Conductor.Definition.TaskType.LlmTasks +{ + /// + /// ChatMessage + /// + public class ChatMessage + { + /// + /// Gets or Sets Role + /// + public string Role { get; set; } + + /// + /// Gets or Sets Message + /// + public string Message { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + public ChatMessage(string role, string message) + { + Role = role; + Message = message; + } + } + + /// + /// LlmChatComplete + /// + public class LlmChatComplete : Task + { + /// + /// Gets or Sets TaskRefName + /// + public string TaskRefName { get; set; } + + /// + /// Gets or Sets LlmProvider + /// + public string LlmProvider { get; set; } + + /// + /// Gets or Sets Model + /// + public string Model { get; set; } + + /// + /// Gets or Sets Messages + /// + public List Messages { get; set; } + + /// + /// Gets or Sets StopWords + /// + public List StopWords { get; set; } + + /// + /// Gets or Sets MaxTokens + /// + public int MaxTokens { get; set; } + + /// + /// Gets or Sets Temperature + /// + public int Temperature { get; set; } + + /// + /// Gets or Sets TopP + /// + public int TopP { get; set; } + + /// + /// Gets or Sets InstructionTemplate + /// + public string InstructionsTemplate { get; set; } + + /// + /// Gets or Sets TemplateVariables + /// + public Dictionary TemplateVariables { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public LlmChatComplete(string taskReferenceName, string llmProvider, string model, List messages, + List stopWords = null, int maxTokens = 100, int temperature = 0, int topP = 1, + string instructionsTemplate = null, Dictionary templateVariables = null) : base(taskReferenceName, WorkflowTask.WorkflowTaskTypeEnum.LLMCHATCOMPLETE) + { + TaskRefName = taskReferenceName; + LlmProvider = llmProvider; + Model = model; + Messages = messages; + StopWords = stopWords ?? new List(); + MaxTokens = maxTokens; + Temperature = temperature; + TopP = topP; + InstructionsTemplate = instructionsTemplate; + TemplateVariables = templateVariables ?? new Dictionary(); + + InitializeInputs(); + } + + /// + /// Adding PromptVariables to InputParams + /// + /// + /// + public LlmChatComplete PromptVariables(Dictionary variables) + { + foreach (var variable in variables) + { + TemplateVariables[variable.Key] = variable.Value; + } + WithInput(Constants.PROMPTVARIABLES, TemplateVariables); + return this; + } + + /// + /// Adding PromptVariable to InputParams + /// + /// + /// + /// + public LlmChatComplete PromptVariable(string variable, object value) + { + TemplateVariables[variable] = value; + WithInput(Constants.PROMPTVARIABLES, TemplateVariables); + return this; + } + + /// + /// Populates inputParams dictionary with LlmChatComplete attributes. + /// + private void InitializeInputs() + { + WithInput(Constants.LLMPROVIDER, LlmProvider); + WithInput(Constants.MODEL, Model); + WithInput(Constants.PROMPTVARIABLES, TemplateVariables); + WithInput(Constants.TEMPERATURE, Temperature); + WithInput(Constants.TOPP, TopP); + WithInput(Constants.INSTRUCTIONTEMPLATE, InstructionsTemplate); + WithInput(Constants.MESSAGES, Messages); + WithInput(Constants.MAXTOKENS, MaxTokens); + WithInput(Constants.MAXTOKENS, StopWords); + WithInput(Constants.PROMPTVARIABLES, TemplateVariables); + } + } +} \ No newline at end of file diff --git a/Conductor/Definition/TaskType/LlmTasks/LlmGenerateEmbeddings.cs b/Conductor/Definition/TaskType/LlmTasks/LlmGenerateEmbeddings.cs new file mode 100644 index 00000000..8a0e8de9 --- /dev/null +++ b/Conductor/Definition/TaskType/LlmTasks/LlmGenerateEmbeddings.cs @@ -0,0 +1,58 @@ +using Conductor.Client; + +namespace Conductor.Definition.TaskType.LlmTasks +{ + /// + /// LlmGenerateEmbeddings + /// + public class LlmGenerateEmbeddings : Task + { + /// + /// Gets or Sets TaskName + /// + public string TaskName { get; set; } + + /// + /// Gets or Sets LlmProvider + /// + public string LlmProvider { get; set; } + + /// + /// Gets or Sets Model + /// + public string Model { get; set; } + + /// + /// Gets or Sets Text + /// + public string Text { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + /// + /// + /// + public LlmGenerateEmbeddings(string taskReferenceName, string llmProvider, string model, string text, string taskName = null) : base(taskReferenceName, WorkflowTaskTypeEnum.LLMGENERATEEMBEDDINGS) + { + TaskName = taskName ?? Constants.LLM_GENERATE_EMBEDDINGS; + LlmProvider = llmProvider; + Model = model; + Text = text; + + InitializeInputs(); + } + + /// + /// Populates inputParams dictionary with LlmGenerateEmbeddings attributes. + /// + private void InitializeInputs() + { + WithInput(Constants.LLMPROVIDER, LlmProvider); + WithInput(Constants.MODEL, Model); + WithInput(Constants.TEXT, Text); + } + } +} \ No newline at end of file diff --git a/Conductor/Definition/TaskType/LlmTasks/LlmIndexDocuments.cs b/Conductor/Definition/TaskType/LlmTasks/LlmIndexDocuments.cs new file mode 100644 index 00000000..3f2a544e --- /dev/null +++ b/Conductor/Definition/TaskType/LlmTasks/LlmIndexDocuments.cs @@ -0,0 +1,142 @@ +using Conductor.Client; +using System.Collections.Generic; + +namespace Conductor.Definition.TaskType.LlmTasks +{ + /// + /// LlmIndexDocuments + /// + public class LlmIndexDocuments : Task + { + /// + /// Gets or Sets TaskRefName + /// + public string TaskRefName { get; set; } + + /// + /// Gets or Sets VectorDB + /// + public string VectorDB { get; set; } + + /// + /// Gets or Sets NameSpace + /// + public string NameSpace { get; set; } + + /// + /// Gets or Sets Index + /// + public string Index { get; set; } + + /// + /// Gets or Sets EmbeddingModelProvider + /// + public string EmbeddingModelProvider { get; set; } + + /// + /// Gets or Sets EmbeddingModel + /// + public string EmbeddingModel { get; set; } + + /// + /// Gets or Sets Url + /// + public string Url { get; set; } + + /// + /// Gets or Sets MediaType + /// + public string MediaType { get; set; } + + /// + /// Gets or Sets MetaData + /// + public Dictionary MetaData { get; set; } + + /// + /// Gets or Sets ChunkSize + /// + public int? ChunkSize { get; set; } + + /// + /// Gets or Sets ChunkOverlap + /// + public int? ChunkOverlap { get; set; } + + /// + /// Gets or Sets DocId + /// + public int? DocId { get; set; } + + /// + /// Gets or Sets TaskName + /// + public string TaskName { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public LlmIndexDocuments(string taskReferenceName, string vectorDB, string nameSpace, string index, string embeddingModelProvider, string embeddingModel, string url, + string mediaType, int? chunkSize, int? chunkOverlap, int? docId, string taskName = null, Dictionary metaData = null) : base(taskReferenceName, WorkflowTaskTypeEnum.LLMINDEXDOCUMENT) + { + TaskRefName = taskReferenceName; + VectorDB = vectorDB; + NameSpace = nameSpace; + Index = index; + EmbeddingModelProvider = embeddingModelProvider; + EmbeddingModel = embeddingModel; + Url = url; + MediaType = mediaType; + MetaData = metaData; + ChunkSize = chunkSize; + ChunkOverlap = chunkOverlap; + DocId = docId; + TaskName = taskName ?? Constants.LLM_INDEX_DOCUMENT_TASKNAME; + + InitializeInputs(); + } + + /// + /// Populates inputParams dictionary with LlmIndexDocuments attributes. + /// + private void InitializeInputs() + { + if (ChunkSize.HasValue) + { + WithInput(Constants.CHUNKSIZE, ChunkSize); + } + + if (ChunkOverlap.HasValue) + { + WithInput(Constants.CHUNKOVERLAP, ChunkOverlap); + } + + if (DocId.HasValue) + { + WithInput(Constants.DOCID, DocId); + } + + WithInput(Constants.VECTORDB, VectorDB); + WithInput(Constants.NAMESPACE, NameSpace); + WithInput(Constants.INDEX, Index); + WithInput(Constants.EMBEDDING_MODEL_PROVIDER, EmbeddingModelProvider); + WithInput(Constants.EMBEDDING_MODEL, EmbeddingModel); + WithInput(Constants.URL, Url); + WithInput(Constants.MEDIATYPE, MediaType); + WithInput(Constants.METADATA, MetaData); + } + } +} diff --git a/Conductor/Definition/TaskType/LlmTasks/LlmIndexText.cs b/Conductor/Definition/TaskType/LlmTasks/LlmIndexText.cs new file mode 100644 index 00000000..a1e4d08e --- /dev/null +++ b/Conductor/Definition/TaskType/LlmTasks/LlmIndexText.cs @@ -0,0 +1,98 @@ +using Conductor.Client; +using Conductor.Definition.TaskType; +using Conductor.Definition.TaskType.LlmTasks.Utils; +using System.Collections.Generic; + +namespace Conductor.DefinitaskNametion.TaskType.LlmTasks +{ + /// + /// LlmIndexText + /// + public class LlmIndexText : Task + { + /// + /// Gets or Sets TaskName + /// + public string TaskName { get; set; } + + /// + /// Gets or Sets NameSpace + /// + public string NameSpace { get; set; } + + /// + /// Gets or Sets VectorDB + /// + public string VectorDB { get; set; } + + /// + /// Gets or Sets Index + /// + public string Index { get; set; } + + /// + /// Gets or Sets EmbeddingModel + /// + public EmbeddingModel EmbeddingModel { get; set; } + + /// + /// Gets or Sets Text + /// + public string Text { get; set; } + + /// + /// Gets or Sets DocId + /// + public string DocId { get; set; } + + /// + /// Gets or Sets MetaData + /// + public Dictionary MetaData { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public LlmIndexText(string taskReferenceName, string vectorDB, string index, EmbeddingModel embeddingModel, + string text, string docid, string nameSpace = null, string taskName = null, Dictionary metaData = null) : base(taskReferenceName, WorkflowTaskTypeEnum.LLMINDEXTEXT) + { + TaskName = taskName ?? Constants.LLM_INDEX_TEXT_TASKNAME; + NameSpace = nameSpace; + VectorDB = vectorDB; + Index = index; + EmbeddingModel = embeddingModel; + Text = text; + DocId = docid; + MetaData = metaData; + + InitializeInputs(); + } + + /// + /// Populates inputParams dictionary with LlmIndexText attributes. + /// + private void InitializeInputs() + { + if (NameSpace != null) + { + WithInput(Constants.NAMESPACE, NameSpace); + } + WithInput(Constants.VECTORDB, VectorDB); + WithInput(Constants.INDEX, Index); + WithInput(Constants.EMBEDDING_MODEL_PROVIDER, EmbeddingModel.Provider); + WithInput(Constants.EMBEDDING_MODEL, EmbeddingModel.Model); + WithInput(Constants.TEXT, Text); + WithInput(Constants.DOCID, DocId); + WithInput(Constants.METADATA, MetaData); + } + } +} diff --git a/Conductor/Definition/TaskType/LlmTasks/LlmQueryEmbeddings.cs b/Conductor/Definition/TaskType/LlmTasks/LlmQueryEmbeddings.cs new file mode 100644 index 00000000..8270aed8 --- /dev/null +++ b/Conductor/Definition/TaskType/LlmTasks/LlmQueryEmbeddings.cs @@ -0,0 +1,73 @@ +using Conductor.Client; +using System.Collections.Generic; + +namespace Conductor.Definition.TaskType.LlmTasks +{ + /// + /// LlmQueryEmbeddings + /// + public class LlmQueryEmbeddings : Task + { + /// + /// Gets or Sets TaskRefName + /// + public string TaskRefName { get; set; } + + /// + /// Gets or Sets VectorDB + /// + public string VectorDB { get; set; } + + /// + /// Gets or Sets Index + /// + public string Index { get; set; } + + /// + /// Gets or Sets Embeddings + /// + public List Embeddings { get; set; } + + /// + /// Gets or Sets TaskName + /// + public string TaskName { get; set; } + + /// + /// Gets or Sets Namespace + /// + public string Namespace { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + /// + /// + /// + /// + public LlmQueryEmbeddings(string taskReferenceName, string vectorDB, string index, List embeddings, string taskName = null, string nameSpace = null) : base(taskReferenceName, WorkflowTaskTypeEnum.LLMGETEMBEDDINGS) + { + TaskRefName = taskReferenceName; + TaskName = taskName ?? Constants.LLM_QUERY_EMBEDDING_TASKNAME; + VectorDB = vectorDB; + Index = index; + Embeddings = embeddings; + Namespace = nameSpace; + + InitializeInputs(); + } + + /// + /// Populates inputParams dictionary with LlmQueryEmbeddings attributes. + /// + private void InitializeInputs() + { + WithInput(Constants.VECTORDB, VectorDB); + WithInput(Constants.NAMESPACE, Namespace); + WithInput(Constants.INDEX, Index); + WithInput(Constants.EMBEDDINGS, Embeddings); + } + } +} \ No newline at end of file diff --git a/Conductor/Definition/TaskType/LlmTasks/LlmSearchIndex.cs b/Conductor/Definition/TaskType/LlmTasks/LlmSearchIndex.cs new file mode 100644 index 00000000..0ef9b885 --- /dev/null +++ b/Conductor/Definition/TaskType/LlmTasks/LlmSearchIndex.cs @@ -0,0 +1,96 @@ +using Conductor.Client; + +namespace Conductor.Definition.TaskType.LlmTasks +{ + /// + /// LlmSearchIndex + /// + public class LlmSearchIndex : Task + { + /// + /// Gets or Sets TaskRefName + /// + public string TaskRefName { get; set; } + + /// + /// Gets or Sets VectorDB + /// + public string VectorDB { get; set; } + + /// + /// Gets or Sets Namespace + /// + public string Namespace { get; set; } + + /// + /// Gets or Sets Index + /// + public string Index { get; set; } + + /// + /// Gets or Sets EmbeddingModelProvider + /// + public string EmbeddingModelProvider { get; set; } + + /// + /// Gets or Sets EmbeddingModel + /// + public string EmbeddingModel { get; set; } + + /// + /// Gets or Sets Query + /// + public string Query { get; set; } + + /// + /// Gets or Sets TaskName + /// + public string TaskName { get; set; } + + /// + /// Gets or Sets MaxResults + /// + public int MaxResults { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public LlmSearchIndex(string taskReferenceName, string vectorDB, string nameSpace, string index, string embeddingModelProvider, + string embeddingModel, string query, string taskName = null, int MaxResults = 1) : base(taskReferenceName, WorkflowTaskTypeEnum.LLMSEARCHINDEX) + { + TaskReferenceName = taskReferenceName; + VectorDB = vectorDB; + Namespace = nameSpace; + Index = index; + EmbeddingModelProvider = embeddingModelProvider; + EmbeddingModel = embeddingModel; + Query = query; + TaskName = taskName; + + InitializeInputs(); + } + + /// + /// Populates inputParams dictionary with LlmSearchIndex attributes. + /// + private void InitializeInputs() + { + WithInput(Constants.VECTORDB, VectorDB); + WithInput(Constants.NAMESPACE, Namespace); + WithInput(Constants.INDEX, Index); + WithInput(Constants.EMBEDDING_MODEL_PROVIDER, EmbeddingModelProvider); + WithInput(Constants.EMBEDDING_MODEL, EmbeddingModel); + WithInput(Constants.QUERY, Query); + WithInput(Constants.MAXRESULTS, MaxResults); + } + } +} \ No newline at end of file diff --git a/Conductor/Definition/TaskType/LlmTasks/LlmTextComplete.cs b/Conductor/Definition/TaskType/LlmTasks/LlmTextComplete.cs new file mode 100644 index 00000000..4f8114ea --- /dev/null +++ b/Conductor/Definition/TaskType/LlmTasks/LlmTextComplete.cs @@ -0,0 +1,143 @@ +using Conductor.Client; +using System.Collections.Generic; + +namespace Conductor.Definition.TaskType.LlmTasks +{ + /// + /// LlmTextCompare + /// + public class LlmTextComplete : Task + { + /// + /// Gets or Sets TaskRefName + /// + public string TaskRefName { get; set; } + + /// + /// Gets or Sets LlmProvider + /// + public string LlmProvider { get; set; } + + /// + /// Gets or Sets Model + /// + public string Model { get; set; } + + /// + /// Gets or Sets PromptName + /// + public string PromptName { get; set; } + + /// + /// Gets or Sets StopWords + /// + public List StopWords { get; set; } + + /// + /// Gets or Sets MaxTokens + /// + public int MaxTokens { get; set; } + + /// + /// Gets or Sets Temperature + /// + public int Temperature { get; set; } + + /// + /// Gets or Sets TopP + /// + public int TopP { get; set; } + + /// + /// Gets or Sets TaskName + /// + public string TaskName { get; set; } + + /// + /// Gets or Sets TemplateVariables + /// + public Dictionary TemplateVariables { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public LlmTextComplete(string taskRefName, string llmProvider, string model, string promptName, + List stopWords = null, int maxTokens = 100, int temperature = 0, int topP = 1, + string taskName = null, Dictionary templateVariables = null) : base(taskRefName, WorkflowTaskTypeEnum.LLMTEXTCOMPLETE) + { + TaskRefName = taskRefName; + LlmProvider = llmProvider; + Model = model; + PromptName = promptName; + StopWords = stopWords; + Temperature = temperature; + TopP = topP; + MaxTokens = maxTokens; + TaskName = taskName ?? Constants.LLM_TEXT_COMPLETE; + TemplateVariables = templateVariables ?? new Dictionary(); + + InitializeInputs(); + } + + /// + /// Adding PromptVariables to InputParams + /// + /// + /// + public LlmTextComplete PromptVariables(Dictionary variables) + { + foreach (var variable in variables) + { + TemplateVariables[variable.Key] = variable.Value; + } + WithInput(Constants.PROMPTVARIABLES, TemplateVariables); + return this; + } + + /// + /// Adding PromptVariable to InputParams + /// + /// + /// + /// + public LlmTextComplete PromptVariable(string variable, object value) + { + TemplateVariables[variable] = value; + WithInput(Constants.PROMPTVARIABLES, TemplateVariables); + return this; + } + + /// + /// Populates inputParams dictionary with LlmTextComplete attributes. + /// + private void InitializeInputs() + { + if (StopWords != null) + { + WithInput(Constants.STOPWORDS, StopWords); + } + + if (MaxTokens != 0) + { + WithInput(Constants.MAXTOKENS, MaxTokens); + } + + WithInput(Constants.LLMPROVIDER, LlmProvider); + WithInput(Constants.MODEL, Model); + WithInput(Constants.PROMPTNAME, PromptName); + WithInput(Constants.TEMPERATURE, Temperature); + WithInput(Constants.TOPP, TopP); + WithInput(Constants.PROMPTVARIABLES, TemplateVariables); + } + } +} diff --git a/Conductor/Definition/TaskType/LlmTasks/Utils/EmbeddingModel.cs b/Conductor/Definition/TaskType/LlmTasks/Utils/EmbeddingModel.cs new file mode 100644 index 00000000..057fdead --- /dev/null +++ b/Conductor/Definition/TaskType/LlmTasks/Utils/EmbeddingModel.cs @@ -0,0 +1,29 @@ +namespace Conductor.Definition.TaskType.LlmTasks.Utils +{ + /// + /// EmbeddingModel + /// + public class EmbeddingModel + { + /// + /// Gets or Sets Provider + /// + public string Provider { get; set; } + + /// + /// Gets or Sets Model + /// + public string Model { get; set; } + + /// + /// Initializes a new instance of the class + /// + /// + /// + public EmbeddingModel(string provider, string model) + { + Provider = provider; + Model = model; + } + } +} diff --git a/Conductor/conductor-csharp.csproj b/Conductor/conductor-csharp.csproj index b0514fb9..1ad67ccb 100644 --- a/Conductor/conductor-csharp.csproj +++ b/Conductor/conductor-csharp.csproj @@ -16,7 +16,7 @@ - + \ No newline at end of file diff --git a/Tests/Api/IntegrationResourceApiTests.cs b/Tests/Api/IntegrationResourceApiTests.cs index 250b5e1d..e9350653 100644 --- a/Tests/Api/IntegrationResourceApiTests.cs +++ b/Tests/Api/IntegrationResourceApiTests.cs @@ -27,7 +27,7 @@ public class IntegrationResourceApiTests : IDisposable /// public IntegrationResourceApiTests(ITestOutputHelper testOutputHelper) { - ////dev local testing + //dev local testing //_orkesApiClient = new OrkesApiClient(new Configuration(), new OrkesAuthenticationSettings(Constants.KEY_ID, Constants.KEY_SECRET)); //_integrationResourceApi = _orkesApiClient.GetClient(); @@ -461,12 +461,12 @@ public async void DeleteTagForIntegrationAsyncTest() public void Dispose() { if (_performCleanup) - IntegrationExtensions.DeleteIntegration(_integrationResourceApi); + _integrationResourceApi.DeleteIntegration(); } private void Setup() { - IntegrationExtensions.CreateIntegration(_integrationResourceApi); + _integrationResourceApi.CreateIntegration(); } } } diff --git a/Tests/Api/PromptResourceApiTest.cs b/Tests/Api/PromptResourceApiTest.cs index 860a4126..49b56356 100644 --- a/Tests/Api/PromptResourceApiTest.cs +++ b/Tests/Api/PromptResourceApiTest.cs @@ -42,7 +42,7 @@ public PromptResourceApiTest(ITestOutputHelper testOutputHelper) [Fact] public void CreatePromptTemplateTest() { - IntegrationExtensions.CreateIntegration(_integrationResourceApi, true); + Setup(); string ModelName = IntegrationExtensions.GetModelName(TestConstants.IntegrationPromptName, TestConstants.ModelPromptName); List model = new List() { ModelName }; Assert.Null(Record.Exception(() => _promptResourceApi.SaveMessageTemplate(TestConstants.TemplateBody, TestConstants.TemplateDescription, TestConstants.PromptName, model))); @@ -55,7 +55,7 @@ public void CreatePromptTemplateTest() [Fact] public async void CreatePromptTemplateAsyncTest() { - IntegrationExtensions.CreateIntegration(_integrationResourceApi, true); + Setup(); string ModelName = IntegrationExtensions.GetModelName(TestConstants.IntegrationPromptName, TestConstants.ModelPromptName); List model = new List() { ModelName }; var saveMessageTemplateException = await Record.ExceptionAsync(async () => @@ -70,7 +70,7 @@ public async void CreatePromptTemplateAsyncTest() [Fact] public void GetMessageTemplatesTest() { - IntegrationExtensions.CreateIntegration(_integrationResourceApi, true); + Setup(); var response = _promptResourceApi.GetMessageTemplates(); AssertExtensions.AssertModelResponse>(response); _performCleanup = true; @@ -82,7 +82,7 @@ public void GetMessageTemplatesTest() [Fact] public async void GetMessageTemplatesAsyncTest() { - IntegrationExtensions.CreateIntegration(_integrationResourceApi, true); + Setup(); var response = await _promptResourceApi.GetMessageTemplatesAsync(); AssertExtensions.AssertModelResponse>(response); _performCleanup = true; @@ -94,7 +94,7 @@ public async void GetMessageTemplatesAsyncTest() [Fact] public void GetMessageTemplateTest() { - IntegrationExtensions.CreateIntegration(_integrationResourceApi, true); + Setup(); var response = _promptResourceApi.GetMessageTemplate(TestConstants.PromptName); AssertExtensions.AssertModelResponse(response); _performCleanup = true; @@ -106,7 +106,7 @@ public void GetMessageTemplateTest() [Fact] public async void GetMessageTemplateAsyncTest() { - IntegrationExtensions.CreateIntegration(_integrationResourceApi, true); + Setup(); var response = await _promptResourceApi.GetMessageTemplateAsync(TestConstants.PromptName); AssertExtensions.AssertModelResponse(response); _performCleanup = true; @@ -118,7 +118,7 @@ public async void GetMessageTemplateAsyncTest() [Fact] public void DeleteMessageTemplateTest() { - IntegrationExtensions.CreateIntegration(_integrationResourceApi, true); + Setup(); Assert.Null(Record.Exception(() => _promptResourceApi.DeleteMessageTemplate(TestConstants.PromptName))); } @@ -128,7 +128,7 @@ public void DeleteMessageTemplateTest() [Fact] public async void DeleteMessageTemplateAsyncTest() { - IntegrationExtensions.CreateIntegration(_integrationResourceApi, true); + Setup(); var deleteMessageTemplateException = await Record.ExceptionAsync(async () => await _promptResourceApi.DeleteMessageTemplateAsync(TestConstants.PromptName)); } @@ -226,7 +226,7 @@ private void CreatePromptTemplate() { try { - IntegrationExtensions.CreateIntegration(_integrationResourceApi, true); + Setup(); var jsonObject = IntegrationExtensions.LoadIntegrationData(); string ModelName = IntegrationExtensions.GetModelName(TestConstants.IntegrationPromptName, TestConstants.ModelPromptName); List model = new List() { ModelName }; @@ -240,5 +240,10 @@ private void CreatePromptTemplate() throw new Exception(errorMessage, ex); } } + + private void Setup() + { + _integrationResourceApi.CreateIntegration(true); + } } } diff --git a/Tests/Extensions/IntegrationExtensions.cs b/Tests/Extensions/IntegrationExtensions.cs index e113c84e..6269658f 100644 --- a/Tests/Extensions/IntegrationExtensions.cs +++ b/Tests/Extensions/IntegrationExtensions.cs @@ -11,7 +11,7 @@ namespace conductor_csharp.test.Extensions /// public static class IntegrationExtensions { - public static void CreateIntegration(IntegrationResourceApi integrationResourceApi, bool isPromptTemplate = false) + public static void CreateIntegration(this IntegrationResourceApi integrationResourceApi, bool isPromptTemplate = false) { try { @@ -31,7 +31,7 @@ public static void CreateIntegration(IntegrationResourceApi integrationResourceA } } - public static void DeleteIntegration(IntegrationResourceApi integrationResourceApi, bool isPromptTemplate = false) + public static void DeleteIntegration(this IntegrationResourceApi integrationResourceApi, bool isPromptTemplate = false) { try {