From 68802d41deb06a4e304e2c5b5e175fd95c6b1252 Mon Sep 17 00:00:00 2001 From: al2na Date: Mon, 29 Jan 2024 23:09:47 +0100 Subject: [PATCH] runCodeInResponse and associated functions in --- DESCRIPTION | 3 +- NAMESPACE | 2 + NEWS.md | 5 + R/runCodeInResponse.R | 130 +++++++++++++++ R/selfcorrect.R | 341 +++++++++++++++++++++++++++++++++++---- man/runCodeInResponse.Rd | 60 +++++++ man/sampleResponse.Rd | 55 +++++++ man/selfcorrect.Rd | 4 + 8 files changed, 566 insertions(+), 34 deletions(-) create mode 100644 R/runCodeInResponse.R create mode 100644 man/runCodeInResponse.Rd create mode 100644 man/sampleResponse.Rd diff --git a/DESCRIPTION b/DESCRIPTION index ed117aa..1a244a2 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -42,5 +42,6 @@ Collate: 'fileHeaderPrompt.R' 'test-helper-test_argument_validation.R' 'promptContext.R' -RoxygenNote: 7.2.3 + 'runCodeInResponse.R' +RoxygenNote: 7.3.0 Config/testthat/edition: 3 diff --git a/NAMESPACE b/NAMESPACE index 24a8d0c..e50d7c5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -8,6 +8,8 @@ export(extractFilenames) export(extractInstallPkg) export(fileHeaderPrompt) export(promptContext) +export(runCodeInResponse) +export(sampleResponse) export(selfcorrect) export(sendPrompt) export(setupAgent) diff --git a/NEWS.md b/NEWS.md index ae305ad..8791125 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,11 @@ agent name as "generic" in setupAgent() function * new feature: clean_code_blocks() now automatically runs when extractCode() is called * new feature: sendPrompt() now can send previous message correspondence with LLM APIs via "previous.msgs" argument + * new feature: selfcorrect() has a new argument called responseWithErrors (default:NULL), via this arguments + users can input problematic responses to the selfcorrect() function instead of letting the getting the initial response and potentially failing. + * new function: runCodeInResponse() can take LLM response and directly run code, and tries + to fix if there are errors. + * new function: sampleResponse() function tries to fix code errors by repatedly sampling new solutions to the existing prompts * fix: typos in setupAgent fixed * fix: sendPrompt fixed which was broken due to openai API changes * fix: extractInstallPkg can now deal with require() calls in code blocks diff --git a/R/runCodeInResponse.R b/R/runCodeInResponse.R new file mode 100644 index 0000000..9ae79d3 --- /dev/null +++ b/R/runCodeInResponse.R @@ -0,0 +1,130 @@ +#' Executes the code in the received response from the agent +#' +#' The function extracts and executes the code in the response. If required it can +#' try to correct errors in the code via different strategies. +#' +#' @param response response to be parsed for code and executed +#' @param prompt prompt for the response, if correction="none" it is not needed +#' @param agent AI agent, if correction="none" it is not needed +#' @param context context for the prompt, if correction="none" it is not needed +#' @param correction "none" no code correction is needed. "selfcorrect" feedback the errors to LLM and ask for fix. +#' "sampleMore" sample responses for the #prompt until an executable code is returned or number of attempts reached. +#' "correctThenSample" first try self-correct "attempt" number of times. +#' If no executable code is returned. It will sample new responses "attempt" number of times or until executable code +#' @param attempts Numeric value denoting how many times the code should be sent back for fixing. +#' @param output.file Optional output file created holding parsed code +#' @param ... arguments to sendPrompt() +#' @return A list containing the following elements: +#' \item{init.response}{A character vector representing the initial prompt response.} +#' \item{init.blocks}{A list of initial blocks.} +#' \item{final.blocks}{A list of final blocks.} +#' \item{code.works}{A boolean value indicating whether the code works.} +#' \item{exec.result}{A character string representing the execution results.} +#' \item{tried.attempts}{An integer representing the number of attempts.} +#' @seealso \code{\link{selfcorrect}},\code{\link{sampleResponse}} +#' @examples +#' \dontrun{ +#' +#' resp.list <- runCodeInResponse(agent,prompt,context=rbionfoExp,correction="sampleMore",attempt=2) +#' } +#' @export +runCodeInResponse<-function( + response, + prompt=NULL, + agent=NULL, + context = NULL, + correction=c("none","selfcorrect","sampleMore","sampleThenCorrect","correctThenSample"), + attempts = 3, + output.file = NULL, + ... +){ + + # check optional arguments + if(any( !is.null(prompt),!is.null(agent),!is.null(context))){ + + if( (is.null(prompt)+is.null(agent)+is.null(context))>1 ){ + stop("\nif any of the following arguments is provided, all of them must be provided:\n", + "'prompt','agent' and 'context'\n", + "no NULL values are allowed in that case for those arguments" + ) + } + } + + + + # Clean the code backtick structure and install.packages calls + response<-clean_code_blocks(response) + + + + # Parse the code + blocks <- extractCode(text=response,delimiter="```") + + #execute response + if(is.null(output.file)){ + res<-executeCode(blocks$code, output = "eval",output.file = output.file ) + + }else{ + res<-executeCode(blocks$code, output = "html",output.file = output.file ) + + } + + first.res<-list(init.response=response, + init.blocks=blocks, + final.response=response, + final.blocks=blocks, + code.works=!(is.list(res) & ("error" %in% names(res) )), + exec.result=res, + tried.attempts=1) + + # if code works or no correction asked + if(correction=="none" | first.res$code.works){ + return(first.res) + + }else if(correction=="selfcorrect"){ + + res.list<-selfcorrect(agent,prompt, + context=context,attempts,output.file, + responseWithError=list(response,res), + ...) + + return(res.list) + }else if(correction=="sampleMore"){ + res.list<-sampleResponse(agent,prompt, + context=context,attempts,output.file, + responseWithError=list(response,res), + ...) + return(res.list) + }else if(correction=="sampleThenCorrect"){ + + res.list<-sampleResponse(agent,prompt, + context=context,attempts,output.file, + responseWithError=list(response,res), + ...) + if(!res.list$code.works){ + res.list<-selfcorrect(agent,prompt, + context=context,attempts,output.file, + responseWithError=list(response,res), + ...) + } + + return(res.list) + }else if(correction=="correctThenSample"){ + + res.list<-selfcorrect(agent,prompt, + context=context,attempts,output.file, + responseWithError=list(response,res), + ...) + + if(!res.list$code.works){ + res.list<-sampleResponse(agent,prompt, + context=context,attempts,output.file, + responseWithError=list(response,res), + ...) + } + + return(res.list) + } + + +} diff --git a/R/selfcorrect.R b/R/selfcorrect.R index 437c2ee..98b00b0 100644 --- a/R/selfcorrect.R +++ b/R/selfcorrect.R @@ -9,6 +9,8 @@ #' @param context Optional context to provide alongside the prompt (default is rbionfoExp). #' @param attempts Numeric value denoting how many times the code should be sent back for fixing. #' @param output.file Optional output file created holding parsed code +#' @param responseWithError a list of response and errors returned from executeCode(). +#' First element is expected to be the response and the second element is the error list returned by executeCode(). #' @param ... Additional arguments to be passed to the \code{\link{sendPrompt}} function. #' @return A list containing the following elements: #' \item{init.response}{A character vector representing the initial prompt response.} @@ -24,7 +26,7 @@ #' response <- selfcorrect(agent,prompt,context=rbionfoExp, max_tokens = 500) #' } #' @export -selfcorrect<-function(agent,prompt,context=rbionfoExp,attempts=3,output.file=NULL,...){ +selfcorrect<-function(agent,prompt,context=rbionfoExp,attempts=3,output.file=NULL,responseWithError=NULL,...){ #--------------------------------------------------------------------------- # Validate arguments @@ -60,53 +62,127 @@ selfcorrect<-function(agent,prompt,context=rbionfoExp,attempts=3,output.file=NUL } } - # Send prompt - response <- sendPrompt(agent,prompt,context,return.type="text",...) + # Define the prompt template to inject the error message + promptTemplate <- "The previous code above returned the following errors and/or warnings,\n \n fix errors and return the fixed code in one block, delimited in triple backticks" + + # Set up the on of the final variables that will be returned in the end + codeWorks=FALSE - # Clean the code backtick structure and install.packages calls - response<-clean_code_blocks(response) + attmpt.st=1 # attempt counter starting point - initial.response <- response + if(is.null(output.file)){output="eval"} + if(is.null(responseWithError)){ - # Parse the code - blocks <- extractCode(text=initial.response,delimiter="```") - initial.blocks <- blocks - # Check if any code is returned - if(blocks$code==""){ - message(response) - stop("no code returned") + # Send prompt + response <- sendPrompt(agent,prompt,context,return.type="text",...) - } + # Clean the code backtick structure and install.packages calls + response<-clean_code_blocks(response) + + initial.response <- response + + + # Parse the code + blocks <- extractCode(text=initial.response,delimiter="```") + initial.blocks <- blocks + + # Check if any code is returned + if(blocks$code==""){ + message(response) + stop("no code returned") + + } - # Extract and install packages if needed - extractInstallPkg(blocks$code) + # Extract and install packages if needed + extractInstallPkg(blocks$code) - # List of messages to the bot - msgs<- list( - list( - "role" = "user", - "content" = paste(context,"\n",prompt) - ), - list( - "role"="assistant", - "content"=initial.response + # List of messages to the bot + msgs<- list( + list( + "role" = "user", + "content" = paste(context,"\n",prompt) + ), + list( + "role"="assistant", + "content"=initial.response + ) ) - ) + }else if(length(responseWithError)==2 && is.list(responseWithError)){ + message("attempt number ",1," returned error, trying to fix the error") + attmpt.st=2 - # Define the prompt template to inject the error message - promptTemplate <- "The previous code returned the following errors and/or warnings,\n \n return fixed code in one block, delimited in triple backticks" + # process errors + # Collapse the character vectors within the list elements + # This is good if we have multiple errors in the list per element + err0<-responseWithError[[2]] + collapsed_list <- lapply(err0, function(x) paste(x, collapse = "\n")) + + # Get error/warning text + errs<- paste(paste0(names(collapsed_list ), ": ", collapsed_list ), collapse = "\n ") + + + # get new prompt + # Use sub() to substitute the replacement string for the wildcard string + promptAddon <- sub("", errs, promptTemplate) + + new.prompt<-promptAddon + + msgs<- list( + list( + "role" = "system", + "content" = context + ), + list( + "role"="user", + "content"= prompt + ), + list( + "role"="assistant", + "content"= responseWithError[[1]] + ) + ) + # send prompt + response <- sendPrompt(agent=agent, prompt=new.prompt, + context=context, + return.type = "text",previous.msgs = msgs ,...) + + msgs<-append(msgs,list(list("role" = "user","content" = new.prompt))) + + # Clean the code backtick structure and install.packages calls + response<-clean_code_blocks(response) + + msgs<-append(msgs,list(list("role" = "assistant","content" = response))) + + initial.response <- response + + + # Parse the code + blocks <- extractCode(text=initial.response,delimiter="```") + initial.blocks <- blocks + + # Check if any code is returned + if(blocks$code==""){ + message(response) + stop("no code returned") + + } + + # Extract and install packages if needed + extractInstallPkg(blocks$code) + + } - # Set up the on of the final variables that will be returned in the end - codeWorks=FALSE # Execute the code up to "attempts" times - for(i in 1:attempts){ + for(i in attmpt.st:attempts){ + + message("attempt number ",i," started for fixing the code in the response") # See if the code runs without errors res<-executeCode(blocks$code, output = "html",output.file = output.file ) @@ -115,6 +191,7 @@ selfcorrect<-function(agent,prompt,context=rbionfoExp,attempts=3,output.file=NUL if(is.list(res) & ("error" %in% names(res) )){ # Get error messages + message("attempt number ",i," returned error, trying again for a fix") # Collapse the character vectors within the list elements # This is good if we have multiple errors in the list per element @@ -132,8 +209,8 @@ selfcorrect<-function(agent,prompt,context=rbionfoExp,attempts=3,output.file=NUL # Send prompt - msgs<-append(msgs,list(list("role" = "user","content" = new.prompt))) - response <- sendPrompt(agent=agent, prompt=paste(response,promptAddon), return.type = "text",messages=msgs) + response <- sendPrompt(agent=agent, prompt=new.prompt,context=context, + return.type = "text",previous.msgs=msgs,...) msgs<-append(msgs,list(list("role" = "assistant","content" = response))) # Clean code from wrong backticks @@ -165,3 +242,201 @@ selfcorrect<-function(agent,prompt,context=rbionfoExp,attempts=3,output.file=NUL exec.result=res, tried.attempts=i)) } + + + +#' Sample more solutions when non-executable code returned by the agent +#' +#' The function attempts to sample more solutions by the agent when +#' the code returned by the agent is faulty. The function simply asks for solutions +#' until an executable code is returned or until number of attempts is reached. +# If there are no error messages, the function returns the original response. +#' +#' @param agent An object containing the agent's information (e.g., type and model). +#' @param prompt The prompt text to send to the language model. +#' @param context Optional context to provide alongside the prompt (default is rbionfoExp). +#' @param attempts Numeric value denoting how many times the code should be sent back for fixing. +#' @param output.file Optional output file created holding parsed code +#' @param responseWithError a list of response and errors returned from executeCode(). +#' First element is expected to be the response and the second element is the error list returned by executeCode(). +#' @param ... Additional arguments to be passed to the \code{\link{sendPrompt}} function. +#' @return A list containing the following elements: +#' \item{init.response}{A character vector representing the initial prompt response.} +#' \item{init.blocks}{A list of initial blocks.} +#' \item{final.blocks}{A list of final blocks.} +#' \item{code.works}{A boolean value indicating whether the code works.} +#' \item{exec.result}{A character string representing the execution results.} +#' \item{tried.attempts}{An integer representing the number of attempts.} +#' @seealso \code{\link{promptContext}} for predefined contexts to use. +#' @examples +#' \dontrun{ +#' +#' resp.list <- sampleResponse(agent,prompt,context=rbionfoExp, max_tokens = 500) +#' } +#' @export +sampleResponse<-function(agent,prompt, + context=rbionfoExp,attempts=3,output.file=NULL,responseWithError=NULL,...){ + + #--------------------------------------------------------------------------- + # Validate arguments + assertthat::assert_that( + assertthat::`%has_name%`(agent,c("name","model","API","url","headers","ai_api_key","type")), + assertthat::noNA(agent) + ) + + assertthat::assert_that( + assertthat::is.string(prompt), + assertthat::noNA(prompt) + ) + + assertthat::assert_that( + assertthat::is.string(context) + ) + + assertthat::assert_that( + assertthat::is.number(attempts), + assertthat::noNA(attempts) + ) + + if (!is.null(output.file)) { + assertthat::assert_that( + assertthat::is.string(output.file), + assertthat::noNA(output.file) + ) + } + #------------------------------------------------------------------------------------------ + if (agent$API =="openai"){ + if (agent$type == "completion"){ + stop("selfcorrect cannot be used with type completion. Can only be used with type chat.") + } + } + + # Define the prompt template to inject the error message + promptTemplate <- "The previous code above returned the following errors and/or warnings,\n \n fix errors and return the fixed code in one block, delimited in triple backticks" + + # Set up the on of the final variables that will be returned in the end + codeWorks=FALSE + attmpt.st=1 # attempt counter starting point + + if(is.null(responseWithError)){ + + + # Send prompt + response <- sendPrompt(agent,prompt,context,return.type="text",...) + + # Clean the code backtick structure and install.packages calls + response<-clean_code_blocks(response) + + initial.response <- response + + + # Parse the code + blocks <- extractCode(text=initial.response,delimiter="```") + initial.blocks <- blocks + + # Check if any code is returned + if(blocks$code==""){ + message(response) + stop("no code returned") + + } + + # Extract and install packages if needed + extractInstallPkg(blocks$code) + + + # List of messages to the bot + msgs<- list( + list( + "role" = "user", + "content" = paste(context,"\n",prompt) + ), + list( + "role"="assistant", + "content"=initial.response + ) + ) + + + }else if(length(responseWithError)==2 && ("error" %in% names(responseWithError[[2]]) ) ){ + + message("attempt number ",1," returned error, sampling LLM response again for a solution") + attmpt.st=2 + # send prompt again for a new and potentially different response + response <- sendPrompt(agent=agent, prompt=prompt,context=context, + return.type = "text",...) + + + # Clean the code backtick structure and install.packages calls + response<-clean_code_blocks(response) + + initial.response <- response + + + # Parse the code + blocks <- extractCode(text=initial.response,delimiter="```") + initial.blocks <- blocks + + # Check if any code is returned + if(blocks$code==""){ + message(response) + stop("no code returned") + + } + + # Extract and install packages if needed + extractInstallPkg(blocks$code) + + } + + + # Execute the code up to "attempts" times + for(i in attmpt.st:attempts){ + + message("attempt number ",i," started: sampling a new and hopefully better LLM response") + # See if the code runs without errors + res<-executeCode(blocks$code, output = "html",output.file = output.file ) + + # If there are errors + if(is.list(res) & ("error" %in% names(res) )){ + + + message("attempt number ",i," returned error, sampling LLM response again for a solution") + + # Send prompt + response <- sendPrompt(agent=agent, prompt=prompt,context=context, + return.type = "text",...) + + # Clean code from wrong backticks + response<-clean_code_blocks(response) + + # Parse the code + blocks<- extractCode(text=response,delimiter="```") + + # Extract and install libs needed + extractInstallPkg(blocks$code) + + + }else{ + # Break the loop if the code works without errors + codeWorks=TRUE + break + + } + + } + + + # Return the latest code, initial prompt, and everything else + return(list(init.response=initial.response, + init.blocks=initial.blocks, + final.response=response, + final.blocks=blocks, + code.works=codeWorks, + exec.result=res, + tried.attempts=i)) +} + + + + diff --git a/man/runCodeInResponse.Rd b/man/runCodeInResponse.Rd new file mode 100644 index 0000000..87bdc0b --- /dev/null +++ b/man/runCodeInResponse.Rd @@ -0,0 +1,60 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/runCodeInResponse.R +\name{runCodeInResponse} +\alias{runCodeInResponse} +\title{Executes the code in the received response from the agent} +\usage{ +runCodeInResponse( + response, + prompt = NULL, + agent = NULL, + context = NULL, + correction = c("none", "selfcorrect", "sampleMore", "sampleThenCorrect", + "correctThenSample"), + attempts = 3, + output.file = NULL, + ... +) +} +\arguments{ +\item{response}{response to be parsed for code and executed} + +\item{prompt}{prompt for the response, if correction="none" it is not needed} + +\item{agent}{AI agent, if correction="none" it is not needed} + +\item{context}{context for the prompt, if correction="none" it is not needed} + +\item{correction}{"none" no code correction is needed. "selfcorrect" feedback the errors to LLM and ask for fix. +"sampleMore" sample responses for the #prompt until an executable code is returned or number of attempts reached. +"correctThenSample" first try self-correct "attempt" number of times. +If no executable code is returned. It will sample new responses "attempt" number of times or until executable code} + +\item{attempts}{Numeric value denoting how many times the code should be sent back for fixing.} + +\item{output.file}{Optional output file created holding parsed code} + +\item{...}{arguments to sendPrompt()} +} +\value{ +A list containing the following elements: +\item{init.response}{A character vector representing the initial prompt response.} +\item{init.blocks}{A list of initial blocks.} +\item{final.blocks}{A list of final blocks.} +\item{code.works}{A boolean value indicating whether the code works.} +\item{exec.result}{A character string representing the execution results.} +\item{tried.attempts}{An integer representing the number of attempts.} +} +\description{ +The function extracts and executes the code in the response. If required it can +try to correct errors in the code via different strategies. +} +\examples{ +\dontrun{ + +resp.list <- runCodeInResponse(agent,prompt,context=rbionfoExp,correction="sampleMore",attempt=2) +} +} +\seealso{ +\code{\link{selfcorrect}},\code{\link{sampleResponse}} +} diff --git a/man/sampleResponse.Rd b/man/sampleResponse.Rd new file mode 100644 index 0000000..4754d05 --- /dev/null +++ b/man/sampleResponse.Rd @@ -0,0 +1,55 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/selfcorrect.R +\name{sampleResponse} +\alias{sampleResponse} +\title{Sample more solutions when non-executable code returned by the agent} +\usage{ +sampleResponse( + agent, + prompt, + context = rbionfoExp, + attempts = 3, + output.file = NULL, + responseWithError = NULL, + ... +) +} +\arguments{ +\item{agent}{An object containing the agent's information (e.g., type and model).} + +\item{prompt}{The prompt text to send to the language model.} + +\item{context}{Optional context to provide alongside the prompt (default is rbionfoExp).} + +\item{attempts}{Numeric value denoting how many times the code should be sent back for fixing.} + +\item{output.file}{Optional output file created holding parsed code} + +\item{responseWithError}{a list of response and errors returned from executeCode(). +First element is expected to be the response and the second element is the error list returned by executeCode().} + +\item{...}{Additional arguments to be passed to the \code{\link{sendPrompt}} function.} +} +\value{ +A list containing the following elements: +\item{init.response}{A character vector representing the initial prompt response.} +\item{init.blocks}{A list of initial blocks.} +\item{final.blocks}{A list of final blocks.} +\item{code.works}{A boolean value indicating whether the code works.} +\item{exec.result}{A character string representing the execution results.} +\item{tried.attempts}{An integer representing the number of attempts.} +} +\description{ +The function attempts to sample more solutions by the agent when +the code returned by the agent is faulty. The function simply asks for solutions +until an executable code is returned or until number of attempts is reached. +} +\examples{ +\dontrun{ + +resp.list <- sampleResponse(agent,prompt,context=rbionfoExp, max_tokens = 500) +} +} +\seealso{ +\code{\link{promptContext}} for predefined contexts to use. +} diff --git a/man/selfcorrect.Rd b/man/selfcorrect.Rd index f4df1a6..223b282 100644 --- a/man/selfcorrect.Rd +++ b/man/selfcorrect.Rd @@ -10,6 +10,7 @@ selfcorrect( context = rbionfoExp, attempts = 3, output.file = NULL, + responseWithError = NULL, ... ) } @@ -24,6 +25,9 @@ selfcorrect( \item{output.file}{Optional output file created holding parsed code} +\item{responseWithError}{a list of response and errors returned from executeCode(). +First element is expected to be the response and the second element is the error list returned by executeCode().} + \item{...}{Additional arguments to be passed to the \code{\link{sendPrompt}} function.} } \value{