diff --git a/docs/docs/guides/function-calling.md b/docs/docs/guides/function-calling.md index 6b9157f18..7725f225d 100644 --- a/docs/docs/guides/function-calling.md +++ b/docs/docs/guides/function-calling.md @@ -63,8 +63,14 @@ tools = [ completion_payload = { "messages": [ - {"role": "system", "content": "You are a helpful customer support assistant. Use the supplied tools to assist the user."}, - {"role": "user", "content": "Hi, can you tell me the delivery date for my order?"}, + { + "role": "system", + "content": 'You have access to the following CUSTOM functions:\n\n\n\nIf a you choose to call a function ONLY reply in the following format:\n<{start_tag}={function_name}>{parameters}{end_tag}\nwhere\n\nstart_tag => ` a JSON dict with the function argument name as key and function argument value as value.\nend_tag => ``\n\nHere is an example,\n{"example_name": "example_value"}\n\nReminder:\n- Function calls MUST follow the specified format\n- Required parameters MUST be specified\n- You can call one or more functions at a time, but remember only chose correct function\n- Put the entire function call reply on one line\n- Always add your sources when using search results to answer the user query\n- If you can not find correct parameters or arguments corresponding to function in the user\'s message, ask user again to provide, do not make assumptions.\n- No explanation are needed when calling a function.\n\nYou are a helpful assistant.', + }, + { + "role": "user", + "content": "Hi, can you tell me the delivery date for my order?" + }, ] } @@ -126,10 +132,22 @@ Once the user provides their order ID: ```python completion_payload = { "messages": [ - {"role": "system", "content": "You are a helpful customer support assistant. Use the supplied tools to assist the user."}, - {"role": "user", "content": "Hi, can you tell me the delivery date for my order?"}, - {"role": "assistant", "content": "Of course! Please provide your order ID so I can look it up."}, - {"role": "user", "content": "i think it is order_70705"}, + { + "role": "system", + "content": 'You have access to the following CUSTOM functions:\n\n\n\nIf a you choose to call a function ONLY reply in the following format:\n<{start_tag}={function_name}>{parameters}{end_tag}\nwhere\n\nstart_tag => ` a JSON dict with the function argument name as key and function argument value as value.\nend_tag => ``\n\nHere is an example,\n{"example_name": "example_value"}\n\nReminder:\n- Function calls MUST follow the specified format\n- Required parameters MUST be specified\n- You can call one or more functions at a time, but remember only chose correct function\n- Put the entire function call reply on one line\n- Always add your sources when using search results to answer the user query\n- If you can not find correct parameters or arguments corresponding to function in the user\'s message, ask user again to provide, do not make assumptions.\n- No explanation are needed when calling a function.\n\nYou are a helpful assistant.', + }, + { + "role": "user", + "content": "Hi, can you tell me the delivery date for my order?" + }, + { + "role": "assistant", + "content": "Of course! Please provide your order ID so I can look it up." + }, + { + "role": "user", + "content": "i think it is order_70705" + }, ] } diff --git a/engine/controllers/server.cc b/engine/controllers/server.cc index 6ea733a70..3ba4aa327 100644 --- a/engine/controllers/server.cc +++ b/engine/controllers/server.cc @@ -179,7 +179,6 @@ void server::ProcessStreamRes(std::function cb, void server::ProcessNonStreamRes(std::function cb, SyncQueue& q) { auto [status, res] = q.wait_and_pop(); - function_calling_utils::PostProcessResponse(res); LOG_DEBUG << "response: " << res.toStyledString(); auto resp = cortex_utils::CreateCortexHttpJsonResponse(res); resp->setStatusCode( diff --git a/engine/extensions/local-engine/local_engine.cc b/engine/extensions/local-engine/local_engine.cc index b769c5e8c..c4d296427 100644 --- a/engine/extensions/local-engine/local_engine.cc +++ b/engine/extensions/local-engine/local_engine.cc @@ -544,6 +544,7 @@ void LocalEngine::LoadModel(std::shared_ptr json_body, params.push_back("--pooling"); params.push_back("mean"); + params.push_back("--jinja"); std::vector v; v.reserve(params.size() + 1); diff --git a/engine/services/inference_service.cc b/engine/services/inference_service.cc index 75d95f06d..e07ed71ba 100644 --- a/engine/services/inference_service.cc +++ b/engine/services/inference_service.cc @@ -13,8 +13,6 @@ cpp::result InferenceService::HandleChatCompletion( engine_type = (*(json_body)).get("engine", kLlamaRepo).asString(); } CTL_DBG("engine_type: " << engine_type); - function_calling_utils::PreprocessRequest(json_body); - CTL_DBG("engine_type: " << engine_type); auto tool_choice = json_body->get("tool_choice", Json::Value::null); auto model_id = json_body->get("model", "").asString(); if (saved_models_.find(model_id) != saved_models_.end()) { @@ -46,51 +44,6 @@ cpp::result InferenceService::HandleChatCompletion( return cpp::fail(std::make_pair(stt, res)); } - if (!model_id.empty()) { - if (auto model_service = model_service_.lock()) { - auto metadata_ptr = model_service->GetCachedModelMetadata(model_id); - if (metadata_ptr != nullptr && - !metadata_ptr->tokenizer->chat_template.empty()) { - auto tokenizer = metadata_ptr->tokenizer; - auto messages = (*json_body)["messages"]; - Json::Value messages_jsoncpp(Json::arrayValue); - for (auto message : messages) { - messages_jsoncpp.append(message); - } - - Json::Value tools(Json::arrayValue); - Json::Value template_data_json; - template_data_json["messages"] = messages_jsoncpp; - // template_data_json["tools"] = tools; - - auto prompt_result = jinja::RenderTemplate( - tokenizer->chat_template, template_data_json, tokenizer->bos_token, - tokenizer->eos_token, tokenizer->add_bos_token, - tokenizer->add_eos_token, tokenizer->add_generation_prompt); - if (prompt_result.has_value()) { - (*json_body)["prompt"] = prompt_result.value(); - if (json_body->isMember("stop")) { - bool need_append = true; - for (auto& s : (*json_body)["stop"]) { - if (s.asString() == tokenizer->eos_token) { - need_append = false; - } - } - if (need_append) { - (*json_body)["stop"].append(tokenizer->eos_token); - } - } else { - Json::Value stops(Json::arrayValue); - stops.append(tokenizer->eos_token); - (*json_body)["stop"] = stops; - } - } else { - CTL_ERR("Failed to render prompt: " + prompt_result.error()); - } - } - } - } - CTL_DBG("Json body inference: " + json_body->toStyledString()); auto cb = [q, tool_choice](Json::Value status, Json::Value res) { diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index d9359b698..68f0fe070 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -691,21 +691,7 @@ cpp::result ModelService::StartModel( auto status = std::get<0>(ir)["status_code"].asInt(); auto data = std::get<1>(ir); - if (status == drogon::k200OK) { - // start model successfully, in case not vision model, we store the metadata so we can use - // for each inference - if (!json_data.isMember("mmproj") || json_data["mmproj"].isNull()) { - auto metadata_res = GetModelMetadata(model_handle); - if (metadata_res.has_value()) { - loaded_model_metadata_map_.emplace(model_handle, - std::move(metadata_res.value())); - CTL_INF("Successfully stored metadata for model " << model_handle); - } else { - CTL_WRN("Failed to get metadata for model " << model_handle << ": " - << metadata_res.error()); - } - } - + if (status == drogon::k200OK) { return StartModelResult{/* .success = */ true, /* .warning = */ may_fallback_res.value()}; } else if (status == drogon::k409Conflict) { @@ -760,8 +746,6 @@ cpp::result ModelService::StopModel( if (bypass_check) { bypass_stop_check_set_.erase(model_handle); } - loaded_model_metadata_map_.erase(model_handle); - CTL_INF("Removed metadata for model " << model_handle); return true; } else { CTL_ERR("Model failed to stop with status code: " << status); @@ -1090,14 +1074,6 @@ ModelService::GetModelMetadata(const std::string& model_id) const { return std::move(*model_metadata_res); } -std::shared_ptr ModelService::GetCachedModelMetadata( - const std::string& model_id) const { - if (loaded_model_metadata_map_.find(model_id) == - loaded_model_metadata_map_.end()) - return nullptr; - return loaded_model_metadata_map_.at(model_id); -} - std::string ModelService::GetEngineByModelId( const std::string& model_id) const { namespace fs = std::filesystem; diff --git a/engine/services/model_service.h b/engine/services/model_service.h index beba91f8c..fa247b954 100644 --- a/engine/services/model_service.h +++ b/engine/services/model_service.h @@ -83,9 +83,6 @@ class ModelService { cpp::result, std::string> GetModelMetadata( const std::string& model_id) const; - std::shared_ptr GetCachedModelMetadata( - const std::string& model_id) const; - std::string GetEngineByModelId(const std::string& model_id) const; private: @@ -104,12 +101,6 @@ class ModelService { std::unordered_set bypass_stop_check_set_; std::shared_ptr engine_svc_ = nullptr; - /** - * Store the chat template of loaded model. - */ - std::unordered_map> - loaded_model_metadata_map_; - std::mutex es_mtx_; std::unordered_map> es_; cortex::TaskQueue& task_queue_; diff --git a/engine/test/components/test_function_calling.cc b/engine/test/components/test_function_calling.cc deleted file mode 100644 index 7a4810b29..000000000 --- a/engine/test/components/test_function_calling.cc +++ /dev/null @@ -1,157 +0,0 @@ -#include -#include "gtest/gtest.h" -#include "json/json.h" -#include "utils/function_calling/common.h" - -class FunctionCallingUtilsTest : public ::testing::Test { - protected: - std::shared_ptr createTestRequest() { - auto request = std::make_shared(); - (*request)["tools"] = Json::Value(Json::arrayValue); - return request; - } -}; - -TEST_F(FunctionCallingUtilsTest, ReplaceCustomFunctions) { - std::string original = "Test placeholder"; - std::string replacement = "Custom function"; - std::string result = - function_calling_utils::ReplaceCustomFunctions(original, replacement); - EXPECT_EQ(result, "Test Custom function placeholder"); -} - -TEST_F(FunctionCallingUtilsTest, HasTools) { - auto request = createTestRequest(); - EXPECT_FALSE(function_calling_utils::HasTools(request)); - - (*request)["tools"].append(Json::Value()); - EXPECT_TRUE(function_calling_utils::HasTools(request)); - - (*request)["tools"] = "random"; - EXPECT_FALSE(function_calling_utils::HasTools(request)); - - (*request)["tools"] = Json::Value::null; - EXPECT_FALSE(function_calling_utils::HasTools(request)); -} - -TEST_F(FunctionCallingUtilsTest, ProcessTools) { - auto request = createTestRequest(); - Json::Value tool; - tool["type"] = "function"; - tool["function"]["name"] = "test_function"; - tool["function"]["description"] = "Test description"; - (*request)["tools"].append(tool); - - std::string result = function_calling_utils::ProcessTools(request); - EXPECT_TRUE( - result.find("Use the function 'test_function' to: Test description") != - std::string::npos); -} - -TEST_F(FunctionCallingUtilsTest, ParseMultipleFunctionStrings) { - std::string input = - "{\"arg\":\"value1\"}{\"arg\":\"value2\"}"; - Json::Value result = - function_calling_utils::ParseMultipleFunctionStrings(input); - - ASSERT_EQ(result.size(), 2); - EXPECT_EQ(result[0]["function"]["name"].asString(), "func1"); - EXPECT_EQ(result[0]["function"]["arguments"].asString(), - "{\"arg\":\"value1\"}"); - EXPECT_EQ(result[1]["function"]["name"].asString(), "func2"); - EXPECT_EQ(result[1]["function"]["arguments"].asString(), - "{\"arg\":\"value2\"}"); -} - -TEST_F(FunctionCallingUtilsTest, ConvertJsonToFunctionStrings) { - Json::Value jsonArray(Json::arrayValue); - Json::Value function1, function2; - function1["function"]["name"] = "func1"; - function1["function"]["arguments"] = "{\"arg\":\"value1\"}"; - function2["function"]["name"] = "func2"; - function2["function"]["arguments"] = "{\"arg\":\"value2\"}"; - jsonArray.append(function1); - jsonArray.append(function2); - - std::string result = - function_calling_utils::ConvertJsonToFunctionStrings(jsonArray); - EXPECT_EQ(result, - "{\"arg\":\"value1\"}{\"arg\":\"value2\"}"); -} - -TEST_F(FunctionCallingUtilsTest, CreateCustomFunctionsString) { - auto request = createTestRequest(); - Json::Value tool; - tool["type"] = "function"; - tool["function"]["name"] = "test_function"; - tool["function"]["description"] = "Test description"; - (*request)["tools"].append(tool); - - std::string result = - function_calling_utils::CreateCustomFunctionsString(request); - EXPECT_TRUE(result.find("```") != std::string::npos); - EXPECT_TRUE( - result.find("Use the function 'test_function' to: Test description") != - std::string::npos); -} - -TEST_F(FunctionCallingUtilsTest, IsValidToolChoiceFormat) { - Json::Value validTool; - validTool["type"] = "function"; - validTool["function"]["name"] = "test_function"; - EXPECT_TRUE(function_calling_utils::IsValidToolChoiceFormat(validTool)); - - Json::Value invalidTool; - EXPECT_FALSE(function_calling_utils::IsValidToolChoiceFormat(invalidTool)); -} - -TEST_F(FunctionCallingUtilsTest, UpdateMessages) { - auto request = createTestRequest(); - std::string system_prompt = "Original prompt"; - (*request)["messages"] = Json::Value(Json::arrayValue); - - function_calling_utils::UpdateMessages(system_prompt, request); - - ASSERT_TRUE((*request)["messages"].isArray()); - EXPECT_EQ((*request)["messages"][0]["role"].asString(), "system"); - EXPECT_EQ((*request)["messages"][0]["content"].asString(), system_prompt); -} - -TEST_F(FunctionCallingUtilsTest, PreprocessRequest) { - auto request = createTestRequest(); - Json::Value tool; - tool["type"] = "function"; - tool["function"]["name"] = "test_function"; - tool["function"]["description"] = "Test description"; - (*request)["tools"].append(tool); - - function_calling_utils::PreprocessRequest(request); - - ASSERT_TRUE((*request)["messages"].isArray()); - EXPECT_TRUE((*request)["messages"][0]["content"].asString().find( - "Test description") != std::string::npos); -} - -TEST_F(FunctionCallingUtilsTest, PostProcessResponse) { - Json::Value response; - response["choices"] = Json::Value(Json::arrayValue); - Json::Value choice; - choice["message"]["content"] = - "{\"arg\":\"value\"}"; - response["choices"].append(choice); - - function_calling_utils::PostProcessResponse(response); - - EXPECT_EQ(response["choices"][0]["message"]["content"].asString(), ""); - EXPECT_TRUE(response["choices"][0]["message"]["tool_calls"].isArray()); - EXPECT_EQ( - response["choices"][0]["message"]["tool_calls"][0]["function"]["name"] - .asString(), - "test_function"); - EXPECT_EQ(response["choices"][0]["message"]["tool_calls"][0]["function"] - ["arguments"] - .asString(), - "{\"arg\":\"value\"}"); -} \ No newline at end of file diff --git a/engine/utils/cli_selection_utils.h b/engine/utils/cli_selection_utils.h index dca6fe675..487c21e6b 100644 --- a/engine/utils/cli_selection_utils.h +++ b/engine/utils/cli_selection_utils.h @@ -27,13 +27,13 @@ inline void PrintMenu( inline std::optional GetNumericValue(const std::string& sval) { try { - return std::stoi(sval); + return std::stoi(sval); } catch (const std::invalid_argument&) { - // Not a valid number - return std::nullopt; + // Not a valid number + return std::nullopt; } catch (const std::out_of_range&) { - // Number out of range - return std::nullopt; + // Number out of range + return std::nullopt; } } @@ -73,14 +73,16 @@ inline std::optional PrintModelSelection( } // Validate if the selection consists solely of numeric characters - if(!std::all_of(selection.begin(), selection.end(), ::isdigit)){ + if (!std::all_of(selection.begin(), selection.end(), ::isdigit)) { return std::nullopt; } // deal with out of range numeric values std::optional numeric_value = GetNumericValue(selection); - - if (!numeric_value.has_value() || (unsigned) numeric_value.value() > availables.size() || numeric_value.value() < 1) { + + if (!numeric_value.has_value() || + (unsigned)numeric_value.value() > availables.size() || + numeric_value.value() < 1) { return std::nullopt; } @@ -101,13 +103,15 @@ inline std::optional PrintSelection( } // Validate if the selection consists solely of numeric characters - if(!std::all_of(selection.begin(), selection.end(), ::isdigit)){ + if (!std::all_of(selection.begin(), selection.end(), ::isdigit)) { return std::nullopt; } - + // deal with out of range numeric values std::optional numeric_value = GetNumericValue(selection); - if (!numeric_value.has_value() ||(unsigned) numeric_value.value() > options.size() || numeric_value.value() < 1) { + if (!numeric_value.has_value() || + (unsigned)numeric_value.value() > options.size() || + numeric_value.value() < 1) { return std::nullopt; } diff --git a/engine/utils/function_calling/common.h b/engine/utils/function_calling/common.h index 34a1c9862..953a9964c 100644 --- a/engine/utils/function_calling/common.h +++ b/engine/utils/function_calling/common.h @@ -129,157 +129,4 @@ inline Json::Value ParseJsonString(const std::string& jsonString) { return root; } -inline std::string CreateCustomFunctionsString( - std::shared_ptr request) { - std::string customFunctions = ProcessTools(request); - if (customFunctions.empty()) { - return ""; // No custom functions found - } - - return "```\n" + customFunctions + "```"; -} -inline bool IsValidToolChoiceFormat(const Json::Value& root) { - return root.isObject() && root.isMember("type") && root["type"].isString() && - root["type"].asString() == "function" && root.isMember("function") && - root["function"].isObject() && root["function"].isMember("name") && - root["function"]["name"].isString(); -} -inline void UpdateMessages(std::string& system_prompt, - std::shared_ptr request) { - Json::Value tool_choice = request->get("tool_choice", "auto"); - if (tool_choice.isString() && tool_choice.asString() == "required") { - system_prompt += - "\n\nYou must call a function to answer the user's question."; - } else if (!tool_choice.isString()) { - - system_prompt += - "\n\nNow this is your first priority: You must call the function '" + - tool_choice["function"]["name"].asString() + - "' to answer the user's question."; - } - bool parallel_tool_calls = request->get("parallel_tool_calls", true).asBool(); - if (!parallel_tool_calls) { - system_prompt += "\n\nNow this is your first priority: You must call the only one function at a time."; - } - - bool tools_call_in_user_message = - request->get("tools_call_in_user_message", false).asBool(); - - bool original_stream_config = (*request).get("stream", false).asBool(); - // (*request)["grammar"] = function_calling_utils::gamma_json; - (*request)["stream"] = - false; //when using function calling, disable stream automatically because we need to parse the response to get function name and params - - if (!request->isMember("messages") || !(*request)["messages"].isArray() || - (*request)["messages"].empty()) { - // If no messages, add the system prompt as the first message - Json::Value systemMessage; - systemMessage["role"] = "system"; - systemMessage["content"] = system_prompt; - (*request)["messages"].append(systemMessage); - } else { - - if (tools_call_in_user_message) { - for (Json::Value& message : (*request)["messages"]) { - if (message["role"] == "user" && message.isMember("tools") && - message["tools"].isArray() && message["tools"].size() > 0) { - message["content"] = system_prompt + "\n User question: " + - message["content"].asString(); - } - } - } else { - Json::Value& firstMessage = (*request)["messages"][0]; - if (firstMessage["role"] == "system") { - bool addCustomPrompt = - request->get("add_custom_system_prompt", true).asBool(); - if (addCustomPrompt) { - firstMessage["content"] = - system_prompt + "\n" + firstMessage["content"].asString(); - } - } else { - // If the first message is not a system message, prepend the system prompt - Json::Value systemMessage; - systemMessage["role"] = "system"; - systemMessage["content"] = system_prompt; - (*request)["messages"].insert(0, systemMessage); - } - } - - // transform last message role to tool if it is a function call - Json::Value& lastMessage = - (*request)["messages"][(*request)["messages"].size() - 1]; - if (lastMessage.get("role", "") == "tool") { - lastMessage["role"] = function_calling_llama3_1_utils::tool_role; - (*request)["stream"] = - original_stream_config; // if role is tool then should restore stream config to original value - } - } - for (Json::Value& message : (*request)["messages"]) { - if (message["role"] == "assistant" && message.isMember("tool_calls")) { - const Json::Value& tool_calls = message["tool_calls"]; - if (!tool_calls.isNull() && tool_calls.isArray() && - tool_calls.size() > 0) { - message["content"] = ConvertJsonToFunctionStrings(tool_calls); - message["tool_calls"] = {}; - } - } - } -} -inline void PreprocessRequest(std::shared_ptr request) { - if (!function_calling_utils::HasTools(request)) { - return; // Exit if no tools present - } - if (request->get("tool_choice", "auto").isString()) { - std::string tool_choice = request->get("tool_choice", "auto").asString(); - if (tool_choice == "none") { - return; // Exit if tool_choice is none - } - } - std::string customFunctionsString = - function_calling_utils::CreateCustomFunctionsString(request); - std::string new_system_prompt = - function_calling_utils::ReplaceCustomFunctions( - function_calling_llama3_1_utils::system_prompt, - customFunctionsString); - UpdateMessages(new_system_prompt, request); -} - -inline void PostProcessResponse(Json::Value& response) { - if (!response.isMember("choices") || !response["choices"].isArray() || - response["choices"].empty()) { - // If there are no choices or the structure is incorrect, do nothing - return; - } - - // Get a reference to the first choice - Json::Value& firstChoice = response["choices"][0]; - - // Check if the choice has a message with content - if (firstChoice.isMember("message") && - firstChoice["message"].isMember("content")) { - std::string content = firstChoice["message"]["content"].asString(); - - // Create a new structure for tool_calls - Json::Value toolCall = ParseMultipleFunctionStrings(content); - if (toolCall.size() > 0) { - // Add tool_calls to the message - if (response.get("tool_choice", "auto").isString()) { - std::string tool_choice = - response.get("tool_choice", "auto").asString(); - if (tool_choice == "auto") { - firstChoice["finish_reason"] = "tool_calls"; - } else { - firstChoice["finish_reason"] = "stop"; - } - } - - firstChoice["message"]["tool_calls"] = toolCall; - - // Clear the content as it's now represented in tool_calls - firstChoice["message"]["content"] = ""; - } - } - - // Add any additional post-processing logic here -} } // namespace function_calling_utils diff --git a/function-calling.py b/function-calling.py new file mode 100644 index 000000000..32ef31752 --- /dev/null +++ b/function-calling.py @@ -0,0 +1,173 @@ +from datetime import datetime +from openai import OpenAI +from pydantic import BaseModel +import json + +# MODEL = "deepseek-r1-distill-qwen-7b:7b" +MODEL = "llama3.1:8b-q8" + +client = OpenAI( + base_url="http://localhost:39281/v1", + api_key="not-needed", # Authentication is not required for local deployment +) + +tools = [ + { + "type": "function", + "function": { + "name": "puppeteer_navigate", + "description": "Navigate to a URL", + "parameters": { + "properties": {"url": {"type": "string"}}, + "required": ["url"], + "type": "object", + }, + "strict": False, + }, + }, + { + "type": "function", + "function": { + "name": "puppeteer_screenshot", + "description": "Take a screenshot of the current page or a specific element", + "parameters": { + "properties": { + "height": { + "description": "Height in pixels (default: 600)", + "type": "number", + }, + "name": { + "description": "Name for the screenshot", + "type": "string", + }, + "selector": { + "description": "CSS selector for element to screenshot", + "type": "string", + }, + "width": { + "description": "Width in pixels (default: 800)", + "type": "number", + }, + }, + "required": ["name"], + "type": "object", + }, + "strict": False, + }, + }, + { + "type": "function", + "function": { + "name": "puppeteer_click", + "description": "Click an element on the page", + "parameters": { + "properties": { + "selector": { + "description": "CSS selector for element to click", + "type": "string", + } + }, + "required": ["selector"], + "type": "object", + }, + "strict": False, + }, + }, + { + "type": "function", + "function": { + "name": "puppeteer_fill", + "description": "Fill out an input field", + "parameters": { + "properties": { + "selector": { + "description": "CSS selector for input field", + "type": "string", + }, + "value": {"description": "Value to fill", "type": "string"}, + }, + "required": ["selector", "value"], + "type": "object", + }, + "strict": False, + }, + }, + { + "type": "function", + "function": { + "name": "puppeteer_select", + "description": "Select an element on the page with Select tag", + "parameters": { + "properties": { + "selector": { + "description": "CSS selector for element to select", + "type": "string", + }, + "value": {"description": "Value to select", "type": "string"}, + }, + "required": ["selector", "value"], + "type": "object", + }, + "strict": False, + }, + }, + { + "type": "function", + "function": { + "name": "puppeteer_hover", + "description": "Hover an element on the page", + "parameters": { + "properties": { + "selector": { + "description": "CSS selector for element to hover", + "type": "string", + } + }, + "required": ["selector"], + "type": "object", + }, + "strict": False, + }, + }, + { + "type": "function", + "function": { + "name": "puppeteer_evaluate", + "description": "Execute JavaScript in the browser console", + "parameters": { + "properties": { + "script": { + "description": "JavaScript code to execute", + "type": "string", + } + }, + "required": ["script"], + "type": "object", + }, + "strict": False, + }, + }, +] + +completion_payload = { + "messages": [ + { + "role": "system", + "content": 'You have access to the following CUSTOM functions:\n\n\n\nIf a you choose to call a function ONLY reply in the following format:\n<{start_tag}={function_name}>{parameters}{end_tag}\nwhere\n\nstart_tag => ` a JSON dict with the function argument name as key and function argument value as value.\nend_tag => ``\n\nHere is an example,\n{"example_name": "example_value"}\n\nReminder:\n- Function calls MUST follow the specified format\n- Required parameters MUST be specified\n- You can call one or more functions at a time, but remember only chose correct function\n- Put the entire function call reply on one line\n- Always add your sources when using search results to answer the user query\n- If you can not find correct parameters or arguments corresponding to function in the user\'s message, ask user again to provide, do not make assumptions.\n- No explanation are needed when calling a function.\n\nYou are a helpful assistant.', + }, + { + "role": "user", + "content": "go to google search", + }, + ] +} + +response = client.chat.completions.create( + top_p=0.9, + temperature=0.6, + model=MODEL, + messages=completion_payload["messages"], + tools=tools, +) + +print(response) \ No newline at end of file