From 373c05b079edace97450af0667e1428f3b448908 Mon Sep 17 00:00:00 2001 From: Randy Mackay Date: Fri, 15 Dec 2023 19:35:40 +0900 Subject: [PATCH 1/5] chat: improve status reporting --- MAVProxy/modules/mavproxy_chat/chat_openai.py | 15 +++++++++++++-- MAVProxy/modules/mavproxy_chat/chat_window.py | 12 +++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/MAVProxy/modules/mavproxy_chat/chat_openai.py b/MAVProxy/modules/mavproxy_chat/chat_openai.py index c276acf899..5f82be2c13 100644 --- a/MAVProxy/modules/mavproxy_chat/chat_openai.py +++ b/MAVProxy/modules/mavproxy_chat/chat_openai.py @@ -20,10 +20,13 @@ exit() class chat_openai(): - def __init__(self, mpstate): + def __init__(self, mpstate, status_cb=None): # keep reference to mpstate self.mpstate = mpstate + # keep reference to status callback + self.status_cb = status_cb + # initialise OpenAI connection self.client = None self.assistant = None @@ -123,9 +126,12 @@ def send_to_assistant(self, text): self.handle_function_call(latest_run) run_done = False else: - print ("chat: unrecognised run status" + latest_run.status) + print("chat: unrecognised run status" + latest_run.status) run_done = True + # send status to status callback + self.send_status(latest_run.status) + # retrieve messages on the thread reply_messages = self.client.beta.threads.messages.list(self.assistant_thread.id, order = "asc", after=input_message.id) if reply_messages is None: @@ -381,3 +387,8 @@ def wrap_longitude(self, longitude_deg): if longitude_deg < -180: return longitude_deg + 360 return longitude_deg + + # send status to chat window via callback + def send_status(self, status): + if self.status_cb is not None: + self.status_cb(status) \ No newline at end of file diff --git a/MAVProxy/modules/mavproxy_chat/chat_window.py b/MAVProxy/modules/mavproxy_chat/chat_window.py index 086bc0970d..a6f64bb066 100644 --- a/MAVProxy/modules/mavproxy_chat/chat_window.py +++ b/MAVProxy/modules/mavproxy_chat/chat_window.py @@ -18,7 +18,7 @@ def __init__(self, mpstate): self.send_lock = Lock() # create chat_openai object - self.chat_openai = chat_openai.chat_openai(self.mpstate) + self.chat_openai = chat_openai.chat_openai(self.mpstate, self.set_status_text) # create chat_voice_to_text object self.chat_voice_to_text = chat_voice_to_text.chat_voice_to_text() @@ -65,9 +65,13 @@ def __init__(self, mpstate): # add a reply box and read-only text box self.text_reply = wx.TextCtrl(self.frame, id=-1, size=(600, 80), style=wx.TE_READONLY | wx.TE_MULTILINE) + # add a read-only status text box at the bottom + self.text_status = wx.TextCtrl(self.frame, id=-1, size=(600, -1), style=wx.TE_READONLY) + # set size hints and add sizer to frame self.vert_sizer.Add(self.horiz_sizer, proportion=0, flag=wx.EXPAND) self.vert_sizer.Add(self.text_reply, proportion=1, flag=wx.EXPAND, border=5) + self.vert_sizer.Add(self.text_status, proportion=0, flag=wx.EXPAND, border=5) self.frame.SetSizer(self.vert_sizer) # show frame @@ -114,12 +118,14 @@ def record_button_click_execute(self, event): rec_filename = self.chat_voice_to_text.record_audio() if rec_filename is None: print("chat: audio recording failed") + self.set_status_text("Audio recording failed") return # convert audio to text and place in input box text = self.chat_voice_to_text.convert_audio_to_text(rec_filename) if text is None: print("chat: audio to text conversion failed") + self.set_status_text("Audio to text conversion failed") return wx.CallAfter(self.text_input.SetValue, text) @@ -164,3 +170,7 @@ def send_text_to_assistant(self): wx.CallAfter(self.record_button.Enable) wx.CallAfter(self.text_input.Enable) wx.CallAfter(self.send_button.Enable) + + # set status text + def set_status_text(self, text): + wx.CallAfter(self.text_status.SetValue, text) From cfe0d8edd0aafd17623e8f39ae6992e70e55c6c6 Mon Sep 17 00:00:00 2001 From: Randy Mackay Date: Fri, 15 Dec 2023 21:11:22 +0900 Subject: [PATCH 2/5] chat: assistant function defs for get_mavlink_messages --- .../get_available_mavlink_messages.json | 12 ++++++++++++ .../assistant_setup/get_mavlink_message.json | 14 ++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 MAVProxy/modules/mavproxy_chat/assistant_setup/get_available_mavlink_messages.json create mode 100644 MAVProxy/modules/mavproxy_chat/assistant_setup/get_mavlink_message.json diff --git a/MAVProxy/modules/mavproxy_chat/assistant_setup/get_available_mavlink_messages.json b/MAVProxy/modules/mavproxy_chat/assistant_setup/get_available_mavlink_messages.json new file mode 100644 index 0000000000..8c97608311 --- /dev/null +++ b/MAVProxy/modules/mavproxy_chat/assistant_setup/get_available_mavlink_messages.json @@ -0,0 +1,12 @@ +{ + "type": "function", + "function": { + "name": "get_available_mavlink_messages", + "description": "Get a list of mavlink message names that can be retrieved using the get_mavlink_message function", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } +} diff --git a/MAVProxy/modules/mavproxy_chat/assistant_setup/get_mavlink_message.json b/MAVProxy/modules/mavproxy_chat/assistant_setup/get_mavlink_message.json new file mode 100644 index 0000000000..eb3817ee1e --- /dev/null +++ b/MAVProxy/modules/mavproxy_chat/assistant_setup/get_mavlink_message.json @@ -0,0 +1,14 @@ +{ + "type": "function", + "function": { + "name": "get_mavlink_message", + "description": "Get a mavlink message including all fields and values sent by the vehicle. The list of available messages can be retrieved using the get_available_mavlink_messages", + "parameters": { + "type": "object", + "properties": { + "message": {"type": "string", "description": "mavlink message name (e.g. HEARTBEAT, VFR_HUD, GLOBAL_POSITION_INT, etc)"} + }, + "required": ["message"] + } + } +} From 7601fb0e09d8f0eb988a230e2e822eb9c8656419 Mon Sep 17 00:00:00 2001 From: Randy Mackay Date: Fri, 15 Dec 2023 21:14:03 +0900 Subject: [PATCH 3/5] chat: support retrieving any mavlink message from vehicle --- MAVProxy/modules/mavproxy_chat/chat_openai.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/MAVProxy/modules/mavproxy_chat/chat_openai.py b/MAVProxy/modules/mavproxy_chat/chat_openai.py index 5f82be2c13..09bc0adceb 100644 --- a/MAVProxy/modules/mavproxy_chat/chat_openai.py +++ b/MAVProxy/modules/mavproxy_chat/chat_openai.py @@ -222,6 +222,22 @@ def handle_function_call(self, run): except: print("chat: send_mavlink_set_position_target_global_int: failed to parse arguments") + + # get a list of mavlink message names that can be retrieved using the get_mavlink_message function + if tool_call.function.name == "get_available_mavlink_messages": + recognised_function = True + output = self.get_available_mavlink_messages() + + # get mavlink message from vehicle + if tool_call.function.name == "get_mavlink_message": + recognised_function = True + try: + arguments = json.loads(tool_call.function.arguments) + output = self.get_mavlink_message(arguments) + except: + output = "get_mavlink_message: failed to retrieve message" + print("chat: get_mavlink_message: failed to retrieve message") + if not recognised_function: print("chat: handle_function_call: unrecognised function call: " + tool_call.function.name) output = "unrecognised function call: " + tool_call.function.name @@ -372,6 +388,46 @@ def send_mavlink_set_position_target_global_int(self, arguments): self.mpstate.master().mav.set_position_target_global_int_send(time_boot_ms, target_system, target_component, coordinate_frame, type_mask, lat_int, lon_int, alt, vx, vy, vz, afx, afy, afz, yaw, yaw_rate) return "set_position_target_global_int sent" + # get a list of mavlink message names that can be retrieved using the get_mavlink_message function + def get_available_mavlink_messages(self): + # check if no messages available + if self.mpstate.master().messages is None or len(self.mpstate.master().messages) == 0: + return "get_available_mavlink_messages: no messages available" + + # retrieve each available message's name + mav_msg_names = [] + for msg in self.mpstate.master().messages: + # append all message names except MAV + if msg != "MAV": + mav_msg_names.append(msg) + + # return list of message names + try: + return json.dumps(mav_msg_names) + except: + return "get_available_mavlink_messages: failed to convert message name list to json" + + # get a mavlink message including all fields and values sent by the vehicle + def get_mavlink_message(self, arguments): + if arguments is None: + return "get_mavlink_message: arguments is None" + + # retrieve requested message's name + mav_msg_name = arguments.get("message", None) + if mav_msg_name is None: + return "get_mavlink_message: message not specified" + + # retrieve message + mav_msg = self.mpstate.master().messages.get(mav_msg_name, None) + if mav_msg is None: + return "get_mavlink_message: message not found" + + # convert message to json + try: + return json.dumps(mav_msg.to_dict()) + except: + return "get_mavlink_message: failed to convert message to json" + # wrap latitude to range -90 to 90 def wrap_latitude(self, latitude_deg): if latitude_deg > 90: From e66a223df6a65280c3422d2a951dc57ba03fa14d Mon Sep 17 00:00:00 2001 From: Randy Mackay Date: Sun, 17 Dec 2023 16:01:49 +0900 Subject: [PATCH 4/5] chat: assistant support for get and set params includes get and set parameter function definitions json files assistant setup script downloads/uploads parameter definition files updated assistant instructions on how to use the parameter definition files --- .../assistant_setup/assistant_instructions.txt | 4 ++++ .../assistant_setup/get_all_parameters.json | 12 ++++++++++++ .../assistant_setup/get_parameter.json | 15 +++++++++++++++ .../assistant_setup/set_parameter.json | 15 +++++++++++++++ .../assistant_setup/setup_assistant.py | 18 +++++++++++++++--- 5 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 MAVProxy/modules/mavproxy_chat/assistant_setup/get_all_parameters.json create mode 100644 MAVProxy/modules/mavproxy_chat/assistant_setup/get_parameter.json create mode 100644 MAVProxy/modules/mavproxy_chat/assistant_setup/set_parameter.json diff --git a/MAVProxy/modules/mavproxy_chat/assistant_setup/assistant_instructions.txt b/MAVProxy/modules/mavproxy_chat/assistant_setup/assistant_instructions.txt index 81da1d5923..5cef7e778f 100644 --- a/MAVProxy/modules/mavproxy_chat/assistant_setup/assistant_instructions.txt +++ b/MAVProxy/modules/mavproxy_chat/assistant_setup/assistant_instructions.txt @@ -34,3 +34,7 @@ The short form of "longitude" is "lat". The words "position" and "location" are often used synonymously. Rovers and Boats cannot control their altitude + +Parameters on the vehicle hold many settings that affect how the vehicle behaves. When responding to users requests to get or set parameter values be sure to check the vehicle specific parameter definition files (e.g. copter_parameter_definitions.xml, plane_parameter_definitions.xml, rover_parameter_definitions.xml, sub_parameter_definitions.xml) to ensure the correct units are used. + +Before any action is taken to set or get vehicle parameters, be sure you know the vehicle type. The easiest way to do this may be to call the get_vehicle_type function. Once you know the vehicle type, the vehicle specific parameter definition file must be accessed and read to confirm the correct parameter names and the expected data types and units. For copters refer to the copter_parameter_definitions.xml file, for planes refer to plane_parameter_definitions.xml, for rovers and boats refer to rover_parameter_definitions.xml, and for subs (aka submarines) refer to sub_parameter_definitions.xml. If the file cannot be found or accessed, please report to the user that the parameter definitions file is required before proceeding. Once the file is accessed, utilize the parameter information within it to validate parameter names and units against any user request for setting or getting vehicle parameter values. Perform the requested action (set or get) only if the parameter definitions have been successfully verified to match the request. diff --git a/MAVProxy/modules/mavproxy_chat/assistant_setup/get_all_parameters.json b/MAVProxy/modules/mavproxy_chat/assistant_setup/get_all_parameters.json new file mode 100644 index 0000000000..283e24fcb8 --- /dev/null +++ b/MAVProxy/modules/mavproxy_chat/assistant_setup/get_all_parameters.json @@ -0,0 +1,12 @@ +{ + "type": "function", + "function": { + "name": "get_all_parameters", + "description": "Get all available parameter names and values", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } +} diff --git a/MAVProxy/modules/mavproxy_chat/assistant_setup/get_parameter.json b/MAVProxy/modules/mavproxy_chat/assistant_setup/get_parameter.json new file mode 100644 index 0000000000..a5a536d1b5 --- /dev/null +++ b/MAVProxy/modules/mavproxy_chat/assistant_setup/get_parameter.json @@ -0,0 +1,15 @@ +{ + "type": "function", + "function": { + "name": "get_parameter", + "description": "Get a vehicle parameter's value. The full list of available parameters and their values is available using the get_all_parameters function", + "parameters": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "parameter name (e.g. ARMING_CHECK, LOG_BITMASK). Regex expressions are supported"}, + "value": {"type": "number", "description": "parameter value"} + }, + "required": ["name"] + } + } +} diff --git a/MAVProxy/modules/mavproxy_chat/assistant_setup/set_parameter.json b/MAVProxy/modules/mavproxy_chat/assistant_setup/set_parameter.json new file mode 100644 index 0000000000..269833c565 --- /dev/null +++ b/MAVProxy/modules/mavproxy_chat/assistant_setup/set_parameter.json @@ -0,0 +1,15 @@ +{ + "type": "function", + "function": { + "name": "set_parameter", + "description": "Set a vehicle parameter's value. The full list of parameters is available using the get_all_parameters function", + "parameters": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "parameter name (e.g. ARMING_CHECK, LOG_BITMASK)"}, + "value": {"type": "number", "description": "parameter value"} + }, + "required": ["name", "value"] + } + } +} diff --git a/MAVProxy/modules/mavproxy_chat/assistant_setup/setup_assistant.py b/MAVProxy/modules/mavproxy_chat/assistant_setup/setup_assistant.py index c8a564486c..6a5d5502fb 100644 --- a/MAVProxy/modules/mavproxy_chat/assistant_setup/setup_assistant.py +++ b/MAVProxy/modules/mavproxy_chat/assistant_setup/setup_assistant.py @@ -87,6 +87,18 @@ def main(openai_api_key=None, assistant_name=None, model_name=None, upgrade=Fals if not download_file("https://raw.githubusercontent.com/ArduPilot/mavlink/master/message_definitions/v1.0/" + mavlink_filename, mavlink_filename): exit() + # download latest vehicle parameter definition files from ardupilot server + paramdef_file_info = [ + {"url": "https://autotest.ardupilot.org/Parameters/ArduCopter/apm.pdef.xml", "filename": "copter_parameter_definitions.xml"}, + {"url": "https://autotest.ardupilot.org/Parameters/ArduPlane/apm.pdef.xml", "filename": "plane_parameter_definitions.xml"}, + {"url": "https://autotest.ardupilot.org/Parameters/APMrover2/apm.pdef.xml", "filename": "rover_parameter_definitions.xml"}, + {"url": "https://autotest.ardupilot.org/Parameters/ArduSub/apm.pdef.xml", "filename": "sub_parameter_definitions.xml"}] + paramdef_filenames = [] + for pdef_file_info in paramdef_file_info: + if not download_file(pdef_file_info["url"], pdef_file_info["filename"]): + exit() + paramdef_filenames.append(pdef_file_info["filename"]) + # variable to hold new assistant assistant = None @@ -116,11 +128,11 @@ def main(openai_api_key=None, assistant_name=None, model_name=None, upgrade=Fals print("setup_assistant: failed to update assistant instructions") exit() - # upload MAVLink and text files + # upload MAVLink, text and parameter definition files # get our organisation's existing list of files on OpenAI existing_files = client.files.list() uploaded_file_ids = [] - for filename in text_filenames + mavlink_filenames: + for filename in text_filenames + mavlink_filenames + paramdef_filenames: try: # open local file as read-only file = open(filename, 'rb') @@ -156,7 +168,7 @@ def main(openai_api_key=None, assistant_name=None, model_name=None, upgrade=Fals exit() # delete downloaded mavlink files - for mavlink_filename in mavlink_filenames: + for mavlink_filename in mavlink_filenames + paramdef_filenames: try: os.remove(mavlink_filename) print("setup_assistant: deleted local file: " + mavlink_filename) From 038d3b278d2ef4e074b143d1886d12c71e1f1f19 Mon Sep 17 00:00:00 2001 From: Randy Mackay Date: Sun, 17 Dec 2023 13:06:51 +0900 Subject: [PATCH 5/5] chat: support get and set parameters --- MAVProxy/modules/mavproxy_chat/chat_openai.py | 98 ++++++++++++++++++- 1 file changed, 96 insertions(+), 2 deletions(-) diff --git a/MAVProxy/modules/mavproxy_chat/chat_openai.py b/MAVProxy/modules/mavproxy_chat/chat_openai.py index 09bc0adceb..d3b0f8c4ae 100644 --- a/MAVProxy/modules/mavproxy_chat/chat_openai.py +++ b/MAVProxy/modules/mavproxy_chat/chat_openai.py @@ -8,7 +8,7 @@ ''' from pymavlink import mavutil -import time +import time, re from datetime import datetime import json import math @@ -238,6 +238,36 @@ def handle_function_call(self, run): output = "get_mavlink_message: failed to retrieve message" print("chat: get_mavlink_message: failed to retrieve message") + # get all parameters from vehicle + if tool_call.function.name == "get_all_parameters": + recognised_function = True + try: + arguments = json.loads(tool_call.function.arguments) + output = self.get_all_parameters(arguments) + except: + output = "get_all_parameters: failed to retrieve parameters" + print("chat: get_all_parameters: failed to retrieve parameters") + + # get a vehicle parameter's value + if tool_call.function.name == "get_parameter": + recognised_function = True + try: + arguments = json.loads(tool_call.function.arguments) + output = self.get_parameter(arguments) + except: + output = "get_parameter: failed to retrieve parameter value" + print("chat: get_parameters: failed to retrieve parameter value") + + # set a vehicle parameter's value + if tool_call.function.name == "set_parameter": + recognised_function = True + try: + arguments = json.loads(tool_call.function.arguments) + output = self.set_parameter(arguments) + except: + output = "set_parameter: failed to set parameter value" + print("chat: set_parameter: failed to set parameter value") + if not recognised_function: print("chat: handle_function_call: unrecognised function call: " + tool_call.function.name) output = "unrecognised function call: " + tool_call.function.name @@ -428,6 +458,62 @@ def get_mavlink_message(self, arguments): except: return "get_mavlink_message: failed to convert message to json" + # get all available parameters names and their values + def get_all_parameters(self, arguments): + # check if any parameters are available + if self.mpstate.mav_param is None or len(self.mpstate.mav_param) == 0: + return "get_all_parameters: no parameters are available" + param_list = {} + for param_name in sorted(self.mpstate.mav_param.keys()): + param_list[param_name] = self.mpstate.mav_param.get(param_name) + try: + return json.dumps(param_list) + except: + return "get_all_parameters: failed to convert parameter list to json" + + # get a vehicle parameter's value + def get_parameter(self, arguments): + param_name = arguments.get("name", None) + if param_name is None: + print("get_parameter: name not specified") + return "get_parameter: name not specified" + + # start with empty parameter list + param_list = {} + + # handle param name containing regex + if self.contains_regex(param_name): + pattern = re.compile(param_name) + for existing_param_name in sorted(self.mpstate.mav_param.keys()): + if pattern.match(existing_param_name) is not None: + param_value = self.mpstate.functions.get_mav_param(existing_param_name, None) + if param_value is None: + print("chat: get_parameter unable to get " + existing_param_name) + else: + param_list[existing_param_name] = param_value + else: + # handle simple case of a single parameter name + param_value = self.mpstate.functions.get_mav_param(param_name, None) + if param_value is None: + return "get_parameter: " + param_name + " parameter not found" + param_list[param_name] = param_value + + try: + return json.dumps(param_list) + except: + return "get_parameter: failed to convert parameter list to json" + + # set a vehicle parameter's value + def set_parameter(self, arguments): + param_name = arguments.get("name", None) + if param_name is None: + return "set_parameter: parameter name not specified" + param_value = arguments.get("value", None) + if param_value is None: + return "set_parameter: value not specified" + self.mpstate.functions.param_set(param_name, param_value, retries=3) + return "set_parameter: parameter value set" + # wrap latitude to range -90 to 90 def wrap_latitude(self, latitude_deg): if latitude_deg > 90: @@ -447,4 +533,12 @@ def wrap_longitude(self, longitude_deg): # send status to chat window via callback def send_status(self, status): if self.status_cb is not None: - self.status_cb(status) \ No newline at end of file + self.status_cb(status) + + # returns true if string contains regex characters + def contains_regex(self, string): + regex_characters = ".^$*+?{}[]\|()" + for x in regex_characters: + if string.count(x): + return True + return False