From 75e7297704dd864d456b75a5414224aab9642ddf Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Wed, 6 Nov 2024 16:48:44 +0100 Subject: [PATCH 01/19] Debug wrapper WIP --- build/docker-compose.leaderboard.yaml | 6 +- code/debug_wrapper.py | 123 +++++++++++++++++++++++ code/perception/launch/perception.launch | 2 +- code/perception/src/debug_wrapper.py | 1 + code/perception/src/lidar_distance.py | 20 +++- 5 files changed, 148 insertions(+), 4 deletions(-) create mode 100755 code/debug_wrapper.py create mode 120000 code/perception/src/debug_wrapper.py diff --git a/build/docker-compose.leaderboard.yaml b/build/docker-compose.leaderboard.yaml index 32fc98fc..126f3d09 100644 --- a/build/docker-compose.leaderboard.yaml +++ b/build/docker-compose.leaderboard.yaml @@ -8,4 +8,8 @@ services: extends: file: agent_service.yaml service: agent - command: bash -c "sleep 10 && sudo chown -R ${USER_UID}:${USER_GID} ../ && sudo chmod -R a+w ../ && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=$${ROUTE} --agent=/workspace/code/agent/src/agent/agent.py --host=$${CARLA_SIM_HOST} --track=MAP" + command: |- + bash -c "sleep 10 && sudo chown -R ${USER_UID}:${USER_GID} ../ && \ + sudo chmod -R a+w ../ && sudo mkdir -p $${XDG_RUNTIME_DIR} && sudo chmod 0700 $${XDG_RUNTIME_DIR} && sudo chown -R ${USER_UID}:${USER_GID} $${XDG_RUNTIME_DIR} && \ + (rqt_console &) && disown -a && \ + python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=$${ROUTE} --agent=/workspace/code/agent/src/agent/agent.py --host=$${CARLA_SIM_HOST} --track=MAP" diff --git a/code/debug_wrapper.py b/code/debug_wrapper.py new file mode 100755 index 00000000..f1cecd9f --- /dev/null +++ b/code/debug_wrapper.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +import importlib +import importlib.util +import os +import sys +import argparse + +import rospy + +DEBUG_WRAPPER_MSG_PREFIX = "[debug_wrapper]:" +TYPE_PARAM = "~debug_type" +PORT_PARAM = "~debug_port" +WAIT_PARAM = "~debug_wait" + + +def eprint(*args, **kwargs): + """Print to stderr""" + print(*args, file=sys.stderr, **kwargs) + + +def logfatal(msg: str): + """Only works after ros node has been initialized""" + rospy.logfatal( + f"""{DEBUG_WRAPPER_MSG_PREFIX} FAILED TO START NODE - NODE WILL NOT SHOW UP: + {msg}""" + ) + + +def logerr(msg: str): + """Only works after ros node has been initialized""" + rospy.logerr(f"{DEBUG_WRAPPER_MSG_PREFIX} {msg}") + + +def loginfo(msg: str): + """Only works after ros node has been initialized""" + rospy.loginfo(f"{DEBUG_WRAPPER_MSG_PREFIX} {msg}") + + +def logdebug(msg: str): + """Only works after ros node has been initialized""" + print(f"{DEBUG_WRAPPER_MSG_PREFIX} {msg}") + + +def get_required_params(): + try: + result = {} + if not rospy.has_param(TYPE_PARAM): + logfatal( + """Missing parameter to start debug wrapper: debug_type + Add it in the launch configuration""" + ) + result["type"] = str(rospy.get_param(TYPE_PARAM)) + if rospy.has_param(PORT_PARAM): + result["port"] = int(rospy.get_param(PORT_PARAM)) + else: + logerr( + """Missing parameter to start debugger: debug_port + Add it in the launch configuration""" + ) + result["port"] = None + result["wait"] = bool(rospy.get_param(WAIT_PARAM, False)) + return result + except BaseException as error: + logerr(f"Failed to get required node parameters: {error}") + raise error + + +def run_node_at(path: str): + try: + runpy.run_path(path, run_name="__main__") + except BaseException as error: + logfatal( + f"""Failed to run node module at {path}: + {error}""" + ) + raise error + + +def start_debugger(port: int, node_module_name: str, wait_for_debugee: bool = False): + debugger_spec = importlib.util.find_spec("debugpy") + if debugger_spec is not None: + try: + import debugpy + + debugpy.listen(("localhost", port)) + loginfo(f"Started debugger on port {port} for {node_module_name}") + if wait_for_debugee: + debugpy.wait_for_client() + except BaseException as error: + # Yes, all exceptions should be catched and sent into rosconsole + logerr(f"Failed to start debugger: {error}") + else: + logerr("debugpy module not found. Unable to start debugger") + + +def main(argv): + """# http://wiki.ros.org/Nodes: 7. Special keys -- __name is the node's name. + # In this case the name is set by the launch file + # Todo: when using rosrun, the name also has to be set somehow + ros_node_name: str = __name + # We have to init the node to be able to log and access parameters + rospy.init_node(name=ros_node_name)""" + node_args = rospy.myargv(argv=sys.argv) + parser = argparse.ArgumentParser( + prog="debug wrapper", + ) + parser.add_argument("--debug_node") + parser.add_argument("--debug_port") + parser.add_argument("--debug_wait") + args, unknown = parser.parse_known_args(node_args) + + base_dir = os.path.abspath(os.path.dirname(__file__)) + logdebug(f"Node {ros_node_name} started at {base_dir}") + params = get_required_params() + if params["port"] is not None: + start_debugger(params["port"], params["type"], wait_for_debugee=params["wait"]) + target_type_path = os.path.join(base_dir, params["type"]) + run_node_at(target_type_path) + + +if __name__ == "__main__": + main(argv=sys.argv) diff --git a/code/perception/launch/perception.launch b/code/perception/launch/perception.launch index 8d6072e7..f194174c 100644 --- a/code/perception/launch/perception.launch +++ b/code/perception/launch/perception.launch @@ -75,7 +75,7 @@ - + diff --git a/code/perception/src/debug_wrapper.py b/code/perception/src/debug_wrapper.py new file mode 120000 index 00000000..e4e2c378 --- /dev/null +++ b/code/perception/src/debug_wrapper.py @@ -0,0 +1 @@ +../../debug_wrapper.py \ No newline at end of file diff --git a/code/perception/src/lidar_distance.py b/code/perception/src/lidar_distance.py index 04394ee2..93ab1bc2 100755 --- a/code/perception/src/lidar_distance.py +++ b/code/perception/src/lidar_distance.py @@ -120,7 +120,6 @@ def listener(self): Initializes the node and it's publishers """ # run simultaneously. - rospy.init_node("lidar_distance") self.bridge = CvBridge() self.pub_pointcloud = rospy.Publisher( @@ -246,6 +245,23 @@ def reconstruct_img_from_lidar(self, coordinates_xyz, focus): return dist_array -if __name__ == "__main__": +def ros_init(): + """Initializes the node for basic ROS functions. + + Must only be called ONCE and not as part of def main() + + Required for debugger entry""" + rospy.init_node("lidar_distance") + + +def main(): + """Main entry point of this node + + Required for debugger entry""" lidar_distance = LidarDistance() lidar_distance.listener() + + +if __name__ == "__main__": + rospy.init_node("lidar_distance") + main() From 4ef21fcc0fd686663ec395a781a949e4cfd9debc Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Thu, 7 Nov 2024 03:16:15 +0100 Subject: [PATCH 02/19] Basic debug wrapper working --- .vscode/launch.json | 23 +++++ code/debug_wrapper.py | 123 ++++++++++++----------- code/perception/launch/perception.launch | 2 +- code/perception/src/lidar_distance.py | 6 +- pyrightconfig.json | 25 +++++ 5 files changed, 114 insertions(+), 65 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 pyrightconfig.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..e7ed6ee0 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,23 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Remote Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}/code", + "remoteRoot": "/internal_workspace/catkin_ws/src" + } + ] + } + ] +} \ No newline at end of file diff --git a/code/debug_wrapper.py b/code/debug_wrapper.py index f1cecd9f..88486091 100755 --- a/code/debug_wrapper.py +++ b/code/debug_wrapper.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import importlib import importlib.util import os import sys @@ -14,9 +13,13 @@ WAIT_PARAM = "~debug_wait" -def eprint(*args, **kwargs): +def printdebug(msg: str): + print(f"{DEBUG_WRAPPER_MSG_PREFIX} {msg}") + + +def printerror(msg: str): """Print to stderr""" - print(*args, file=sys.stderr, **kwargs) + print(f"{DEBUG_WRAPPER_MSG_PREFIX} {msg}", file=sys.stderr) def logfatal(msg: str): @@ -37,47 +40,25 @@ def loginfo(msg: str): rospy.loginfo(f"{DEBUG_WRAPPER_MSG_PREFIX} {msg}") -def logdebug(msg: str): - """Only works after ros node has been initialized""" - print(f"{DEBUG_WRAPPER_MSG_PREFIX} {msg}") - - -def get_required_params(): - try: - result = {} - if not rospy.has_param(TYPE_PARAM): - logfatal( - """Missing parameter to start debug wrapper: debug_type - Add it in the launch configuration""" - ) - result["type"] = str(rospy.get_param(TYPE_PARAM)) - if rospy.has_param(PORT_PARAM): - result["port"] = int(rospy.get_param(PORT_PARAM)) - else: - logerr( - """Missing parameter to start debugger: debug_port - Add it in the launch configuration""" - ) - result["port"] = None - result["wait"] = bool(rospy.get_param(WAIT_PARAM, False)) - return result - except BaseException as error: - logerr(f"Failed to get required node parameters: {error}") - raise error - - -def run_node_at(path: str): - try: - runpy.run_path(path, run_name="__main__") - except BaseException as error: - logfatal( - f"""Failed to run node module at {path}: - {error}""" - ) - raise error +def import_module_at(path: str): + basename = os.path.basename(path) + module_dir = os.path.dirname(path) + module_name = os.path.splitext(basename)[0] + # Based on https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly + """ spec = importlib.util.spec_from_file_location( + name=module_name, location=path, submodule_search_locations=[module_dir] + ) + if spec is None: + raise Exception(f"Failed to load {path} as module {module_name}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) """ + sys.path.append(module_dir) + module = importlib.import_module(module_name) + return module -def start_debugger(port: int, node_module_name: str, wait_for_debugee: bool = False): +def start_debugger(port: int, node_module_name: str, wait_for_client: bool = False): debugger_spec = importlib.util.find_spec("debugpy") if debugger_spec is not None: try: @@ -85,7 +66,7 @@ def start_debugger(port: int, node_module_name: str, wait_for_debugee: bool = Fa debugpy.listen(("localhost", port)) loginfo(f"Started debugger on port {port} for {node_module_name}") - if wait_for_debugee: + if wait_for_client: debugpy.wait_for_client() except BaseException as error: # Yes, all exceptions should be catched and sent into rosconsole @@ -94,29 +75,49 @@ def start_debugger(port: int, node_module_name: str, wait_for_debugee: bool = Fa logerr("debugpy module not found. Unable to start debugger") +class ArgumentParserError(Exception): + pass + + +class ThrowingArgumentParser(argparse.ArgumentParser): + def error(self, message): + raise ArgumentParserError(message) + + def main(argv): - """# http://wiki.ros.org/Nodes: 7. Special keys -- __name is the node's name. - # In this case the name is set by the launch file - # Todo: when using rosrun, the name also has to be set somehow - ros_node_name: str = __name - # We have to init the node to be able to log and access parameters - rospy.init_node(name=ros_node_name)""" - node_args = rospy.myargv(argv=sys.argv) - parser = argparse.ArgumentParser( + node_args = rospy.myargv(argv=argv) + parser = ThrowingArgumentParser( prog="debug wrapper", ) - parser.add_argument("--debug_node") - parser.add_argument("--debug_port") - parser.add_argument("--debug_wait") - args, unknown = parser.parse_known_args(node_args) + parser.add_argument("--debug_node", required=True) + parser.add_argument("--debug_port", required=False, type=int) + parser.add_argument("--debug_wait", default=False, type=bool) + args, unknown_args = parser.parse_known_args(node_args) + debug_node = args.debug_node base_dir = os.path.abspath(os.path.dirname(__file__)) - logdebug(f"Node {ros_node_name} started at {base_dir}") - params = get_required_params() - if params["port"] is not None: - start_debugger(params["port"], params["type"], wait_for_debugee=params["wait"]) - target_type_path = os.path.join(base_dir, params["type"]) - run_node_at(target_type_path) + printdebug(f"Node {args.debug_node} starting at {base_dir}") + + target_type_path = os.path.join(base_dir, debug_node) + module = import_module_at(target_type_path) + + module.init_ros() + + if args.debug_port is not None: + start_debugger( + args.debug_port, args.debug_node, wait_for_client=args.debug_wait + ) + else: + logerr( + """Missing parameter to start debugger: --debug_port + Add it in the launch configuration""" + ) + + try: + module.main(unknown_args) + except BaseException as error: + logfatal(f"Failed to run node {debug_node}: {error}") + raise error if __name__ == "__main__": diff --git a/code/perception/launch/perception.launch b/code/perception/launch/perception.launch index f194174c..39d97263 100644 --- a/code/perception/launch/perception.launch +++ b/code/perception/launch/perception.launch @@ -75,7 +75,7 @@ - + diff --git a/code/perception/src/lidar_distance.py b/code/perception/src/lidar_distance.py index 93ab1bc2..84afa1b2 100755 --- a/code/perception/src/lidar_distance.py +++ b/code/perception/src/lidar_distance.py @@ -245,7 +245,7 @@ def reconstruct_img_from_lidar(self, coordinates_xyz, focus): return dist_array -def ros_init(): +def init_ros(): """Initializes the node for basic ROS functions. Must only be called ONCE and not as part of def main() @@ -254,7 +254,7 @@ def ros_init(): rospy.init_node("lidar_distance") -def main(): +def main(argv=None): """Main entry point of this node Required for debugger entry""" @@ -263,5 +263,5 @@ def main(): if __name__ == "__main__": - rospy.init_node("lidar_distance") + init_ros() main() diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 00000000..5cafbb7c --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,25 @@ +{ + "include": ["code/src"], + + "exclude": [], + + "ignore": [], + + "defineConstant": { + "DEBUG": true + }, + + "typeCheckingMode": "standard", + "strictListInference": true, + "strictDictionaryInference": true, + "strictSetInference": true, + + "pythonVersion": "3.8", + "pythonPlatform": "Linux", + + "executionEnvironments": [ + { + "root": "code/src" + } + ] +} From 4a55b03832f96879d7847f779b8907533359c217 Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Thu, 7 Nov 2024 15:32:08 +0100 Subject: [PATCH 03/19] Revert lidar distance changes --- code/perception/src/lidar_distance.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/code/perception/src/lidar_distance.py b/code/perception/src/lidar_distance.py index 84afa1b2..04394ee2 100755 --- a/code/perception/src/lidar_distance.py +++ b/code/perception/src/lidar_distance.py @@ -120,6 +120,7 @@ def listener(self): Initializes the node and it's publishers """ # run simultaneously. + rospy.init_node("lidar_distance") self.bridge = CvBridge() self.pub_pointcloud = rospy.Publisher( @@ -245,23 +246,6 @@ def reconstruct_img_from_lidar(self, coordinates_xyz, focus): return dist_array -def init_ros(): - """Initializes the node for basic ROS functions. - - Must only be called ONCE and not as part of def main() - - Required for debugger entry""" - rospy.init_node("lidar_distance") - - -def main(argv=None): - """Main entry point of this node - - Required for debugger entry""" +if __name__ == "__main__": lidar_distance = LidarDistance() lidar_distance.listener() - - -if __name__ == "__main__": - init_ros() - main() From b159c55d9b3f6b63cf8790b08985a1a6be1b199e Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Thu, 7 Nov 2024 18:22:18 +0100 Subject: [PATCH 04/19] Debugger working with docker + logging improvements --- .vscode/launch.json | 8 +- build/agent_service.yaml | 1 + build/docker-compose.leaderboard.yaml | 2 + build/docker/agent/Dockerfile | 3 +- code/debug_logger_node.py | 77 +++++++++++++++ code/debug_wrapper.py | 114 ++++++++++++++--------- code/perception/launch/perception.launch | 2 +- code/requirements.txt | 1 + doc/development/debugging.md | 39 ++++++++ 9 files changed, 198 insertions(+), 49 deletions(-) create mode 100755 code/debug_logger_node.py create mode 100644 doc/development/debugging.md diff --git a/.vscode/launch.json b/.vscode/launch.json index e7ed6ee0..29e9ce53 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,19 +5,19 @@ "version": "0.2.0", "configurations": [ { - "name": "Python Debugger: Remote Attach", + "name": "53000 leaderboard attach", "type": "debugpy", "request": "attach", "connect": { "host": "localhost", - "port": 5678 + "port": 53000 }, "pathMappings": [ { "localRoot": "${workspaceFolder}/code", - "remoteRoot": "/internal_workspace/catkin_ws/src" + "remoteRoot": "${env:PAF_CODE_ROOT}" } ] } - ] + ], } \ No newline at end of file diff --git a/build/agent_service.yaml b/build/agent_service.yaml index 530eb208..5255aae4 100644 --- a/build/agent_service.yaml +++ b/build/agent_service.yaml @@ -30,6 +30,7 @@ services: - ROS_HOSTNAME=agent - XDG_RUNTIME_DIR=/tmp/runtime-carla - ROUTE=/opt/leaderboard/data/routes_devtest.xml + - DEBUG_WRAPPER_DEFAULT_HOST=0.0.0.0 # Simple route without special scenarios # - ROUTE=/workspace/code/routes/routes_simple.xml volumes: diff --git a/build/docker-compose.leaderboard.yaml b/build/docker-compose.leaderboard.yaml index 126f3d09..8d6633ee 100644 --- a/build/docker-compose.leaderboard.yaml +++ b/build/docker-compose.leaderboard.yaml @@ -8,6 +8,8 @@ services: extends: file: agent_service.yaml service: agent + ports: + - "53000-53100:53000-53100" command: |- bash -c "sleep 10 && sudo chown -R ${USER_UID}:${USER_GID} ../ && \ sudo chmod -R a+w ../ && sudo mkdir -p $${XDG_RUNTIME_DIR} && sudo chmod 0700 $${XDG_RUNTIME_DIR} && sudo chown -R ${USER_UID}:${USER_GID} $${XDG_RUNTIME_DIR} && \ diff --git a/build/docker/agent/Dockerfile b/build/docker/agent/Dockerfile index 408fae6c..1ac2cebc 100644 --- a/build/docker/agent/Dockerfile +++ b/build/docker/agent/Dockerfile @@ -171,7 +171,8 @@ RUN source /opt/ros/noetic/setup.bash && catkin_make ADD ./build/docker/agent/entrypoint.sh /entrypoint.sh # set the default working directory to the code -WORKDIR /workspace/code +ENV PAF_CODE_ROOT=/workspace/code +WORKDIR ${PAF_CODE_ROOT} RUN echo "source /opt/ros/noetic/setup.bash" >> ~/.bashrc RUN echo "source /catkin_ws/devel/setup.bash" >> ~/.bashrc diff --git a/code/debug_logger_node.py b/code/debug_logger_node.py new file mode 100755 index 00000000..202bcfc6 --- /dev/null +++ b/code/debug_logger_node.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +""" +This is a dedicated debug logger node + +Without a fully initialized node, log messages do not show up in the rqt_console + +This is why this node is used by the debug_wrapper for logging +before it has initialized it's node +""" + +import time +import sys +from multiprocessing.connection import Listener + +import rospy + + +def eprint(msg: str): + print(f"[debug_logger_node]: {msg}", file=sys.stderr) + + +def log(name: str, msg: str, level: str): + msg = f"[debug_logger for {name}]: {msg}" + + level = level.lower() + if level == "debug": + rospy.logdebug(msg) + elif level == "info": + rospy.loginfo(msg) + elif level == "warn": + rospy.logwarn(msg) + elif level == "error": + rospy.logerr(msg) + else: + rospy.logfatal(msg) + + +def main(): + try: + address = ("localhost", 52999) # family is deduced to be 'AF_INET' + listener = Listener( + address, authkey=b"debug_logger" + ) # Only one listerner can be active at once + eprint("Debug logger node started") + except OSError as error: + eprint(f"Failed to start listener: {error}. Exiting...") + exit(0) + + rospy.init_node(name="debug_logger") + time.sleep( + 5.0 + ) # We need to wait a bit until the node is fully initialized to send log messages + + # Based on + # https://stackoverflow.com/questions/6920858/interprocess-communication-in-python + while not rospy.is_shutdown(): + conn = listener.accept() + print(f"[debug_logger]: connection accepted from {listener.last_accepted}") + msg = conn.recv() + conn.close() + log_msg = "Wrong log message format" + log_name = "NAMERR" + log_level = "fatal" + if isinstance(msg, dict): + if "name" in msg: + log_name = msg["name"] + if "msg" in msg: + log_msg = msg["msg"] + if "level" in msg: + log_level = msg["level"] + log(log_name, log_msg, log_level) + + listener.close() + + +if __name__ == "__main__": + main() diff --git a/code/debug_wrapper.py b/code/debug_wrapper.py index 88486091..993b216d 100755 --- a/code/debug_wrapper.py +++ b/code/debug_wrapper.py @@ -2,70 +2,89 @@ import importlib.util import os +import subprocess +import runpy import sys +import shutil import argparse +from multiprocessing.connection import Client import rospy -DEBUG_WRAPPER_MSG_PREFIX = "[debug_wrapper]:" -TYPE_PARAM = "~debug_type" -PORT_PARAM = "~debug_port" -WAIT_PARAM = "~debug_wait" +NODE_NAME = "NAMEERR" +LOGGER_STARTED = False -def printdebug(msg: str): - print(f"{DEBUG_WRAPPER_MSG_PREFIX} {msg}") +def start_logger(): + global LOGGER_STARTED + # read directory of this python file with resolved symlinks + real_dir = os.path.dirname(os.path.realpath(__file__)) + logger_path = os.path.join(real_dir, "debug_logger_node.py") + eprint(f"Starting logger at {logger_path}") + try: + python_path: str = shutil.which("python3") + subprocess.Popen( + [python_path, logger_path], + start_new_session=True, + ) + LOGGER_STARTED = True + except BaseException as error: + eprint(f"Failed to start logger: {error}") + + +def eprint(msg: str): + print(f"[debug_wrapper]: {msg}", file=sys.stderr) -def printerror(msg: str): - """Print to stderr""" - print(f"{DEBUG_WRAPPER_MSG_PREFIX} {msg}", file=sys.stderr) +def log(msg: str, level: str): + if LOGGER_STARTED: + try: + address = ("localhost", 52999) + conn = Client(address, authkey=b"debug_logger") + conn.send({"name": NODE_NAME, "msg": msg, "level": level}) + conn.close() + except BaseException as error: + eprint(msg) + eprint(f"Failed to send to logger: {error}") + else: + eprint(msg) def logfatal(msg: str): - """Only works after ros node has been initialized""" - rospy.logfatal( - f"""{DEBUG_WRAPPER_MSG_PREFIX} FAILED TO START NODE - NODE WILL NOT SHOW UP: - {msg}""" - ) + log(f"FAILED TO START NODE - NODE WILL NOT SHOW UP: {msg}", "fatal") def logerr(msg: str): - """Only works after ros node has been initialized""" - rospy.logerr(f"{DEBUG_WRAPPER_MSG_PREFIX} {msg}") + log(msg, "error") + + +def logwarn(msg: str): + log(msg, "warn") def loginfo(msg: str): - """Only works after ros node has been initialized""" - rospy.loginfo(f"{DEBUG_WRAPPER_MSG_PREFIX} {msg}") + log(msg, "info") -def import_module_at(path: str): +def run_module_at(path: str): basename = os.path.basename(path) module_dir = os.path.dirname(path) module_name = os.path.splitext(basename)[0] - # Based on https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly - """ spec = importlib.util.spec_from_file_location( - name=module_name, location=path, submodule_search_locations=[module_dir] - ) - if spec is None: - raise Exception(f"Failed to load {path} as module {module_name}") - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) """ sys.path.append(module_dir) - module = importlib.import_module(module_name) - return module + runpy.run_module(module_name, run_name="__main__") -def start_debugger(port: int, node_module_name: str, wait_for_client: bool = False): +def start_debugger( + node_module_name: str, host: str, port: int, wait_for_client: bool = False +): debugger_spec = importlib.util.find_spec("debugpy") if debugger_spec is not None: try: import debugpy - debugpy.listen(("localhost", port)) - loginfo(f"Started debugger on port {port} for {node_module_name}") + debugpy.configure(subProcess=False) + debugpy.listen((host, port)) + logwarn(f"Started debugger on {host}:{port} for {node_module_name}") if wait_for_client: debugpy.wait_for_client() except BaseException as error: @@ -81,31 +100,38 @@ class ArgumentParserError(Exception): class ThrowingArgumentParser(argparse.ArgumentParser): def error(self, message): + logfatal(f"Wrong node arguments. Check launch config. : {message}") raise ArgumentParserError(message) def main(argv): + global NODE_NAME + start_logger() + + default_host = "localhost" + if "DEBUG_WRAPPER_DEFAULT_HOST" in os.environ: + default_host = os.environ["DEBUG_WRAPPER_DEFAULT_HOST"] + node_args = rospy.myargv(argv=argv) parser = ThrowingArgumentParser( prog="debug wrapper", ) - parser.add_argument("--debug_node", required=True) + parser.add_argument("--debug_node", required=True, type=str) parser.add_argument("--debug_port", required=False, type=int) + parser.add_argument("--debug_host", default=default_host, type=str) parser.add_argument("--debug_wait", default=False, type=bool) args, unknown_args = parser.parse_known_args(node_args) - debug_node = args.debug_node + debug_node = args.debug_node + NODE_NAME = debug_node base_dir = os.path.abspath(os.path.dirname(__file__)) - printdebug(f"Node {args.debug_node} starting at {base_dir}") - - target_type_path = os.path.join(base_dir, debug_node) - module = import_module_at(target_type_path) - - module.init_ros() if args.debug_port is not None: start_debugger( - args.debug_port, args.debug_node, wait_for_client=args.debug_wait + args.debug_node, + args.debug_host, + args.debug_port, + wait_for_client=args.debug_wait, ) else: logerr( @@ -113,8 +139,10 @@ def main(argv): Add it in the launch configuration""" ) + target_type_path = os.path.join(base_dir, debug_node) + loginfo(f"Node {args.debug_node} starting at {base_dir}") try: - module.main(unknown_args) + run_module_at(target_type_path) except BaseException as error: logfatal(f"Failed to run node {debug_node}: {error}") raise error diff --git a/code/perception/launch/perception.launch b/code/perception/launch/perception.launch index 39d97263..b7a78c48 100644 --- a/code/perception/launch/perception.launch +++ b/code/perception/launch/perception.launch @@ -75,7 +75,7 @@ - + diff --git a/code/requirements.txt b/code/requirements.txt index b9b6155d..55de87f0 100644 --- a/code/requirements.txt +++ b/code/requirements.txt @@ -16,3 +16,4 @@ numpy==1.23.5 ultralytics==8.1.11 scikit-learn>=0.18 pandas==2.0.3 +debugpy==1.8.7 \ No newline at end of file diff --git a/doc/development/debugging.md b/doc/development/debugging.md new file mode 100644 index 00000000..45a3a6f5 --- /dev/null +++ b/doc/development/debugging.md @@ -0,0 +1,39 @@ +# Debugging + +**Summary:** This page explains multiple debugging methods for ROS nodes. + +- [Debugging possibilities](#debugging-possibilities) + - [Message based debugging](#message-based-debugging) + - [VS Code debugger](#vs-code-debugger) +- [Rebuild docker containers](#rebuild-docker-containers) +- [Sources](#sources) + +## Debugging possibilities + +There are two main debugging possibilities + +### Message based debugging + +Messages can be logged into the ROS console vie the `rospy.logdebug`, `rospy.loginfo`, `rospy.logwarn`, `rospy.logerror` and `rospy.logfatal` functions. + +### VS Code debugger + + + +## Rebuild docker containers + +The docker images on your pc might not match the latest Dockerfile in the repository + +To update them, open a terminal and change into the *build* directory. Execute: + +```bash +export USER_UID=$(id -u) +export USER_GID=$(id -g) +export USERNAME=$(id -u -n) +docker compose -f ./docker-compose.leaderboard.yaml up -d --build +docker compose -f ./docker-compose.dev.yaml up -d --build +``` + +## Sources + + From 3d43f017bc12e045b46bc297b3813664883217cf Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Fri, 8 Nov 2024 15:53:06 +0100 Subject: [PATCH 05/19] Debug logger node added to launch config + general logging improvements --- code/agent/launch/agent.launch | 4 + code/debug_logger_node.py | 77 --------- code/debug_wrapper.py | 41 ++--- code/debugging/CMakeLists.txt | 204 ++++++++++++++++++++++++ code/debugging/launch/debugging.launch | 4 + code/debugging/package.xml | 62 +++++++ code/debugging/src/debug_logger_node.py | 126 +++++++++++++++ 7 files changed, 412 insertions(+), 106 deletions(-) delete mode 100755 code/debug_logger_node.py create mode 100644 code/debugging/CMakeLists.txt create mode 100644 code/debugging/launch/debugging.launch create mode 100644 code/debugging/package.xml create mode 100755 code/debugging/src/debug_logger_node.py diff --git a/code/agent/launch/agent.launch b/code/agent/launch/agent.launch index fb9ef983..7861a38f 100644 --- a/code/agent/launch/agent.launch +++ b/code/agent/launch/agent.launch @@ -21,5 +21,9 @@ + + + + diff --git a/code/debug_logger_node.py b/code/debug_logger_node.py deleted file mode 100755 index 202bcfc6..00000000 --- a/code/debug_logger_node.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -""" -This is a dedicated debug logger node - -Without a fully initialized node, log messages do not show up in the rqt_console - -This is why this node is used by the debug_wrapper for logging -before it has initialized it's node -""" - -import time -import sys -from multiprocessing.connection import Listener - -import rospy - - -def eprint(msg: str): - print(f"[debug_logger_node]: {msg}", file=sys.stderr) - - -def log(name: str, msg: str, level: str): - msg = f"[debug_logger for {name}]: {msg}" - - level = level.lower() - if level == "debug": - rospy.logdebug(msg) - elif level == "info": - rospy.loginfo(msg) - elif level == "warn": - rospy.logwarn(msg) - elif level == "error": - rospy.logerr(msg) - else: - rospy.logfatal(msg) - - -def main(): - try: - address = ("localhost", 52999) # family is deduced to be 'AF_INET' - listener = Listener( - address, authkey=b"debug_logger" - ) # Only one listerner can be active at once - eprint("Debug logger node started") - except OSError as error: - eprint(f"Failed to start listener: {error}. Exiting...") - exit(0) - - rospy.init_node(name="debug_logger") - time.sleep( - 5.0 - ) # We need to wait a bit until the node is fully initialized to send log messages - - # Based on - # https://stackoverflow.com/questions/6920858/interprocess-communication-in-python - while not rospy.is_shutdown(): - conn = listener.accept() - print(f"[debug_logger]: connection accepted from {listener.last_accepted}") - msg = conn.recv() - conn.close() - log_msg = "Wrong log message format" - log_name = "NAMERR" - log_level = "fatal" - if isinstance(msg, dict): - if "name" in msg: - log_name = msg["name"] - if "msg" in msg: - log_msg = msg["msg"] - if "level" in msg: - log_level = msg["level"] - log(log_name, log_msg, log_level) - - listener.close() - - -if __name__ == "__main__": - main() diff --git a/code/debug_wrapper.py b/code/debug_wrapper.py index 993b216d..91b3d961 100755 --- a/code/debug_wrapper.py +++ b/code/debug_wrapper.py @@ -2,34 +2,15 @@ import importlib.util import os -import subprocess import runpy import sys -import shutil import argparse +import time from multiprocessing.connection import Client import rospy NODE_NAME = "NAMEERR" -LOGGER_STARTED = False - - -def start_logger(): - global LOGGER_STARTED - # read directory of this python file with resolved symlinks - real_dir = os.path.dirname(os.path.realpath(__file__)) - logger_path = os.path.join(real_dir, "debug_logger_node.py") - eprint(f"Starting logger at {logger_path}") - try: - python_path: str = shutil.which("python3") - subprocess.Popen( - [python_path, logger_path], - start_new_session=True, - ) - LOGGER_STARTED = True - except BaseException as error: - eprint(f"Failed to start logger: {error}") def eprint(msg: str): @@ -37,17 +18,22 @@ def eprint(msg: str): def log(msg: str, level: str): - if LOGGER_STARTED: + error = None + success = False + start_time = time.monotonic() + while not success and start_time + 5.0 > time.monotonic(): try: address = ("localhost", 52999) conn = Client(address, authkey=b"debug_logger") conn.send({"name": NODE_NAME, "msg": msg, "level": level}) conn.close() - except BaseException as error: - eprint(msg) - eprint(f"Failed to send to logger: {error}") - else: + success = True + except BaseException as e: + error = e + if not success: eprint(msg) + if error is not None: + eprint(f"Failed to send to logger: {error}") def logfatal(msg: str): @@ -82,7 +68,6 @@ def start_debugger( try: import debugpy - debugpy.configure(subProcess=False) debugpy.listen((host, port)) logwarn(f"Started debugger on {host}:{port} for {node_module_name}") if wait_for_client: @@ -105,9 +90,6 @@ def error(self, message): def main(argv): - global NODE_NAME - start_logger() - default_host = "localhost" if "DEBUG_WRAPPER_DEFAULT_HOST" in os.environ: default_host = os.environ["DEBUG_WRAPPER_DEFAULT_HOST"] @@ -123,6 +105,7 @@ def main(argv): args, unknown_args = parser.parse_known_args(node_args) debug_node = args.debug_node + global NODE_NAME NODE_NAME = debug_node base_dir = os.path.abspath(os.path.dirname(__file__)) diff --git a/code/debugging/CMakeLists.txt b/code/debugging/CMakeLists.txt new file mode 100644 index 00000000..8cc214d8 --- /dev/null +++ b/code/debugging/CMakeLists.txt @@ -0,0 +1,204 @@ +cmake_minimum_required(VERSION 3.0.2) +project(debugging) + +## Compile as C++11, supported in ROS Kinetic and newer +# add_compile_options(-std=c++11) + +## Find catkin macros and libraries +## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz) +## is used, also find other catkin packages +find_package(catkin REQUIRED COMPONENTS + rospy +) + +## System dependencies are found with CMake's conventions +# find_package(Boost REQUIRED COMPONENTS system) + + +## Uncomment this if the package has a setup.py. This macro ensures +## modules and global scripts declared therein get installed +## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html +# catkin_python_setup() + +################################################ +## Declare ROS messages, services and actions ## +################################################ + +## To declare and build messages, services or actions from within this +## package, follow these steps: +## * Let MSG_DEP_SET be the set of packages whose message types you use in +## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...). +## * In the file package.xml: +## * add a build_depend tag for "message_generation" +## * add a build_depend and a exec_depend tag for each package in MSG_DEP_SET +## * If MSG_DEP_SET isn't empty the following dependency has been pulled in +## but can be declared for certainty nonetheless: +## * add a exec_depend tag for "message_runtime" +## * In this file (CMakeLists.txt): +## * add "message_generation" and every package in MSG_DEP_SET to +## find_package(catkin REQUIRED COMPONENTS ...) +## * add "message_runtime" and every package in MSG_DEP_SET to +## catkin_package(CATKIN_DEPENDS ...) +## * uncomment the add_*_files sections below as needed +## and list every .msg/.srv/.action file to be processed +## * uncomment the generate_messages entry below +## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...) + +## Generate messages in the 'msg' folder +# add_message_files( +# FILES +# Message1.msg +# Message2.msg +# ) + +## Generate services in the 'srv' folder +# add_service_files( +# FILES +# Service1.srv +# Service2.srv +# ) + +## Generate actions in the 'action' folder +# add_action_files( +# FILES +# Action1.action +# Action2.action +# ) + +## Generate added messages and services with any dependencies listed here +# generate_messages( +# DEPENDENCIES +# std_msgs # Or other packages containing msgs +# ) + +################################################ +## Declare ROS dynamic reconfigure parameters ## +################################################ + +## To declare and build dynamic reconfigure parameters within this +## package, follow these steps: +## * In the file package.xml: +## * add a build_depend and a exec_depend tag for "dynamic_reconfigure" +## * In this file (CMakeLists.txt): +## * add "dynamic_reconfigure" to +## find_package(catkin REQUIRED COMPONENTS ...) +## * uncomment the "generate_dynamic_reconfigure_options" section below +## and list every .cfg file to be processed + +## Generate dynamic reconfigure parameters in the 'cfg' folder +# generate_dynamic_reconfigure_options( +# cfg/DynReconf1.cfg +# cfg/DynReconf2.cfg +# ) + +################################### +## catkin specific configuration ## +################################### +## The catkin_package macro generates cmake config files for your package +## Declare things to be passed to dependent projects +## INCLUDE_DIRS: uncomment this if your package contains header files +## LIBRARIES: libraries you create in this project that dependent projects also need +## CATKIN_DEPENDS: catkin_packages dependent projects also need +## DEPENDS: system dependencies of this project that dependent projects also need +catkin_package( +# INCLUDE_DIRS include +# LIBRARIES debugging +# CATKIN_DEPENDS rospy +# DEPENDS system_lib +) + +########### +## Build ## +########### + +## Specify additional locations of header files +## Your package locations should be listed before other locations +include_directories( +# include + ${catkin_INCLUDE_DIRS} +) + +## Declare a C++ library +# add_library(${PROJECT_NAME} +# src/${PROJECT_NAME}/debugging.cpp +# ) + +## Add cmake target dependencies of the library +## as an example, code may need to be generated before libraries +## either from message generation or dynamic reconfigure +# add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) + +## Declare a C++ executable +## With catkin_make all packages are built within a single CMake context +## The recommended prefix ensures that target names across packages don't collide +# add_executable(${PROJECT_NAME}_node src/debugging_node.cpp) + +## Rename C++ executable without prefix +## The above recommended prefix causes long target names, the following renames the +## target back to the shorter version for ease of user use +## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node" +# set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "") + +## Add cmake target dependencies of the executable +## same as for the library above +# add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) + +## Specify libraries to link a library or executable target against +# target_link_libraries(${PROJECT_NAME}_node +# ${catkin_LIBRARIES} +# ) + +############# +## Install ## +############# + +# all install targets should use catkin DESTINATION variables +# See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html + +## Mark executable scripts (Python etc.) for installation +## in contrast to setup.py, you can choose the destination +# catkin_install_python(PROGRAMS +# scripts/my_python_script +# DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} +# ) + +## Mark executables for installation +## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_executables.html +# install(TARGETS ${PROJECT_NAME}_node +# RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} +# ) + +## Mark libraries for installation +## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_libraries.html +# install(TARGETS ${PROJECT_NAME} +# ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} +# LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} +# RUNTIME DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION} +# ) + +## Mark cpp header files for installation +# install(DIRECTORY include/${PROJECT_NAME}/ +# DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION} +# FILES_MATCHING PATTERN "*.h" +# PATTERN ".svn" EXCLUDE +# ) + +## Mark other files for installation (e.g. launch and bag files, etc.) +# install(FILES +# # myfile1 +# # myfile2 +# DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} +# ) + +############# +## Testing ## +############# + +## Add gtest based cpp test target and link libraries +# catkin_add_gtest(${PROJECT_NAME}-test test/test_debugging.cpp) +# if(TARGET ${PROJECT_NAME}-test) +# target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME}) +# endif() + +## Add folders to be run by python nosetests +# catkin_add_nosetests(test) diff --git a/code/debugging/launch/debugging.launch b/code/debugging/launch/debugging.launch new file mode 100644 index 00000000..4388cbbf --- /dev/null +++ b/code/debugging/launch/debugging.launch @@ -0,0 +1,4 @@ + + + + diff --git a/code/debugging/package.xml b/code/debugging/package.xml new file mode 100644 index 00000000..5598d5ed --- /dev/null +++ b/code/debugging/package.xml @@ -0,0 +1,62 @@ + + + debugging + 0.0.0 + The debugging package + + + + + peter + + + + + + TODO + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + catkin + rospy + rospy + rospy + + + + + + + + diff --git a/code/debugging/src/debug_logger_node.py b/code/debugging/src/debug_logger_node.py new file mode 100755 index 00000000..68bc4b6c --- /dev/null +++ b/code/debugging/src/debug_logger_node.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +""" +This is a dedicated debug logger node + +Without a fully initialized node, log messages only show up in the logfile. +But not in the /rosout topic and thus not in the rqt_console + +This is why this node is used by the debug_wrapper for logging +before it has initialized its own node +""" + +import time +import sys +from multiprocessing.connection import Listener, Client +import threading +import signal + +import rospy + +ADDRESS = ("localhost", 52999) +AUTHKEY = b"debug_logger" +CLOSE_MSG = "close" +LISTENER_THREAD = None +NODE_RUNNING = True + +MESSAGES_MUTEX = threading.Lock() +MESSAGES = [] + + +def eprint(msg: str): + print(f"[debug_logger_node]: {msg}", file=sys.stderr) + + +def log(name: str, msg: str, level: str): + msg = f"[debug_logger for {name}]: {msg}" + + level = level.lower() + if level == "debug": + rospy.logdebug(msg) + elif level == "info": + rospy.loginfo(msg) + elif level == "warn": + rospy.logwarn(msg) + elif level == "error": + rospy.logerr(msg) + else: + rospy.logfatal(msg) + + +def run_listener(listener: Listener): + running = True + while running: + conn = listener.accept() + print(f"[debug_logger]: connection accepted from {listener.last_accepted}") + msg = None + if conn.poll(timeout=2.0): + msg = conn.recv() + if isinstance(msg, str): + if msg.lower() == CLOSE_MSG: + running = False + msg = None + conn.close() + if msg is not None: + with MESSAGES_MUTEX: + MESSAGES.append(msg) + listener.close() + + +def close_listener(): + try: + conn = Client(ADDRESS, authkey=AUTHKEY) + conn.send(CLOSE_MSG) + conn.close() + except BaseException: + pass + + +def exit_cleanup(signum=None, frame=None): + close_listener() + if LISTENER_THREAD is not None: + LISTENER_THREAD.join() + global NODE_RUNNING + NODE_RUNNING = False + + +def main(): + try: + # Based on + # https://stackoverflow.com/questions/6920858/interprocess-communication-in-python + # Only one listener can be active at once + listener = Listener(ADDRESS, authkey=AUTHKEY) + eprint("Debug logger node started") + except OSError as error: + eprint(f"Failed to run listener: {error}. Exiting...") + exit(0) + signal.signal(signal.SIGINT, exit_cleanup) + signal.signal(signal.SIGTERM, exit_cleanup) + global LISTENER_THREAD + LISTENER_THREAD = threading.Thread(target=run_listener, args=(listener,)) + LISTENER_THREAD.start() + + rospy.init_node(name="debug_logger") + # We need to wait a bit until the node is fully initialized to send log messages + time.sleep(5.0) + rospy.on_shutdown(exit_cleanup) + + while NODE_RUNNING: + with MESSAGES_MUTEX: + while len(MESSAGES) > 0: + msg = MESSAGES.pop() + log_msg = "Wrong log message format" + log_name = "NAMERR" + log_level = "fatal" + if isinstance(msg, dict): + if "name" in msg: + log_name = msg["name"] + if "msg" in msg: + log_msg = msg["msg"] + if "level" in msg: + log_level = msg["level"] + log(log_name, log_msg, log_level) + time.sleep(0.5) + + +if __name__ == "__main__": + main() From 37c308a2d59271f3c758c7471d23b8b2aa8dd864 Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Fri, 8 Nov 2024 16:39:14 +0100 Subject: [PATCH 06/19] Update remoteRoot source mapping variable PAF_CATKIN_CODE_ROOT --- .vscode/launch.json | 2 +- build/docker/agent/Dockerfile | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 29e9ce53..68420ca7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -15,7 +15,7 @@ "pathMappings": [ { "localRoot": "${workspaceFolder}/code", - "remoteRoot": "${env:PAF_CODE_ROOT}" + "remoteRoot": "${env:PAF_CATKIN_CODE_ROOT}" } ] } diff --git a/build/docker/agent/Dockerfile b/build/docker/agent/Dockerfile index 1ac2cebc..48818408 100644 --- a/build/docker/agent/Dockerfile +++ b/build/docker/agent/Dockerfile @@ -165,14 +165,16 @@ COPY --chown=$USERNAME:$USERNAME ./code /workspace/code/ # Link code into catkin workspace RUN ln -s /workspace/code /catkin_ws/src +# For debugger +ENV PAF_CATKIN_CODE_ROOT=/catkin_ws/src + # re-make the catkin workspace RUN source /opt/ros/noetic/setup.bash && catkin_make ADD ./build/docker/agent/entrypoint.sh /entrypoint.sh # set the default working directory to the code -ENV PAF_CODE_ROOT=/workspace/code -WORKDIR ${PAF_CODE_ROOT} +WORKDIR /workspace/code RUN echo "source /opt/ros/noetic/setup.bash" >> ~/.bashrc RUN echo "source /catkin_ws/devel/setup.bash" >> ~/.bashrc From b5cc5e4d1d7a7f8e4c5a88052fee8b683ccb8b49 Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Fri, 8 Nov 2024 21:25:21 +0100 Subject: [PATCH 07/19] Revert perception.launch --- code/perception/launch/perception.launch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/perception/launch/perception.launch b/code/perception/launch/perception.launch index b7a78c48..10f79a80 100644 --- a/code/perception/launch/perception.launch +++ b/code/perception/launch/perception.launch @@ -75,7 +75,7 @@ - + From 6fad77251a403f501121a41110188255623d4449 Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Fri, 8 Nov 2024 21:55:36 +0100 Subject: [PATCH 08/19] Add detailed documentation --- code/debug_wrapper.py | 63 +++++++++++ code/debugging/launch/debugging.launch | 2 +- .../{debug_logger_node.py => debug_logger.py} | 39 ++++++- doc/development/debugging.md | 103 +++++++++++++++++- 4 files changed, 198 insertions(+), 9 deletions(-) rename code/debugging/src/{debug_logger_node.py => debug_logger.py} (73%) diff --git a/code/debug_wrapper.py b/code/debug_wrapper.py index 91b3d961..c02d8d0a 100755 --- a/code/debug_wrapper.py +++ b/code/debug_wrapper.py @@ -1,4 +1,39 @@ #!/usr/bin/env python3 +"""Debug wrapper node + +Node that wraps a python ros node +and is able to open a debugpy remote debugger instance. + +Logs any exceptions from the node and other information +into the ros console via the debug_logger node. + +Always tries to at least start the node, +even if dependencies like debugpy are missing. + +Usage: + Already done for existing ros packages: symlink this file into the package. Example: + `cd code/perception/src && ln -s ../../debug_wrapper.py debug_wrapper.py` + + Adjust the launch configuration to use the debug_wrapper.py + instead of the node file and set the required args + + More info in [debugging.md](doc/development/debugging.md) + +Arguments: + --debug_node: Required: The filename of the node to debug + --debug_port: The port the debugger listens on. If not set, + --debug_host: The host the debugger binds to. + Defaults to the environment variable `DEBUG_WRAPPER_DEFAULT_HOST` if set, + otherwise `localhost` + --debug_wait: If True, the wrapper waits until a client (VS Code) is connected + and only then starts node execution + +Raises: + ArgumentParserError: Missing required parameters + error: If the started debug_node raises an exception, + it is logged and then raised again +""" + import importlib.util import os @@ -14,10 +49,24 @@ def eprint(msg: str): + """Log msg into stderr. + + Used instead of print, because only stderr seems to reliably land in agent.log + + Args: + msg (str): Log message + """ print(f"[debug_wrapper]: {msg}", file=sys.stderr) def log(msg: str, level: str): + """Log msg via the debug_logger node + + Args: + msg (str): Log message + level (str): Log level. One of (debug), info, warn, error, fatal. + debug level not recommended, because these are not published to /rosout + """ error = None success = False start_time = time.monotonic() @@ -53,6 +102,11 @@ def loginfo(msg: str): def run_module_at(path: str): + """Runs a python module based on its file path + + Args: + path (str): python file path to run + """ basename = os.path.basename(path) module_dir = os.path.dirname(path) module_name = os.path.splitext(basename)[0] @@ -63,6 +117,15 @@ def run_module_at(path: str): def start_debugger( node_module_name: str, host: str, port: int, wait_for_client: bool = False ): + """_summary_ + + Args: + node_module_name (str): Name of the underlying node. Only used for logging + host (str): host the debugger binds to + port (int): debugger port + wait_for_client (bool, optional): If the debugger should wait + for a client to attach. Defaults to False. + """ debugger_spec = importlib.util.find_spec("debugpy") if debugger_spec is not None: try: diff --git a/code/debugging/launch/debugging.launch b/code/debugging/launch/debugging.launch index 4388cbbf..330263e8 100644 --- a/code/debugging/launch/debugging.launch +++ b/code/debugging/launch/debugging.launch @@ -1,4 +1,4 @@ - + diff --git a/code/debugging/src/debug_logger_node.py b/code/debugging/src/debug_logger.py similarity index 73% rename from code/debugging/src/debug_logger_node.py rename to code/debugging/src/debug_logger.py index 68bc4b6c..6778531d 100755 --- a/code/debugging/src/debug_logger_node.py +++ b/code/debugging/src/debug_logger.py @@ -1,9 +1,16 @@ #!/usr/bin/env python3 -""" -This is a dedicated debug logger node +"""Dedicated debug logger node. + +It can receive messages via a multiprocessing ipc socket on localhost:52999. +Message format is a python dict with keys {name, msg, level}. +It will then send those messages into the ros console via rospy.log*() -Without a fully initialized node, log messages only show up in the logfile. -But not in the /rosout topic and thus not in the rqt_console +Main usecase: it enables not initialized python nodes to publish log messages +to the /rosout topic. + +Without a fully initialized node, +log messages from python files only show up in the ros logfile, +but not in the /rosout topic and thus not in the rqt_console This is why this node is used by the debug_wrapper for logging before it has initialized its own node @@ -28,10 +35,25 @@ def eprint(msg: str): - print(f"[debug_logger_node]: {msg}", file=sys.stderr) + """Log msg into stderr. + + Used instead of print, because only stderr seems to reliably land in agent.log + + Args: + msg (str): Log message + """ + print(f"[debug_logger]: {msg}", file=sys.stderr) def log(name: str, msg: str, level: str): + """Log to the ros console + + Args: + name (str): Node name this message should be associated with + msg (str): Log message + level (str): Log level. One of (debug), info, warn, error, fatal. + debug level not recommended, because these are not published to /rosout + """ msg = f"[debug_logger for {name}]: {msg}" level = level.lower() @@ -48,6 +70,13 @@ def log(name: str, msg: str, level: str): def run_listener(listener: Listener): + """Run the multiprocessing listener + + The listener exits when it receives CLOSE_MSG + + Args: + listener (Listener): Listener to run + """ running = True while running: conn = listener.accept() diff --git a/doc/development/debugging.md b/doc/development/debugging.md index 45a3a6f5..9d364145 100644 --- a/doc/development/debugging.md +++ b/doc/development/debugging.md @@ -4,27 +4,124 @@ - [Debugging possibilities](#debugging-possibilities) - [Message based debugging](#message-based-debugging) + - [Viewing the messages](#viewing-the-messages) + - [Problems of message based debugging](#problems-of-message-based-debugging) - [VS Code debugger](#vs-code-debugger) + - [Setup steps required once](#setup-steps-required-once) + - [Required once for the node you want to debug](#required-once-for-the-node-you-want-to-debug) + - [Debugging workflow after setup](#debugging-workflow-after-setup) + - [Known problems of debugging with VS Code](#known-problems-of-debugging-with-vs-code) - [Rebuild docker containers](#rebuild-docker-containers) - [Sources](#sources) ## Debugging possibilities -There are two main debugging possibilities +There are two main debugging possibilities: Message based and VS Code based ### Message based debugging Messages can be logged into the ROS console vie the `rospy.logdebug`, `rospy.loginfo`, `rospy.logwarn`, `rospy.logerror` and `rospy.logfatal` functions. +Most messages are then published to the /rosout topic. + +`rospy.logdebug` is not recommended, because these messages are not published to /rosout by default. + +Note that the node has to be initialized with `rospy.init_node()` before logging, otherwise the messages are not published to /rosout and just end up in stdout/stderr. + +#### Viewing the messages + +There are several ways to view the ROS log messages + +- The most convenient way to view logs is via the **rqt_console** GUI, which starts when the leaderboard is started. + It allows filtering for debug levels, nodes and message content. + + Caveat: It only includes messages published to the /rosout topic. + +- Execute `rosconsole echo` inside the *build-agent* container. This shows the /rosout messages from this point on. + +- The leaderboard logs usually end up in [code/log/ros](../../code/log/ros). + + This includes the [agent.log](../../code/log/ros/agent.log) file where most of all the nodes output is captured. + It seems to exclude stdout of the nodes and excludes the debug log level. stderr and all other levels are included. + Exceptions that occur at node initialization can also be found here. + +- More accurate "per-node" logs including the debug level and stdout end up in the individual node log files. + These can be found in the directory returned by running `roslaunch-logs` inside the *build-agent* container. (Usually `~/.ros/log`) + +- Manually starting a node with `rosrun ` inside the *build-agent* container helps to view stdout, stderr and exceptions. + +More information can be found in the [Wiki](https://wiki.ros.org/rospy/Overview/Logging) + +#### Problems of message based debugging + +- Time intensive and cumbersome: leaderboard has to be restarted to add a new message +- Frequent log messages "spam" the log +- Exceptions on node initialization do not show up in the /rosout topic +- If messages are sent right after `rospy.init_node()`, they might not be published to the /rosout topic ### VS Code debugger +Debug individual nodes with the VS Code debugger. This allows usage of breakpoints and code stepping. + +#### Setup steps required once + +1. Make sure the docker images are up-to-date: [Rebuild docker containers](#rebuild-docker-containers) +2. Start the `build/docker-compose.dev.yaml` containers and attach VS Code to the **build-agent-dev** container. Always use the VS Code remote for debugging +3. Make sure the recommended extensions are installed in the VS Code remote. + +#### Required once for the node you want to debug + +Adjust the launch configuration of the node to use the [debug_wrapper.py](../../code/debug_wrapper.py) node. + +Change the type to **debug_wrapper.py** and set `args="--debug_node= --debug_port="`. +*debug_port* can be a port in range **53000-53100**, because those are exposed from the [leaderboard docker configuration](../../build/docker-compose.leaderboard.yaml). + +Configuration example for the [lidar_distance.py](../../code/perception/src/lidar_distance.py) node: [launch configuration](../../code/perception/launch/perception.launch) + +- Original entry: + + ```launch + + + + + ``` + +- Entry modified for debugging: + + ```launch + + + + + ``` + +By default, the affected node will start up as usual. When you attach the debugger later, you are then only able to debug the callbacks the node receives. +But if you set `--debug_wait=True`, it will block the node from starting until VS Code attaches and allow you to debug the initialization of the node. + +If the leaderboard hangs on `Setting up the agent` after the configuration has been adjusted, there most likely is a mistake in the launch configuration. + +More usage information can be found in the [debug_wrapper](../../code/debug_wrapper.py) and the [debug_logger](../../code/debugging/src/debug_logger.py) + +#### Debugging workflow after setup + +1. Start/Restart the [leaderboard](../../build/docker-compose.leaderboard.yaml) as usual +2. Wait for the application to start up +3. Add breakpoints to your node. +4. Start the VS Code debugger (debugging tab). A default launch configuration named *53000 leaderboard attach* is available. It uses port **53000**. + +Any errors/exceptions the debug_wrapper encounters are published to /rosout via the [debug_logger](../../code/debugging/src/debug_logger.py). +With the **rqt_console** GUI you can filter by node *debug_logger* to see all messages related to the wrapper. + +#### Known problems of debugging with VS Code +- Adjusting the launch configurations is a bit cumbersome +- The side effects of just "stopping" a node with the debugger are unknown ## Rebuild docker containers The docker images on your pc might not match the latest Dockerfile in the repository -To update them, open a terminal and change into the *build* directory. Execute: +To update them, open a terminal, change into the *build* directory and execute: ```bash export USER_UID=$(id -u) @@ -36,4 +133,4 @@ docker compose -f ./docker-compose.dev.yaml up -d --build ## Sources - + From 1dd75132524fa979e1dc386971a8bdac4c9914bc Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Fri, 8 Nov 2024 22:00:57 +0100 Subject: [PATCH 09/19] Add debugger comment --- build/docker-compose.leaderboard.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/build/docker-compose.leaderboard.yaml b/build/docker-compose.leaderboard.yaml index 8d6633ee..4c43bd87 100644 --- a/build/docker-compose.leaderboard.yaml +++ b/build/docker-compose.leaderboard.yaml @@ -9,6 +9,7 @@ services: file: agent_service.yaml service: agent ports: + # Reserved ports for the debugger - "53000-53100:53000-53100" command: |- bash -c "sleep 10 && sudo chown -R ${USER_UID}:${USER_GID} ../ && \ From 5753cad1a1cc5810e0802d39eea5c90489324f06 Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Fri, 8 Nov 2024 22:09:18 +0100 Subject: [PATCH 10/19] Add missing symlinks --- code/acting/src/debug_wrapper.py | 1 + code/mock/src/debug_wrapper.py | 1 + code/planning/src/debug_wrapper.py | 1 + 3 files changed, 3 insertions(+) create mode 120000 code/acting/src/debug_wrapper.py create mode 120000 code/mock/src/debug_wrapper.py create mode 120000 code/planning/src/debug_wrapper.py diff --git a/code/acting/src/debug_wrapper.py b/code/acting/src/debug_wrapper.py new file mode 120000 index 00000000..e4e2c378 --- /dev/null +++ b/code/acting/src/debug_wrapper.py @@ -0,0 +1 @@ +../../debug_wrapper.py \ No newline at end of file diff --git a/code/mock/src/debug_wrapper.py b/code/mock/src/debug_wrapper.py new file mode 120000 index 00000000..e4e2c378 --- /dev/null +++ b/code/mock/src/debug_wrapper.py @@ -0,0 +1 @@ +../../debug_wrapper.py \ No newline at end of file diff --git a/code/planning/src/debug_wrapper.py b/code/planning/src/debug_wrapper.py new file mode 120000 index 00000000..e4e2c378 --- /dev/null +++ b/code/planning/src/debug_wrapper.py @@ -0,0 +1 @@ +../../debug_wrapper.py \ No newline at end of file From 95912ca96c153cdc706758ce02024824d8cd6d98 Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Sat, 9 Nov 2024 14:59:44 +0100 Subject: [PATCH 11/19] Add setup.py --- code/debugging/CMakeLists.txt | 2 +- code/debugging/setup.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 code/debugging/setup.py diff --git a/code/debugging/CMakeLists.txt b/code/debugging/CMakeLists.txt index 8cc214d8..a857d84f 100644 --- a/code/debugging/CMakeLists.txt +++ b/code/debugging/CMakeLists.txt @@ -18,7 +18,7 @@ find_package(catkin REQUIRED COMPONENTS ## Uncomment this if the package has a setup.py. This macro ensures ## modules and global scripts declared therein get installed ## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html -# catkin_python_setup() +catkin_python_setup() ################################################ ## Declare ROS messages, services and actions ## diff --git a/code/debugging/setup.py b/code/debugging/setup.py new file mode 100644 index 00000000..d320ac00 --- /dev/null +++ b/code/debugging/setup.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +from distutils.core import setup +from catkin_pkg.python_setup import generate_distutils_setup + +setup_args = generate_distutils_setup(packages=["debugging"], package_dir={"": "src"}) +setup(**setup_args) From 6ed2ae2c722ae017d9a24ec6f0c068cb53334b85 Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Tue, 12 Nov 2024 15:02:26 +0100 Subject: [PATCH 12/19] Improve dockerfile ENV var usage --- build/docker/agent/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build/docker/agent/Dockerfile b/build/docker/agent/Dockerfile index 48818408..fb6d9202 100644 --- a/build/docker/agent/Dockerfile +++ b/build/docker/agent/Dockerfile @@ -162,11 +162,10 @@ RUN source ~/.bashrc && pip install -r /workspace/requirements.txt # Add agent code COPY --chown=$USERNAME:$USERNAME ./code /workspace/code/ -# Link code into catkin workspace -RUN ln -s /workspace/code /catkin_ws/src - # For debugger ENV PAF_CATKIN_CODE_ROOT=/catkin_ws/src +# Link code into catkin workspace +RUN ln -s /workspace/code ${PAF_CATKIN_CODE_ROOT} # re-make the catkin workspace RUN source /opt/ros/noetic/setup.bash && catkin_make From 96240644773f57c45a927e0bbf107b2ef6e9618d Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Tue, 12 Nov 2024 15:04:27 +0100 Subject: [PATCH 13/19] Remove unnecessary uses of BaseException and fix debug_wait flag --- code/debug_wrapper.py | 8 +++++--- code/debugging/src/debug_logger.py | 2 +- doc/development/debugging.md | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/code/debug_wrapper.py b/code/debug_wrapper.py index c02d8d0a..924f7037 100755 --- a/code/debug_wrapper.py +++ b/code/debug_wrapper.py @@ -77,7 +77,7 @@ def log(msg: str, level: str): conn.send({"name": NODE_NAME, "msg": msg, "level": level}) conn.close() success = True - except BaseException as e: + except Exception as e: error = e if not success: eprint(msg) @@ -135,7 +135,7 @@ def start_debugger( logwarn(f"Started debugger on {host}:{port} for {node_module_name}") if wait_for_client: debugpy.wait_for_client() - except BaseException as error: + except Exception as error: # Yes, all exceptions should be catched and sent into rosconsole logerr(f"Failed to start debugger: {error}") else: @@ -164,7 +164,7 @@ def main(argv): parser.add_argument("--debug_node", required=True, type=str) parser.add_argument("--debug_port", required=False, type=int) parser.add_argument("--debug_host", default=default_host, type=str) - parser.add_argument("--debug_wait", default=False, type=bool) + parser.add_argument("--debug_wait", action="store_true") args, unknown_args = parser.parse_known_args(node_args) debug_node = args.debug_node @@ -190,6 +190,8 @@ def main(argv): try: run_module_at(target_type_path) except BaseException as error: + # Yes, all exceptions including SystemExit should be catched. + # We want to always know when a node exits logfatal(f"Failed to run node {debug_node}: {error}") raise error diff --git a/code/debugging/src/debug_logger.py b/code/debugging/src/debug_logger.py index 6778531d..317843ce 100755 --- a/code/debugging/src/debug_logger.py +++ b/code/debugging/src/debug_logger.py @@ -100,7 +100,7 @@ def close_listener(): conn = Client(ADDRESS, authkey=AUTHKEY) conn.send(CLOSE_MSG) conn.close() - except BaseException: + except Exception: pass diff --git a/doc/development/debugging.md b/doc/development/debugging.md index 9d364145..42003d96 100644 --- a/doc/development/debugging.md +++ b/doc/development/debugging.md @@ -96,7 +96,7 @@ Configuration example for the [lidar_distance.py](../../code/perception/src/lida ``` By default, the affected node will start up as usual. When you attach the debugger later, you are then only able to debug the callbacks the node receives. -But if you set `--debug_wait=True`, it will block the node from starting until VS Code attaches and allow you to debug the initialization of the node. +But if you set `--debug_wait`, it will block the node from starting until VS Code attaches and allow you to debug the initialization of the node. If the leaderboard hangs on `Setting up the agent` after the configuration has been adjusted, there most likely is a mistake in the launch configuration. From b4c3ed8843cfeaa829745bf38da25716b9ea961b Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Tue, 12 Nov 2024 15:06:41 +0100 Subject: [PATCH 14/19] Remove pyrightconfig (unnecessary for this issue) --- pyrightconfig.json | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 pyrightconfig.json diff --git a/pyrightconfig.json b/pyrightconfig.json deleted file mode 100644 index 5cafbb7c..00000000 --- a/pyrightconfig.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "include": ["code/src"], - - "exclude": [], - - "ignore": [], - - "defineConstant": { - "DEBUG": true - }, - - "typeCheckingMode": "standard", - "strictListInference": true, - "strictDictionaryInference": true, - "strictSetInference": true, - - "pythonVersion": "3.8", - "pythonPlatform": "Linux", - - "executionEnvironments": [ - { - "root": "code/src" - } - ] -} From 2302dc9d968cba94a4c22d367b4c161555cbea6c Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Tue, 12 Nov 2024 15:16:57 +0100 Subject: [PATCH 15/19] Add rough architecture documentation based on CodeRabbit --- doc/development/debugging.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/development/debugging.md b/doc/development/debugging.md index 42003d96..3e26453f 100644 --- a/doc/development/debugging.md +++ b/doc/development/debugging.md @@ -11,6 +11,7 @@ - [Required once for the node you want to debug](#required-once-for-the-node-you-want-to-debug) - [Debugging workflow after setup](#debugging-workflow-after-setup) - [Known problems of debugging with VS Code](#known-problems-of-debugging-with-vs-code) + - [Debug Components Architecture](#debug-components-architecture) - [Rebuild docker containers](#rebuild-docker-containers) - [Sources](#sources) @@ -100,8 +101,6 @@ But if you set `--debug_wait`, it will block the node from starting until VS Cod If the leaderboard hangs on `Setting up the agent` after the configuration has been adjusted, there most likely is a mistake in the launch configuration. -More usage information can be found in the [debug_wrapper](../../code/debug_wrapper.py) and the [debug_logger](../../code/debugging/src/debug_logger.py) - #### Debugging workflow after setup 1. Start/Restart the [leaderboard](../../build/docker-compose.leaderboard.yaml) as usual @@ -117,6 +116,15 @@ With the **rqt_console** GUI you can filter by node *debug_logger* to see all me - Adjusting the launch configurations is a bit cumbersome - The side effects of just "stopping" a node with the debugger are unknown +#### Debug Components Architecture + +More technical usage information can be found in the: + +- [debug_wrapper](../../code/debug_wrapper.py): Acts as a proxy that wraps the target node and enables remote debugging capabilities. +- [debug_logger](../../code/debugging/src/debug_logger.py): Handles the logging of debugging-related events and exceptions, ensuring they are properly published to the ROS ecosystem. + +The wrapper uses the logger to ensure that any debugging-related issues (connection problems, initialization errors, etc.) are visible through the /rosout topic. + ## Rebuild docker containers The docker images on your pc might not match the latest Dockerfile in the repository From 4db8bdaa4ff8fac8812e10194f3e6bab3cdd8d09 Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Tue, 12 Nov 2024 15:30:51 +0100 Subject: [PATCH 16/19] Fix message ordering in logger --- code/debugging/src/debug_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/debugging/src/debug_logger.py b/code/debugging/src/debug_logger.py index 317843ce..4d45b6f5 100755 --- a/code/debugging/src/debug_logger.py +++ b/code/debugging/src/debug_logger.py @@ -136,7 +136,7 @@ def main(): while NODE_RUNNING: with MESSAGES_MUTEX: while len(MESSAGES) > 0: - msg = MESSAGES.pop() + msg = MESSAGES.pop(0) log_msg = "Wrong log message format" log_name = "NAMERR" log_level = "fatal" From 09342c27c09b301e6af828ebbc70063b320237fc Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Tue, 12 Nov 2024 15:50:42 +0100 Subject: [PATCH 17/19] Add exception handling to the receiver in the debug_logger --- code/debugging/src/debug_logger.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/code/debugging/src/debug_logger.py b/code/debugging/src/debug_logger.py index 4d45b6f5..62f7eb4b 100755 --- a/code/debugging/src/debug_logger.py +++ b/code/debugging/src/debug_logger.py @@ -79,16 +79,21 @@ def run_listener(listener: Listener): """ running = True while running: - conn = listener.accept() - print(f"[debug_logger]: connection accepted from {listener.last_accepted}") - msg = None - if conn.poll(timeout=2.0): - msg = conn.recv() - if isinstance(msg, str): - if msg.lower() == CLOSE_MSG: - running = False - msg = None - conn.close() + try: + conn = listener.accept() + print(f"[debug_logger]: connection accepted from {listener.last_accepted}") + msg = None + if conn.poll(timeout=2.0): + msg = conn.recv() + if isinstance(msg, str): + if msg.lower() == CLOSE_MSG: + running = False + msg = None + conn.close() + except Exception as error: + eprint(f"Failed to receive message: {error}") + continue + if msg is not None: with MESSAGES_MUTEX: MESSAGES.append(msg) From bda286aae58d6adeeedaa376fbfaefe0e285ada5 Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Tue, 12 Nov 2024 16:28:31 +0100 Subject: [PATCH 18/19] Fix PAF_CATKIN_CODE_ROOT path --- build/docker/agent/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/docker/agent/Dockerfile b/build/docker/agent/Dockerfile index fb6d9202..e2176159 100644 --- a/build/docker/agent/Dockerfile +++ b/build/docker/agent/Dockerfile @@ -163,9 +163,9 @@ RUN source ~/.bashrc && pip install -r /workspace/requirements.txt COPY --chown=$USERNAME:$USERNAME ./code /workspace/code/ # For debugger -ENV PAF_CATKIN_CODE_ROOT=/catkin_ws/src +ENV PAF_CATKIN_CODE_ROOT=/catkin_ws/src/code # Link code into catkin workspace -RUN ln -s /workspace/code ${PAF_CATKIN_CODE_ROOT} +RUN ln -s -T /workspace/code ${PAF_CATKIN_CODE_ROOT} # re-make the catkin workspace RUN source /opt/ros/noetic/setup.bash && catkin_make From 5716ef77d2853b2ca988a1feeb6d17153cf90a01 Mon Sep 17 00:00:00 2001 From: Peter Viechter Date: Tue, 12 Nov 2024 16:28:52 +0100 Subject: [PATCH 19/19] Add warning when debugging with debug_wait --- code/debug_wrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/code/debug_wrapper.py b/code/debug_wrapper.py index 924f7037..cafd493f 100755 --- a/code/debug_wrapper.py +++ b/code/debug_wrapper.py @@ -134,6 +134,7 @@ def start_debugger( debugpy.listen((host, port)) logwarn(f"Started debugger on {host}:{port} for {node_module_name}") if wait_for_client: + logwarn("Waiting until debugging client is attached...") debugpy.wait_for_client() except Exception as error: # Yes, all exceptions should be catched and sent into rosconsole