From b1d5948d6957f3e18079c02557e53ace9f1d7709 Mon Sep 17 00:00:00 2001 From: HanchuZhou Date: Tue, 20 Aug 2024 21:12:15 -0700 Subject: [PATCH 1/3] Add error wpt --- car_dreamer/configs/common.yaml | 14 +++++++++ car_dreamer/configs/tasks.yaml | 8 ++--- .../observer/handlers/birdeye_handler.py | 2 ++ .../handlers/renderer/birdeye_renderer.py | 29 +++++++++++++++++++ .../observer/handlers/renderer/constants.py | 1 + 5 files changed, 50 insertions(+), 4 deletions(-) diff --git a/car_dreamer/configs/common.yaml b/car_dreamer/configs/common.yaml index 0f4fe4a..3322662 100644 --- a/car_dreamer/configs/common.yaml +++ b/car_dreamer/configs/common.yaml @@ -113,6 +113,20 @@ env: extend_wpt: False entities: [roadmap, waypoints, background_waypoints, ego_vehicle, background_vehicles, traffic_lights, stop_signs] + birdeye_error_wpt: + handler: birdeye + key: birdeye_wpt + shape: [128, 128, 3] + obs_range: 32 + ego_offset: 12 + camera_fov: 150 + observability: full + color_by_obs: False + waypoint_obs: visible + extend_wpt: False + error_rate: 0.1 + entities: [roadmap, waypoints, error_background_waypoints, ego_vehicle, background_vehicles] + display: enable: True render_keys: [camera, birdeye_wpt] diff --git a/car_dreamer/configs/tasks.yaml b/car_dreamer/configs/tasks.yaml index 155ec6d..1e398d5 100644 --- a/car_dreamer/configs/tasks.yaml +++ b/car_dreamer/configs/tasks.yaml @@ -111,7 +111,7 @@ carla_right_turn_simple: &carla_right_turn_simple name: CarlaRightTurnEnv-v0 action: discrete_steer: [-0.9, -0.3, 0.0, 0.3, 0.9] - observation.enabled: [camera, collision, birdeye_wpt] + observation.enabled: [camera, collision, birdeye_error_wpt] <<: *carla_wpt lane_start_point: [-33.8, -135.1, 0.1, 0.0] ego_path: [[-33.8, -135.1, 0.1], [-5.0, -110.3, 0.1]] @@ -119,9 +119,9 @@ carla_right_turn_simple: &carla_right_turn_simple flow_spawn_point: [-3.4, -151.2, 0.1, 90.0] dreamerv3: - encoder.cnn_keys: "birdeye_wpt" - decoder.cnn_keys: "birdeye_wpt" - run.log_keys_video: [camera, birdeye_wpt] + encoder.cnn_keys: "birdeye_error_wpt" + decoder.cnn_keys: "birdeye_error_wpt" + run.log_keys_video: [camera, birdeye_error_wpt] dreamerv2: encoder.cnn_keys: "birdeye_wpt" diff --git a/car_dreamer/toolkit/observer/handlers/birdeye_handler.py b/car_dreamer/toolkit/observer/handlers/birdeye_handler.py index 083cb5b..58887db 100644 --- a/car_dreamer/toolkit/observer/handlers/birdeye_handler.py +++ b/car_dreamer/toolkit/observer/handlers/birdeye_handler.py @@ -61,6 +61,8 @@ def get_observation(self, env_state: Dict) -> Tuple[Dict, Dict]: 'messages_color': messages_color, 'extend_waypoints': self._config.extend_wpt } + if hasattr(self._config, 'error_rate'): + env_state['error_rate'] = self._config.error_rate self._birdeye_render.render(self.surface, entities, env_state) birdeye = self.surface diff --git a/car_dreamer/toolkit/observer/handlers/renderer/birdeye_renderer.py b/car_dreamer/toolkit/observer/handlers/renderer/birdeye_renderer.py index 396c984..948c3e9 100644 --- a/car_dreamer/toolkit/observer/handlers/renderer/birdeye_renderer.py +++ b/car_dreamer/toolkit/observer/handlers/renderer/birdeye_renderer.py @@ -3,6 +3,7 @@ import math import numpy as np import cv2 +import random from ....carla_manager import WorldManager, Command, ActorPolygon from .constants import BirdeyeEntity, Color @@ -161,6 +162,33 @@ def _render_background_waypoints(self, **env_state): last = path[-1] path.append((last[0], last[1] - 10.0)) self._render_path(self._surface, vehicle_polygon, path, waypoint_color) + + def _render_error_background_waypoints(self, **env_state): + """Render the waypoints with error for background actors on the surface.""" + color = env_state.get('background_waypoints_color') + extend_waypoints = env_state.get('extend_waypoints', False) + error_rate = env_state.get('error_rate') + background_waypoints = self._world_manager.actor_actions + background_waypoints = { + id: [(action[1].transform.location.x, action[1].transform.location.y) for action in actions] + for id, actions in background_waypoints.items() if actions + } + vehicle_polygons = self._world_manager.actor_polygons + + for vehicle_id, path in background_waypoints.items(): + if vehicle_id == self._ego.id or should_filter(self._ego.get_transform(), self._world_manager.actor_transforms[vehicle_id]): + continue + vehicle_polygon = vehicle_polygons.get(vehicle_id, None) + if vehicle_polygon is None: + continue + waypoint_color = color.get(vehicle_id, None) + if waypoint_color is None: + continue + if extend_waypoints: + last = path[-1] + path.append((last[0], last[1] - 10.0)) + if random.random() > error_rate: + self._render_path(self._surface, vehicle_polygon, path, waypoint_color) def _render_messages(self, **env_state): """ @@ -329,4 +357,5 @@ def _render_path(self, surface: np.ndarray, vehicle_polygon: ActorPolygon, path: BirdeyeEntity.TRAFFIC_LIGHTS: _render_traffic_lights, BirdeyeEntity.STOP_SIGNS: _render_stop_signs, BirdeyeEntity.MESSAGES: _render_messages, + BirdeyeEntity.ERROR_BACKGROUND_WAYPOINTS: _render_error_background_waypoints, } diff --git a/car_dreamer/toolkit/observer/handlers/renderer/constants.py b/car_dreamer/toolkit/observer/handlers/renderer/constants.py index 860ede0..63ee874 100644 --- a/car_dreamer/toolkit/observer/handlers/renderer/constants.py +++ b/car_dreamer/toolkit/observer/handlers/renderer/constants.py @@ -80,3 +80,4 @@ class BirdeyeEntity(Enum): TRAFFIC_LIGHTS = "traffic_lights" STOP_SIGNS = "stop_signs" MESSAGES = "messages" + ERROR_BACKGROUND_WAYPOINTS = "error_background_vehicles" From 7bbaeb7d3c7e055016b5d029e7f574387ea4f230 Mon Sep 17 00:00:00 2001 From: "zhouhanchu@hotmail.com" Date: Tue, 20 Aug 2024 21:24:35 -0700 Subject: [PATCH 2/3] fix error bug --- car_dreamer/toolkit/observer/handlers/renderer/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/car_dreamer/toolkit/observer/handlers/renderer/constants.py b/car_dreamer/toolkit/observer/handlers/renderer/constants.py index 63ee874..bcb9b55 100644 --- a/car_dreamer/toolkit/observer/handlers/renderer/constants.py +++ b/car_dreamer/toolkit/observer/handlers/renderer/constants.py @@ -80,4 +80,4 @@ class BirdeyeEntity(Enum): TRAFFIC_LIGHTS = "traffic_lights" STOP_SIGNS = "stop_signs" MESSAGES = "messages" - ERROR_BACKGROUND_WAYPOINTS = "error_background_vehicles" + ERROR_BACKGROUND_WAYPOINTS = "error_background_waypoints" From d2f91317d45da992beecd1998c04e0f46c5acdee Mon Sep 17 00:00:00 2001 From: HanchuZhou Date: Tue, 20 Aug 2024 21:36:32 -0700 Subject: [PATCH 3/3] fixed error bug --- car_dreamer/configs/common.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/car_dreamer/configs/common.yaml b/car_dreamer/configs/common.yaml index 3322662..eeafa5c 100644 --- a/car_dreamer/configs/common.yaml +++ b/car_dreamer/configs/common.yaml @@ -115,7 +115,7 @@ env: birdeye_error_wpt: handler: birdeye - key: birdeye_wpt + key: birdeye_error_wpt shape: [128, 128, 3] obs_range: 32 ego_offset: 12 @@ -124,7 +124,7 @@ env: color_by_obs: False waypoint_obs: visible extend_wpt: False - error_rate: 0.1 + error_rate: 0.2 entities: [roadmap, waypoints, error_background_waypoints, ego_vehicle, background_vehicles] display: