From 5f1a829624e53140382df4aa55e39543690e194f Mon Sep 17 00:00:00 2001 From: "Huller, Anna Elizabeth" Date: Tue, 3 Aug 2021 16:52:42 -0400 Subject: [PATCH 1/5] Update eval_tracking.py makes changes to make data entries compatable with motmetrics version 1.2 for faster runtime --- argoverse/evaluation/eval_tracking.py | 43 ++++++++++++++++----------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/argoverse/evaluation/eval_tracking.py b/argoverse/evaluation/eval_tracking.py index 53911cf3..40582e2e 100644 --- a/argoverse/evaluation/eval_tracking.py +++ b/argoverse/evaluation/eval_tracking.py @@ -24,7 +24,6 @@ """ Computes multiple object tracking (MOT) metrics. Please refer to the following papers for definitions of the following metrics: - #FRAG, #IDSW: Milan et al., MOT16, https://arxiv.org/pdf/1603.00831.pdf MT, ML: Leal-Taixe et al., MOT15, https://arxiv.org/pdf/1504.01942.pdf MOTA: Bernardin et al. https://link.springer.com/article/10.1155/2008/246309 @@ -34,13 +33,11 @@ def in_distance_range_pose(ego_center: np.ndarray, pose: np.ndarray, d_min: float, d_max: float) -> bool: """Determine whether a pose in the ego-vehicle frame falls within a specified distance range of the egovehicle's origin. - Args: ego_center: ego center pose (zero if bbox is in ego frame). pose: pose to test. d_min: minimum distance range d_max: maximum distance range - Returns: A boolean saying if input pose is with specified distance range. """ @@ -80,12 +77,10 @@ def get_distance_iou_3d(x1: Dict[str, np.ndarray], x2: Dict[str, np.ndarray], na def get_distance(x1: Dict[str, np.ndarray], x2: Dict[str, np.ndarray], name: str) -> float: """Get the distance between two poses, returns nan if distance is larger than detection threshold. - Args: x1: first pose x2: second pose name: name of the field to test - Returns: A distance value or NaN """ @@ -103,7 +98,6 @@ def get_distance(x1: Dict[str, np.ndarray], x2: Dict[str, np.ndarray], name: str else: raise NotImplementedError("Not implemented..") - def eval_tracks( path_tracker_output_root: _PathLike, path_dataset_root: _PathLike, @@ -115,7 +109,6 @@ def eval_tracks( category: str = "VEHICLE", ) -> None: """Evaluate tracking output. - Args: path_tracker_output_root: path to tracker output root, containing log_id subdirs path_dataset_root: path to dataset root, containing log_id subdirs @@ -132,6 +125,7 @@ def eval_tracks( However, `mostly tracked` and `mostly lost` will be accurate. category: such as "VEHICLE" "PEDESTRIAN" """ + acc_c = mm.MOTAccumulator(auto_id=True) acc_i = mm.MOTAccumulator(auto_id=True) acc_o = mm.MOTAccumulator(auto_id=True) @@ -154,8 +148,9 @@ def eval_tracks( path_datasets = glob.glob(os.path.join(path_dataset_root, "*")) num_total_gt = 0 + path_counter = 0 for path_dataset in path_datasets: - + path_counter += 1 log_id = pathlib.Path(path_dataset).name if len(log_id) == 0 or log_id.startswith("_"): continue @@ -165,7 +160,7 @@ def eval_tracks( path_track_data = sorted( glob.glob(os.path.join(os.fspath(path_tracker_output), "per_sweep_annotations_amodal", "*")) ) - + all_uuids = {} logger.info("log_id = %s", log_id) for ind_frame in range(len(path_track_data)): @@ -187,8 +182,8 @@ def eval_tracks( gt: Dict[str, Dict[str, Any]] = {} id_gts = [] + track_label_uuids = {} for i in range(len(gt_data)): - if gt_data[i]["label_class"] != category: continue @@ -206,8 +201,14 @@ def eval_tracks( gt_data[i]["center"]["z"], ] ) + if bbox[3] > 0 and in_distance_range_pose(np.zeros(3), center, d_min, d_max): - track_label_uuid = gt_data[i]["track_label_uuid"] + if gt_data[i]["track_label_uuid"] not in track_label_uuids: + track_label_uuids[gt_data[i]["track_label_uuid"]] = i + track_label_uuid = i + else: + track_label_uuid = track_label_uuids[gt_data[i]["track_label_uuid"]] + gt[track_label_uuid] = {} gt[track_label_uuid]["centroid"] = center @@ -229,7 +230,11 @@ def eval_tracks( track_data = read_json_file(path_track_data[ind_frame]) for track in track_data: - key = track["track_label_uuid"] + + if track["track_label_uuid"] not in all_uuids: + all_uuids[track["track_label_uuid"]] = len(all_uuids) + 1 + key = all_uuids[track["track_label_uuid"]] + if track["label_class"] != category or track["height"] == 0: continue @@ -259,14 +264,18 @@ def eval_tracks( dists_o.append(gt_track_data_o) for track_key, track_value in tracks.items(): count_all += 1 - gt_track_data_c.append(get_distance(gt_value, track_value, "centroid")) - gt_track_data_i.append(get_distance(gt_value, track_value, "iou")) - gt_track_data_o.append(get_distance(gt_value, track_value, "orientation")) - + centroid_distance = get_distance(gt_value, track_value, "centroid") + if np.isfinite(centroid_distance): + gt_track_data_c.append(centroid_distance) + gt_track_data_i.append(get_distance(gt_value, track_value, "iou")) + gt_track_data_o.append(get_distance(gt_value, track_value, "orientation")) + else: + gt_track_data_c.append(float(np.nan)) + gt_track_data_i.append(float(np.nan)) + gt_track_data_o.append(float(np.nan)) acc_c.update(id_gts, id_tracks, dists_c) acc_i.update(id_gts, id_tracks, dists_i) acc_o.update(id_gts, id_tracks, dists_o) - # print(count_all) if count_all == 0: # fix for when all hypothesis is empty, # pymotmetric currently doesn't support this, see https://github.com/cheind/py-motmetrics/issues/49 From eb57a328840d5502ee1c10a04fd791325297c840 Mon Sep 17 00:00:00 2001 From: "Huller, Anna Elizabeth" Date: Tue, 3 Aug 2021 17:35:51 -0400 Subject: [PATCH 2/5] Update eval_tracking.py --- argoverse/evaluation/eval_tracking.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/argoverse/evaluation/eval_tracking.py b/argoverse/evaluation/eval_tracking.py index 40582e2e..4cd068c5 100644 --- a/argoverse/evaluation/eval_tracking.py +++ b/argoverse/evaluation/eval_tracking.py @@ -98,6 +98,7 @@ def get_distance(x1: Dict[str, np.ndarray], x2: Dict[str, np.ndarray], name: str else: raise NotImplementedError("Not implemented..") + def eval_tracks( path_tracker_output_root: _PathLike, path_dataset_root: _PathLike, @@ -160,7 +161,7 @@ def eval_tracks( path_track_data = sorted( glob.glob(os.path.join(os.fspath(path_tracker_output), "per_sweep_annotations_amodal", "*")) ) - all_uuids = {} + all_uuids = {} logger.info("log_id = %s", log_id) for ind_frame in range(len(path_track_data)): @@ -182,7 +183,7 @@ def eval_tracks( gt: Dict[str, Dict[str, Any]] = {} id_gts = [] - track_label_uuids = {} + track_label_uuids = {} for i in range(len(gt_data)): if gt_data[i]["label_class"] != category: continue @@ -230,12 +231,10 @@ def eval_tracks( track_data = read_json_file(path_track_data[ind_frame]) for track in track_data: - if track["track_label_uuid"] not in all_uuids: all_uuids[track["track_label_uuid"]] = len(all_uuids) + 1 key = all_uuids[track["track_label_uuid"]] - if track["label_class"] != category or track["height"] == 0: continue @@ -265,6 +264,7 @@ def eval_tracks( for track_key, track_value in tracks.items(): count_all += 1 centroid_distance = get_distance(gt_value, track_value, "centroid") + # appends distances if centroid distance is valid if np.isfinite(centroid_distance): gt_track_data_c.append(centroid_distance) gt_track_data_i.append(get_distance(gt_value, track_value, "iou")) From d3ae182e0e11b5efcf3c7c195552856f1105eb0d Mon Sep 17 00:00:00 2001 From: "Huller, Anna Elizabeth" Date: Wed, 4 Aug 2021 11:55:20 -0400 Subject: [PATCH 3/5] Update eval_tracking.py --- argoverse/evaluation/eval_tracking.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/argoverse/evaluation/eval_tracking.py b/argoverse/evaluation/eval_tracking.py index 4cd068c5..9a52dc0e 100644 --- a/argoverse/evaluation/eval_tracking.py +++ b/argoverse/evaluation/eval_tracking.py @@ -131,7 +131,7 @@ def eval_tracks( acc_i = mm.MOTAccumulator(auto_id=True) acc_o = mm.MOTAccumulator(auto_id=True) - ID_gt_all: List[str] = [] + ID_gt_all: List[int] = [] count_all: int = 0 if diffatt is not None: @@ -149,9 +149,7 @@ def eval_tracks( path_datasets = glob.glob(os.path.join(path_dataset_root, "*")) num_total_gt = 0 - path_counter = 0 for path_dataset in path_datasets: - path_counter += 1 log_id = pathlib.Path(path_dataset).name if len(log_id) == 0 or log_id.startswith("_"): continue @@ -161,7 +159,7 @@ def eval_tracks( path_track_data = sorted( glob.glob(os.path.join(os.fspath(path_tracker_output), "per_sweep_annotations_amodal", "*")) ) - all_uuids = {} + all_uuids: Dict[str, int] = {} logger.info("log_id = %s", log_id) for ind_frame in range(len(path_track_data)): @@ -181,9 +179,9 @@ def eval_tracks( gt_data = read_json_file(path_gt) - gt: Dict[str, Dict[str, Any]] = {} + gt: Dict[int, Dict[str, Any]] = {} id_gts = [] - track_label_uuids = {} + track_label_uuids: Dict[str, int] = {} for i in range(len(gt_data)): if gt_data[i]["label_class"] != category: continue @@ -225,8 +223,8 @@ def eval_tracks( id_gts.append(track_label_uuid) num_total_gt += 1 - tracks: Dict[str, Dict[str, Any]] = {} - id_tracks: List[str] = [] + tracks: Dict[int, Dict[str, Any]] = {} + id_tracks: List[int] = [] track_data = read_json_file(path_track_data[ind_frame]) From 6e5072cce3fd314f934abbd7706bebd859a478b2 Mon Sep 17 00:00:00 2001 From: "Huller, Anna Elizabeth" Date: Wed, 4 Aug 2021 12:00:46 -0400 Subject: [PATCH 4/5] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1c706ed5..03dadd5c 100755 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ "hydra-core==1.1.0", "lap", "matplotlib", - "motmetrics==1.1.3", + "motmetrics==1.2", "numba", "numpy==1.19", "omegaconf==2.1.0", From db024efdfa8f2e1a939b15c429fc2d16e265f66b Mon Sep 17 00:00:00 2001 From: "Huller, Anna Elizabeth" Date: Wed, 25 Aug 2021 17:56:58 -0400 Subject: [PATCH 5/5] Update eval_tracking.py --- argoverse/evaluation/eval_tracking.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/argoverse/evaluation/eval_tracking.py b/argoverse/evaluation/eval_tracking.py index 9a52dc0e..712f9e81 100644 --- a/argoverse/evaluation/eval_tracking.py +++ b/argoverse/evaluation/eval_tracking.py @@ -159,7 +159,7 @@ def eval_tracks( path_track_data = sorted( glob.glob(os.path.join(os.fspath(path_tracker_output), "per_sweep_annotations_amodal", "*")) ) - all_uuids: Dict[str, int] = {} + all_uuids: Dict[str, int] = {} # stores all uuids attached to unique integers logger.info("log_id = %s", log_id) for ind_frame in range(len(path_track_data)): @@ -181,7 +181,7 @@ def eval_tracks( gt: Dict[int, Dict[str, Any]] = {} id_gts = [] - track_label_uuids: Dict[str, int] = {} + track_label_uuids: Dict[str, int] = {} # stores uuids in frame with unique ints for i in range(len(gt_data)): if gt_data[i]["label_class"] != category: continue @@ -202,6 +202,7 @@ def eval_tracks( ) if bbox[3] > 0 and in_distance_range_pose(np.zeros(3), center, d_min, d_max): + # assign a unique int value to each uuid in the frame if gt_data[i]["track_label_uuid"] not in track_label_uuids: track_label_uuids[gt_data[i]["track_label_uuid"]] = i track_label_uuid = i