Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Motmetrics upgrade #263

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Motmetrics upgrade #263

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5f1a829
Update eval_tracking.py
ahuller3 Aug 3, 2021
eb57a32
Update eval_tracking.py
ahuller3 Aug 3, 2021
d3ae182
Update eval_tracking.py
ahuller3 Aug 4, 2021
6e5072c
Update setup.py
ahuller3 Aug 4, 2021
db024ef
Update eval_tracking.py
ahuller3 Aug 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 27 additions & 20 deletions argoverse/evaluation/eval_tracking.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
"""
Computes multiple object tracking (MOT) metrics. Please refer to the following
papers for definitions of the following metrics:

#FRAG, #IDSW: Milan et al., MOT16, https://arxiv.org/pdf/1603.00831.pdf
MT, ML: Leal-Taixe et al., MOT15, https://arxiv.org/pdf/1504.01942.pdf
MOTA: Bernardin et al. https://link.springer.com/article/10.1155/2008/246309
Expand All @@ -34,13 +33,11 @@
def in_distance_range_pose(ego_center: np.ndarray, pose: np.ndarray, d_min: float, d_max: float) -> bool:
"""Determine whether a pose in the ego-vehicle frame falls within a specified distance range
of the egovehicle's origin.

Args:
ego_center: ego center pose (zero if bbox is in ego frame).
pose: pose to test.
d_min: minimum distance range
d_max: maximum distance range

Returns:
A boolean saying if input pose is with specified distance range.
"""
Expand Down Expand Up @@ -80,12 +77,10 @@ def get_distance_iou_3d(x1: Dict[str, np.ndarray], x2: Dict[str, np.ndarray], na

def get_distance(x1: Dict[str, np.ndarray], x2: Dict[str, np.ndarray], name: str) -> float:
"""Get the distance between two poses, returns nan if distance is larger than detection threshold.

Args:
x1: first pose
x2: second pose
name: name of the field to test

Returns:
A distance value or NaN
"""
Expand Down Expand Up @@ -115,7 +110,6 @@ def eval_tracks(
category: str = "VEHICLE",
) -> None:
"""Evaluate tracking output.

Args:
path_tracker_output_root: path to tracker output root, containing log_id subdirs
path_dataset_root: path to dataset root, containing log_id subdirs
Expand All @@ -132,11 +126,12 @@ def eval_tracks(
However, `mostly tracked` and `mostly lost` will be accurate.
category: such as "VEHICLE" "PEDESTRIAN"
"""

acc_c = mm.MOTAccumulator(auto_id=True)
acc_i = mm.MOTAccumulator(auto_id=True)
acc_o = mm.MOTAccumulator(auto_id=True)

ID_gt_all: List[str] = []
ID_gt_all: List[int] = []

count_all: int = 0
if diffatt is not None:
Expand All @@ -155,7 +150,6 @@ def eval_tracks(
num_total_gt = 0

for path_dataset in path_datasets:

log_id = pathlib.Path(path_dataset).name
if len(log_id) == 0 or log_id.startswith("_"):
continue
Expand All @@ -165,7 +159,7 @@ def eval_tracks(
path_track_data = sorted(
glob.glob(os.path.join(os.fspath(path_tracker_output), "per_sweep_annotations_amodal", "*"))
)

all_uuids: Dict[str, int] = {}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could we add a brief inline comment here explaining what the all_uuids dictionary is for?

logger.info("log_id = %s", log_id)

for ind_frame in range(len(path_track_data)):
Expand All @@ -185,10 +179,10 @@ def eval_tracks(

gt_data = read_json_file(path_gt)

gt: Dict[str, Dict[str, Any]] = {}
gt: Dict[int, Dict[str, Any]] = {}
id_gts = []
track_label_uuids: Dict[str, int] = {}
for i in range(len(gt_data)):

if gt_data[i]["label_class"] != category:
continue

Expand All @@ -206,8 +200,14 @@ def eval_tracks(
gt_data[i]["center"]["z"],
]
)

if bbox[3] > 0 and in_distance_range_pose(np.zeros(3), center, d_min, d_max):
track_label_uuid = gt_data[i]["track_label_uuid"]
if gt_data[i]["track_label_uuid"] not in track_label_uuids:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi Anna, could you add a brief inline comment here explaining the logic for what we are doing here?

track_label_uuids[gt_data[i]["track_label_uuid"]] = i
track_label_uuid = i
else:
track_label_uuid = track_label_uuids[gt_data[i]["track_label_uuid"]]

gt[track_label_uuid] = {}
gt[track_label_uuid]["centroid"] = center

Expand All @@ -223,13 +223,15 @@ def eval_tracks(
id_gts.append(track_label_uuid)
num_total_gt += 1

tracks: Dict[str, Dict[str, Any]] = {}
id_tracks: List[str] = []
tracks: Dict[int, Dict[str, Any]] = {}
id_tracks: List[int] = []

track_data = read_json_file(path_track_data[ind_frame])

for track in track_data:
key = track["track_label_uuid"]
if track["track_label_uuid"] not in all_uuids:
all_uuids[track["track_label_uuid"]] = len(all_uuids) + 1
key = all_uuids[track["track_label_uuid"]]

if track["label_class"] != category or track["height"] == 0:
continue
Expand Down Expand Up @@ -259,14 +261,19 @@ def eval_tracks(
dists_o.append(gt_track_data_o)
for track_key, track_value in tracks.items():
count_all += 1
gt_track_data_c.append(get_distance(gt_value, track_value, "centroid"))
gt_track_data_i.append(get_distance(gt_value, track_value, "iou"))
gt_track_data_o.append(get_distance(gt_value, track_value, "orientation"))

centroid_distance = get_distance(gt_value, track_value, "centroid")
# appends distances if centroid distance is valid
if np.isfinite(centroid_distance):
gt_track_data_c.append(centroid_distance)
gt_track_data_i.append(get_distance(gt_value, track_value, "iou"))
gt_track_data_o.append(get_distance(gt_value, track_value, "orientation"))
else:
gt_track_data_c.append(float(np.nan))
gt_track_data_i.append(float(np.nan))
gt_track_data_o.append(float(np.nan))
acc_c.update(id_gts, id_tracks, dists_c)
acc_i.update(id_gts, id_tracks, dists_i)
acc_o.update(id_gts, id_tracks, dists_o)
# print(count_all)
if count_all == 0:
# fix for when all hypothesis is empty,
# pymotmetric currently doesn't support this, see https://github.com/cheind/py-motmetrics/issues/49
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
"hydra-core==1.1.0",
"lap",
"matplotlib",
"motmetrics==1.1.3",
"motmetrics==1.2",
"numba",
"numpy==1.19",
"omegaconf==2.1.0",
Expand Down