diff --git a/.gitignore b/.gitignore index 29c8f5c..2f87dfa 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ data/ .DS_store __pycache__ -Monoloco/*.pyc +monstereo/*.pyc .pytest* build/ dist/ diff --git a/monstereo/__init__.py b/monstereo/__init__.py index d64e70b..dace57d 100644 --- a/monstereo/__init__.py +++ b/monstereo/__init__.py @@ -1,4 +1,4 @@ """Open implementation of MonoLoco++ / MonStereo.""" -__version__ = '0.2.0' +__version__ = '0.2.1' diff --git a/monstereo/activity.py b/monstereo/activity.py index 0ea288b..e123dfc 100644 --- a/monstereo/activity.py +++ b/monstereo/activity.py @@ -111,18 +111,11 @@ def show_social(args, image_t, output_path, annotations, dic_out): assert 'front' in args.output_types or 'bird' in args.output_types, "outputs allowed: front and/or bird" angles = dic_out['angles'] - dds = dic_out['dds_pred'] stds = dic_out['stds_ale'] xz_centers = [[xx[0], xx[2]] for xx in dic_out['xyz_pred']] # Prepare color for social distancing - colors = ['r' if social_interactions(idx, xz_centers, angles, dds, - stds=stds, - threshold_prob=args.threshold_prob, - threshold_dist=args.threshold_dist, - radii=args.radii) - else 'deepskyblue' - for idx, _ in enumerate(dic_out['xyz_pred'])] + colors = ['r' if flag else 'deepskyblue' for flag in dic_out['social_distance']] # Draw keypoints and orientation if 'front' in args.output_types: diff --git a/monstereo/network/net.py b/monstereo/network/net.py index 67b7ac9..d7e5d85 100644 --- a/monstereo/network/net.py +++ b/monstereo/network/net.py @@ -14,6 +14,7 @@ from ..utils import get_iou_matches, reorder_matches, get_keypoints, pixel_to_camera, xyz_from_distance from .process import preprocess_monstereo, preprocess_monoloco, extract_outputs, extract_outputs_mono,\ filter_outputs, cluster_outputs, unnormalize_bi +from ..activity import social_interactions from .architectures import MonolocoModel, MonStereoModel @@ -172,7 +173,7 @@ def post_process(dic_in, boxes, keypoints, kk, dic_gt=None, iou_min=0.3, reorder if verbose: print("NO ground-truth associated") - if reorder: + if reorder and matches: matches = reorder_matches(matches, boxes, mode='left_right') all_idxs = [idx for idx, _ in matches] + not_matches @@ -233,6 +234,23 @@ def post_process(dic_in, boxes, keypoints, kk, dic_gt=None, iou_min=0.3, reorder dic_out['xyz_real'].append(xyz_real.squeeze().tolist()) return dic_out + @staticmethod + def social_distance(dic_out, args): + + angles = dic_out['angles'] + dds = dic_out['dds_pred'] + stds = dic_out['stds_ale'] + xz_centers = [[xx[0], xx[2]] for xx in dic_out['xyz_pred']] + + # Prepare color for social distancing + dic_out['social_distance'] = [bool(social_interactions(idx, xz_centers, angles, dds, + stds=stds, + threshold_prob=args.threshold_prob, + threshold_dist=args.threshold_dist, + radii=args.radii)) + for idx, _ in enumerate(dic_out['xyz_pred'])] + return dic_out + def median_disparity(dic_out, keypoints, keypoints_r, mask): """ diff --git a/monstereo/network/process.py b/monstereo/network/process.py index 9b12378..a79e6b9 100644 --- a/monstereo/network/process.py +++ b/monstereo/network/process.py @@ -1,11 +1,15 @@ import json import os +import logging import numpy as np import torch import torchvision +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + from ..utils import get_keypoints, pixel_to_camera, to_cartesian, back_correct_angles BF = 0.54 * 721 @@ -13,6 +17,9 @@ z_max = 60 D_MIN = BF / z_max D_MAX = BF / z_min +FL = 5.7 # nuScenes focal length (mm) +Sx = 7.2 # nuScenes sensor size (mm) +Sy = 5.4 # nuScenes sensor size (mm) def preprocess_monstereo(keypoints, keypoints_r, kk): @@ -67,31 +74,28 @@ def factory_for_gt(im_size, name=None, path_gt=None, verbose=True): with open(path_gt, 'r') as f: dic_names = json.load(f) if verbose: - print('-' * 120 + "\nGround-truth file opened") + logger.info('-' * 120 + "\nGround-truth file opened") except (FileNotFoundError, TypeError): if verbose: - print('-' * 120 + "\nGround-truth file not found") + logger.info('-' * 120 + "\nGround-truth file not found") dic_names = {} try: kk = dic_names[name]['K'] dic_gt = dic_names[name] if verbose: - print("Matched ground-truth file!") + logger.info("Matched ground-truth file!") except KeyError: dic_gt = None - x_factor = im_size[0] / 1600 - y_factor = im_size[1] / 900 - pixel_factor = (x_factor + y_factor) / 1.75 # 1.75 for MOT - # pixel_factor = 1 - if im_size[0] / im_size[1] > 2.5: + if im_size[0] / im_size[1] > 2.5: # KITTI default kk = [[718.3351, 0., 600.3891], [0., 718.3351, 181.5122], [0., 0., 1.]] # Kitti calibration - else: - kk = [[1266.4 * pixel_factor, 0., 816.27 * x_factor], - [0, 1266.4 * pixel_factor, 491.5 * y_factor], - [0., 0., 1.]] # nuScenes calibration + else: # nuScenes camera parameters + kk = [ + [im_size[0]*FL/Sx, 0., im_size[0]/2], + [0., im_size[1]*FL/Sy, im_size[1]/2], + [0., 0., 1.]] if verbose: - print("Using a standard calibration matrix...") + logger.info("Using a standard calibration matrix...") return kk, dic_gt diff --git a/monstereo/predict.py b/monstereo/predict.py index 25338cd..5b4f443 100644 --- a/monstereo/predict.py +++ b/monstereo/predict.py @@ -27,6 +27,7 @@ OPENPIFPAF_PATH = 'data/models/shufflenetv2k30-201104-224654-cocokp-d75ed641.pkl' # Default model + def factory_from_args(args): # Data @@ -40,7 +41,7 @@ def factory_from_args(args): if os.path.exists(OPENPIFPAF_PATH): args.checkpoint = OPENPIFPAF_PATH else: - print("Checkpoint for OpenPifPaf not specified and default model not found in 'data/models'. " + LOG.info("Checkpoint for OpenPifPaf not specified and default model not found in 'data/models'. " "Using a ShuffleNet backbone") args.checkpoint = 'shufflenetv2k30' @@ -64,7 +65,7 @@ def factory_from_args(args): # Make default pifpaf argument args.force_complete_pose = True - print("Force complete pose is active") + LOG.info("Force complete pose is active") # Configure decoder.configure(args) @@ -82,7 +83,6 @@ def predict(args): # Load Models assert args.net in ('monoloco_pp', 'monstereo', 'pifpaf') - if args.net in ('monoloco_pp', 'monstereo'): net = Loco(model=args.model, net=args.net, device=args.device, n_dropout=args.n_dropout, p_dropout=args.dropout) @@ -99,15 +99,12 @@ def predict(args): data, batch_size=args.batch_size, shuffle=False, pin_memory=False, collate_fn=datasets.collate_images_anns_meta) - # visualizers - annotation_painter = openpifpaf.show.AnnotationPainter() - for batch_i, (image_tensors_batch, _, meta_batch) in enumerate(data_loader): pred_batch = processor.batch(model, image_tensors_batch, device=args.device) # unbatch (only for MonStereo) for idx, (pred, meta) in enumerate(zip(pred_batch, meta_batch)): - LOG.info('batch %d: %s', batch_i, meta['file_name']) + print('batch %d: %s', batch_i, meta['file_name']) pred = preprocess.annotations_inverse(pred, meta) if args.output_directory is None: @@ -117,15 +114,16 @@ def predict(args): file_name = os.path.basename(meta['file_name']) output_path = os.path.join(args.output_directory, 'out_' + file_name) print('image', batch_i, meta['file_name'], output_path) - pifpaf_out = [ann.json_data() for ann in pred] if idx == 0: - pifpaf_outputs = pred # to only print left image for stereo - pifpaf_outs = {'left': pifpaf_out} with open(meta_batch[0]['file_name'], 'rb') as f: cpu_image = PIL.Image.open(f).convert('RGB') + pifpaf_outs = { + 'pred': pred, + 'left': [ann.json_data() for ann in pred], + 'image': cpu_image} else: - pifpaf_outs['right'] = pifpaf_out + pifpaf_outs['right'] = [ann.json_data() for ann in pred] # 3D Predictions if args.net in ('monoloco_pp', 'monstereo'): @@ -138,15 +136,14 @@ def predict(args): boxes, keypoints = preprocess_pifpaf(pifpaf_outs['left'], im_size, enlarge_boxes=False) if args.net == 'monoloco_pp': - print("Prediction with MonoLoco++") + LOG.info("Prediction with MonoLoco++") dic_out = net.forward(keypoints, kk) - dic_out = net.post_process(dic_out, boxes, keypoints, kk, dic_gt, reorder=not args.social_distance) - + dic_out = net.post_process(dic_out, boxes, keypoints, kk, dic_gt) if args.social_distance: - show_social(args, cpu_image, output_path, pifpaf_out, dic_out) + dic_out = net.social_distance(dic_out, args) else: - print("Prediction with MonStereo") + LOG.info("Prediction with MonStereo") boxes_r, keypoints_r = preprocess_pifpaf(pifpaf_outs['right'], im_size) dic_out = net.forward(keypoints, kk, keypoints_r=keypoints_r) dic_out = net.post_process(dic_out, boxes, keypoints, kk, dic_gt) @@ -155,28 +152,38 @@ def predict(args): dic_out = defaultdict(list) kk = None - if not args.social_distance: - factory_outputs(args, annotation_painter, cpu_image, output_path, pifpaf_outputs, - dic_out=dic_out, kk=kk) - print('Image {}\n'.format(cnt) + '-' * 120) + # Outputs + factory_outputs(args, pifpaf_outs, dic_out, output_path, kk=kk) + LOG.info('Image {}\n'.format(cnt) + '-' * 120) cnt += 1 -def factory_outputs(args, annotation_painter, cpu_image, output_path, pred, dic_out=None, kk=None): +def factory_outputs(args, pifpaf_outs, dic_out, output_path, kk=None): """Output json files or images according to the choice""" - # Save json file + # Verify conflicting options + if any((xx in args.output_types for xx in ['front', 'bird', 'multi'])): + assert args.net != 'pifpaf', "please use pifpaf original arguments" + if args.social_distance: + assert args.net == 'monoloco_pp', "Social distancing only works with MonoLoco++ network" + if args.net == 'pifpaf': - with openpifpaf.show.image_canvas(cpu_image, output_path) as ax: - annotation_painter.annotations(ax, pred) - - if any((xx in args.output_types for xx in ['front', 'bird', 'multi'])): - print(output_path) - if dic_out['boxes']: # Only print in case of detections - printer = Printer(cpu_image, output_path, kk, args) - figures, axes = printer.factory_axes(dic_out) - printer.draw(figures, axes, cpu_image) - - if 'json' in args.output_types: - with open(os.path.join(output_path + '.monoloco.json'), 'w') as ff: - json.dump(dic_out, ff) + annotation_painter = openpifpaf.show.AnnotationPainter() + with openpifpaf.show.image_canvas(pifpaf_outs['image'], output_path) as ax: + annotation_painter.annotations(ax, pifpaf_outs['pred']) + + elif any((xx in args.output_types for xx in ['front', 'bird', 'multi'])): + LOG.info(output_path) + if args.social_distance: + show_social(args, pifpaf_outs['image'], output_path, pifpaf_outs['left'], dic_out) + else: + printer = Printer(pifpaf_outs['image'], output_path, kk, args) + figures, axes = printer.factory_axes(dic_out) + printer.draw(figures, axes, pifpaf_outs['image']) + + elif 'json' in args.output_types: + with open(os.path.join(output_path + '.monoloco.json'), 'w') as ff: + json.dump(dic_out, ff) + + else: + LOG.info("No output saved, please select one among front, bird, multi, or pifpaf options")