From d3d0d0eccf155daa0e1aab95b219f5f0c3f39a44 Mon Sep 17 00:00:00 2001 From: MedericFourmy Date: Wed, 18 Oct 2023 06:25:23 -0400 Subject: [PATCH 1/7] Use a more logical rule for selecting detections + code cleaning --- .../megapose/evaluation/bop.py | 81 +++++++------------ .../megapose/evaluation/prediction_runner.py | 28 +++---- 2 files changed, 40 insertions(+), 69 deletions(-) diff --git a/happypose/pose_estimators/megapose/evaluation/bop.py b/happypose/pose_estimators/megapose/evaluation/bop.py index 0729e167..975444c3 100644 --- a/happypose/pose_estimators/megapose/evaluation/bop.py +++ b/happypose/pose_estimators/megapose/evaluation/bop.py @@ -262,19 +262,15 @@ def run_evaluation(cfg: BOPEvalConfig) -> None: return scores_pose_path, scores_detection_path -def load_sam_predictions(ds_dir_name, scene_ds_dir): - ds_name = ds_dir_name - detections_path = CNOS_SUBMISSION_PATHS[ds_name] +def load_external_detections(ds_name, scene_ds_dir): """ - # dets_lst: list of dictionary, each element = detection of one object in an image - $ df_all_dets[0].keys() - > ['scene_id', 'image_id', 'category_id', 'bbox', 'score', 'time', 'segmentation'] - - For the evaluation of Megapose, we only need the 'scene_id', 'image_id', 'category_id', 'score', 'time' and 'bbox' - - We also need need to change the format of bounding boxes as explained below + Loads external detections """ + detections_path = CNOS_SUBMISSION_PATHS[ds_name] + dets_lst = [] for det in json.loads(detections_path.read_text()): - # We don't need the segmentation mask (not always present in the submissions) + # Segmentation mask not needed if 'segmentation' in det: del det['segmentation'] # Bounding box formats: @@ -293,61 +289,42 @@ def load_sam_predictions(ds_dir_name, scene_ds_dir): dets_lst.append(det) df_all_dets = pd.DataFrame.from_records(dets_lst) - df_targets = pd.read_json(scene_ds_dir / "test_targets_bop19.json") - return df_all_dets, df_targets -def get_sam_detections(data, df_all_dets, df_targets, dt_det): - # We assume a unique image ("view") associated with a unique scene_id is - im_info = data['im_infos'][0] - scene_id, view_id = im_info['scene_id'], im_info['view_id'] +def filter_detections_scene_view(scene_id, view_id, df_all_dets, df_targets): + """ + Retrieve detections associated to + + img_data: contains + """ df_dets_scene_img = df_all_dets.loc[(df_all_dets['scene_id'] == scene_id) & (df_all_dets['image_id'] == view_id)] df_targets_scene_img = df_targets[(df_targets['scene_id'] == scene_id) & (df_targets['im_id'] == view_id)] - dt_det += df_dets_scene_img.time.iloc[0] - - ################# - # Filter detections based on 2 criteria - # - 1) Localization 6D task: we can assume that we know which object category and how many instances - # are present in the image - obj_ids = df_targets_scene_img.obj_id.to_list() - df_dets_scene_img_obj_filt = df_dets_scene_img[df_dets_scene_img['category_id'].isin(obj_ids)] - # In case none of the detections category ids match the ones present in the scene, - # keep only one detection to avoid downstream error - if len(df_dets_scene_img_obj_filt) > 0: - df_dets_scene_img = df_dets_scene_img_obj_filt - else: - df_dets_scene_img = df_dets_scene_img[:1] - - # TODO: retain only corresponding inst_count number for each detection category_id - - # - 2) Retain detections with best cnos scores (kind of redundant with finalized 1) ) - # based on expected number of objects in the scene (from groundtruth) - nb_gt_dets = df_targets_scene_img.inst_count.sum() - - # TODO: put that as a parameter somewhere? - MARGIN = 1 # if 0, some images will have no detections - K_MULT = 1 - nb_det = K_MULT*nb_gt_dets + MARGIN - df_dets_scene_img = df_dets_scene_img.sort_values('score', ascending=False).head(nb_det) - ################# - + # Keep only best detections for objects ("targets") given in bop target file + lst_df_target = [] + nb_targets = len(df_targets_scene_img) + MARGIN = 0 + for it in range(nb_targets): + target = df_targets_scene_img.iloc[it] + n_best = target.inst_count + MARGIN + df_filt_target = df_dets_scene_img[df_dets_scene_img['category_id'] == target.obj_id].sort_values('score', ascending=False)[:n_best] + lst_df_target.append(df_filt_target) + + # if missing dets, keep only one detection to avoid downstream error + df_dets_scene_img = pd.concat(lst_df_target) if len(lst_df_target) > 0 else df_dets_scene_img[:1] lst_dets_scene_img = df_dets_scene_img.to_dict('records') - if len(lst_dets_scene_img) == 0: - raise(ValueError('lst_dets_scene_img empty!: ', f'scene_id: {scene_id}, image_id/view_id: {view_id}')) - - # Do not forget the scores that are not present in object data - scores = [] - list_object_data = [] + # Do not forget the scores that are not present in object img_data + scores, list_object_data = [], [] for det in lst_dets_scene_img: list_object_data.append(ObjectData.from_json(det)) scores.append(det['score']) - sam_detections = make_detections_from_object_data(list_object_data).to(device) - sam_detections.infos['score'] = scores - return sam_detections + detections = make_detections_from_object_data(list_object_data).to(device) + detections.infos['score'] = scores + detections.infos['time'] = df_dets_scene_img.time.iloc[0] + return detections if __name__ == "__main__": diff --git a/happypose/pose_estimators/megapose/evaluation/prediction_runner.py b/happypose/pose_estimators/megapose/evaluation/prediction_runner.py index 4c4e0ec3..c1ba6e0f 100644 --- a/happypose/pose_estimators/megapose/evaluation/prediction_runner.py +++ b/happypose/pose_estimators/megapose/evaluation/prediction_runner.py @@ -44,8 +44,8 @@ BOP_DS_DIR ) from happypose.pose_estimators.megapose.evaluation.bop import ( - get_sam_detections, - load_sam_predictions + filter_detections_scene_view, + load_external_detections ) from happypose.pose_estimators.megapose.training.utils import CudaTimer @@ -99,7 +99,7 @@ def run_inference_pipeline( pose_estimator: PoseEstimator, obs_tensor: ObservationTensor, gt_detections: DetectionsType, - sam_detections: DetectionsType, + exte_detections: DetectionsType, initial_estimates: Optional[PoseEstimatesType] = None, ) -> Dict[str, PoseEstimatesType]: """Runs inference pipeline, extracts the results. @@ -117,8 +117,8 @@ def run_inference_pipeline( detections = gt_detections run_detector = False elif self.inference_cfg.detection_type == "sam": - # print("sam_detections =", sam_detections.bboxes) - detections = sam_detections + # print("exte_detections =", exte_detections.bboxes) + detections = exte_detections run_detector = False elif self.inference_cfg.detection_type == "detector": detections = None @@ -185,8 +185,6 @@ def get_predictions(self, pose_estimator: PoseEstimator) -> Dict[str, PoseEstima - 'depth_refiner' With the predictions at the various settings/iterations. - - """ predictions_list = defaultdict(list) @@ -197,7 +195,7 @@ def get_predictions(self, pose_estimator: PoseEstimator) -> Dict[str, PoseEstima ###### # Temporary solution if self.inference_cfg.detection_type == "sam": - df_all_dets, df_targets = load_sam_predictions(self.scene_ds.ds_dir.name, self.scene_ds.ds_dir) + df_all_dets, df_targets = load_external_detections(self.scene_ds.ds_dir.name, self.scene_ds.ds_dir) for n, data in enumerate(tqdm(self.dataloader)): # data is a dict @@ -209,16 +207,12 @@ def get_predictions(self, pose_estimator: PoseEstimator) -> Dict[str, PoseEstima # Dirty but avoids creating error when running with real detector dt_det = 0 - ###### - # Filter the dataframe according to scene id and view id - # Transform the data in ObjectData and then Detections - ###### # Temporary solution if self.inference_cfg.detection_type == "sam": - # We assume a unique image ("view") associated with a unique scene_id is - sam_detections = get_sam_detections(data=data, df_all_dets=df_all_dets, df_targets=df_targets, dt_det=dt_det) + exte_detections = filter_detections_scene_view(scene_id, view_id, df_all_dets, df_targets) + dt_det += exte_detections.infos['time'].iloc[0] else: - sam_detections = None + exte_detections = None gt_detections = data["gt_detections"].cuda() initial_data = None if data["initial_data"]: @@ -231,14 +225,14 @@ def get_predictions(self, pose_estimator: PoseEstimator) -> Dict[str, PoseEstima if n == 0: with torch.no_grad(): self.run_inference_pipeline( - pose_estimator, obs_tensor, gt_detections, sam_detections, initial_estimates=initial_data + pose_estimator, obs_tensor, gt_detections, exte_detections, initial_estimates=initial_data ) cuda_timer = CudaTimer() cuda_timer.start() with torch.no_grad(): all_preds = self.run_inference_pipeline( - pose_estimator, obs_tensor, gt_detections, sam_detections, initial_estimates=initial_data + pose_estimator, obs_tensor, gt_detections, exte_detections, initial_estimates=initial_data ) cuda_timer.end() duration = cuda_timer.elapsed() From ec19c14ad9cb8f28a5c84983ea8f23e2374ce8b2 Mon Sep 17 00:00:00 2001 From: MedericFourmy Date: Wed, 18 Oct 2023 08:32:50 -0400 Subject: [PATCH 2/7] Use external json file to define detection filenames --- .../megapose/evaluation/bop.py | 130 ++++++++---------- .../megapose/evaluation/prediction_runner.py | 2 +- 2 files changed, 62 insertions(+), 70 deletions(-) diff --git a/happypose/pose_estimators/megapose/evaluation/bop.py b/happypose/pose_estimators/megapose/evaluation/bop.py index 975444c3..ef2f863d 100644 --- a/happypose/pose_estimators/megapose/evaluation/bop.py +++ b/happypose/pose_estimators/megapose/evaluation/bop.py @@ -46,47 +46,7 @@ DUMMY_EVAL_SCRIPT_PATH = BOP_TOOLKIT_DIR / "scripts/eval_bop19_dummy.py" -################################## -################################## -import os - -# Official Task 4 detections (CNOS fastSAM) -EXTERNAL_DETECTIONS_FILES = { - "ycbv": 'cnos-fastsam_ycbv-test_f4f2127c-6f59-447c-95b3-28e1e591f1a1.json', - "lmo": 'cnos-fastsam_lmo-test_3cb298ea-e2eb-4713-ae9e-5a7134c5da0f.json', - "tless": 'cnos-fastsam_tless-test_8ca61cb0-4472-4f11-bce7-1362a12d396f.json', - "tudl": 'cnos-fastsam_tudl-test_c48a2a95-1b41-4a51-9920-a667cb3d7149.json', - "icbin": 'cnos-fastsam_icbin-test_f21a9faf-7ef2-4325-885f-f4b6460f4432.json', - "itodd": 'cnos-fastsam_itodd-test_df32d45b-301c-4fc9-8769-797904dd9325.json', - "hb": 'cnos-fastsam_hb-test_db836947-020a-45bd-8ec5-c95560b68011.json', -} - - -# # Official Task 1 detections (gdrnppdet-pbrreal) -# EXTERNAL_DETECTIONS_FILES = { -# "ycbv": 'gdrnppdet-pbrreal_ycbv-test_abe6c5f1-cb26-4bbd-addc-bb76dd722a96.json', -# "lmo": 'gdrnppdet-pbrreal_lmo-test_202a2f15-cbd0-49df-90de-650428c6d157.json', -# "tless": 'gdrnppdet-pbrreal_tless-test_e112ecb4-7f56-4107-8a21-945bc7661267.json', -# "tudl": 'gdrnppdet-pbrreal_tudl-test_66fd26f1-bebf-493b-a42a-d71e8d10c479.json', -# "icbin": 'gdrnppdet-pbrreal_icbin-test_a46668ed-f76b-40ca-9954-708b198c2ab0.json', -# "itodd": 'gdrnppdet-pbrreal_itodd-test_9559c160-9507-4d09-94a5-ef0d6e8f22ce.json', -# "hb": 'gdrnppdet-pbrreal_hb-test_94485f5a-98ea-48f1-9472-06f4ceecad41.json', -# } - - -EXTERNAL_DETECTIONS_DIR = os.environ.get('EXTERNAL_DETECTIONS_DIR') -assert(EXTERNAL_DETECTIONS_DIR is not None) -EXTERNAL_DETECTIONS_DIR = Path(EXTERNAL_DETECTIONS_DIR) - -CNOS_SUBMISSION_PATHS = {ds_name: EXTERNAL_DETECTIONS_DIR / fname for ds_name, fname in EXTERNAL_DETECTIONS_FILES.items()} -# Check if all paths exist -assert( sum(p.exists() for p in CNOS_SUBMISSION_PATHS.values()) == len(EXTERNAL_DETECTIONS_FILES)) -################################## -################################## - - # Third Party -import bop_toolkit_lib from bop_toolkit_lib import inout # noqa @@ -262,30 +222,19 @@ def run_evaluation(cfg: BOPEvalConfig) -> None: return scores_pose_path, scores_detection_path -def load_external_detections(ds_name, scene_ds_dir): + +def load_external_detections(scene_ds_dir: Path): """ Loads external detections """ - detections_path = CNOS_SUBMISSION_PATHS[ds_name] + ds_name = scene_ds_dir.name + + bop_detections_paths = get_external_detections_paths() + detections_path = bop_detections_paths[ds_name] dets_lst = [] for det in json.loads(detections_path.read_text()): - # Segmentation mask not needed - if 'segmentation' in det: - del det['segmentation'] - # Bounding box formats: - # - BOP format: [xmin, ymin, width, height] - # - Megapose expects: [xmin, ymin, xmax, ymax] - x, y, w, h = det['bbox'] - det['bbox'] = [float(v) for v in [x, y, x+w, y+h]] - det['bbox_modal'] = det['bbox'] - - # HACK: object models are same in lm and lmo -> obj labels start with 'lm' - if ds_name == 'lmo': - ds_name = 'lm' - - det['label'] = '{}-obj_{}'.format(ds_name, str(det["category_id"]).zfill(6)) - + det = format_det_bop2megapose(det, ds_name) dets_lst.append(det) df_all_dets = pd.DataFrame.from_records(dets_lst) @@ -293,6 +242,43 @@ def load_external_detections(ds_name, scene_ds_dir): return df_all_dets, df_targets +def get_external_detections_paths(): + EXTERNAL_DETECTIONS_DIR = os.environ.get('EXTERNAL_DETECTIONS_DIR') + assert(EXTERNAL_DETECTIONS_DIR is not None) + EXTERNAL_DETECTIONS_DIR = Path(EXTERNAL_DETECTIONS_DIR) + + files_name_path = EXTERNAL_DETECTIONS_DIR / 'bop_detections_filenames.json' + try: + bop_detections_filenames = json.loads(files_name_path.read_text()) + except json.decoder.JSONDecodeError as e: + print('Check json formatting {files_name_path.as_posix()}') + raise e + bop_detections_paths = {ds_name: EXTERNAL_DETECTIONS_DIR / fname + for ds_name, fname in bop_detections_filenames.items()} + + return bop_detections_paths + + +def format_det_bop2megapose(det, ds_name): + # Segmentation mask not needed + if 'segmentation' in det: + del det['segmentation'] + # Bounding box formats: + # - BOP format: [xmin, ymin, width, height] + # - Megapose expects: [xmin, ymin, xmax, ymax] + x, y, w, h = det['bbox'] + det['bbox'] = [float(v) for v in [x, y, x+w, y+h]] + det['bbox_modal'] = det['bbox'] + + # HACK: object models are same in lm and lmo -> obj labels start with 'lm' + if ds_name == 'lmo': + ds_name = 'lm' + + det['label'] = '{}-obj_{}'.format(ds_name, str(det["category_id"]).zfill(6)) + + return det + + def filter_detections_scene_view(scene_id, view_id, df_all_dets, df_targets): """ Retrieve detections associated to @@ -302,18 +288,9 @@ def filter_detections_scene_view(scene_id, view_id, df_all_dets, df_targets): df_dets_scene_img = df_all_dets.loc[(df_all_dets['scene_id'] == scene_id) & (df_all_dets['image_id'] == view_id)] df_targets_scene_img = df_targets[(df_targets['scene_id'] == scene_id) & (df_targets['im_id'] == view_id)] - # Keep only best detections for objects ("targets") given in bop target file - lst_df_target = [] - nb_targets = len(df_targets_scene_img) - MARGIN = 0 - for it in range(nb_targets): - target = df_targets_scene_img.iloc[it] - n_best = target.inst_count + MARGIN - df_filt_target = df_dets_scene_img[df_dets_scene_img['category_id'] == target.obj_id].sort_values('score', ascending=False)[:n_best] - lst_df_target.append(df_filt_target) + df_dets_scene_img = keep_best_detections(df_dets_scene_img, df_targets_scene_img) - # if missing dets, keep only one detection to avoid downstream error - df_dets_scene_img = pd.concat(lst_df_target) if len(lst_df_target) > 0 else df_dets_scene_img[:1] + # Keep only best detections for objects ("targets") given in bop target file lst_dets_scene_img = df_dets_scene_img.to_dict('records') # Do not forget the scores that are not present in object img_data @@ -327,5 +304,20 @@ def filter_detections_scene_view(scene_id, view_id, df_all_dets, df_targets): return detections +def keep_best_detections(df_dets_scene_img, df_targets_scene_img): + lst_df_target = [] + nb_targets = len(df_targets_scene_img) + MARGIN = 0 + for it in range(nb_targets): + target = df_targets_scene_img.iloc[it] + n_best = target.inst_count + MARGIN + df_filt_target = df_dets_scene_img[df_dets_scene_img['category_id'] == target.obj_id].sort_values('score', ascending=False)[:n_best] + lst_df_target.append(df_filt_target) + + # if missing dets, keep only one detection to avoid downstream error + df_dets_scene_img = pd.concat(lst_df_target) if len(lst_df_target) > 0 else df_dets_scene_img[:1] + + return df_dets_scene_img + if __name__ == "__main__": main() diff --git a/happypose/pose_estimators/megapose/evaluation/prediction_runner.py b/happypose/pose_estimators/megapose/evaluation/prediction_runner.py index c1ba6e0f..72f95bb0 100644 --- a/happypose/pose_estimators/megapose/evaluation/prediction_runner.py +++ b/happypose/pose_estimators/megapose/evaluation/prediction_runner.py @@ -195,7 +195,7 @@ def get_predictions(self, pose_estimator: PoseEstimator) -> Dict[str, PoseEstima ###### # Temporary solution if self.inference_cfg.detection_type == "sam": - df_all_dets, df_targets = load_external_detections(self.scene_ds.ds_dir.name, self.scene_ds.ds_dir) + df_all_dets, df_targets = load_external_detections(self.scene_ds.ds_dir) for n, data in enumerate(tqdm(self.dataloader)): # data is a dict From 0aeb3db80c4e40be57cfd6d25a6eedf4e3948fdc Mon Sep 17 00:00:00 2001 From: MedericFourmy Date: Thu, 19 Oct 2023 04:18:18 -0400 Subject: [PATCH 3/7] Replace confusing "sam" label by "exte" for external detections, documented --- docs/book/megapose/evaluate.md | 25 +++++++++++++++---- .../megapose/evaluation/evaluation.py | 2 +- .../megapose/evaluation/prediction_runner.py | 6 ++--- .../megapose/inference/pose_estimator.py | 2 +- .../scripts/run_full_megapose_eval.py | 2 +- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/docs/book/megapose/evaluate.md b/docs/book/megapose/evaluate.md index 0dc893ef..35c732ed 100644 --- a/docs/book/megapose/evaluate.md +++ b/docs/book/megapose/evaluate.md @@ -4,15 +4,30 @@ Please make sure you followed the steps relative to the evaluation in the main r An example to run the evaluation on `YCBV` dataset. Several datasets can be added to the list. +## Evaluating with Megapose detector +Run a detector part of Megapose pipeline to detect bounding boxes in the image dataset at run-time. + ``` -python -m happypose.pose_estimators.megapose.src.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19] result_id=fastsam_kbestdet_1posehyp detection_coarse_types=[["sam","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=true run_bop_eval=true +python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19] result_id=detector_1posehyp detection_coarse_types=[["detector","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=true run_bop_eval=true ``` +## Evaluating with external detections + +Read + + + +``` +python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19] result_id=exte_det_1posehyp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=true run_bop_eval=true +``` + +python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[tudl.bop19] result_id=results_rgb detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=false run_bop_eval=true + To reproduce the results we obtained for the BOP-Challenge, please run the following commands : ```sh # RGB 1 hyp -python -m happypose.pose_estimators.megapose.src.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19,lmo.bop19,tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=fastsam_kbestdet_1posehyp detection_coarse_types=[["sam","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=False run_bop_eval=true +python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19,lmo.bop19,tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=exte_det_1posehyp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=False run_bop_eval=true ``` Results : @@ -22,7 +37,7 @@ Results : ```sh # RGB 5 hyp -python -m happypose.pose_estimators.megapose.src.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19,lmo.bop19,tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=fastsam_kbestdet_5posehyp detection_coarse_types=[["sam","SO3_grid"]] inference.n_pose_hypotheses=5 skip_inference=False run_bop_eval=true +python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19,lmo.bop19,tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=exte_det_5posehyp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=5 skip_inference=False run_bop_eval=true ``` Results : @@ -31,7 +46,7 @@ Results : ```sh # RGB-D 5 hyp -python -m torch.distributed.run --nproc_per_node gpu -m happypose.pose_estimators.megapose.src.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=fastsam_kbestdet_5posehyp_teaserpp detection_coarse_types=[["sam","SO3_grid"]] inference.n_pose_hypotheses=5 inference.run_depth_refiner=true inference.depth_refiner=teaserpp skip_inference=False run_bop_eval=True +python -m torch.distributed.run --nproc_per_node gpu -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=exte_det_5posehyp_teaserpp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=5 inference.run_depth_refiner=true inference.depth_refiner=teaserpp skip_inference=False run_bop_eval=True ``` Results : @@ -104,5 +119,5 @@ conda activate happypose_pytorch3d cd happypose -python -m torch.distributed.run --nproc_per_node gpu -m happypose.pose_estimators.megapose.src.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[lmo.bop19] result_id=fastsam_kbestdet_1posehyp detection_coarse_types=[["sam","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=False run_bop_eval=true +python -m torch.distributed.run --nproc_per_node gpu -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[lmo.bop19] result_id=exte_det_1posehyp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=False run_bop_eval=true ``` \ No newline at end of file diff --git a/happypose/pose_estimators/megapose/evaluation/evaluation.py b/happypose/pose_estimators/megapose/evaluation/evaluation.py index 1c010525..29551d06 100644 --- a/happypose/pose_estimators/megapose/evaluation/evaluation.py +++ b/happypose/pose_estimators/megapose/evaluation/evaluation.py @@ -126,7 +126,7 @@ def run_eval( detector_model = happypose.toolbox.inference.utils.load_detector(cfg.detector_run_id) elif cfg.inference.detection_type == "gt": detector_model = None - elif cfg.inference.detection_type == "sam": + elif cfg.inference.detection_type == "exte": detector_model = None else: raise ValueError(f"Unknown detection_type={cfg.inference.detection_type}") diff --git a/happypose/pose_estimators/megapose/evaluation/prediction_runner.py b/happypose/pose_estimators/megapose/evaluation/prediction_runner.py index 72f95bb0..83c4d1b6 100644 --- a/happypose/pose_estimators/megapose/evaluation/prediction_runner.py +++ b/happypose/pose_estimators/megapose/evaluation/prediction_runner.py @@ -116,7 +116,7 @@ def run_inference_pipeline( if self.inference_cfg.detection_type == "gt": detections = gt_detections run_detector = False - elif self.inference_cfg.detection_type == "sam": + elif self.inference_cfg.detection_type == "exte": # print("exte_detections =", exte_detections.bboxes) detections = exte_detections run_detector = False @@ -194,7 +194,7 @@ def get_predictions(self, pose_estimator: PoseEstimator) -> Dict[str, PoseEstima # format it and store it in a dataframe that will be accessed later ###### # Temporary solution - if self.inference_cfg.detection_type == "sam": + if self.inference_cfg.detection_type == "exte": df_all_dets, df_targets = load_external_detections(self.scene_ds.ds_dir) for n, data in enumerate(tqdm(self.dataloader)): @@ -208,7 +208,7 @@ def get_predictions(self, pose_estimator: PoseEstimator) -> Dict[str, PoseEstima dt_det = 0 # Temporary solution - if self.inference_cfg.detection_type == "sam": + if self.inference_cfg.detection_type == "exte": exte_detections = filter_detections_scene_view(scene_id, view_id, df_all_dets, df_targets) dt_det += exte_detections.infos['time'].iloc[0] else: diff --git a/happypose/pose_estimators/megapose/inference/pose_estimator.py b/happypose/pose_estimators/megapose/inference/pose_estimator.py index e54af605..6c3da7c4 100644 --- a/happypose/pose_estimators/megapose/inference/pose_estimator.py +++ b/happypose/pose_estimators/megapose/inference/pose_estimator.py @@ -568,7 +568,7 @@ def run_inference_pipeline( start_time = time.time() detections = self.forward_detection_model(observation) detections = detections.cuda() - print("detections detector =", detections.bboxes) + # print("detections detector =", detections.bboxes) elapsed = time.time() - start_time timing_str += f"detection={elapsed:.2f}, " diff --git a/happypose/pose_estimators/megapose/scripts/run_full_megapose_eval.py b/happypose/pose_estimators/megapose/scripts/run_full_megapose_eval.py index 70313440..e5565bfa 100644 --- a/happypose/pose_estimators/megapose/scripts/run_full_megapose_eval.py +++ b/happypose/pose_estimators/megapose/scripts/run_full_megapose_eval.py @@ -101,7 +101,7 @@ def create_eval_cfg( cfg.detector_run_id = PBR_DETECTORS[ds_name_root] elif detection_type == "gt": pass - elif detection_type == "sam": + elif detection_type == "exte": pass else: raise ValueError(f"Unknown detector type {cfg.detector_type}") From 3eb26031f7c6a87c2512b5f82d67f4d9601f8c61 Mon Sep 17 00:00:00 2001 From: MedericFourmy Date: Thu, 19 Oct 2023 04:38:41 -0400 Subject: [PATCH 4/7] Finish documentation evaluation in book --- docs/book/megapose/evaluate.md | 35 +++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/docs/book/megapose/evaluate.md b/docs/book/megapose/evaluate.md index 35c732ed..323a7043 100644 --- a/docs/book/megapose/evaluate.md +++ b/docs/book/megapose/evaluate.md @@ -2,32 +2,41 @@ Please make sure you followed the steps relative to the evaluation in the main readme. -An example to run the evaluation on `YCBV` dataset. Several datasets can be added to the list. - ## Evaluating with Megapose detector Run a detector part of Megapose pipeline to detect bounding boxes in the image dataset at run-time. ``` -python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19] result_id=detector_1posehyp detection_coarse_types=[["detector","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=true run_bop_eval=true +python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19,lmo.bop19,tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=detector_1posehyp detection_coarse_types=[["detector","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=false run_bop_eval=true ``` ## Evaluating with external detections -Read - - +First step: download external detections from bop website (such as default detections for task 1 and 4). You should have one csv file for every bop dataset. +Place these in a directory of your choice and define the environment variable. -``` -python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19] result_id=exte_det_1posehyp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=true run_bop_eval=true +```sh +EXTERNAL_DETECTIONS_DIR = /path/to/saved/detections/ ``` -python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[tudl.bop19] result_id=results_rgb detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=false run_bop_eval=true +Megapose expects a json file named `bop_detections_filenames.json` placed in `EXTERNAL_DETECTIONS_DIR` mapping bop dataset names to the csv file names, e.g. for CNOS detection (default detection for task 4, bop23): + +```json +{ + "ycbv": "cnos-fastsam_ycbv-test_f4f2127c-6f59-447c-95b3-28e1e591f1a1.json", + "lmo": "cnos-fastsam_lmo-test_3cb298ea-e2eb-4713-ae9e-5a7134c5da0f.json", + "tless": "cnos-fastsam_tless-test_8ca61cb0-4472-4f11-bce7-1362a12d396f.json", + "tudl": "cnos-fastsam_tudl-test_c48a2a95-1b41-4a51-9920-a667cb3d7149.json", + "icbin": "cnos-fastsam_icbin-test_f21a9faf-7ef2-4325-885f-f4b6460f4432.json", + "itodd": "cnos-fastsam_itodd-test_df32d45b-301c-4fc9-8769-797904dd9325.json", + "hb": "cnos-fastsam_hb-test_db836947-020a-45bd-8ec5-c95560b68011.json" +} +``` To reproduce the results we obtained for the BOP-Challenge, please run the following commands : ```sh # RGB 1 hyp -python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19,lmo.bop19,tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=exte_det_1posehyp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=False run_bop_eval=true +python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19,lmo.bop19,tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=exte_det_1posehyp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=False run_bop_eval=true ``` Results : @@ -37,7 +46,7 @@ Results : ```sh # RGB 5 hyp -python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19,lmo.bop19,tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=exte_det_5posehyp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=5 skip_inference=False run_bop_eval=true +python -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[ycbv.bop19,lmo.bop19,tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=exte_det_5posehyp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=5 skip_inference=False run_bop_eval=true ``` Results : @@ -46,7 +55,7 @@ Results : ```sh # RGB-D 5 hyp -python -m torch.distributed.run --nproc_per_node gpu -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=exte_det_5posehyp_teaserpp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=5 inference.run_depth_refiner=true inference.depth_refiner=teaserpp skip_inference=False run_bop_eval=True +python -m torch.distributed.run --nproc_per_node gpu -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[tless.bop19,tudl.bop19,icbin.bop19,hb.bop19,itodd.bop19] result_id=exte_det_5posehyp_teaserpp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=5 inference.run_depth_refiner=true inference.depth_refiner=teaserpp skip_inference=False run_bop_eval=True ``` Results : @@ -119,5 +128,5 @@ conda activate happypose_pytorch3d cd happypose -python -m torch.distributed.run --nproc_per_node gpu -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval detector_run_id=bop_pbr coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[lmo.bop19] result_id=exte_det_1posehyp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=False run_bop_eval=true +python -m torch.distributed.run --nproc_per_node gpu -m happypose.pose_estimators.megapose.scripts.run_full_megapose_eval coarse_run_id=coarse-rgb-906902141 refiner_run_id=refiner-rgb-653307694 ds_names=[lmo.bop19] result_id=exte_det_1posehyp detection_coarse_types=[["exte","SO3_grid"]] inference.n_pose_hypotheses=1 skip_inference=False run_bop_eval=true ``` \ No newline at end of file From 3c1ab23544832d145a56da1977f8b27f03706fa7 Mon Sep 17 00:00:00 2001 From: MedericFourmy Date: Thu, 19 Oct 2023 05:05:22 -0400 Subject: [PATCH 5/7] Add instruction to download all pre-trained detectors --- docs/book/megapose/download_data.md | 18 +++++++++++++++--- happypose/toolbox/utils/download.py | 6 +++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/book/megapose/download_data.md b/docs/book/megapose/download_data.md index 5902ac36..58b2a5f7 100644 --- a/docs/book/megapose/download_data.md +++ b/docs/book/megapose/download_data.md @@ -1,6 +1,6 @@ # Download example data for minimal testing -``` +```sh cd $HAPPYPOSE_DATA_DIR wget https://memmo-data.laas.fr/static/examples.tar.xz tar xf examples.tar.xz @@ -8,12 +8,24 @@ tar xf examples.tar.xz # Download pre-trained pose estimation models -Download pose estimation models to $HAPPYPOSE_DATA_DIR/megapose-models: +Download pose estimation models to `$HAPPYPOSE_DATA_DIR/megapose-models`: -``` +```sh python -m happypose.toolbox.utils.download --megapose_models ``` +# Download pre-trained detection models +Megapose can use pretrained detectors from CosyPose, which can be downloaded to `$HAPPYPOSE_DATA_DIR/experiments`: + +python -m happypose.toolbox.utils.download --cosypose_model detector-bop-hb-pbr--497808 +python -m happypose.toolbox.utils.download --cosypose_model detector-bop-hope-pbr--15246 +python -m happypose.toolbox.utils.download --cosypose_model detector-bop-icbin-pbr--947409 +python -m happypose.toolbox.utils.download --cosypose_model detector-bop-itodd-pbr--509908 +python -m happypose.toolbox.utils.download --cosypose_model detector-bop-lmo-pbr--517542 +python -m happypose.toolbox.utils.download --cosypose_model detector-bop-tless-pbr--873074 +python -m happypose.toolbox.utils.download --cosypose_model detector-bop-tudl-pbr--728047 +python -m happypose.toolbox.utils.download --cosypose_model detector-bop-ycbv-pbr--970850 + # Dataset ## Dataset information diff --git a/happypose/toolbox/utils/download.py b/happypose/toolbox/utils/download.py index bd0e1848..f64ba1f1 100644 --- a/happypose/toolbox/utils/download.py +++ b/happypose/toolbox/utils/download.py @@ -64,7 +64,7 @@ def main(): parser.add_argument('--bop_dataset', default='', type=str, choices=BOP_DS_NAMES) parser.add_argument('--bop_src', default='bop', type=str, choices=['bop', 'gdrive']) parser.add_argument('--bop_extra_files', default='', type=str, choices=['ycbv', 'tless']) - parser.add_argument('--cosypose_models', default='', type=str) + parser.add_argument('--cosypose_model', default='', type=str) parser.add_argument("--megapose_models", action="store_true") parser.add_argument('--urdf_models', default='', type=str) parser.add_argument('--ycbv_compat_models', action='store_true') @@ -111,8 +111,8 @@ def main(): download(f'cosypose/bop_datasets/ycbv/models_bop-compat', BOP_DS_DIR / 'ycbv') download(f'cosypose/bop_datasets/ycbv/models_bop-compat_eval', BOP_DS_DIR / 'ycbv') - if args.cosypose_models: - download(f'cosypose/experiments/{args.cosypose_models}', LOCAL_DATA_DIR / 'experiments') + if args.cosypose_model: + download(f'cosypose/experiments/{args.cosypose_model}', LOCAL_DATA_DIR / 'experiments') if args.megapose_models: # rclone copyto inria_data:megapose-models/ megapose-models/ From 87e3c281b6824df9f16c921c68ae445cce099c5c Mon Sep 17 00:00:00 2001 From: MedericFourmy Date: Thu, 19 Oct 2023 08:43:39 -0400 Subject: [PATCH 6/7] Document bug zero length detections, to fix in other issue --- happypose/pose_estimators/megapose/inference/pose_estimator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/happypose/pose_estimators/megapose/inference/pose_estimator.py b/happypose/pose_estimators/megapose/inference/pose_estimator.py index 6c3da7c4..d8701047 100644 --- a/happypose/pose_estimators/megapose/inference/pose_estimator.py +++ b/happypose/pose_estimators/megapose/inference/pose_estimator.py @@ -568,12 +568,13 @@ def run_inference_pipeline( start_time = time.time() detections = self.forward_detection_model(observation) detections = detections.cuda() - # print("detections detector =", detections.bboxes) + print("# detections =", len(detections.bboxes)) elapsed = time.time() - start_time timing_str += f"detection={elapsed:.2f}, " # Ensure that detections has the instance_id column assert detections is not None + assert len(detections) > 0, 'TOFIX: currently, dealing with absence of detections is not supported' detections = happypose.toolbox.inference.utils.add_instance_id(detections) # Filter detections From 8ada35c5e7c44e335fbba964ad55850532315dee Mon Sep 17 00:00:00 2001 From: Mederic Fourmy Date: Tue, 31 Oct 2023 11:05:05 +0100 Subject: [PATCH 7/7] Update bop.py hotfix --- happypose/pose_estimators/megapose/evaluation/bop.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/happypose/pose_estimators/megapose/evaluation/bop.py b/happypose/pose_estimators/megapose/evaluation/bop.py index ef2f863d..7f88773c 100644 --- a/happypose/pose_estimators/megapose/evaluation/bop.py +++ b/happypose/pose_estimators/megapose/evaluation/bop.py @@ -307,12 +307,12 @@ def filter_detections_scene_view(scene_id, view_id, df_all_dets, df_targets): def keep_best_detections(df_dets_scene_img, df_targets_scene_img): lst_df_target = [] nb_targets = len(df_targets_scene_img) - MARGIN = 0 for it in range(nb_targets): target = df_targets_scene_img.iloc[it] - n_best = target.inst_count + MARGIN + n_best = target.inst_count df_filt_target = df_dets_scene_img[df_dets_scene_img['category_id'] == target.obj_id].sort_values('score', ascending=False)[:n_best] - lst_df_target.append(df_filt_target) + if len(df_filt_target) > 0: + lst_df_target.append(df_filt_target) # if missing dets, keep only one detection to avoid downstream error df_dets_scene_img = pd.concat(lst_df_target) if len(lst_df_target) > 0 else df_dets_scene_img[:1]