[PR] Saving Inferred 3D Hand Keypoints

.devcontainer/devcontainer.json

@@ -0,0 +1,36 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+    "name": "Existing Dockerfile",
+    "build": {
+        // Sets the run context to one level up instead of the .devcontainer folder.
+        "context": "..",
+        // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+        "dockerfile": "../docker/hamer-dev.Dockerfile"
+    },
+    "runArgs": [
+        "--gpus",
+        "all"
+    ], // Ensure GPU access
+    // "remoteEnv": {
+    //     "PATH": "${containerEnv:PATH}:/usr/local/cuda/bin",
+    //     "LD_LIBRARY_PATH": "$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64",
+    //     "XLA_FLAGS": "--xla_gpu_cuda_data_dir=/usr/local/cuda"
+    // },
+    // Features to add to the dev container. More info: https://containers.dev/features.
+    // "features": {},
+    // Use 'forwardPorts' to make a list of ports inside the container available locally.
+    // "forwardPorts": [],
+    // Uncomment the next line to run commands after the container is created.
+    // "postCreateCommand": "cat /etc/os-release",
+    // Configure tool-specific properties.
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "ms-python.python"
+            ]
+        }
+    }
+    // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+    // "remoteUser": "devcontainer"
+}

.vscode/launch.json

@@ -0,0 +1,18 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Current File",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "cwd": "${fileDirname}",
+            "console": "integratedTerminal",
+            "args": "--img_folder example_data --out_folder demo_out --batch_size=48 --side_view --save_mesh --full_frame --save_keypoints",
+            "justMyCode": false
+        }
+    ]
+}

demo.py

@@ -6,7 +6,7 @@
 import numpy as np
 
 from hamer.configs import CACHE_DIR_HAMER
-from hamer.models import HAMER, download_models, load_hamer, DEFAULT_CHECKPOINT
+from hamer.models import download_models, load_hamer, DEFAULT_CHECKPOINT
 from hamer.utils import recursive_to
 from hamer.datasets.vitdet_dataset import ViTDetDataset, DEFAULT_MEAN, DEFAULT_STD
 from hamer.utils.renderer import Renderer, cam_crop_to_full
@@ -15,8 +15,6 @@
 
 from vitpose_model import ViTPoseModel
 
-import json
-from typing import Dict, Optional
 
 def main():
     parser = argparse.ArgumentParser(description='HaMeR demo code')
@@ -26,6 +24,7 @@ def main():
     parser.add_argument('--side_view', dest='side_view', action='store_true', default=False, help='If set, render side view also')
     parser.add_argument('--full_frame', dest='full_frame', action='store_true', default=True, help='If set, render all people together also')
     parser.add_argument('--save_mesh', dest='save_mesh', action='store_true', default=False, help='If set, save meshes to disk also')
+    parser.add_argument('--save_keypoints', dest='save_keypoints', action='store_true', default=False, help='If set, save keypoints to disk alongside the mesh.')
     parser.add_argument('--batch_size', type=int, default=1, help='Batch size for inference/fitting')
     parser.add_argument('--rescale_factor', type=float, default=2.0, help='Factor for padding the bbox')
     parser.add_argument('--body_detector', type=str, default='vitdet', choices=['vitdet', 'regnety'], help='Using regnety improves runtime and reduces memory')
@@ -188,6 +187,12 @@ def main():
                     tmesh = renderer.vertices_to_trimesh(verts, camera_translation, LIGHT_BLUE, is_right=is_right)
                     tmesh.export(os.path.join(args.out_folder, f'{img_fn}_{person_id}.obj'))
 
+                if args.save_keypoints:
+                    pred_keypoints_3d = out['pred_keypoints_3d'].detach().cpu().numpy().copy()
+                    pred_keypoints_3d += pred_cam_t_full[:, np.newaxis, :]
+
+                    np.save(os.path.join(args.out_folder, f'{img_fn}_{person_id}_3dkeypoints.npy'), pred_keypoints_3d)
+
         # Render front view
         if args.full_frame and len(all_verts) > 0:
             misc_args = dict(

hamer/utils/renderer.py

@@ -9,7 +9,7 @@
 from yacs.config import CfgNode
 from typing import List, Optional
 
-def cam_crop_to_full(cam_bbox, box_center, box_size, img_size, focal_length=5000.):
+def cam_crop_to_full(cam_bbox, box_center, box_size, img_size, focal_length=5000.) -> torch.Tensor:
     # Convert cam_bbox to full image
     img_w, img_h = img_size[:, 0], img_size[:, 1]
     cx, cy, b = box_center[:, 0], box_center[:, 1], box_size