forked from CalciferZh/minimal-hand
-
Notifications
You must be signed in to change notification settings - Fork 11
/
app.py
121 lines (100 loc) · 3.52 KB
/
app.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import cv2
import keyboard
import numpy as np
import open3d as o3d
import pygame
from transforms3d.axangles import axangle2mat
import config
from capture import OpenCVCapture
from hand_mesh import HandMesh
from kinematics import mpii_to_mano
from utils import OneEuroFilter, imresize
from wrappers import ModelPipeline
from utils import *
def live_application(capture):
"""
Launch an application that reads from a webcam and estimates hand pose at
real-time.
The captured hand must be the right hand, but will be flipped internally
and rendered.
Parameters
----------
capture : object
An object from `capture.py` to read capture stream from.
"""
############ output visualization ############
view_mat = axangle2mat([1, 0, 0], np.pi) # align different coordinate systems
window_size = 1080
hand_mesh = HandMesh(config.HAND_MESH_MODEL_PATH)
mesh = o3d.geometry.TriangleMesh()
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = \
o3d.utility.Vector3dVector(np.matmul(view_mat, hand_mesh.verts.T).T * 1000)
mesh.compute_vertex_normals()
viewer = o3d.visualization.Visualizer()
viewer.create_window(
width=window_size + 1, height=window_size + 1,
window_name='Minimal Hand - output'
)
viewer.add_geometry(mesh)
view_control = viewer.get_view_control()
cam_params = view_control.convert_to_pinhole_camera_parameters()
extrinsic = cam_params.extrinsic.copy()
extrinsic[0:3, 3] = 0
cam_params.extrinsic = extrinsic
cam_params.intrinsic.set_intrinsics(
window_size + 1, window_size + 1, config.CAM_FX, config.CAM_FY,
window_size // 2, window_size // 2
)
view_control.convert_from_pinhole_camera_parameters(cam_params)
view_control.set_constant_z_far(1000)
render_option = viewer.get_render_option()
render_option.load_from_json('./render_option.json')
viewer.update_renderer()
############ input visualization ############
pygame.init()
display = pygame.display.set_mode((window_size, window_size))
pygame.display.set_caption('Minimal Hand - input')
############ misc ############
mesh_smoother = OneEuroFilter(4.0, 0.0)
clock = pygame.time.Clock()
model = ModelPipeline()
while True:
frame_large = capture.read()
if frame_large is None:
continue
if frame_large.shape[0] > frame_large.shape[1]:
margin = int((frame_large.shape[0] - frame_large.shape[1]) / 2)
frame_large = frame_large[margin:-margin]
else:
margin = int((frame_large.shape[1] - frame_large.shape[0]) / 2)
frame_large = frame_large[:, margin:-margin]
frame_large = np.flip(frame_large, axis=1).copy()
frame = imresize(frame_large, (128, 128))
_, theta_mpii = model.process(frame)
theta_mano = mpii_to_mano(theta_mpii)
v = hand_mesh.set_abs_quat(theta_mano)
v *= 2 # for better visualization
v = v * 1000 + np.array([0, 0, 400])
v = mesh_smoother.process(v)
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = o3d.utility.Vector3dVector(np.matmul(view_mat, v.T).T)
mesh.paint_uniform_color(config.HAND_COLOR)
mesh.compute_triangle_normals()
mesh.compute_vertex_normals()
viewer.update_geometry(mesh)
viewer.poll_events()
display.blit(
pygame.surfarray.make_surface(
np.transpose(
imresize(frame_large, (window_size, window_size)
), (1, 0, 2))
),
(0, 0)
)
pygame.display.update()
if keyboard.is_pressed("esc"):
break
clock.tick(30)
if __name__ == '__main__':
live_application(OpenCVCapture())