test.py

import cv2
import mediapipe as mp
import numpy as np
from tensorflow.keras.models import load_model

actions = ['rewind', 'advance','stop']
seq_length = 45 #1.5seconds
model = load_model('models/gesture.h5')
# MediaPipe hands model
mp_hands = mp.solutions.hands
mp_drawing = mp.solutions.drawing_utils
hands = mp_hands.Hands(
    max_num_hands=1,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)
cap = cv2.VideoCapture(0)
seq = [] #flatten(연산값)
action_seq = []
this_action = 0
while cap.isOpened():
    ret, img = cap.read()
    img = cv2.flip(img, 1)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    result = hands.process(img)
    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
    if result.multi_hand_landmarks is not None:
        for res in result.multi_hand_landmarks:
            joint = np.zeros((21, 4))
            for j, lm in enumerate(res.landmark):
                joint[j] = [lm.x, lm.y, lm.z, lm.visibility]
            # Compute angles between joints
            v1 = joint[[0,1,2,3,0,5,6,7,0,9,10,11,0,13,14,15,0,17,18,19], :3] # Parent joint
            v2 = joint[[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20], :3] # Child joint
            v = v2 - v1 # [20, 3]
            # Normalize v
            v = v / np.linalg.norm(v, axis=1)[:, np.newaxis]
            # Get angle using arcos of dot product
            angle = np.arccos(np.einsum('nt,nt->n',
                v[[0,1,2,4,5,6,8,9,10,12,13,14,16,17,18],:], 
                v[[1,2,3,5,6,7,9,10,11,13,14,15,17,18,19],:])) # [15,]
            angle = np.degrees(angle) # Convert radian to degree
            d = np.concatenate([joint.flatten(), angle])
            seq.append(d)
            mp_drawing.draw_landmarks(img, res, mp_hands.HAND_CONNECTIONS)
            if len(seq) < seq_length:
                continue
            if len(seq) % seq_length == 0:
                input_data = np.expand_dims(np.array(seq[-seq_length:], dtype=np.float32), axis=0)
                y_pred = model.predict(input_data).squeeze()
            cv2.putText(img,str('%4.1f'%(y_pred[0]*100)), (10,40), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.4, color=(255, 255, 255), thickness=1)
            cv2.putText(img,str('%4.1f'%(y_pred[1]*100)), (60,40), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.4, color=(255, 255, 255), thickness=1)
            cv2.putText(img,str('%4.1f'%(y_pred[2]*100)), (110,40), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.4, color=(255, 255, 255), thickness=1)
            i_pred = int(np.argmax(y_pred))
            conf = y_pred[i_pred]
            if conf < 0.99: #(~1)
                this_action=("unknown")
                continue
            action = actions[i_pred]
            action_seq.append(action)
            if len(action_seq)<4:
                continue
            if action_seq[-1]==action_seq[-3]==action_seq[-2]==action_seq[-4]:
                this_action=action
            #this_action=action
            cv2.putText(img,f'{this_action.upper()}', (10,20), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.4, color=(255, 255, 255), thickness=1)
    cv2.imshow('img', img)
    if cv2.waitKey(1) == (ord('q') or ord('Q')):
        break
cap.release()
cv2.destroyAllWindows()