diff --git a/README.md b/README.md new file mode 100644 index 0000000..4cef6b9 --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ +# ICL-Gait Dataset 1.0 +Release Date: Sept 2021 + +## Primary Contact +**Xiao Gu**, xiao.gu17@imperial.ac.uk Imperial College London + +**Yao Guo**, yao.guo@sjtu.edu.cn Shanghai Jiao Tong University + +## Citing +Please cite the following paper for the use of this dataset + +Gu, Xiao, et al. "Occlusion-Invariant Rotation-Equivariant Semi-Supervised Depth Based Cross-View Gait Pose Estimation." arXiv preprint arXiv:2109.01397 (2021). + +## Supplementary Code +* `vis_demo` provides script to visualize the data from different modalities +* `syn` provides script to generate synthetic data based on SMPL + +## Dataset Details +The dataset contains a real-world gait dataset collected from multiple viewpoints. +Please see our [paper](https://arxiv.org/pdf/2109.01397) and the [website](https://xiaogu.site/ICL_gait) for more details. + +## Data-Split Experiment Settings +Please follow the settings in our [paper](https://arxiv.org/pdf/2109.01397) to benchmark your algorithms. + +### Cross-Subject (CS) Validation (2 loops) +For cross-subject validation, the data were split to two groups {S01, S02, S04, S07}, {S03, S05, S06, S8}. + +Each loop, use one group as training, and the other group as testing set. + +### Cross-View (CV) Validation (5 loops) +For cross-view validation, the data were split based on the five views. + +Each loop, use the data from one view as training, the data from the other views as testing. +### Cross-Subject Cross-View (CS-CV) Validation (10 loops) +For cross-subject & cross-view validation, the data was split to ten subgroups as a combination of CS and CV validation. + +For example, one group is {S01-V01, S02-V01, S04-V01, S07-V01}. In each loop, use one group as the training set, and report the results on the remaining nine groups. + +**You can further split some proportion from the training set as a validation set, but any use of the testing data during training is not allowed.** + + +## Folder Details +### Dataset folder format +S##_V##_C## refers to the data of the trial per subject, condition, and viewpoint. +S##: subject id +V##: viewpoint +C##: walking condition + +Each folder contains 300 consecutive samples from one trial (the remaining samples leading to a much large data volume will be released in the future). +Missing trials (S1-C1-V1, S1-C2-V2, S1-C4-V1, S3-C3-V3, S5-C4-V4, S8-C2-V3, S8-C2-V4, S8-C4-V3, S8-C5-V3) + +* **depth**: contains the depth images recorded by RealSense D435 + + ``` + scale = 0.0010000000474974513; + fx = 928.108; fy = 927.443; + cx = 647.394; cy = 361.699 + ``` +* **mask**: contains the segmentation mask predicted from RGB images (access suspended) by CDCL + ``` + ROI (lower-limb) RGB Value [255,127,127; 0,127,255; 0,0,255; 255,255,127; 127,255,127; 0,255,0] + ``` +* **point cloud**: contains the point cloud converted from depth data, corresponding 3D keypoint, and root orientation + +* **pose_2d**: contains the 2D keypoints predicted by OpenPose + +* **kinematics**: contains the kinematics (randomly picked, not synchronized with the modalities above) which can be used synthetic data generation + + + + diff --git a/syn/README.md b/syn/README.md new file mode 100644 index 0000000..9c5c062 --- /dev/null +++ b/syn/README.md @@ -0,0 +1,19 @@ +# ICL-Gait to SMPL + +We provide the script [utils.py](smpl_webuser/utils.py) for transferring the gait parameter from our dataset to SMPL + +## Usage +* Please download the repository of [SMPL](smpl_webuser/utils.py) + +* Add `models/pose.mat` to `models` of SMPL respository + +* Add `smpl_webuser/hello_world/demo_gait.py` to `smpl_webuser/hello_world/` + +* (optional) replace the `smpl_webuser/serialization.py` by a modified `smpl_webuser/serialization.py` compatible with python3 + +* Running `demo_gait.py`, a synthetic model will be generated automatically + + + + + diff --git a/syn/models/pose.mat b/syn/models/pose.mat new file mode 100644 index 0000000..4f3f6e8 Binary files /dev/null and b/syn/models/pose.mat differ diff --git a/syn/smpl_webuser/hello_world/demo_gait.py b/syn/smpl_webuser/hello_world/demo_gait.py new file mode 100644 index 0000000..e2e75e8 --- /dev/null +++ b/syn/smpl_webuser/hello_world/demo_gait.py @@ -0,0 +1,32 @@ +''' +Modified from hellp_smpl.py with mocap2smpl function added +''' + +from smpl_webuser.serialization import load_model +import numpy as np +from smpl_webuser.utils import mocap2smpl + +## Load SMPL model (here we load the female model) +## Make sure path is correct +m = load_model( '../../models/basicmodel_m_lbs_10_207_0_v1.1.0.pkl' ) + +## Assign random pose and shape parameters +# m.pose[:] = np.random.rand(m.pose.size) * .2 +m.betas[:] = np.random.rand(m.betas.size) * .03 + +## assign poses from ICL +import scipy.io as sio +data = sio.loadmat('../../models/pose.mat')['pose'][0] +mocap2smpl(data, m) + +## Write to an .obj file +outmesh_path = './gait_smpl.obj' +with open( outmesh_path, 'w') as fp: + for v in m.r: + fp.write( 'v %f %f %f\n' % ( v[0], v[1], v[2]) ) + + for f in m.f+1: # Faces are 1-based, not 0-based in obj files + fp.write( 'f %d %d %d\n' % (f[0], f[1], f[2]) ) + +## Print message +print '..Output mesh saved to: ', outmesh_path diff --git a/syn/smpl_webuser/serialization_new.py b/syn/smpl_webuser/serialization_new.py new file mode 100644 index 0000000..e69de29 diff --git a/syn/smpl_webuser/utils.py b/syn/smpl_webuser/utils.py new file mode 100644 index 0000000..290f46b --- /dev/null +++ b/syn/smpl_webuser/utils.py @@ -0,0 +1,59 @@ +import transforms3d +import numpy as np + +def mocap2smpl(data, m, global_rotation=[0,0,0.00001]): + ### global transform + p = data[89,90,91] # Pelvis + p_eulr = transforms3d.euler.axangle2euler(p/np.linalg.norm(p), np.linalg.norm(p)/180*np.pi, 'sxyz') + p_transform_woz = transforms3d.euler.euler2mat(p_eulr[0], p_eulr[1], 0) + p_transform = transforms3d.axangles.axangle2mat(p/np.linalg.norm(p), np.linalg.norm(p)/180*np.pi) + global_transform = transforms3d.axangles.axangle2mat(global_rotation, np.linalg.norm(global_rotation)) + body_rotation = transforms3d.axangles.mat2axangle(np.dot(global_transform, p_transform_woz)) + m.pose[0:3] = body_rotation[0][[1, 2, 0]]*body_rotation[1] + + ### Left_UpperLeg transform + lfe = data[17:20] # LFemur + lfe_transform = transforms3d.axangles.axangle2mat(lfe/np.linalg.norm(lfe), np.linalg.norm(lfe)/180*np.pi) + lhip_transform = np.dot(np.linalg.inv(p_transform), lfe_transform) # relative rotation + lhip = transforms3d.axangles.mat2axangle(lhip_transform) + m.pose[3:6] = lhip[0][[1, 2, 0]]*lhip[1] + + ### Right_UpperLeg transform + rfe = data[113:116] # RFemur + rfe_transform = transforms3d.axangles.axangle2mat(rfe/np.linalg.norm(rfe), np.linalg.norm(rfe)/180*np.pi) + rhip_transform = np.dot(np.linalg.inv(p_transform), rfe_transform) + rhip = transforms3d.axangles.mat2axangle(rhip_transform) + m.pose[6:9] = rhip[0][[1, 2, 0]]*rhip[1] + + ### Left_LowerLeg transform + lti = data[71:74] # LTibia + lti_transform = transforms3d.axangles.axangle2mat(lti/np.linalg.norm(lti), np.linalg.norm(lti)/180*np.pi) + lk_transform = np.dot(np.linalg.inv(lfe_transform), lti_transform) + lk = transforms3d.axangles.mat2axangle(lk_transform) + m.pose[12:15] = lk[0][[1, 2, 0]]*lk[1] + + ### Right_LowerLeg transform + rti = data[167:170] # RTibia + rti_transform = transforms3d.axangles.axangle2mat(rti/np.linalg.norm(rti), np.linalg.norm(rti)/180*np.pi) + rk_transform = np.dot(np.linalg.inv(rfe_transform), rti_transform) + rk = transforms3d.axangles.mat2axangle(rk_transform) + m.pose[15:18] = rk[0][[1, 2, 0]]*rk[1] + + ### Left_Foot (90 offset) + lfo = data[26:29] + lfo_transform = transforms3d.axangles.axangle2mat(lfo/np.linalg.norm(lfo), np.linalg.norm(lfo)/180*np.pi) + lfo_transform = np.dot(lfo_transform, transforms3d.axangles.axangle2mat([0,1,0], np.pi/2)) + la_transform = np.dot(np.linalg.inv(lti_transform), lfo_transform) + la = transforms3d.axangles.mat2axangle(la_transform) + m.pose[21:24] = la[0][[1, 2, 0]]*la[1] + + ### Right_Foot (90 offset) + rfo = data[122:125] + rfo_transform = transforms3d.axangles.axangle2mat(rfo/np.linalg.norm(lfo), np.linalg.norm(rfo)/180*np.pi) + rfo_transform = np.dot(rfo_transform, transforms3d.axangles.axangle2mat([0,1,0], np.pi/2)) + ra_transform = np.dot(np.linalg.inv(rti_transform), rfo_transform) + ra = transforms3d.axangles.mat2axangle(ra_transform) + m.pose[24:27] = ra[0][[1, 2, 0]]*ra[1] + + ### + diff --git a/vis_demo/README.md b/vis_demo/README.md new file mode 100644 index 0000000..53f3fb7 --- /dev/null +++ b/vis_demo/README.md @@ -0,0 +1,5 @@ +# Code for visualization data from different modalities + +Run `visualize.m` to visualize the data. + +![alt text](demo_misc.png) diff --git a/vis_demo/demo_misc.png b/vis_demo/demo_misc.png new file mode 100644 index 0000000..2a6bf81 Binary files /dev/null and b/vis_demo/demo_misc.png differ diff --git a/vis_demo/depth.png b/vis_demo/depth.png new file mode 100644 index 0000000..13e0f35 Binary files /dev/null and b/vis_demo/depth.png differ diff --git a/vis_demo/keypoint.json b/vis_demo/keypoint.json new file mode 100644 index 0000000..56f0dc4 --- /dev/null +++ b/vis_demo/keypoint.json @@ -0,0 +1 @@ +{"version":1.3,"people":[{"person_id":[-1],"pose_keypoints_2d":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,758.695,170.933,0.461265,786.089,174.877,0.409076,758.723,351.216,0.728424,774.32,515.742,0.714466,733.233,165.084,0.390594,652.908,333.579,0.693749,617.664,507.932,0.813623,0,0,0,0,0,0,0,0,0,0,0,0,551.009,511.862,0.657727,564.709,523.518,0.702532,627.433,525.489,0.676285,723.417,537.273,0.355216,729.271,527.554,0.331402,784.131,533.386,0.575139],"face_keypoints_2d":[],"hand_left_keypoints_2d":[],"hand_right_keypoints_2d":[],"pose_keypoints_3d":[],"face_keypoints_3d":[],"hand_left_keypoints_3d":[],"hand_right_keypoints_3d":[]}]} \ No newline at end of file diff --git a/vis_demo/mask.png b/vis_demo/mask.png new file mode 100644 index 0000000..bd06238 Binary files /dev/null and b/vis_demo/mask.png differ diff --git a/vis_demo/pointcloud.mat b/vis_demo/pointcloud.mat new file mode 100644 index 0000000..8d34385 Binary files /dev/null and b/vis_demo/pointcloud.mat differ diff --git a/vis_demo/visualize.m b/vis_demo/visualize.m new file mode 100644 index 0000000..1d0c85e --- /dev/null +++ b/vis_demo/visualize.m @@ -0,0 +1,62 @@ +% this is the script used for visualizing samples from ICL-Gait Dataset +% https://xiaogu.site/ICL_gait/ +% Contact: xiao.gu17@imperial.ac.uk + +clear all; +close all; +clc; + +figure(); +%% visualize depth +depth = imread('depth.png'); +subplot(2,2,1); +imshow(depth); +colormap('parula'); +caxis([0,3000]); +title('depth image'); + +%% visualize mask +mask = imread('mask.png'); +subplot(2,2,2); +imshow(mask); +title('segmentation mask'); + +%% visualize skeleton from openpose +% load openpose result +fid = fopen('keypoint.json'); +raw = fread(fid,inf); +str = char(raw'); +fclose(fid); +val = jsondecode(str); +pose2d = reshape(val.people.pose_keypoints_2d, 3, 25)'; + +% visualize 2d result +j_index = [9,10,11,12,13,14,15,20,21,22,23,24,25]; +j_tree = [9,10;10,11;11,12;12,23;12,25;23,24; + 9,13;13,14;14,15;15,20;15,22;20,21]; +subplot(2,2,3); +plot(pose2d(j_index,1), pose2d(j_index,2),'.','MarkerSize',25); +hold on; + +for j = 1:size(j_tree,1) + plot(pose2d(j_tree(j,:),1), pose2d(j_tree(j,:),2),'-', 'LineWidth', 3); +end + +imshow([]); +xlim([400, 1000]); +ylim([50, 600]); +title('2D Keypoint'); + +%% visualize point cloud +load('pointcloud.mat'); +subplot(2,2,4); +pcshow(pc_2048,'VerticalAxis','Y','VerticalAxisDir','down') +hold on; +pcshow(keypoint,'VerticalAxis','Y','VerticalAxisDir','down', 'MarkerSize', 300) +title('\color{black}Point Cloud'); +set(gca, 'color', 'w'); +set(gcf, 'color', 'w'); +axis off; + +% visualize the body_orientation (of pelvis) +plotTransforms((keypoint(1,:)+keypoint(5,:))/2, body_orientation, 'FrameSize',0.2);