From bb20bed0b79ef20a80ab30f6b971b5e976f0692c Mon Sep 17 00:00:00 2001 From: 0b5vr <0b5vr@0b5vr.com> Date: Fri, 13 Dec 2024 14:41:27 +0900 Subject: [PATCH 1/4] fix: fix removeUnnecessaryJoints it processes a same bone index attribute twice or more, which collapses the mesh We are going to fix combineSkeletons in the next commit --- .../src/VRMUtils/removeUnnecessaryJoints.ts | 58 +++++++++++-------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/packages/three-vrm/src/VRMUtils/removeUnnecessaryJoints.ts b/packages/three-vrm/src/VRMUtils/removeUnnecessaryJoints.ts index 2dca72475..e28af2f64 100644 --- a/packages/three-vrm/src/VRMUtils/removeUnnecessaryJoints.ts +++ b/packages/three-vrm/src/VRMUtils/removeUnnecessaryJoints.ts @@ -46,41 +46,42 @@ export function removeUnnecessaryJoints( skinnedMeshes.push(obj as THREE.SkinnedMesh); }); - // A map from meshes to bones and boneInverses + // A map from meshes to new-to-old bone index map // some meshes might share a same skinIndex attribute, and this map also prevents to convert the attribute twice - const bonesList: Map< - THREE.SkinnedMesh, - { - bones: THREE.Bone[]; - boneInverses: THREE.Matrix4[]; - } + const attributeToBoneIndexMapMap: Map< + THREE.BufferAttribute | THREE.InterleavedBufferAttribute, + Map > = new Map(); // A maximum number of bones let maxBones = 0; - // Iterate over all skinned meshes and collect bones and boneInverses + // Iterate over all skinned meshes and remap bones for each skin index attribute for (const mesh of skinnedMeshes) { const geometry = mesh.geometry; const attribute = geometry.getAttribute('skinIndex'); - const bones: THREE.Bone[] = []; // new list of bone - const boneInverses: THREE.Matrix4[] = []; // new list of boneInverse - const boneIndexMap: { [index: number]: number } = {}; // map of old bone index vs. new bone index + if (attributeToBoneIndexMapMap.has(attribute)) { + continue; + } + + const oldToNew = new Map(); // map of old bone index vs. new bone index + const newToOld = new Map(); // map of new bone index vs. old bone index // create a new bone map for (let i = 0; i < attribute.count; i++) { for (let j = 0; j < attribute.itemSize; j++) { - const index = attribute.getComponent(i, j); + const oldIndex = attribute.getComponent(i, j); + let newIndex = oldToNew.get(oldIndex); // new skinIndex buffer - if (boneIndexMap[index] == null) { - boneIndexMap[index] = bones.length; - bones.push(mesh.skeleton.bones[index]); - boneInverses.push(mesh.skeleton.boneInverses[index]); + if (newIndex == null) { + newIndex = oldToNew.size; + oldToNew.set(oldIndex, newIndex); + newToOld.set(newIndex, oldIndex); } - attribute.setComponent(i, j, boneIndexMap[index]); + attribute.setComponent(i, j, newIndex); } } @@ -88,22 +89,29 @@ export function removeUnnecessaryJoints( attribute.needsUpdate = true; // update boneList - bonesList.set(mesh, { bones, boneInverses }); + attributeToBoneIndexMapMap.set(attribute, newToOld); // update max bones count - maxBones = Math.max(maxBones, bones.length); + maxBones = Math.max(maxBones, oldToNew.size); } // Let's actually set the skeletons for (const mesh of skinnedMeshes) { - const { bones, boneInverses } = bonesList.get(mesh)!; + const geometry = mesh.geometry; + const attribute = geometry.getAttribute('skinIndex'); + const newToOld = attributeToBoneIndexMapMap.get(attribute)!; + + const bones: THREE.Bone[] = []; + const boneInverses: THREE.Matrix4[] = []; // if `experimentalSameBoneCounts` is `true`, compensate skeletons with dummy bones to keep the bone count same between skeletons - if (experimentalSameBoneCounts) { - for (let i = bones.length; i < maxBones; i++) { - bones[i] = bones[0]; - boneInverses[i] = boneInverses[0]; - } + const nBones = experimentalSameBoneCounts ? maxBones : newToOld.size; + + for (let newIndex = 0; newIndex < nBones; newIndex++) { + const oldIndex = newToOld.get(newIndex) ?? 0; + + bones.push(mesh.skeleton.bones[oldIndex]); + boneInverses.push(mesh.skeleton.boneInverses[oldIndex]); } const skeleton = new THREE.Skeleton(bones, boneInverses); From 25b7c73f373de2a84d2ae5f79b5df2f00a803cdf Mon Sep 17 00:00:00 2001 From: 0b5vr <0b5vr@0b5vr.com> Date: Fri, 13 Dec 2024 18:24:14 +0900 Subject: [PATCH 2/4] fix: Fix combineSkeletons It sometimes modified a skin index attribute twice or more, which collapses the skinned mesh rendering I'm sorry that it became a huge refactor ._. --- .../src/VRMUtils/combineSkeletons.ts | 231 ++++++++++++------ 1 file changed, 163 insertions(+), 68 deletions(-) diff --git a/packages/three-vrm/src/VRMUtils/combineSkeletons.ts b/packages/three-vrm/src/VRMUtils/combineSkeletons.ts index 26a14e1bd..57cd66637 100644 --- a/packages/three-vrm/src/VRMUtils/combineSkeletons.ts +++ b/packages/three-vrm/src/VRMUtils/combineSkeletons.ts @@ -9,98 +9,193 @@ import * as THREE from 'three'; * @param root Root object that will be traversed */ export function combineSkeletons(root: THREE.Object3D): void { - const skinnedMeshes = new Set(); - const geometryToSkinnedMesh = new Map(); + const skinnedMeshes = collectSkinnedMeshes(root); + + // List all bones and boneInverses for each meshes + const attributeUsedIndexSetMap = new Map>(); + const meshBoneInverseMapMap = new Map>(); + for (const mesh of skinnedMeshes) { + const boneInverseMap = listUsedBones(mesh, attributeUsedIndexSetMap); + meshBoneInverseMapMap.set(mesh, boneInverseMap); + } - // Traverse entire tree and collect skinned meshes - root.traverse((obj) => { - if (obj.type !== 'SkinnedMesh') { - return; - } + // Group meshes by bone sets + const groups: { boneInverseMap: Map; meshes: Set }[] = []; + for (const [mesh, boneInverseMap] of meshBoneInverseMapMap) { + let foundMergeableGroup = false; + for (const candidate of groups) { + // check if the candidate group is mergeable + const isMergeable = boneInverseMapIsMergeable(boneInverseMap, candidate.boneInverseMap); + + // if we found a mergeable group, add the mesh to the group + if (isMergeable) { + foundMergeableGroup = true; + candidate.meshes.add(mesh); + + // add lacking bones to the group + for (const [bone, boneInverse] of boneInverseMap) { + candidate.boneInverseMap.set(bone, boneInverse); + } - const skinnedMesh = obj as THREE.SkinnedMesh; + break; + } + } - // Check if the geometry has already been encountered - const previousSkinnedMesh = geometryToSkinnedMesh.get(skinnedMesh.geometry); - if (previousSkinnedMesh) { - // Skinned meshes that share their geometry with other skinned meshes can't be processed. - // The skinnedMeshes already contain previousSkinnedMesh, so remove it now. - skinnedMeshes.delete(previousSkinnedMesh); - } else { - geometryToSkinnedMesh.set(skinnedMesh.geometry, skinnedMesh); - skinnedMeshes.add(skinnedMesh); + // if we couldn't find a mergeable group, create a new group + if (!foundMergeableGroup) { + groups.push({ boneInverseMap, meshes: new Set([mesh]) }); } - }); + } + + // prepare new skeletons for each group, and bind them to the meshes + for (const { boneInverseMap, meshes } of groups) { + // create a new skeleton + const newBones = Array.from(boneInverseMap.keys()); + const newBoneInverses = Array.from(boneInverseMap.values()); + const newSkeleton = new THREE.Skeleton(newBones, newBoneInverses); - // Prepare new skeletons for the skinned meshes - const newSkeletons: Array<{ bones: THREE.Bone[]; boneInverses: THREE.Matrix4[]; meshes: THREE.SkinnedMesh[] }> = []; - skinnedMeshes.forEach((skinnedMesh) => { - const skeleton = skinnedMesh.skeleton; + const attributeProcessedSet = new Set(); + + for (const mesh of meshes) { + const attribute = mesh.geometry.getAttribute('skinIndex'); + + if (!attributeProcessedSet.has(attribute)) { + // remap skin index attribute + remapSkinIndexAttribute(attribute, mesh.skeleton.bones, newBones); + attributeProcessedSet.add(attribute); + } - // Find suitable skeleton - let newSkeleton = newSkeletons.find((candidate) => skeletonMatches(skeleton, candidate)); - if (!newSkeleton) { - newSkeleton = { bones: [], boneInverses: [], meshes: [] }; - newSkeletons.push(newSkeleton); + // bind the new skeleton to the mesh + mesh.bind(newSkeleton, new THREE.Matrix4()); } + } +} - // Add skinned mesh to the new skeleton - newSkeleton.meshes.push(skinnedMesh); +/** + * Traverse an entire tree and collect skinned meshes. + */ +function collectSkinnedMeshes(scene: THREE.Object3D): Set { + const skinnedMeshes = new Set(); - // Determine bone index mapping from skeleton -> newSkeleton - const boneIndexMap: number[] = skeleton.bones.map((bone) => newSkeleton.bones.indexOf(bone)); + scene.traverse((obj) => { + if (!(obj as any).isSkinnedMesh) { + return; + } - // Update skinIndex attribute - const geometry = skinnedMesh.geometry; - const attribute = geometry.getAttribute('skinIndex'); - const weightAttribute = geometry.getAttribute('skinWeight'); + const skinnedMesh = obj as THREE.SkinnedMesh; + skinnedMeshes.add(skinnedMesh); + }); - for (let i = 0; i < attribute.count; i++) { - for (let j = 0; j < attribute.itemSize; j++) { - // check bone weight - const weight = weightAttribute.getComponent(i, j); - if (weight === 0) { - continue; - } + return skinnedMeshes; +} + +/** + * List all skin indices used by the given geometry. + * If the skin weight is 0, the index won't be considered as used. + * @param skinIndexAttr The skin index attribute + * @param skinWeightAttr The skin weight attribute + */ +function listUsedIndices(geometry: THREE.BufferGeometry): Set { + const skinIndexAttr = geometry.getAttribute('skinIndex'); + const skinWeightAttr = geometry.getAttribute('skinWeight'); - const index = attribute.getComponent(i, j); + const usedIndices = new Set(); - // new skinIndex buffer - if (boneIndexMap[index] === -1) { - boneIndexMap[index] = newSkeleton.bones.length; - newSkeleton.bones.push(skeleton.bones[index]); - newSkeleton.boneInverses.push(skeleton.boneInverses[index]); - } + for (let i = 0; i < skinIndexAttr.count; i++) { + for (let j = 0; j < skinIndexAttr.itemSize; j++) { + const index = skinIndexAttr.getComponent(i, j); + const weight = skinWeightAttr.getComponent(i, j); - attribute.setComponent(i, j, boneIndexMap[index]); + if (weight !== 0) { + usedIndices.add(index); } } + } - attribute.needsUpdate = true; - }); + return usedIndices; +} + +/** + * List all bones used by the given skinned mesh. + * @param mesh The skinned mesh to list used bones + * @param attributeUsedIndexSetMap A map from skin index attribute to the set of used skin indices + * @returns A map from used bone to the corresponding bone inverse matrix + */ +function listUsedBones( + mesh: THREE.SkinnedMesh, + attributeUsedIndexSetMap: Map>, +): Map { + const boneInverseMap = new Map(); + + const skeleton = mesh.skeleton; - // Bind new skeleton to the meshes - for (const { bones, boneInverses, meshes } of newSkeletons) { - const newSkeleton = new THREE.Skeleton(bones, boneInverses); - meshes.forEach((mesh) => mesh.bind(newSkeleton, new THREE.Matrix4())); + const geometry = mesh.geometry; + const skinIndexAttr = geometry.getAttribute('skinIndex'); + let usedIndicesSet = attributeUsedIndexSetMap.get(skinIndexAttr); + + if (!usedIndicesSet) { + usedIndicesSet = listUsedIndices(geometry); + attributeUsedIndexSetMap.set(skinIndexAttr, usedIndicesSet); } + + for (const index of usedIndicesSet) { + boneInverseMap.set(skeleton.bones[index], skeleton.boneInverses[index]); + } + + return boneInverseMap; } /** - * Checks if a given skeleton matches a candidate skeleton. For the skeletons to match, - * all bones must either be in the candidate skeleton with the same boneInverse OR - * not part of the candidate skeleton (as it can be added to it). - * @param skeleton The skeleton to check. - * @param candidate The candidate skeleton to match against. + * Check if the given bone inverse map is mergeable to the candidate bone inverse map. + * @param toCheck The bone inverse map to check + * @param candidate The candidate bone inverse map + * @returns True if the bone inverse map is mergeable to the candidate bone inverse map */ -function skeletonMatches(skeleton: THREE.Skeleton, candidate: { bones: THREE.Bone[]; boneInverses: THREE.Matrix4[] }) { - return skeleton.bones.every((bone, index) => { - const candidateIndex = candidate.bones.indexOf(bone); - if (candidateIndex !== -1) { - return matrixEquals(skeleton.boneInverses[index], candidate.boneInverses[candidateIndex]); +function boneInverseMapIsMergeable( + toCheck: Map, + candidate: Map, +): boolean { + for (const [bone, boneInverse] of toCheck.entries()) { + // if the bone is in the candidate group and the boneInverse is different, it's not mergeable + const candidateBoneInverse = candidate.get(bone); + if (candidateBoneInverse != null) { + if (!matrixEquals(boneInverse, candidateBoneInverse)) { + return false; + } } - return true; - }); + } + + return true; +} + +function remapSkinIndexAttribute( + attribute: THREE.BufferAttribute | THREE.InterleavedBufferAttribute, + oldBones: THREE.Bone[], + newBones: THREE.Bone[], +): void { + // a map from bone to old index + const boneOldIndexMap = new Map(); + for (const bone of oldBones) { + boneOldIndexMap.set(bone, boneOldIndexMap.size); + } + + // a map from old skin index to new skin index + const oldToNew = new Map(); + for (const [i, bone] of newBones.entries()) { + const oldIndex = boneOldIndexMap.get(bone)!; + oldToNew.set(oldIndex, i); + } + + // replace the skin index attribute with new indices + for (let i = 0; i < attribute.count; i++) { + for (let j = 0; j < attribute.itemSize; j++) { + const oldIndex = attribute.getComponent(i, j); + const newIndex = oldToNew.get(oldIndex)!; + attribute.setComponent(i, j, newIndex); + } + } + + attribute.needsUpdate = true; } // https://github.com/mrdoob/three.js/blob/r170/test/unit/src/math/Matrix4.tests.js#L12 From ff87e66dc0c542f569fb184d595c5e1c753a7cfc Mon Sep 17 00:00:00 2001 From: 0b5vr <0b5vr@0b5vr.com> Date: Wed, 18 Dec 2024 10:44:43 +0900 Subject: [PATCH 3/4] refactor: combineSkeletons, trivial fix doc comment of listUsedIndices See: https://github.com/pixiv/three-vrm/pull/1557#discussion_r1886452375 --- packages/three-vrm/src/VRMUtils/combineSkeletons.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/three-vrm/src/VRMUtils/combineSkeletons.ts b/packages/three-vrm/src/VRMUtils/combineSkeletons.ts index 57cd66637..f3556358b 100644 --- a/packages/three-vrm/src/VRMUtils/combineSkeletons.ts +++ b/packages/three-vrm/src/VRMUtils/combineSkeletons.ts @@ -92,8 +92,7 @@ function collectSkinnedMeshes(scene: THREE.Object3D): Set { /** * List all skin indices used by the given geometry. * If the skin weight is 0, the index won't be considered as used. - * @param skinIndexAttr The skin index attribute - * @param skinWeightAttr The skin weight attribute + * @param geometry The geometry to list used skin indices */ function listUsedIndices(geometry: THREE.BufferGeometry): Set { const skinIndexAttr = geometry.getAttribute('skinIndex'); From 78e3695fd9a9ee1455625aa53051c48c4601f6ce Mon Sep 17 00:00:00 2001 From: 0b5vr <0b5vr@0b5vr.com> Date: Wed, 18 Dec 2024 17:58:11 +0900 Subject: [PATCH 4/4] refactor: combineSkeletons, refrain using side effect inside listUsedBones See: https://github.com/pixiv/three-vrm/pull/1557#discussion_r1886463948 --- .../src/VRMUtils/combineSkeletons.ts | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/three-vrm/src/VRMUtils/combineSkeletons.ts b/packages/three-vrm/src/VRMUtils/combineSkeletons.ts index f3556358b..fcc3887d4 100644 --- a/packages/three-vrm/src/VRMUtils/combineSkeletons.ts +++ b/packages/three-vrm/src/VRMUtils/combineSkeletons.ts @@ -11,8 +11,17 @@ import * as THREE from 'three'; export function combineSkeletons(root: THREE.Object3D): void { const skinnedMeshes = collectSkinnedMeshes(root); - // List all bones and boneInverses for each meshes + // List all used skin indices for each skin index attribute const attributeUsedIndexSetMap = new Map>(); + for (const mesh of skinnedMeshes) { + const geometry = mesh.geometry; + const skinIndexAttr = geometry.getAttribute('skinIndex'); + const skinWeightAttr = geometry.getAttribute('skinWeight'); + const usedIndicesSet = listUsedIndices(skinIndexAttr, skinWeightAttr); + attributeUsedIndexSetMap.set(skinIndexAttr, usedIndicesSet); + } + + // List all bones and boneInverses for each meshes const meshBoneInverseMapMap = new Map>(); for (const mesh of skinnedMeshes) { const boneInverseMap = listUsedBones(mesh, attributeUsedIndexSetMap); @@ -92,12 +101,13 @@ function collectSkinnedMeshes(scene: THREE.Object3D): Set { /** * List all skin indices used by the given geometry. * If the skin weight is 0, the index won't be considered as used. - * @param geometry The geometry to list used skin indices + * @param skinIndexAttr The skin index attribute to list used indices + * @param skinWeightAttr The skin weight attribute corresponding to the skin index attribute */ -function listUsedIndices(geometry: THREE.BufferGeometry): Set { - const skinIndexAttr = geometry.getAttribute('skinIndex'); - const skinWeightAttr = geometry.getAttribute('skinWeight'); - +function listUsedIndices( + skinIndexAttr: THREE.BufferAttribute | THREE.InterleavedBufferAttribute, + skinWeightAttr: THREE.BufferAttribute | THREE.InterleavedBufferAttribute, +): Set { const usedIndices = new Set(); for (let i = 0; i < skinIndexAttr.count; i++) { @@ -130,11 +140,10 @@ function listUsedBones( const geometry = mesh.geometry; const skinIndexAttr = geometry.getAttribute('skinIndex'); - let usedIndicesSet = attributeUsedIndexSetMap.get(skinIndexAttr); + const usedIndicesSet = attributeUsedIndexSetMap.get(skinIndexAttr); if (!usedIndicesSet) { - usedIndicesSet = listUsedIndices(geometry); - attributeUsedIndexSetMap.set(skinIndexAttr, usedIndicesSet); + throw new Error('Unreachable. attributeUsedIndexSetMap does not know the skin index attribute'); } for (const index of usedIndicesSet) {