From daf7ffc1f77853dc719e7f82822ae294f34080fb Mon Sep 17 00:00:00 2001 From: Manuel Martin Date: Wed, 29 Nov 2023 18:06:28 +0100 Subject: [PATCH] Local media new loader issues fix --- src/bit-components.js | 1 + src/bit-systems/media-loading.ts | 13 +- .../object-menu-transform-system.ts | 3 +- src/bit-systems/object-menu.ts | 3 +- src/bit-systems/pdf-menu-system.ts | 17 +-- src/bit-systems/video-menu-system.ts | 123 ++++++++---------- src/bit-systems/video-system.ts | 7 +- src/inflators/media-loader.ts | 3 +- src/systems/bit-media-frames.js | 1 - src/utils/bit-pinning-helper.ts | 2 +- 10 files changed, 74 insertions(+), 99 deletions(-) diff --git a/src/bit-components.js b/src/bit-components.js index dd423a1cb8..a96dc66f63 100644 --- a/src/bit-components.js +++ b/src/bit-components.js @@ -405,5 +405,6 @@ export const MediaLink = defineComponent({ MediaLink.src[$isStringType] = true; export const ObjectMenuTransform = defineComponent({ targetObjectRef: Types.eid, + prevObjectRef: Types.eid, flags: Types.ui8 }); diff --git a/src/bit-systems/media-loading.ts b/src/bit-systems/media-loading.ts index 44df8c28f5..8ced7b53c7 100644 --- a/src/bit-systems/media-loading.ts +++ b/src/bit-systems/media-loading.ts @@ -43,7 +43,7 @@ import { MediaType, mediaTypeName, resolveMediaInfo } from "../utils/media-utils import { EntityID } from "../utils/networking-types"; import { LinkType, inflateLink } from "../inflators/link"; import { inflateGrabbable } from "../inflators/grabbable"; -import { findAncestorsWithComponent } from "../utils/bit-utils"; +import { findAncestorsWithComponent, findChildWithComponent } from "../utils/bit-utils"; import { setMatrixWorld } from "../utils/three-utils"; import { computeObjectAABB, getScaleCoefficient } from "../utils/auto-box-collider"; @@ -129,14 +129,15 @@ function resizeAndRecenter(world: HubsWorld, mediaLoaderEid: EntityID, box: Box3 } export function* animateScale(world: HubsWorld, mediaLoaderEid: EntityID) { - const mediaLoaderRootObj = world.eid2obj.get(mediaLoaderEid)!; + const mediaLoadedEid = findChildWithComponent(world, LoadedByMediaLoader, mediaLoaderEid)!; + const mediaLoadedObj = world.eid2obj.get(mediaLoadedEid)!; const onAnimate = ([scale]: [Vector3]) => { - mediaLoaderRootObj.scale.copy(scale); - mediaLoaderRootObj.matrixNeedsUpdate = true; + mediaLoadedObj.scale.copy(scale); + mediaLoadedObj.matrixNeedsUpdate = true; }; const scalar = 0.001; - const startScale = new Vector3().copy(mediaLoaderRootObj.scale).multiplyScalar(scalar); - const endScale = new Vector3().copy(mediaLoaderRootObj.scale); + const startScale = new Vector3().copy(mediaLoadedObj.scale).multiplyScalar(scalar); + const endScale = new Vector3().copy(mediaLoadedObj.scale); // Animate once to set the initial state, then yield one frame // because the first render of the new object may be slow // TODO: We could move uploading textures to the GPU to the loader, diff --git a/src/bit-systems/object-menu-transform-system.ts b/src/bit-systems/object-menu-transform-system.ts index 699eabd9e2..907e1ba2a1 100644 --- a/src/bit-systems/object-menu-transform-system.ts +++ b/src/bit-systems/object-menu-transform-system.ts @@ -47,7 +47,7 @@ function transformMenu(world: HubsWorld, menu: EntityID) { // Calculate the menu offset based on visible elements const center = (ObjectMenuTransform.flags[menu] & ObjectMenuTransformFlags.Center) !== 0 ? true : false; - if (center) { + if (center && ObjectMenuTransform.targetObjectRef[menu] !== ObjectMenuTransform.prevObjectRef[menu]) { getAABB(menuObj, aabb); aabb.getCenter(tmpVec1); getAABB(menuObj, aabb, true); @@ -100,6 +100,7 @@ function transformMenu(world: HubsWorld, menu: EntityID) { setMatrixWorld(menuObj, tmpMat4); } + ObjectMenuTransform.prevObjectRef[menu] = ObjectMenuTransform.targetObjectRef[menu]; } const menuQuery = defineQuery([ObjectMenuTransform]); diff --git a/src/bit-systems/object-menu.ts b/src/bit-systems/object-menu.ts index b8a0089920..0dfbf2869d 100644 --- a/src/bit-systems/object-menu.ts +++ b/src/bit-systems/object-menu.ts @@ -13,7 +13,8 @@ import { RemoteRight, Rigidbody, Deleting, - Deletable + Deletable, + MediaContentBounds } from "../bit-components"; import { anyEntityWith, findAncestorWithComponent, findAncestorWithComponents } from "../utils/bit-utils"; import { createNetworkedEntity } from "../utils/create-networked-entity"; diff --git a/src/bit-systems/pdf-menu-system.ts b/src/bit-systems/pdf-menu-system.ts index 2be21803c0..5d07aa933f 100644 --- a/src/bit-systems/pdf-menu-system.ts +++ b/src/bit-systems/pdf-menu-system.ts @@ -2,10 +2,10 @@ import { addComponent, defineQuery, entityExists, hasComponent } from "bitecs"; import { Text } from "troika-three-text"; import type { HubsWorld } from "../app"; import { + Deleting, EntityStateDirty, HoveredRemoteRight, Interacted, - MediaContentBounds, MediaPDF, NetworkedPDF, ObjectMenuTransform, @@ -16,7 +16,6 @@ import type { EntityID } from "../utils/networking-types"; import { takeOwnership } from "../utils/take-ownership"; import { PDFResourcesMap } from "./pdf-system"; import { ObjectMenuTransformFlags } from "../inflators/object-menu-transform"; -import { canPin } from "../utils/bit-pinning-helper"; function clicked(world: HubsWorld, eid: EntityID) { return hasComponent(world, Interacted, eid); @@ -90,23 +89,11 @@ function flushToObject3Ds(world: HubsWorld, menu: EntityID, frozen: boolean) { ObjectMenuTransform.flags[menu] &= ~ObjectMenuTransformFlags.Enabled; } - // The media loader entity is the entity that's is actually pinned and decides - // the pinnable state of the pdf component so we need to check the media loader entity pin - // state to show/hide certain buttons. The media loader component is not present anymore after - // the media has been loaded but it will always have a MediaContentBounds. - // TODO We should use something more meaningful than MediaContentBounds for the media loader root entity - // or rename it to something like MediaRoot. - let canIPin = false; - const mediaLoader = findAncestorWithComponent(world, MediaContentBounds, target); - if (mediaLoader && canPin(APP.hubChannel!, mediaLoader)) { - canIPin = true; - } - [PDFMenu.prevButtonRef[menu], PDFMenu.nextButtonRef[menu]].forEach(buttonRef => { const buttonObj = world.eid2obj.get(buttonRef)!; // Parent visibility doesn't block raycasting, so we must set each button to be invisible // TODO: Ensure that children of invisible entities aren't raycastable - buttonObj.visible = visible && canIPin; + buttonObj.visible = visible; }); if (target) { diff --git a/src/bit-systems/video-menu-system.ts b/src/bit-systems/video-menu-system.ts index 75425c4e6d..ea10ee90b2 100644 --- a/src/bit-systems/video-menu-system.ts +++ b/src/bit-systems/video-menu-system.ts @@ -9,12 +9,10 @@ import { Held, HeldRemoteRight, HoveredRemoteRight, - MediaContentBounds, MediaVideo, MediaVideoData, NetworkedVideo, ObjectMenuTransform, - Owned, VideoMenu } from "../bit-components"; import { timeFmt } from "../components/media-video"; @@ -23,11 +21,11 @@ import { paths } from "../systems/userinput/paths"; import { animate } from "../utils/animate"; import { coroutine } from "../utils/coroutine"; import { easeOutQuadratic } from "../utils/easing"; +import { isFacingCamera } from "../utils/three-utils"; import { Emitter2Audio } from "./audio-emitter-system"; import { EntityID } from "../utils/networking-types"; import { findAncestorWithComponent, hasAnyComponent } from "../utils/bit-utils"; import { ObjectMenuTransformFlags } from "../inflators/object-menu-transform"; -import { isPinned } from "./networking"; const videoMenuQuery = defineQuery([VideoMenu]); const hoveredQuery = defineQuery([HoveredRemoteRight]); @@ -121,87 +119,70 @@ export function videoMenuSystem(world: HubsWorld, userinput: any, sceneIsFrozen: videoMenuQuery(world).forEach(function (eid) { const videoEid = VideoMenu.videoRef[eid]; if (!videoEid) return; + const menuObj = world.eid2obj.get(eid)!; const video = MediaVideoData.get(videoEid)!; - const ratio = MediaVideo.ratio[videoEid]; + const togglePlayVideo = userinput.get(paths.actions.cursor.right.togglePlayVideo); + if (togglePlayVideo) { + if (hasComponent(world, NetworkedVideo, videoEid)) { + takeOwnership(world, videoEid); + addComponent(world, EntityStateDirty, videoEid); + } + + const playIndicatorObj = world.eid2obj.get(VideoMenu.playIndicatorRef[eid])!; + const pauseIndicatorObj = world.eid2obj.get(VideoMenu.pauseIndicatorRef[eid])!; + + const audioEid = Emitter2Audio.get(videoEid)!; + if (video.paused) { + video.play(); + APP.isAudioPaused.delete(audioEid); + playIndicatorObj.visible = true; + pauseIndicatorObj.visible = false; + rightMenuIndicatorCoroutine = coroutine(animateIndicator(world, VideoMenu.playIndicatorRef[eid])); + } else { + video.pause(); + APP.isAudioPaused.add(audioEid); + playIndicatorObj.visible = false; + pauseIndicatorObj.visible = true; + rightMenuIndicatorCoroutine = coroutine(animateIndicator(world, VideoMenu.pauseIndicatorRef[eid])); + } + } - // The media loader entity is the entity that's is actually pinned and decides - // the pinnable state of the video component so we need to check the media loader entity pin - // state to show/hide certain buttons. The media loader component is not present anymore after - // the media has been loaded but it will always have a MediaContentBounds. - // TODO We should use something more meaningful than MediaContentBounds for the media loader root entity - // or rename it to something like MediaRoot. - let canIPin = false; - const mediaRoot = findAncestorWithComponent(world, MediaContentBounds, videoEid)!; - if (isPinned(mediaRoot) && !hasComponent(world, Owned, mediaRoot)) { - canIPin = true; + const videoIsFacingCamera = isFacingCamera(world.eid2obj.get(videoEid)!); + const yRot = videoIsFacingCamera ? 0 : Math.PI; + if (menuObj.rotation.y !== yRot) { + menuObj.rotation.y = yRot; + menuObj.matrixNeedsUpdate = true; } const headObj = world.eid2obj.get(VideoMenu.headRef[eid])!; - const slider = world.eid2obj.get(VideoMenu.sliderRef[eid])!; - const playIndicatorObj = world.eid2obj.get(VideoMenu.playIndicatorRef[eid])!; - const pauseIndicatorObj = world.eid2obj.get(VideoMenu.pauseIndicatorRef[eid])!; - if (canIPin) { - const togglePlayVideo = userinput.get(paths.actions.cursor.right.togglePlayVideo); - if (togglePlayVideo) { - if (hasComponent(world, NetworkedVideo, videoEid)) { - takeOwnership(world, videoEid); - addComponent(world, EntityStateDirty, videoEid); - } - - const audioEid = Emitter2Audio.get(videoEid)!; - if (video.paused) { - video.play(); - APP.isAudioPaused.delete(audioEid); - playIndicatorObj.visible = true; - pauseIndicatorObj.visible = false; - rightMenuIndicatorCoroutine = coroutine(animateIndicator(world, VideoMenu.playIndicatorRef[eid])); - } else { - video.pause(); - APP.isAudioPaused.add(audioEid); - playIndicatorObj.visible = false; - pauseIndicatorObj.visible = true; - rightMenuIndicatorCoroutine = coroutine(animateIndicator(world, VideoMenu.pauseIndicatorRef[eid])); - } + if (hasComponent(world, HeldRemoteRight, VideoMenu.trackRef[eid])) { + const trackObj = world.eid2obj.get(VideoMenu.trackRef[eid])!; + intersectInThePlaneOf(trackObj, userinput.get(paths.actions.cursor.right.pose), intersectionPoint); + if (intersectionPoint) { + const newPosition = headObj.parent!.worldToLocal(intersectionPoint); + video.currentTime = + mapLinear(clamp(newPosition.x, -sliderHalfWidth, sliderHalfWidth), -sliderHalfWidth, sliderHalfWidth, 0, 1) * + video.duration; } - - if (hasComponent(world, HeldRemoteRight, VideoMenu.trackRef[eid])) { - const trackObj = world.eid2obj.get(VideoMenu.trackRef[eid])!; - intersectInThePlaneOf(trackObj, userinput.get(paths.actions.cursor.right.pose), intersectionPoint); - if (intersectionPoint) { - const newPosition = headObj.parent!.worldToLocal(intersectionPoint); - video.currentTime = - mapLinear( - clamp(newPosition.x, -sliderHalfWidth, sliderHalfWidth), - -sliderHalfWidth, - sliderHalfWidth, - 0, - 1 - ) * video.duration; - } - if (hasComponent(world, NetworkedVideo, videoEid)) { - takeOwnership(world, videoEid); - addComponent(world, EntityStateDirty, videoEid); - } + if (hasComponent(world, NetworkedVideo, videoEid)) { + takeOwnership(world, videoEid); + addComponent(world, EntityStateDirty, videoEid); } - headObj.visible = true; - headObj.position.x = mapLinear(video.currentTime, 0, video.duration, -sliderHalfWidth, sliderHalfWidth); - headObj.matrixNeedsUpdate = true; - - slider.visible = true; - slider.position.setY(-(ratio / 2) + 0.025); - slider.matrixNeedsUpdate = true; - } else { - headObj.visible = false; - slider.visible = false; - playIndicatorObj.visible = false; - pauseIndicatorObj.visible = false; } + headObj.position.x = mapLinear(video.currentTime, 0, video.duration, -sliderHalfWidth, sliderHalfWidth); + headObj.matrixNeedsUpdate = true; + + const ratio = MediaVideo.ratio[videoEid]; const timeLabel = world.eid2obj.get(VideoMenu.timeLabelRef[eid])! as TroikaText; timeLabel.text = `${timeFmt(video.currentTime)} / ${timeFmt(video.duration)}`; timeLabel.position.setY(ratio / 2 - 0.02); timeLabel.matrixNeedsUpdate = true; + const slider = world.eid2obj.get(VideoMenu.sliderRef[eid])!; + slider.position.setY(-(ratio / 2) + 0.025); + slider.matrixNeedsUpdate = true; + if (rightMenuIndicatorCoroutine && rightMenuIndicatorCoroutine().done) { rightMenuIndicatorCoroutine = null; } @@ -214,7 +195,6 @@ const START_SCALE = new Vector3().setScalar(0.05); const END_SCALE = new Vector3().setScalar(0.25); function* animateIndicator(world: HubsWorld, eid: number) { const obj = world.eid2obj.get(eid)!; - obj.visible = true; yield* animate({ properties: [ [START_SCALE, END_SCALE], @@ -228,5 +208,4 @@ function* animateIndicator(world: HubsWorld, eid: number) { ((obj as Mesh).material as MeshBasicMaterial).opacity = opacity; } }); - obj.visible = false; } diff --git a/src/bit-systems/video-system.ts b/src/bit-systems/video-system.ts index 615128cdfe..b41be3b886 100644 --- a/src/bit-systems/video-system.ts +++ b/src/bit-systems/video-system.ts @@ -80,7 +80,12 @@ export function videoSystem(world: HubsWorld, audioSystem: AudioSystem) { } else { const networkedPauseState = !!(NetworkedVideo.flags[eid] & Flags.PAUSED); if (networkedPauseState !== video.paused) { - video.paused ? video.play() : video.pause(); + video.paused + ? video.play().catch(() => { + // Need to deal with the fact play() may fail if user has not interacted with browser yet. + console.error("Error playing video."); + }) + : video.pause(); } if (networkedPauseState || Math.abs(NetworkedVideo.time[eid] - video.currentTime) > OUT_OF_SYNC_SEC) { video.currentTime = NetworkedVideo.time[eid]; diff --git a/src/inflators/media-loader.ts b/src/inflators/media-loader.ts index 00d041c7b8..f111d67aaa 100644 --- a/src/inflators/media-loader.ts +++ b/src/inflators/media-loader.ts @@ -1,6 +1,6 @@ import { addComponent } from "bitecs"; import { HubsWorld } from "../app"; -import { MediaLoader, MediaLoading } from "../bit-components"; +import { MediaLoader, MediaLoading, Networked } from "../bit-components"; import { MEDIA_LOADER_FLAGS } from "../bit-systems/media-loading"; export type MediaLoaderParams = { @@ -18,6 +18,7 @@ export function inflateMediaLoader( eid: number, { src, recenter, resize, animateLoad, fileId, isObjectMenuTarget, moveParentNotObject }: MediaLoaderParams ) { + addComponent(world, Networked, eid); addComponent(world, MediaLoader, eid); addComponent(world, MediaLoading, eid); let flags = 0; diff --git a/src/systems/bit-media-frames.js b/src/systems/bit-media-frames.js index b2d2a55de4..0181708cac 100644 --- a/src/systems/bit-media-frames.js +++ b/src/systems/bit-media-frames.js @@ -207,7 +207,6 @@ function createPreview(world, capturableEid) { return mat; }); }); - // } setMatrixWorld(previewObj, capturableObj.matrixWorld); world.scene.add(previewObj); return previewObj; diff --git a/src/utils/bit-pinning-helper.ts b/src/utils/bit-pinning-helper.ts index 1ff5b710c3..bc48a14d7d 100644 --- a/src/utils/bit-pinning-helper.ts +++ b/src/utils/bit-pinning-helper.ts @@ -42,7 +42,7 @@ const _signInAndPinOrUnpinElement = (hubChannel: HubChannel, world: HubsWorld, e export const canPin = (hubChannel: HubChannel, eid: EntityID): boolean => { const createMessageData = createMessageDatas.get(eid)!; - if (createMessageData.prefabName !== "media") { + if (createMessageData && createMessageData.prefabName !== "media") { return false; } return isNetworkInstantiated(eid) && hubChannel.can("pin_objects");