diff --git a/demo/scripts/components/ThumbnailPreview.tsx b/demo/scripts/components/ThumbnailPreview.tsx new file mode 100644 index 0000000000..3e87576f2f --- /dev/null +++ b/demo/scripts/components/ThumbnailPreview.tsx @@ -0,0 +1,212 @@ +import * as React from "react"; +import useModuleState from "../lib/useModuleState"; +import { IPlayerModule } from "../modules/player"; +import { IThumbnailTrackInfo } from "../../../src/public_types"; + +const DIV_SPINNER_STYLE = { + backgroundColor: "gray", + position: "absolute", + width: "100%", + height: "100%", + opacity: "50%", + display: "flex", + justifyContent: "center", + alignItems: "center", +} as const; + +const IMG_SPINNER_STYLE = { + width: "50%", + margin: "auto", +} as const; + +export default function ThumbnailPreview({ + xPosition, + time, + player, + showVideoThumbnail, +}: { + player: IPlayerModule; + xPosition: number | null; + time: number; + showVideoThumbnail: boolean; +}): JSX.Element { + const videoThumbnailLoader = useModuleState(player, "videoThumbnailLoader"); + const videoElement = useModuleState(player, "videoThumbnailsElement"); + const imageThumbnailElement = useModuleState(player, "imageThumbnailContainerElement"); + const parentElementRef = React.useRef(null); + const [shouldDisplaySpinner, setShouldDisplaySpinner] = React.useState(true); + const ceiledTime = Math.ceil(time); + + // Insert the div element containing the image thumbnail + React.useEffect(() => { + if (showVideoThumbnail) { + return; + } + + if (parentElementRef.current !== null) { + parentElementRef.current.appendChild(imageThumbnailElement); + } + return () => { + if ( + parentElementRef.current !== null && + parentElementRef.current.contains(imageThumbnailElement) + ) { + parentElementRef.current.removeChild(imageThumbnailElement); + } + }; + }, [showVideoThumbnail]); + + // OR insert the video element containing the thumbnail + React.useEffect(() => { + if (!showVideoThumbnail) { + return; + } + if (videoElement !== null && parentElementRef.current !== null) { + parentElementRef.current.appendChild(videoElement); + } + return () => { + if ( + videoElement !== null && + parentElementRef.current !== null && + parentElementRef.current.contains(videoElement) + ) { + parentElementRef.current.removeChild(videoElement); + } + }; + }, [videoElement, showVideoThumbnail]); + + React.useEffect(() => { + if (!showVideoThumbnail) { + return; + } + player.actions.attachVideoThumbnailLoader(); + return () => { + player.actions.dettachVideoThumbnailLoader(); + }; + }, [showVideoThumbnail]); + + // Change the thumbnail when a new time is wanted + React.useEffect(() => { + let spinnerTimeout: number | null = null; + let loadThumbnailTimeout: number | null = null; + + startSpinnerTimeoutIfNotAlreadyStarted(); + + // load thumbnail after a timer to avoid doing too many requests when the + // user quickly moves its pointer or whatever is calling this + loadThumbnailTimeout = window.setTimeout(() => { + loadThumbnailTimeout = null; + if (showVideoThumbnail) { + if (videoThumbnailLoader === null) { + return; + } + videoThumbnailLoader + .setTime(ceiledTime) + .then(hideSpinner) + .catch((err) => { + if ( + typeof err === "object" && + err !== null && + (err as Partial>).code === "ABORTED" + ) { + return; + } else { + hideSpinner(); + + // eslint-disable-next-line no-console + console.error("Error while loading thumbnails:", err); + } + }); + } else { + const metadata = player.actions.getAvailableThumbnailTracks(ceiledTime); + const thumbnailTrack = metadata.reduce((acc: IThumbnailTrackInfo | null, t) => { + if (acc === null || acc.height === undefined) { + return t; + } + if (t.height === undefined) { + return acc; + } + if (acc.height > t.height) { + return t.height > 100 ? t : acc; + } else { + return acc.height > 100 ? acc : t; + } + }, null); + if (thumbnailTrack === null) { + hideSpinner(); + return; + } + player.actions + .renderThumbnail(ceiledTime, thumbnailTrack.id) + .then(hideSpinner) + .catch((err) => { + if ( + typeof err === "object" && + err !== null && + (err as Partial>).code === "ABORTED" + ) { + return; + } else { + hideSpinner(); + // eslint-disable-next-line no-console + console.warn("Error while loading thumbnails:", err); + } + }); + } + }, 30); + + return () => { + if (loadThumbnailTimeout !== null) { + clearTimeout(loadThumbnailTimeout); + } + hideSpinner(); + }; + + /** + * Display a spinner after some delay if `stopSpinnerTimeout` hasn't been + * called since. + * This function allows to schedule a spinner if the request to display a + * thumbnail takes too much time. + */ + function startSpinnerTimeoutIfNotAlreadyStarted() { + if (spinnerTimeout !== null) { + return; + } + + // Wait a little before displaying spinner, to + // be sure loading takes time + spinnerTimeout = window.setTimeout(() => { + spinnerTimeout = null; + setShouldDisplaySpinner(true); + }, 150); + } + + /** + * Hide the spinner if one is active and stop the last started spinner + * timeout. + * Allow to avoid showing a spinner when the thumbnail we were waiting for + * was succesfully loaded. + */ + function hideSpinner() { + if (spinnerTimeout !== null) { + clearTimeout(spinnerTimeout); + spinnerTimeout = null; + } + setShouldDisplaySpinner(false); + } + }, [ceiledTime, videoThumbnailLoader, parentElementRef]); + + return ( +
+ {shouldDisplaySpinner ? ( +
+ +
+ ) : null} +
+ ); +} diff --git a/demo/scripts/components/VideoThumbnail.tsx b/demo/scripts/components/VideoThumbnail.tsx deleted file mode 100644 index 1a4cf94ae6..0000000000 --- a/demo/scripts/components/VideoThumbnail.tsx +++ /dev/null @@ -1,152 +0,0 @@ -import * as React from "react"; -import useModuleState from "../lib/useModuleState"; -import { IPlayerModule } from "../modules/player"; - -const DIV_SPINNER_STYLE = { - backgroundColor: "gray", - position: "absolute", - width: "100%", - height: "100%", - opacity: "50%", - display: "flex", - justifyContent: "center", - alignItems: "center", -} as const; - -const IMG_SPINNER_STYLE = { - width: "50%", - margin: "auto", -} as const; - -export default function VideoThumbnail({ - xPosition, - time, - player, -}: { - player: IPlayerModule; - xPosition: number | null; - time: number; -}): JSX.Element { - const videoThumbnailLoader = useModuleState(player, "videoThumbnailLoader"); - const videoElement = useModuleState(player, "videoThumbnailsElement"); - - React.useEffect(() => { - player.actions.attachVideoThumbnailLoader(); - return () => { - player.actions.dettachVideoThumbnailLoader(); - }; - }, []); - - const elementRef = React.useRef(null); - const [shouldDisplaySpinner, setShouldDisplaySpinner] = React.useState(true); - const roundedTime = Math.round(time); - - // Insert the video element containing the thumbnail when it changes - React.useEffect(() => { - if (videoElement !== null && elementRef.current !== null) { - elementRef.current.appendChild(videoElement); - } - return () => { - if ( - videoElement !== null && - elementRef.current !== null && - elementRef.current.contains(videoElement) - ) { - elementRef.current.removeChild(videoElement); - } - }; - }, [videoElement]); - - // Change the thumbnail when a new time is wanted - React.useEffect(() => { - let spinnerTimeout: number | null = null; - let loadThumbnailTimeout: number | null = null; - - if (videoThumbnailLoader === null) { - return; - } - - startSpinnerTimeoutIfNotAlreadyStarted(); - - if (loadThumbnailTimeout !== null) { - clearTimeout(loadThumbnailTimeout); - } - - // load thumbnail after a 40ms timer to avoid doing too many requests - // when the user quickly moves its pointer or whatever is calling this - loadThumbnailTimeout = window.setTimeout(() => { - loadThumbnailTimeout = null; - videoThumbnailLoader - .setTime(roundedTime) - .then(hideSpinner) - .catch((err) => { - if ( - typeof err === "object" && - err !== null && - (err as Partial>).code === "ABORTED" - ) { - return; - } else { - hideSpinner(); - - // eslint-disable-next-line no-console - console.error("Error while loading thumbnails:", err); - } - }); - }, 40); - return () => { - if (loadThumbnailTimeout !== null) { - clearTimeout(loadThumbnailTimeout); - } - hideSpinner(); - }; - - /** - * Display a spinner after some delay if `stopSpinnerTimeout` hasn't been - * called since. - * This function allows to schedule a spinner if the request to display a - * thumbnail takes too much time. - */ - function startSpinnerTimeoutIfNotAlreadyStarted() { - if (spinnerTimeout !== null) { - return; - } - - // Wait a little before displaying spinner, to - // be sure loading takes time - spinnerTimeout = window.setTimeout(() => { - spinnerTimeout = null; - setShouldDisplaySpinner(true); - }, 150); - } - - /** - * Hide the spinner if one is active and stop the last started spinner - * timeout. - * Allow to avoid showing a spinner when the thumbnail we were waiting for - * was succesfully loaded. - */ - function hideSpinner() { - if (spinnerTimeout !== null) { - clearTimeout(spinnerTimeout); - spinnerTimeout = null; - } - - setShouldDisplaySpinner(false); - } - }, [roundedTime, videoThumbnailLoader]); - - return ( -
- {shouldDisplaySpinner ? ( -
- -
- ) : null} -
- ); -} diff --git a/demo/scripts/contents.ts b/demo/scripts/contents.ts index 39ef0246b5..d4c5b5bb1d 100644 --- a/demo/scripts/contents.ts +++ b/demo/scripts/contents.ts @@ -22,6 +22,12 @@ const DEFAULT_CONTENTS: IDefaultContent[] = [ transport: "dash", live: false, }, + { + name: "Live with thumbnail track", + url: "https://livesim2.dashif.org/livesim2/testpic_2s/Manifest_thumbs.mpd", + transport: "dash", + live: true, + }, { name: "Axinom CMAF multiple Audio and Text tracks Tears of steel", url: "https://media.axprod.net/TestVectors/Cmaf/clear_1080p_h264/manifest.mpd", @@ -64,6 +70,12 @@ const DEFAULT_CONTENTS: IDefaultContent[] = [ transport: "dash", live: true, }, + { + name: "VOD with thumbnail track", + url: "https://dash.akamaized.net/akamai/bbb_30fps/bbb_with_tiled_thumbnails.mpd", + transport: "dash", + live: false, + }, { name: "Super SpeedWay", url: "https://playready.directtaps.net/smoothstreaming/SSWSS720H264/SuperSpeedway_720.ism/Manifest", diff --git a/demo/scripts/controllers/ProgressBar.tsx b/demo/scripts/controllers/ProgressBar.tsx index c891175056..a9bcafb008 100644 --- a/demo/scripts/controllers/ProgressBar.tsx +++ b/demo/scripts/controllers/ProgressBar.tsx @@ -1,7 +1,7 @@ import * as React from "react"; import ProgressbarComponent from "../components/ProgressBar"; import ToolTip from "../components/ToolTip"; -import VideoThumbnail from "../components/VideoThumbnail"; +import ThumbnailPreview from "../components/ThumbnailPreview"; import useModuleState from "../lib/useModuleState"; import type { IPlayerModule } from "../modules/player/index"; @@ -66,23 +66,14 @@ function ProgressBar({ setTimeIndicatorText(""); }, [isLive]); - const showVideoTumbnail = React.useCallback((ts: number, clientX: number): void => { + const showThumbnail = React.useCallback((ts: number, clientX: number): void => { const timestampToMs = ts; setThumbnailIsVisible(true); setTipPosition(clientX); setImageTime(timestampToMs); }, []); - const showThumbnail = React.useCallback( - (ts: number, clientX: number): void => { - if (enableVideoThumbnails) { - showVideoTumbnail(ts, clientX); - } - }, - [showVideoTumbnail, enableVideoThumbnails], - ); - - const hideTumbnail = React.useCallback((): void => { + const hideThumbnail = React.useCallback((): void => { setThumbnailIsVisible(false); setTipPosition(0); setImageTime(null); @@ -98,8 +89,8 @@ function ProgressBar({ const hideToolTips = React.useCallback(() => { hideTimeIndicator(); - hideTumbnail(); - }, [hideTumbnail, hideTimeIndicator]); + hideThumbnail(); + }, [hideThumbnail, hideTimeIndicator]); const onMouseMove = React.useCallback( (position: number, event: React.MouseEvent) => { @@ -127,9 +118,14 @@ function ProgressBar({ let thumbnailElement: JSX.Element | null = null; if (thumbnailIsVisible) { const xThumbnailPosition = tipPosition - toolTipOffset; - if (enableVideoThumbnails && imageTime !== null) { + if (imageTime !== null) { thumbnailElement = ( - + ); } } diff --git a/demo/scripts/modules/player/index.ts b/demo/scripts/modules/player/index.ts index aae24a8e88..172f1ec524 100644 --- a/demo/scripts/modules/player/index.ts +++ b/demo/scripts/modules/player/index.ts @@ -40,6 +40,7 @@ import type { ITextTrack, IVideoRepresentation, IVideoTrack, + IThumbnailTrackInfo, } from "../../../../src/public_types"; RxPlayer.addFeatures([ @@ -127,6 +128,7 @@ export interface IPlayerModuleState { isContentLoaded: boolean; isLive: boolean; isLoading: boolean; + imageThumbnailContainerElement: HTMLElement; isPaused: boolean; isReloading: boolean; isSeeking: boolean; @@ -185,6 +187,7 @@ const PlayerModule = declareModule( isContentLoaded: false, isLive: false, isLoading: false, + imageThumbnailContainerElement: document.createElement("div"), isPaused: false, isReloading: false, isSeeking: false, @@ -339,6 +342,19 @@ const PlayerModule = declareModule( player.unMute(); }, + getAvailableThumbnailTracks(time: number): IThumbnailTrackInfo[] { + const metadata = player.getAvailableThumbnailTracks({ time }); + return metadata ?? []; + }, + + renderThumbnail(time: number, thumbnailTrackId: string): Promise { + return player.renderThumbnail({ + container: state.get("imageThumbnailContainerElement"), + time, + thumbnailTrackId, + }); + }, + setDefaultVideoRepresentationSwitchingMode( mode: IVideoRepresentationsSwitchingMode, ): void { diff --git a/demo/styles/style.css b/demo/styles/style.css index 7477db48d7..8f61553447 100644 --- a/demo/styles/style.css +++ b/demo/styles/style.css @@ -368,7 +368,7 @@ header .right { } .progress-bar-wrapper:hover { - transform: scaleY(2); + transform: scaleY(2.5); } .progress-bar-current { diff --git a/src/core/fetchers/index.ts b/src/core/fetchers/index.ts index 85c6951f9a..e1bed395e6 100644 --- a/src/core/fetchers/index.ts +++ b/src/core/fetchers/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import CdnPrioritizer from "./cdn_prioritizer"; import type { IManifestFetcherSettings, IManifestFetcherEvent, @@ -22,6 +23,8 @@ import type { import ManifestFetcher from "./manifest"; import type { SegmentQueue, ISegmentQueueCreatorBackoffOptions } from "./segment"; import SegmentQueueCreator from "./segment"; +import createThumbnailFetcher, { getThumbnailFetcherRequestOptions } from "./thumbnails"; +import type { IThumbnailFetcher } from "./thumbnails"; export type { IManifestFetcherSettings, @@ -29,5 +32,12 @@ export type { IManifestRefreshSettings, ISegmentQueueCreatorBackoffOptions, SegmentQueue, + IThumbnailFetcher, +}; +export { + CdnPrioritizer, + ManifestFetcher, + SegmentQueueCreator, + createThumbnailFetcher, + getThumbnailFetcherRequestOptions, }; -export { ManifestFetcher, SegmentQueueCreator }; diff --git a/src/core/fetchers/segment/segment_queue_creator.ts b/src/core/fetchers/segment/segment_queue_creator.ts index 66fc1d9f39..79425c018a 100644 --- a/src/core/fetchers/segment/segment_queue_creator.ts +++ b/src/core/fetchers/segment/segment_queue_creator.ts @@ -17,10 +17,9 @@ import config from "../../../config"; import type { ISegmentPipeline, ITransportPipelines } from "../../../transports"; import type SharedReference from "../../../utils/reference"; -import type { CancellationSignal } from "../../../utils/task_canceller"; import type CmcdDataBuilder from "../../cmcd"; import type { IBufferType } from "../../segment_sinks"; -import CdnPrioritizer from "../cdn_prioritizer"; +import type CdnPrioritizer from "../cdn_prioritizer"; import applyPrioritizerToSegmentFetcher from "./prioritized_segment_fetcher"; import type { ISegmentFetcherLifecycleCallbacks } from "./segment_fetcher"; import createSegmentFetcher, { getSegmentFetcherRequestOptions } from "./segment_fetcher"; @@ -61,17 +60,16 @@ export default class SegmentQueueCreator { /** * @param {Object} transport + * @param {Object} cdnPrioritizer + * @param {Object|null} cmcdDataBuilder * @param {Object} options - * @param {Object} cancelSignal */ constructor( transport: ITransportPipelines, + cdnPrioritizer: CdnPrioritizer, cmcdDataBuilder: CmcdDataBuilder | null, options: ISegmentQueueCreatorBackoffOptions, - cancelSignal: CancellationSignal, ) { - const cdnPrioritizer = new CdnPrioritizer(cancelSignal); - const { MIN_CANCELABLE_PRIORITY, MAX_HIGH_PRIORITY_LEVEL } = config.getCurrent(); this._transport = transport; this._prioritizer = new TaskPrioritizer({ diff --git a/src/core/fetchers/thumbnails/index.ts b/src/core/fetchers/thumbnails/index.ts new file mode 100644 index 0000000000..5af224a5c6 --- /dev/null +++ b/src/core/fetchers/thumbnails/index.ts @@ -0,0 +1,8 @@ +import createThumbnailFetcher, { + getThumbnailFetcherRequestOptions, +} from "./thumbnail_fetcher"; +import type { IThumbnailFetcher } from "./thumbnail_fetcher"; + +export default createThumbnailFetcher; +export { getThumbnailFetcherRequestOptions }; +export type { IThumbnailFetcher }; diff --git a/src/core/fetchers/thumbnails/thumbnail_fetcher.ts b/src/core/fetchers/thumbnails/thumbnail_fetcher.ts new file mode 100644 index 0000000000..6284c89912 --- /dev/null +++ b/src/core/fetchers/thumbnails/thumbnail_fetcher.ts @@ -0,0 +1,233 @@ +import config from "../../../config"; +import { formatError } from "../../../errors"; +import log from "../../../log"; +import type { ISegment, IThumbnailTrack } from "../../../manifest"; +import type { ICdnMetadata } from "../../../parsers/manifest"; +import type { + IThumbnailLoader, + IThumbnailLoaderOptions, + IThumbnailPipeline, + IThumbnailResponse, +} from "../../../transports"; +import objectAssign from "../../../utils/object_assign"; +import type { CancellationSignal } from "../../../utils/task_canceller"; +import { CancellationError } from "../../../utils/task_canceller"; +import type CdnPrioritizer from "../cdn_prioritizer"; +import errorSelector from "../utils/error_selector"; +import { scheduleRequestWithCdns } from "../utils/schedule_request"; + +/** + * Create an `IThumbnailFetcher` object which will allow to easily fetch and parse + * segments. + * An `IThumbnailFetcher` also implements a retry mechanism, based on the given + * `requestOptions` argument, which may retry a segment request when it fails. + * + * @param {Object} pipeline + * @param {Object|null} cdnPrioritizer + * @returns {Function} + */ +export default function createThumbnailFetcher( + /** The transport-specific logic allowing to load thumbnails. */ + pipeline: IThumbnailPipeline, + /** + * Abstraction allowing to synchronize, update and keep track of the + * priorization of the CDN to use to load any given segment, in cases where + * multiple ones are available. + * + * Can be set to `null` in which case a minimal priorization logic will be used + * instead. + */ + cdnPrioritizer: CdnPrioritizer | null, + // TODO CMCD? +): IThumbnailFetcher { + const { loadThumbnail } = pipeline; + + // TODO short-lived cache? + + /** + * Fetch a specific segment. + * @param {Object} thumbnail + * @param {Object} thumbnailTrack + * @param {Object} requestOptions + * @param {Object} cancellationSignal + * @returns {Promise} + */ + return async function fetchThumbnail( + thumbnail: ISegment, + thumbnailTrack: IThumbnailTrack, + requestOptions: IThumbnailFetcherOptions, + cancellationSignal: CancellationSignal, + ): Promise { + let connectionTimeout; + if ( + requestOptions.connectionTimeout === undefined || + requestOptions.connectionTimeout < 0 + ) { + connectionTimeout = undefined; + } else { + connectionTimeout = requestOptions.connectionTimeout; + } + const pipelineRequestOptions: IThumbnailLoaderOptions = { + timeout: + requestOptions.requestTimeout < 0 ? undefined : requestOptions.requestTimeout, + connectionTimeout, + cmcdPayload: undefined, + }; + + log.debug("TF: Beginning thumbnail request", thumbnail.time); + cancellationSignal.register(onCancellation); + let res; + try { + res = await scheduleRequestWithCdns( + thumbnailTrack.cdnMetadata, + cdnPrioritizer, + callLoaderWithUrl, + objectAssign({ onRetry }, requestOptions), + cancellationSignal, + ); + + if (cancellationSignal.isCancelled()) { + return Promise.reject(cancellationSignal.cancellationError); + } + + log.debug("TF: Thumbnail request ended with success", thumbnail.time); + cancellationSignal.deregister(onCancellation); + } catch (err) { + cancellationSignal.deregister(onCancellation); + if (err instanceof CancellationError) { + log.debug("TF: Thumbnail request aborted", thumbnail.time); + throw err; + } + log.debug("TF: Thumbnail request failed", thumbnail.time); + throw errorSelector(err); + } + + try { + const parsed = pipeline.parseThumbnail(res.responseData, { + thumbnail, + thumbnailTrack, + }); + return parsed; + } catch (error) { + throw formatError(error, { + defaultCode: "PIPELINE_PARSE_ERROR", + defaultReason: "Unknown parsing error", + }); + } + function onCancellation() { + log.debug("TF: Thumbnail request cancelled", thumbnail.time); + } + + /** + * Call a segment loader for the given URL with the right arguments. + * @param {Object|null} cdnMetadata + * @returns {Promise} + */ + function callLoaderWithUrl( + cdnMetadata: ICdnMetadata | null, + ): ReturnType { + return loadThumbnail( + cdnMetadata, + thumbnail, + pipelineRequestOptions, + cancellationSignal, + ); + } + + /** + * Function called when the function request is retried. + * @param {*} err + */ + function onRetry(err: unknown): void { + const formattedErr = errorSelector(err); + log.warn("TF: Thumbnail request retry ", thumbnail.time, formattedErr); + } + }; +} + +/** + * Defines the `IThumbnailFetcher` function which allows to load a single segment. + * + * Loaded data is entirely communicated through callbacks present in the + * `callbacks` arguments. + * + * The returned Promise only gives an indication of if the request ended with + * success or on error. + */ +export type IThumbnailFetcher = ( + /** Actual thumbnail you want to load */ + thumbnail: ISegment, + /** Metadata on the linked thumbnails track. */ + thumbnailTrack: IThumbnailTrack, + /** + * Various tweaking requestOptions allowing to configure the behavior of the returned + * `IThumbnailFetcher` regarding segment requests. + */ + requestOptions: IThumbnailFetcherOptions, + /** CancellationSignal allowing to cancel the request. */ + cancellationSignal: CancellationSignal, +) => Promise; + +/** requestOptions allowing to configure an `IThumbnailFetcher`'s behavior. */ +export interface IThumbnailFetcherOptions { + /** + * Initial delay to wait if a request fails before making a new request, in + * milliseconds. + */ + baseDelay: number; + /** + * Maximum delay to wait if a request fails before making a new request, in + * milliseconds. + */ + maxDelay: number; + /** + * Maximum number of retries to perform on "regular" errors (e.g. due to HTTP + * status, integrity errors, timeouts...). + */ + maxRetry: number; + /** + * Timeout after which request are aborted and, depending on other requestOptions, + * retried. + * To set to `-1` for no timeout. + */ + requestTimeout: number; + /** + * Connection timeout, in milliseconds, after which the request is canceled + * if the responses headers has not being received. + * Do not set or set to "undefined" to disable it. + */ + connectionTimeout: number | undefined; +} + +/** + * @param {Object} baseOptions + * @returns {Object} + */ +export function getThumbnailFetcherRequestOptions({ + maxRetry, + requestTimeout, + connectionTimeout, +}: { + maxRetry?: number | undefined; + requestTimeout?: number | undefined; + connectionTimeout?: number | undefined; +}): IThumbnailFetcherOptions { + const { + DEFAULT_MAX_THUMBNAIL_REQUESTS_RETRY_ON_ERROR, + DEFAULT_THUMBNAIL_REQUEST_TIMEOUT, + DEFAULT_THUMBNAIL_CONNECTION_TIMEOUT, + INITIAL_BACKOFF_DELAY_BASE, + MAX_BACKOFF_DELAY_BASE, + } = config.getCurrent(); + return { + maxRetry: maxRetry ?? DEFAULT_MAX_THUMBNAIL_REQUESTS_RETRY_ON_ERROR, + baseDelay: INITIAL_BACKOFF_DELAY_BASE.REGULAR, + maxDelay: MAX_BACKOFF_DELAY_BASE.REGULAR, + requestTimeout: + requestTimeout === undefined ? DEFAULT_THUMBNAIL_REQUEST_TIMEOUT : requestTimeout, + connectionTimeout: + connectionTimeout === undefined + ? DEFAULT_THUMBNAIL_CONNECTION_TIMEOUT + : connectionTimeout, + }; +} diff --git a/src/core/main/common/get_thumbnail_data.ts b/src/core/main/common/get_thumbnail_data.ts new file mode 100644 index 0000000000..6e13327979 --- /dev/null +++ b/src/core/main/common/get_thumbnail_data.ts @@ -0,0 +1,43 @@ +import type { IManifest } from "../../../manifest"; +import type { IThumbnailResponse } from "../../../transports"; +import arrayFind from "../../../utils/array_find"; +import TaskCanceller from "../../../utils/task_canceller"; +import { getThumbnailFetcherRequestOptions } from "../../fetchers"; +import type { IThumbnailFetcher } from "../../fetchers"; + +/** + * @param {function} fetchThumbnails + * @param {Object} manifest + * @param {string} periodId + * @param {string} thumbnailTrackId + * @param {number} time + * @returns {Promise.} + */ +export default async function getThumbnailData( + fetchThumbnails: IThumbnailFetcher, + manifest: IManifest, + periodId: string, + thumbnailTrackId: string, + time: number, +): Promise { + const period = manifest.getPeriod(periodId); + if (period === undefined) { + throw new Error("Wanted Period not found."); + } + const thumbnailTrack = arrayFind(period.thumbnailTracks, (t) => { + return t.id === thumbnailTrackId; + }); + if (thumbnailTrack === undefined) { + throw new Error("Wanted Period has no thumbnail track."); + } + const wantedThumbnail = thumbnailTrack.index.getSegments(time, 1)[0]; + if (wantedThumbnail === undefined) { + throw new Error("No thumbnail for the given timestamp"); + } + return fetchThumbnails( + wantedThumbnail, + thumbnailTrack, + getThumbnailFetcherRequestOptions({}), + new TaskCanceller().signal, + ); +} diff --git a/src/core/main/worker/content_preparer.ts b/src/core/main/worker/content_preparer.ts index 92a12af54d..9ff6b846f8 100644 --- a/src/core/main/worker/content_preparer.ts +++ b/src/core/main/worker/content_preparer.ts @@ -24,6 +24,9 @@ import createAdaptiveRepresentationSelector from "../../adaptive"; import CmcdDataBuilder from "../../cmcd"; import type { IManifestRefreshSettings } from "../../fetchers"; import { ManifestFetcher, SegmentQueueCreator } from "../../fetchers"; +import CdnPrioritizer from "../../fetchers/cdn_prioritizer"; +import createThumbnailFetcher from "../../fetchers/thumbnails/thumbnail_fetcher"; +import type { IThumbnailFetcher } from "../../fetchers/thumbnails/thumbnail_fetcher"; import SegmentSinksStore from "../../segment_sinks"; import type { INeedsMediaSourceReloadPayload } from "../../stream"; import DecipherabilityFreezeDetector from "../common/DecipherabilityFreezeDetector"; @@ -129,11 +132,16 @@ export default class ContentPreparer { }, ); + const cdnPrioritizer = new CdnPrioritizer(contentCanceller.signal); const segmentQueueCreator = new SegmentQueueCreator( dashPipelines, + cdnPrioritizer, cmcdDataBuilder, context.segmentRetryOptions, - contentCanceller.signal, + ); + const fetchThumbnailData = createThumbnailFetcher( + dashPipelines.thumbnails, + cdnPrioritizer, ); const trackChoiceSetter = new TrackChoiceSetter(); @@ -161,6 +169,7 @@ export default class ContentPreparer { representationEstimator, segmentSinksStore, segmentQueueCreator, + fetchThumbnailData, workerTextSender, trackChoiceSetter, }; @@ -363,6 +372,8 @@ export interface IPreparedContentData { * fetching. */ segmentQueueCreator: SegmentQueueCreator; + /** Allows to load image thumbnails. */ + fetchThumbnailData: IThumbnailFetcher; /** * Allows to store and update the wanted tracks and Representation inside that * track. diff --git a/src/core/main/worker/worker_main.ts b/src/core/main/worker/worker_main.ts index 6672776704..5f1b943ce4 100644 --- a/src/core/main/worker/worker_main.ts +++ b/src/core/main/worker/worker_main.ts @@ -8,6 +8,7 @@ import type { IDiscontinuityUpdateWorkerMessagePayload, IMainThreadMessage, IReferenceUpdateMessage, + IThumbnailDataRequestMainMessage, } from "../../../multithread_types"; import { MainThreadMessageType, WorkerMessageType } from "../../../multithread_types"; import DashFastJsParser from "../../../parsers/manifest/dash/fast-js-parser"; @@ -35,6 +36,7 @@ import type { import StreamOrchestrator from "../../stream"; import createContentTimeBoundariesObserver from "../common/create_content_time_boundaries_observer"; import getBufferedDataPerMediaBuffer from "../common/get_buffered_data_per_media_buffer"; +import getThumbnailData from "../common/get_thumbnail_data"; import ContentPreparer from "./content_preparer"; import { limitVideoResolution, @@ -401,7 +403,12 @@ export default function initializeWorkerMain() { } case MainThreadMessageType.PullSegmentSinkStoreInfos: { - sendSegmentSinksStoreInfos(contentPreparer, msg.value.messageId); + sendSegmentSinksStoreInfos(contentPreparer, msg.value.requestId); + break; + } + + case MainThreadMessageType.ThumbnailDataRequest: { + sendThumbnailData(contentPreparer, msg); break; } @@ -947,7 +954,7 @@ function updateLoggerLevel( */ function sendSegmentSinksStoreInfos( contentPreparer: ContentPreparer, - messageId: number, + requestId: number, ): void { const currentContent = contentPreparer.getCurrentContent(); if (currentContent === null) { @@ -957,6 +964,63 @@ function sendSegmentSinksStoreInfos( sendMessage({ type: WorkerMessageType.SegmentSinkStoreUpdate, contentId: currentContent.contentId, - value: { segmentSinkMetrics: segmentSinksMetrics, messageId }, + value: { segmentSinkMetrics: segmentSinksMetrics, requestId }, }); } + +/** + * Handles thumbnail requests and send back the result to the main thread. + * @param {ContentPreparer} contentPreparer + * @returns {void} + */ +function sendThumbnailData( + contentPreparer: ContentPreparer, + msg: IThumbnailDataRequestMainMessage, +): void { + const preparedContent = contentPreparer.getCurrentContent(); + const respondWithError = (err: unknown) => { + sendMessage({ + type: WorkerMessageType.ThumbnailDataResponse, + contentId: msg.contentId, + value: { + status: "error", + requestId: msg.value.requestId, + error: formatErrorForSender(err), + }, + }); + }; + + if ( + preparedContent === null || + preparedContent.manifest === null || + preparedContent.contentId !== msg.contentId + ) { + return respondWithError(new Error("Content changed")); + } + + getThumbnailData( + preparedContent.fetchThumbnailData, + preparedContent.manifest, + msg.value.periodId, + msg.value.thumbnailTrackId, + msg.value.time, + ).then( + (result) => { + sendMessage( + { + type: WorkerMessageType.ThumbnailDataResponse, + contentId: msg.contentId, + value: { + status: "success", + requestId: msg.value.requestId, + data: result, + }, + }, + [result.data], + ); + }, + (err) => { + return respondWithError(err); + }, + ); +} diff --git a/src/default_config.ts b/src/default_config.ts index 5878e6460f..1a7f9abd9f 100644 --- a/src/default_config.ts +++ b/src/default_config.ts @@ -1183,6 +1183,31 @@ const DEFAULT_CONFIG = { * one. */ DEFAULT_AUDIO_TRACK_SWITCHING_MODE: "seamless" as const, + + /** + * The default number of times a thumbnail request will be re-performed when + * on error which justify a retry. + * + * Note that some errors do not use this counter: + * - if the error is not due to the xhr, no retry will be peformed + * - if the error is an HTTP error code, but not a 500-smthg or a 404, no + * retry will be performed. + * @type Number + */ + DEFAULT_MAX_THUMBNAIL_REQUESTS_RETRY_ON_ERROR: 1, + + /** + * Default time interval after which a thumbnail request will timeout, in ms. + * @type {Number} + */ + DEFAULT_THUMBNAIL_REQUEST_TIMEOUT: 10 * 1000, + + /** + * Default connection time after which a thumbnail request conncection will + * timeout, in ms. + * @type {Number} + */ + DEFAULT_THUMBNAIL_CONNECTION_TIMEOUT: 7 * 1000, }; export type IDefaultConfig = typeof DEFAULT_CONFIG; diff --git a/src/main_thread/api/public_api.ts b/src/main_thread/api/public_api.ts index 57c820db34..28ae0679ae 100644 --- a/src/main_thread/api/public_api.ts +++ b/src/main_thread/api/public_api.ts @@ -62,6 +62,7 @@ import { getMinimumSafePosition, ManifestMetadataFormat, createRepresentationFilterFromFnString, + getPeriodForTime, } from "../../manifest"; import type { IWorkerMessage } from "../../multithread_types"; import { MainThreadMessageType, WorkerMessageType } from "../../multithread_types"; @@ -101,7 +102,10 @@ import type { ITrackType, IModeInformation, IWorkerSettings, + IThumbnailTrackInfo, + IThumbnailRenderingOptions, } from "../../public_types"; +import type { IThumbnailResponse } from "../../transports"; import arrayFind from "../../utils/array_find"; import arrayIncludes from "../../utils/array_includes"; import assert, { assertUnreachable } from "../../utils/assert"; @@ -123,6 +127,7 @@ import { getKeySystemConfiguration, } from "../decrypt"; import type { ContentInitializer } from "../init"; +import renderThumbnail from "../render_thumbnail"; import type { IMediaElementTracksStore, ITSPeriodObject } from "../tracks_store"; import TracksStore from "../tracks_store"; import type { IParsedLoadVideoOptions, IParsedStartAtOption } from "./option_utils"; @@ -383,14 +388,6 @@ class Player extends EventEmitter { } } - /** - * Function passed from the ContentInitializer that return segment sinks metrics. - * This is used for monitor and debugging. - */ - private _priv_segmentSinkMetricsCallback: - | null - | (() => Promise); - /** * @constructor * @param {Object} options @@ -467,8 +464,6 @@ class Player extends EventEmitter { this._priv_worker = null; - this._priv_segmentSinkMetricsCallback = null; - const onVolumeChange = () => { this.trigger("volumeChange", { volume: videoElement.volume, @@ -758,6 +753,55 @@ class Player extends EventEmitter { }; } + /** + * Returns either an array decribing the various thumbnail tracks that can be + * encountered at the given time, or `null` if no thumbnail track is available + * at that time. + * @param {number} time - The position to check for thumbnail tracks, in + * seconds. + * @returns {Array.|null} + */ + public getAvailableThumbnailTracks({ + time, + }: { + time: number; + }): IThumbnailTrackInfo[] | null { + if (this._priv_contentInfos === null || this._priv_contentInfos.manifest === null) { + return null; + } + const period = getPeriodForTime(this._priv_contentInfos.manifest, time); + if (period === undefined || period.thumbnailTracks.length === 0) { + return null; + } + return period.thumbnailTracks.map((t) => { + return { + id: t.id, + width: Math.floor(t.width / t.horizontalTiles), + height: Math.floor(t.height / t.verticalTiles), + mimeType: t.mimeType, + }; + }); + } + + /** + * Render inside the given `container` the thumbnail corresponding to the + * given time. + * + * If no thumbnail is available at that time or if the RxPlayer does not succeed + * to load or render it, reject the corresponding Promise and remove the + * potential previous thumbnail from the container. + * + * If a new `renderThumbnail` call is made with the same `container` before it + * had time to finish, the Promise is also rejected but the previous thumbnail + * potentially found in the container is untouched. + * + * @param {Object|undefined} options + * @returns {Promise} + */ + public async renderThumbnail(options: IThumbnailRenderingOptions): Promise { + return renderThumbnail(this._priv_contentInfos, options); + } + /** * From given options, initialize content playback. * @param {Object} options @@ -1029,6 +1073,12 @@ class Player extends EventEmitter { tracksStore: null, mediaElementTracksStore, useWorker, + segmentSinkMetricsCallback: null, + fetchThumbnailDataCallback: null, + thumbnailRequestsInfo: { + pendingRequests: new Map(), + lastResponse: null, + }, }; // Bind events @@ -1047,7 +1097,9 @@ class Player extends EventEmitter { if (contentInfos.tracksStore !== null) { contentInfos.tracksStore.resetPeriodObjects(); } - this._priv_segmentSinkMetricsCallback = null; + if (this._priv_contentInfos !== null) { + this._priv_contentInfos.segmentSinkMetricsCallback = null; + } this._priv_lastAutoPlay = payload.autoPlay; }); initializer.addEventListener("inbandEvents", (inbandEvents) => @@ -1091,7 +1143,10 @@ class Player extends EventEmitter { this._priv_onDecipherabilityUpdate(contentInfos, updates), ); initializer.addEventListener("loaded", (evt) => { - this._priv_segmentSinkMetricsCallback = evt.getSegmentSinkMetrics; + if (this._priv_contentInfos !== null) { + this._priv_contentInfos.segmentSinkMetricsCallback = evt.getSegmentSinkMetrics; + this._priv_contentInfos.fetchThumbnailDataCallback = evt.getThumbnailData; + } }); // Now, that most events are linked, prepare the next content. @@ -2451,11 +2506,7 @@ class Player extends EventEmitter { * @returns */ async __priv_getSegmentSinkMetrics(): Promise { - if (this._priv_segmentSinkMetricsCallback === null) { - return undefined; - } else { - return this._priv_segmentSinkMetricsCallback(); - } + return this._priv_contentInfos?.segmentSinkMetricsCallback?.(); } /** @@ -2523,7 +2574,6 @@ class Player extends EventEmitter { this._priv_contentInfos?.tracksStore?.dispose(); this._priv_contentInfos?.mediaElementTracksStore?.dispose(); this._priv_contentInfos = null; - this._priv_segmentSinkMetricsCallback = null; this._priv_contentEventsMemory = {}; @@ -3365,7 +3415,7 @@ interface IPublicAPIEvent { } /** State linked to a particular contents loaded by the public API. */ -interface IPublicApiContentInfos { +export interface IPublicApiContentInfos { /** * Unique identifier for this `IPublicApiContentInfos` object. * Allows to identify and thus compare this `contentInfos` object with another @@ -3427,6 +3477,45 @@ interface IPublicApiContentInfos { * content. */ useWorker: boolean; + /** + * Function passed from the ContentInitializer that return segment sinks metrics. + * This is used for monitor and debugging. + */ + segmentSinkMetricsCallback: null | (() => Promise); + /** + * Function allowing to retrieve thumbnails from a content. + */ + fetchThumbnailDataCallback: + | null + | (( + periodId: string, + thumbnailTrackId: string, + time: number, + ) => Promise); + /** Metadata related to thumbnail rendering for the current content. */ + thumbnailRequestsInfo: { + /** + * Thumbnail requests that are still pending, identified by the thumbnail + * container. + * The value allows to cancel that task. + */ + pendingRequests: Map; + /** + * Metadata about the last requested thumbnails. + * + * This is an optimization to avoid an unnecessary request and round-trip to + * the core code as many times thumbnail previews asked by applications are + * really close to the last asked one, often in the same thumbnail resource. + */ + lastResponse: { + /** Actual thumbnail data response from core RxPlayer code. */ + response: IThumbnailResponse; + /** The identifier for the Period for which that request was made. */ + periodId: string; + /** The identifier for the thumbnail track for which that request was made. */ + thumbnailTrackId: string; + } | null; + }; } export default Player; diff --git a/src/main_thread/init/directfile_content_initializer.ts b/src/main_thread/init/directfile_content_initializer.ts index 2d20cb3dd7..3f230ebbbd 100644 --- a/src/main_thread/init/directfile_content_initializer.ts +++ b/src/main_thread/init/directfile_content_initializer.ts @@ -231,6 +231,10 @@ export default class DirectFileContentInitializer extends ContentInitializer { stopListening(); this.trigger("loaded", { getSegmentSinkMetrics: null, + getThumbnailData: () => + Promise.reject( + new Error("Thumbnail data not available with directfile contents"), + ), }); } }, diff --git a/src/main_thread/init/media_source_content_initializer.ts b/src/main_thread/init/media_source_content_initializer.ts index 81767de45c..21c146c967 100644 --- a/src/main_thread/init/media_source_content_initializer.ts +++ b/src/main_thread/init/media_source_content_initializer.ts @@ -25,9 +25,15 @@ import type { } from "../../core/adaptive"; import AdaptiveRepresentationSelector from "../../core/adaptive"; import CmcdDataBuilder from "../../core/cmcd"; -import { ManifestFetcher, SegmentQueueCreator } from "../../core/fetchers"; +import { + CdnPrioritizer, + createThumbnailFetcher, + ManifestFetcher, + SegmentQueueCreator, +} from "../../core/fetchers"; import createContentTimeBoundariesObserver from "../../core/main/common/create_content_time_boundaries_observer"; import DecipherabilityFreezeDetector from "../../core/main/common/DecipherabilityFreezeDetector"; +import getThumbnailData from "../../core/main/common/get_thumbnail_data"; import SegmentSinksStore from "../../core/segment_sinks"; import type { IStreamOrchestratorOptions, @@ -49,7 +55,7 @@ import type { IKeySystemOption, IPlayerError, } from "../../public_types"; -import type { ITransportPipelines } from "../../transports"; +import type { IThumbnailResponse, ITransportPipelines } from "../../transports"; import areArraysOfNumbersEqual from "../../utils/are_arrays_of_numbers_equal"; import assert from "../../utils/assert"; import createCancellablePromise from "../../utils/create_cancellable_promise"; @@ -443,11 +449,12 @@ export default class MediaSourceContentInitializer extends ContentInitializer { bufferOptions, ); + const cdnPrioritizer = new CdnPrioritizer(initCanceller.signal); const segmentQueueCreator = new SegmentQueueCreator( transport, + cdnPrioritizer, this._cmcdDataBuilder, segmentRequestOptions, - initCanceller.signal, ); this._refreshManifestCodecSupport(manifest); @@ -466,6 +473,7 @@ export default class MediaSourceContentInitializer extends ContentInitializer { autoPlay, manifest, representationEstimator, + cdnPrioritizer, segmentQueueCreator, speed, bufferOptions: subBufferOptions, @@ -556,9 +564,11 @@ export default class MediaSourceContentInitializer extends ContentInitializer { mediaSource, playbackObserver, representationEstimator, + cdnPrioritizer, segmentQueueCreator, speed, } = args; + const { transport } = this._initSettings; const initialPeriod = manifest.getPeriodForTime(initialTime) ?? manifest.getNextPeriod(initialTime); @@ -758,6 +768,23 @@ export default class MediaSourceContentInitializer extends ContentInitializer { resolve(segmentSinksStore.getSegmentSinksMetrics()), ); }, + getThumbnailData: async ( + periodId: string, + thumbnailTrackId: string, + time: number, + ): Promise => { + const fetchThumbnails = createThumbnailFetcher( + transport.thumbnails, + cdnPrioritizer, + ); + return getThumbnailData( + fetchThumbnails, + manifest, + periodId, + thumbnailTrackId, + time, + ); + }, }); } }, @@ -1247,6 +1274,11 @@ interface IBufferingMediaSettings { playbackObserver: IMediaElementPlaybackObserver; /** Estimate the right Representation. */ representationEstimator: IRepresentationEstimator; + /** + * Interface allowing to prioritize CDN between one another depending on past + * performances, content steering, etc. + */ + cdnPrioritizer: CdnPrioritizer; /** Module to facilitate segment fetching. */ segmentQueueCreator: SegmentQueueCreator; /** Last wanted playback rate. */ diff --git a/src/main_thread/init/multi_thread_content_initializer.ts b/src/main_thread/init/multi_thread_content_initializer.ts index 89ca5a1e56..e072d7d632 100644 --- a/src/main_thread/init/multi_thread_content_initializer.ts +++ b/src/main_thread/init/multi_thread_content_initializer.ts @@ -40,7 +40,7 @@ import type { IKeySystemOption, IPlayerError, } from "../../public_types"; -import type { ITransportOptions } from "../../transports"; +import type { IThumbnailResponse, ITransportOptions } from "../../transports"; import arrayFind from "../../utils/array_find"; import assert, { assertUnreachable } from "../../utils/assert"; import idGenerator from "../../utils/id_generator"; @@ -115,14 +115,30 @@ export default class MultiThreadContentInitializer extends ContentInitializer { */ private _currentMediaSourceCanceller: TaskCanceller; - /** - * Stores the resolvers and the current messageId that is sent to the web worker to - * receive segment sink metrics. - * The purpose of collecting metrics is for monitoring and debugging. - */ - private _segmentMetrics: { - lastMessageId: number; - resolvers: Map void>; + private _awaitingRequests: { + nextRequestId: number; + /** + * Stores the resolvers and the current messageId that is sent to the web worker to + * receive segment sink metrics. + * The purpose of collecting metrics is for monitoring and debugging. + */ + pendingSinkMetrics: Map< + number /* request id */, + { + resolve: (value: ISegmentSinkMetrics | undefined) => void; + } + >; + /** + * Stores the resolvers and the current messageId that is sent to the web worker to + * receive image thumbnails. + */ + pendingThumbnailFetching: Map< + number /* request id */, + { + resolve: (value: IThumbnailResponse) => void; + reject: (error: Error) => void; + } + >; }; /** @@ -137,9 +153,10 @@ export default class MultiThreadContentInitializer extends ContentInitializer { this._currentMediaSourceCanceller = new TaskCanceller(); this._currentMediaSourceCanceller.linkToSignal(this._initCanceller.signal); this._currentContentInfo = null; - this._segmentMetrics = { - lastMessageId: 0, - resolvers: new Map(), + this._awaitingRequests = { + nextRequestId: 0, + pendingSinkMetrics: new Map(), + pendingThumbnailFetching: new Map(), }; this._queuedWorkerMessages = null; } @@ -1135,9 +1152,11 @@ export default class MultiThreadContentInitializer extends ContentInitializer { if (this._currentContentInfo?.contentId !== msgData.contentId) { return; } - const resolveFn = this._segmentMetrics.resolvers.get(msgData.value.messageId); - if (resolveFn !== undefined) { - resolveFn(msgData.value.segmentSinkMetrics); + const sinkObj = this._awaitingRequests.pendingSinkMetrics.get( + msgData.value.requestId, + ); + if (sinkObj !== undefined) { + sinkObj.resolve(msgData.value.segmentSinkMetrics); } else { log.error("MTCI: Failed to send segment sink store update"); } @@ -1152,6 +1171,23 @@ export default class MultiThreadContentInitializer extends ContentInitializer { case WorkerMessageType.LogMessage: // Already handled by prepare's handler break; + case WorkerMessageType.ThumbnailDataResponse: + if (this._currentContentInfo?.contentId !== msgData.contentId) { + return; + } + const tObj = this._awaitingRequests.pendingThumbnailFetching.get( + msgData.value.requestId, + ); + if (tObj !== undefined) { + if (msgData.value.status === "error") { + tObj.reject(formatWorkerError(msgData.value.error)); + } else { + tObj.resolve(msgData.value.data); + } + } else { + log.error("MTCI: Failed to send segment sink store update"); + } + break; default: assertUnreachable(msgData); } @@ -1592,29 +1628,65 @@ export default class MultiThreadContentInitializer extends ContentInitializer { { clearSignal: cancelSignal, emitCurrentValue: true }, ); - const _getSegmentSinkMetrics: () => Promise< - ISegmentSinkMetrics | undefined - > = async () => { - this._segmentMetrics.lastMessageId++; - const messageId = this._segmentMetrics.lastMessageId; + const _getSegmentSinkMetrics = async (): Promise => { + this._awaitingRequests.nextRequestId++; + const requestId = this._awaitingRequests.nextRequestId; sendMessage(this._settings.worker, { type: MainThreadMessageType.PullSegmentSinkStoreInfos, - value: { messageId }, + value: { requestId }, }); return new Promise((resolve, reject) => { const rejectFn = (err: CancellationError) => { cancelSignal.deregister(rejectFn); - this._segmentMetrics.resolvers.delete(messageId); + this._awaitingRequests.pendingSinkMetrics.delete(requestId); return reject(err); }; - this._segmentMetrics.resolvers.set( - messageId, - (value: ISegmentSinkMetrics | undefined) => { + this._awaitingRequests.pendingSinkMetrics.set(requestId, { + resolve: (value: ISegmentSinkMetrics | undefined) => { cancelSignal.deregister(rejectFn); - this._segmentMetrics.resolvers.delete(messageId); + this._awaitingRequests.pendingSinkMetrics.delete(requestId); resolve(value); }, - ); + }); + cancelSignal.register(rejectFn); + }); + }; + const _getThumbnailsData = async ( + periodId: string, + thumbnailTrackId: string, + time: number, + ): Promise => { + if (this._currentContentInfo === null) { + return Promise.reject(new Error("Cannot fetch thumbnails: No content loaded.")); + } + this._awaitingRequests.nextRequestId++; + const requestId = this._awaitingRequests.nextRequestId; + sendMessage(this._settings.worker, { + type: MainThreadMessageType.ThumbnailDataRequest, + contentId: this._currentContentInfo.contentId, + value: { requestId, periodId, thumbnailTrackId, time }, + }); + + return new Promise((resolve, reject) => { + const rejectFn = (err: CancellationError) => { + cleanUp(); + reject(err); + }; + const cleanUp = () => { + cancelSignal.deregister(rejectFn); + this._awaitingRequests.pendingThumbnailFetching.delete(requestId); + }; + + this._awaitingRequests.pendingThumbnailFetching.set(requestId, { + resolve: (value: IThumbnailResponse) => { + cleanUp(); + resolve(value); + }, + reject: (value: unknown) => { + cleanUp(); + reject(value); + }, + }); cancelSignal.register(rejectFn); }); }; @@ -1631,6 +1703,7 @@ export default class MultiThreadContentInitializer extends ContentInitializer { stopListening(); this.trigger("loaded", { getSegmentSinkMetrics: _getSegmentSinkMetrics, + getThumbnailData: _getThumbnailsData, }); } }, diff --git a/src/main_thread/init/types.ts b/src/main_thread/init/types.ts index 46e5a25dbc..be2325e38e 100644 --- a/src/main_thread/init/types.ts +++ b/src/main_thread/init/types.ts @@ -27,6 +27,7 @@ import type { } from "../../manifest"; import type { IMediaElementPlaybackObserver } from "../../playback_observer"; import type { IPlayerError } from "../../public_types"; +import type { IThumbnailResponse } from "../../transports"; import EventEmitter from "../../utils/event_emitter"; import type SharedReference from "../../utils/reference"; import type { @@ -146,6 +147,17 @@ export interface IContentInitializerEvents { */ loaded: { getSegmentSinkMetrics: null | (() => Promise); + /** + * Fetch the thumbnail data of the given Period for the corresponding time. + * If there's no thumbnail for that Period or if the request fails, reject + * the Promise with a given reason. + * @param {number} time + */ + getThumbnailData: ( + periodId: string, + thumbnailTrackId: string, + time: number, + ) => Promise; }; /** Event emitted when a stream event is encountered. */ streamEvent: IPublicStreamEvent | IPublicNonFiniteStreamEvent; diff --git a/src/main_thread/render_thumbnail.ts b/src/main_thread/render_thumbnail.ts new file mode 100644 index 0000000000..a9b7d355ee --- /dev/null +++ b/src/main_thread/render_thumbnail.ts @@ -0,0 +1,253 @@ +import { formatError } from "../errors"; +import errorMessage from "../errors/error_message"; +import { getPeriodForTime } from "../manifest"; +import type { IThumbnailRenderingOptions } from "../public_types"; +import type { IThumbnailResponse } from "../transports"; +import arrayFind from "../utils/array_find"; +import TaskCanceller from "../utils/task_canceller"; +import type { IPublicApiContentInfos } from "./api/public_api"; + +/** + * Render thumbnail available at `time` in the given `container` (in place of + * a potential previously-rendered thumbnail in that container). + * + * If there is no thumbnail at this time, or if there is but it fails to + * load/render, also removes the previously displayed thumbnail, unless + * `options.keepPreviousThumbnailOnError` is set to `true`. + * + * Returns a Promise which resolves when the thumbnail is rendered successfully, + * rejects if anything prevented a thumbnail to be rendered. + * + * A newer `renderThumbnail` call performed while a previous `renderThumbnail` + * call on the same container did not yet finish will abort that previous call, + * rejecting the old call's returned promise. + * + * You may know if the promise returned by `renderThumbnail` rejected due to it + * being aborted, by checking the `code` property on the rejected error: Error + * due to aborting have their `code` property set to `ABORTED`. + * + * @param {Object} contentInfos + * @param {Object} options + * @returns {Object} + */ +export default async function renderThumbnail( + contentInfos: IPublicApiContentInfos | null, + options: IThumbnailRenderingOptions, +): Promise { + const { time, container } = options; + if ( + contentInfos === null || + contentInfos.fetchThumbnailDataCallback === null || + contentInfos.manifest === null + ) { + return Promise.reject( + new ThumbnailRenderingError( + "NO_CONTENT", + "Cannot get thumbnail: no content loaded", + ), + ); + } + + const { thumbnailRequestsInfo, currentContentCanceller } = contentInfos; + const canceller = new TaskCanceller(); + canceller.linkToSignal(currentContentCanceller.signal); + + let imageUrl: string | undefined; + + const olderTaskSameContainer = thumbnailRequestsInfo.pendingRequests.get(container); + olderTaskSameContainer?.cancel(); + + thumbnailRequestsInfo.pendingRequests.set(container, canceller); + + const onFinished = () => { + canceller.cancel(); + thumbnailRequestsInfo.pendingRequests.delete(container); + + // Let's revoke the URL after a round-trip to the event loop just in case + // to prevent revoking before the browser use it. + // This is normally not necessary, but better safe than sorry. + setTimeout(() => { + if (imageUrl !== undefined) { + URL.revokeObjectURL(imageUrl); + } + }, 0); + }; + + try { + const period = getPeriodForTime(contentInfos.manifest, time); + if (period === undefined) { + throw new ThumbnailRenderingError("NO_THUMBNAIL", "Wanted Period not found."); + } + const thumbnailTracks = period.thumbnailTracks; + const thumbnailTrack = + options.thumbnailTrackId !== undefined + ? arrayFind(thumbnailTracks, (t) => t.id === options.thumbnailTrackId) + : thumbnailTracks[0]; + if (thumbnailTrack === undefined) { + if (options.thumbnailTrackId !== undefined) { + throw new ThumbnailRenderingError( + "NO_THUMBNAIL", + "Given `thumbnailTrackId` not found", + ); + } else { + throw new ThumbnailRenderingError( + "NO_THUMBNAIL", + "Wanted Period has no thumbnail track.", + ); + } + } + + const { lastResponse } = thumbnailRequestsInfo; + let res: IThumbnailResponse | undefined; + if ( + lastResponse !== null && + lastResponse.thumbnailTrackId === thumbnailTrack.id && + lastResponse.periodId === period.id + ) { + const previousThumbs = lastResponse.response.thumbnails; + if ( + previousThumbs.length > 0 && + time >= previousThumbs[0].start && + time < previousThumbs[previousThumbs.length - 1].end + ) { + res = lastResponse.response; + } + } + + if (res === undefined) { + res = await contentInfos.fetchThumbnailDataCallback( + period.id, + thumbnailTrack.id, + time, + ); + thumbnailRequestsInfo.lastResponse = { + response: res, + periodId: period.id, + thumbnailTrackId: thumbnailTrack.id, + }; + } + const canvas = document.createElement("canvas"); + const context = canvas.getContext("2d"); + if (context === null) { + throw new ThumbnailRenderingError( + "RENDERING", + "Cannot display thumbnail: cannot create canvas context", + ); + } + let foundIdx: number | undefined; + for (let i = 0; i < res.thumbnails.length; i++) { + if (res.thumbnails[i].start <= time && res.thumbnails[i].end > time) { + foundIdx = i; + break; + } + } + if (foundIdx === undefined) { + throw new Error("Cannot display thumbnail: time not found in fetched data"); + } + const image = new Image(); + const blob = new Blob([res.data], { type: res.mimeType }); + imageUrl = URL.createObjectURL(blob); + image.src = imageUrl; + canvas.height = res.thumbnails[foundIdx].height; + canvas.width = res.thumbnails[foundIdx].width; + return new Promise((resolve, reject) => { + image.onload = () => { + try { + context.drawImage( + image, + res.thumbnails[foundIdx].offsetX, + res.thumbnails[foundIdx].offsetY, + res.thumbnails[foundIdx].width, + res.thumbnails[foundIdx].height, + 0, + 0, + res.thumbnails[foundIdx].width, + res.thumbnails[foundIdx].height, + ); + canvas.style.width = "100%"; + canvas.style.height = "100%"; + canvas.className = "__rx-thumbnail__"; + clearPreviousThumbnails(); + container.appendChild(canvas); + resolve(); + } catch (srcError) { + reject( + new ThumbnailRenderingError( + "RENDERING", + "Could not draw the image in a canvas", + ), + ); + } + onFinished(); + }; + + image.onerror = () => { + if (options.keepPreviousThumbnailOnError !== true) { + clearPreviousThumbnails(); + } + reject( + new ThumbnailRenderingError( + "RENDERING", + "Could not load the corresponding image in the DOM", + ), + ); + onFinished(); + }; + }); + } catch (srcError) { + if (options.keepPreviousThumbnailOnError !== true) { + clearPreviousThumbnails(); + } + if (srcError !== null && srcError === canceller.signal.cancellationError) { + const error = new ThumbnailRenderingError( + "ABORTED", + "Thumbnail rendering has been aborted", + ); + throw error; + } + const formattedErr = formatError(srcError, { + defaultCode: "NONE", + defaultReason: "Unknown error", + }); + + let returnedError; + if (formattedErr.type === "NETWORK_ERROR") { + returnedError = new ThumbnailRenderingError("LOADING", formattedErr.message); + } else { + returnedError = new ThumbnailRenderingError("NOT_FOUND", formattedErr.message); + } + onFinished(); + throw returnedError; + } + + function clearPreviousThumbnails() { + for (let i = container.children.length - 1; i >= 0; i--) { + const child = container.children[i]; + if (child.className === "__rx-thumbnail__") { + container.removeChild(child); + } + } + } +} + +/** + * Error specifcically defined for the thumbnail rendering API. + * A caller is then supposed to programatically classify the type of error + * by checking the `code` property from such an error. + * @class ThumbnailRenderingError + */ +class ThumbnailRenderingError extends Error { + public readonly name: "ThumbnailRenderingError"; + public readonly code: string; + + /** + * @param {string} code + * @param {string} message + */ + constructor(code: string, message: string) { + super(errorMessage(code, message)); + Object.setPrototypeOf(this, ThumbnailRenderingError.prototype); + this.name = "ThumbnailRenderingError"; + this.code = code; + } +} diff --git a/src/manifest/classes/__tests__/manifest.test.ts b/src/manifest/classes/__tests__/manifest.test.ts index 9cd9b78778..d37f824f34 100644 --- a/src/manifest/classes/__tests__/manifest.test.ts +++ b/src/manifest/classes/__tests__/manifest.test.ts @@ -24,6 +24,7 @@ function generateParsedPeriod( adaptations, duration: end === undefined ? undefined : end - start, streamEvents: [], + thumbnailTracks: [], }; } function generateParsedAudioAdaptation(id: string): IParsedAdaptation { @@ -221,8 +222,8 @@ describe("Manifest - Manifest", () => { it("should expose the adaptations of the first period if set", async () => { const adapP1 = {}; const adapP2 = {}; - const period1 = { id: "0", start: 4, adaptations: adapP1 }; - const period2 = { id: "1", start: 12, adaptations: adapP2 }; + const period1 = { id: "0", start: 4, adaptations: adapP1, thumbnailTracks: [] }; + const period2 = { id: "1", start: 12, adaptations: adapP2, thumbnailTracks: [] }; const simpleFakeManifest = { id: "man", isDynamic: false, @@ -267,8 +268,8 @@ describe("Manifest - Manifest", () => { ); expect(manifest.periods).toEqual([ - { id: "foo0", start: 4, adaptations: adapP1 }, - { id: "foo1", start: 12, adaptations: adapP2 }, + { id: "foo0", start: 4, adaptations: adapP1, thumbnailTracks: [] }, + { id: "foo1", start: 12, adaptations: adapP2, thumbnailTracks: [] }, ]); expect(manifest.adaptations).toBe(adapP1); @@ -412,8 +413,20 @@ describe("Manifest - Manifest", () => { expect(manifest.getMaximumSafePosition()).toEqual(10); expect(manifest.getMinimumSafePosition()).toEqual(5); expect(manifest.periods).toEqual([ - { id: "foo0", adaptations: oldPeriod1.adaptations, start: 4, streamEvents: [] }, - { id: "foo1", adaptations: oldPeriod2.adaptations, start: 12, streamEvents: [] }, + { + id: "foo0", + adaptations: oldPeriod1.adaptations, + start: 4, + streamEvents: [], + thumbnailTracks: [], + }, + { + id: "foo1", + adaptations: oldPeriod2.adaptations, + start: 12, + streamEvents: [], + thumbnailTracks: [], + }, ]); expect(manifest.suggestedPresentationDelay).toEqual(99); expect(manifest.uris).toEqual(["url1", "url2"]); @@ -478,8 +491,8 @@ describe("Manifest - Manifest", () => { isLastPeriodKnown: true, lifetime: 13, periods: [ - { id: "0", start: 4, adaptations: oldPeriod1.adaptations }, - { id: "1", start: 12, adaptations: oldPeriod2.adaptations }, + { id: "0", start: 4, adaptations: oldPeriod1.adaptations, thumbnailTracks: [] }, + { id: "1", start: 12, adaptations: oldPeriod2.adaptations, thumbnailTracks: [] }, ], suggestedPresentationDelay: 99, timeBounds: { diff --git a/src/manifest/classes/__tests__/period.test.ts b/src/manifest/classes/__tests__/period.test.ts index 371cd9c19f..40c86117ed 100644 --- a/src/manifest/classes/__tests__/period.test.ts +++ b/src/manifest/classes/__tests__/period.test.ts @@ -24,7 +24,7 @@ describe("Manifest - Period", () => { })); const Period = (await vi.importActual("../period")).default as typeof IPeriod; - const args = { id: "12", adaptations: {}, start: 0 }; + const args = { id: "12", adaptations: {}, start: 0, thumbnailTracks: [] }; let period: IPeriod | null = null; let errorReceived: unknown = null; const unsupportedAdaptations: Adaptation[] = []; @@ -83,6 +83,7 @@ describe("Manifest - Period", () => { id: "12", adaptations: { foo }, start: 0, + thumbnailTracks: [], } as unknown as IParsedPeriod; let period: IPeriod | null = null; let errorReceived: unknown = null; @@ -131,7 +132,12 @@ describe("Manifest - Period", () => { })); const Period = (await vi.importActual("../period")).default as typeof IPeriod; - const args = { id: "12", adaptations: { video: [], audio: [] }, start: 0 }; + const args = { + id: "12", + thumbnailTracks: [], + adaptations: { video: [], audio: [] }, + start: 0, + }; let period: IPeriod | null = null; let errorReceived: unknown = null; const unsupportedAdaptations: Adaptation[] = []; @@ -223,6 +229,7 @@ describe("Manifest - Period", () => { const args: IParsedPeriod = { id: "12", adaptations: { video, audio }, + thumbnailTracks: [], start: 0, } as unknown as IParsedPeriod; let period: IPeriod | null = null; @@ -324,6 +331,7 @@ describe("Manifest - Period", () => { id: "12", adaptations: { video, audio }, start: 0, + thumbnailTracks: [], }; let period: IPeriod | null = null; let errorReceived: unknown = null; @@ -422,7 +430,12 @@ describe("Manifest - Period", () => { audioAda1, audioAda2, ] as unknown as IParsedAdaptation[]; - const args: IParsedPeriod = { id: "12", adaptations: { video, audio }, start: 0 }; + const args: IParsedPeriod = { + id: "12", + adaptations: { video, audio }, + start: 0, + thumbnailTracks: [], + }; let period: IPeriod | null = null; let errorReceived: unknown = null; const unsupportedAdaptations: Adaptation[] = []; @@ -515,7 +528,12 @@ describe("Manifest - Period", () => { }, }; const audio = [audioAda1, audioAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video, audio }, start: 0 }; + const args = { + id: "12", + adaptations: { video, audio }, + start: 0, + thumbnailTracks: [], + }; let period: IPeriod | null = null; let errorReceived: unknown = null; const unsupportedAdaptations: Adaptation[] = []; @@ -580,7 +598,12 @@ describe("Manifest - Period", () => { }, }; const video2 = [videoAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video, video2 }, start: 0 }; + const args = { + id: "12", + adaptations: { video, video2 }, + start: 0, + thumbnailTracks: [], + }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period(args, unsupportedAdaptations, codecSupportCache); @@ -626,7 +649,7 @@ describe("Manifest - Period", () => { }; const video = [videoAda1] as unknown as IParsedAdaptation[]; const bar = undefined; - const args = { id: "12", adaptations: { bar, video }, start: 0 }; + const args = { id: "12", thumbnailTracks: [], adaptations: { bar, video }, start: 0 }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period(args, unsupportedAdaptations, codecSupportCache); @@ -682,7 +705,7 @@ describe("Manifest - Period", () => { }, }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video }, start: 0 }; + const args = { id: "12", adaptations: { video }, start: 0, thumbnailTracks: [] }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period( @@ -746,7 +769,7 @@ describe("Manifest - Period", () => { }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; const foo = [fooAda1]; - const args = { id: "12", adaptations: { video, foo }, start: 0 }; + const args = { id: "12", thumbnailTracks: [], adaptations: { video, foo }, start: 0 }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); new Period(args, unsupportedAdaptations, codecSupportCache); @@ -798,7 +821,7 @@ describe("Manifest - Period", () => { }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; const foo = [fooAda1]; - const args = { id: "12", adaptations: { video, foo }, start: 0 }; + const args = { id: "12", thumbnailTracks: [], adaptations: { video, foo }, start: 0 }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); new Period(args, unsupportedAdaptations, codecSupportCache); @@ -840,7 +863,7 @@ describe("Manifest - Period", () => { }, }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video }, start: 72 }; + const args = { id: "12", adaptations: { video }, start: 72, thumbnailTracks: [] }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period(args, unsupportedAdaptations, codecSupportCache); @@ -885,7 +908,13 @@ describe("Manifest - Period", () => { }, }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video }, start: 0, duration: 12 }; + const args = { + id: "12", + adaptations: { video }, + start: 0, + duration: 12, + thumbnailTracks: [], + }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period(args, unsupportedAdaptations, codecSupportCache); @@ -930,7 +959,13 @@ describe("Manifest - Period", () => { }, }; const video = [videoAda1, videoAda2] as unknown as IParsedAdaptation[]; - const args = { id: "12", adaptations: { video }, start: 50, duration: 12 }; + const args = { + id: "12", + adaptations: { video }, + start: 50, + duration: 12, + thumbnailTracks: [], + }; const unsupportedAdaptations: Adaptation[] = []; const codecSupportCache = new CodecSupportCache([]); const period = new Period(args, unsupportedAdaptations, codecSupportCache); @@ -988,6 +1023,7 @@ describe("Manifest - Period", () => { const args = { id: "12", + thumbnailTracks: [], adaptations: { video, audio }, start: 50, duration: 12, @@ -1050,6 +1086,7 @@ describe("Manifest - Period", () => { const args = { id: "12", + thumbnailTracks: [], adaptations: { video, audio }, start: 50, duration: 12, @@ -1129,6 +1166,7 @@ describe("Manifest - Period", () => { const args = { id: "12", + thumbnailTracks: [], adaptations: { video, audio }, start: 50, duration: 12, diff --git a/src/manifest/classes/__tests__/update_period_in_place.test.ts b/src/manifest/classes/__tests__/update_period_in_place.test.ts index a61ff14a2e..3ddb4da617 100644 --- a/src/manifest/classes/__tests__/update_period_in_place.test.ts +++ b/src/manifest/classes/__tests__/update_period_in_place.test.ts @@ -166,6 +166,25 @@ function generateFakeAdaptation({ }; } +function generateFakeThumbnailTrack({ id }: { id: string }) { + return { + id, + mimeType: "image/png", + height: 100, + width: 200, + horizontalTiles: 5, + verticalTiles: 3, + index: { + _update() { + /* noop */ + }, + _replace() { + /* noop */ + }, + }, + }; +} + describe("Manifest - updatePeriodInPlace", () => { let mockOldVideoRepresentation1Replace: MockInstance | undefined; let mockOldVideoRepresentation2Replace: MockInstance | undefined; @@ -308,6 +327,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 5, end: 15, duration: 10, + thumbnailTracks: [], adaptations: { video: [oldVideoAdaptation1, oldVideoAdaptation2], audio: [oldAudioAdaptation], @@ -335,6 +355,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 500, end: 520, duration: 20, + thumbnailTracks: [], adaptations: { video: [newVideoAdaptation1, newVideoAdaptation2], audio: [newAudioAdaptation], @@ -355,6 +376,9 @@ describe("Manifest - updatePeriodInPlace", () => { ); expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [], + removedThumbnailTracks: [], addedAdaptations: [], removedAdaptations: [], updatedAdaptations: [ @@ -464,6 +488,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 5, end: 15, duration: 10, + thumbnailTracks: [], adaptations: { video: [oldVideoAdaptation1, oldVideoAdaptation2], audio: [oldAudioAdaptation], @@ -491,6 +516,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 500, end: 520, duration: 20, + thumbnailTracks: [], adaptations: { video: [newVideoAdaptation1, newVideoAdaptation2], audio: [newAudioAdaptation], @@ -510,6 +536,9 @@ describe("Manifest - updatePeriodInPlace", () => { MANIFEST_UPDATE_TYPE.Partial, ); expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [], + removedThumbnailTracks: [], addedAdaptations: [], removedAdaptations: [], updatedAdaptations: [ @@ -614,6 +643,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 5, end: 15, duration: 10, + thumbnailTracks: [], adaptations: { video: [oldVideoAdaptation1], audio: [oldAudioAdaptation], @@ -644,6 +674,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 500, end: 520, duration: 20, + thumbnailTracks: [], adaptations: { video: [newVideoAdaptation1, newVideoAdaptation2], audio: [newAudioAdaptation], @@ -663,6 +694,9 @@ describe("Manifest - updatePeriodInPlace", () => { MANIFEST_UPDATE_TYPE.Full, ); expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [], + removedThumbnailTracks: [], addedAdaptations: [newVideoAdaptation2.getMetadataSnapshot()], removedAdaptations: [], updatedAdaptations: [ @@ -709,6 +743,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 5, end: 15, duration: 10, + thumbnailTracks: [], adaptations: { video: [oldVideoAdaptation1], audio: [oldAudioAdaptation], @@ -736,6 +771,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 500, end: 520, duration: 20, + thumbnailTracks: [], adaptations: { video: [newVideoAdaptation1, newVideoAdaptation2], audio: [newAudioAdaptation], @@ -755,6 +791,9 @@ describe("Manifest - updatePeriodInPlace", () => { MANIFEST_UPDATE_TYPE.Partial, ); expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [], + removedThumbnailTracks: [], addedAdaptations: [newVideoAdaptation2.getMetadataSnapshot()], removedAdaptations: [], updatedAdaptations: [ @@ -806,6 +845,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 5, end: 15, duration: 10, + thumbnailTracks: [], adaptations: { video: [oldVideoAdaptation1, oldVideoAdaptation2], audio: [oldAudioAdaptation], @@ -828,6 +868,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 500, end: 520, duration: 20, + thumbnailTracks: [], adaptations: { video: [newVideoAdaptation1], audio: [newAudioAdaptation], @@ -847,6 +888,9 @@ describe("Manifest - updatePeriodInPlace", () => { MANIFEST_UPDATE_TYPE.Full, ); expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [], + removedThumbnailTracks: [], addedAdaptations: [], removedAdaptations: [ { @@ -903,6 +947,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 5, end: 15, duration: 10, + thumbnailTracks: [], adaptations: { video: [oldVideoAdaptation1, oldVideoAdaptation2], audio: [oldAudioAdaptation], @@ -925,6 +970,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 500, end: 520, duration: 20, + thumbnailTracks: [], adaptations: { video: [newVideoAdaptation1], audio: [newAudioAdaptation], @@ -944,6 +990,9 @@ describe("Manifest - updatePeriodInPlace", () => { MANIFEST_UPDATE_TYPE.Partial, ); expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [], + removedThumbnailTracks: [], addedAdaptations: [], removedAdaptations: [ { @@ -995,6 +1044,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 5, end: 15, duration: 10, + thumbnailTracks: [], adaptations: { video: [oldVideoAdaptation1], audio: [oldAudioAdaptation], @@ -1018,6 +1068,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 500, end: 520, duration: 20, + thumbnailTracks: [], adaptations: { video: [newVideoAdaptation1], audio: [newAudioAdaptation], @@ -1037,6 +1088,9 @@ describe("Manifest - updatePeriodInPlace", () => { MANIFEST_UPDATE_TYPE.Full, ); expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [], + removedThumbnailTracks: [], addedAdaptations: [], removedAdaptations: [], updatedAdaptations: [ @@ -1079,6 +1133,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 5, end: 15, duration: 10, + thumbnailTracks: [], adaptations: { video: [oldVideoAdaptation1], audio: [oldAudioAdaptation], @@ -1102,6 +1157,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 500, end: 520, duration: 20, + thumbnailTracks: [], adaptations: { video: [newVideoAdaptation1], audio: [newAudioAdaptation], @@ -1121,6 +1177,9 @@ describe("Manifest - updatePeriodInPlace", () => { MANIFEST_UPDATE_TYPE.Partial, ); expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [], + removedThumbnailTracks: [], addedAdaptations: [], removedAdaptations: [], updatedAdaptations: [ @@ -1163,6 +1222,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 500, end: 520, duration: 20, + thumbnailTracks: [], adaptations: { video: [oldVideoAdaptation1], audio: [oldAudioAdaptation], @@ -1185,6 +1245,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 5, end: 15, duration: 10, + thumbnailTracks: [], adaptations: { video: [newVideoAdaptation1], audio: [newAudioAdaptation], @@ -1204,6 +1265,9 @@ describe("Manifest - updatePeriodInPlace", () => { MANIFEST_UPDATE_TYPE.Full, ); expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [], + removedThumbnailTracks: [], addedAdaptations: [], removedAdaptations: [], updatedAdaptations: [ @@ -1246,6 +1310,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 500, end: 520, duration: 20, + thumbnailTracks: [], adaptations: { video: [oldVideoAdaptation1], audio: [oldAudioAdaptation], @@ -1268,6 +1333,7 @@ describe("Manifest - updatePeriodInPlace", () => { start: 5, end: 15, duration: 10, + thumbnailTracks: [], adaptations: { video: [newVideoAdaptation1], audio: [newAudioAdaptation], @@ -1287,6 +1353,9 @@ describe("Manifest - updatePeriodInPlace", () => { MANIFEST_UPDATE_TYPE.Partial, ); expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [], + removedThumbnailTracks: [], addedAdaptations: [], removedAdaptations: [], updatedAdaptations: [ @@ -1313,4 +1382,302 @@ describe("Manifest - updatePeriodInPlace", () => { expect(oldVideoAdaptation1.representations).toHaveLength(1); mockLog.mockRestore(); }); + + it("should add new Thumbnail Track in Full mode", () => { + const oldThumbnailTrack1 = generateFakeThumbnailTrack({ + id: "thumb-1", + }); + const oldPeriod = { + start: 5, + end: 15, + duration: 10, + thumbnailTracks: [oldThumbnailTrack1], + adaptations: { + video: [], + audio: [], + }, + getAdaptations() { + return []; + }, + }; + const newThumbnailTrack1 = generateFakeThumbnailTrack({ + id: "thumb-1", + }); + const newThumbnailTrack2 = generateFakeThumbnailTrack({ + id: "thumb-2", + }); + const newPeriod = { + start: 500, + end: 520, + duration: 20, + thumbnailTracks: [newThumbnailTrack1, newThumbnailTrack2], + adaptations: { + video: [], + audio: [], + }, + getAdaptations() { + return []; + }, + }; + const mockLog = vi.spyOn(log, "warn"); + const res = updatePeriodInPlace( + oldPeriod as unknown as Period, + newPeriod as unknown as Period, + MANIFEST_UPDATE_TYPE.Full, + ); + expect(res).toEqual({ + addedThumbnailTracks: [ + { + height: 100, + horizontalTiles: 5, + id: "thumb-2", + mimeType: "image/png", + verticalTiles: 3, + width: 200, + }, + ], + updatedThumbnailTracks: [ + { + height: 100, + horizontalTiles: 5, + id: "thumb-1", + mimeType: "image/png", + verticalTiles: 3, + width: 200, + }, + ], + removedThumbnailTracks: [], + addedAdaptations: [], + removedAdaptations: [], + updatedAdaptations: [], + }); + expect(mockLog).toHaveBeenCalled(); + expect(mockLog).toHaveBeenNthCalledWith( + 1, + "Manifest: 1 new Thumbnail tracks found when merging.", + ); + expect(oldPeriod.thumbnailTracks).toHaveLength(2); + mockLog.mockRestore(); + }); + + it("should add new Thumbnail Track in Partial mode", () => { + const oldThumbnailTrack1 = generateFakeThumbnailTrack({ + id: "thumb-1", + }); + const oldPeriod = { + start: 5, + end: 15, + duration: 10, + thumbnailTracks: [oldThumbnailTrack1], + adaptations: { + video: [], + audio: [], + }, + getAdaptations() { + return []; + }, + }; + const newThumbnailTrack1 = generateFakeThumbnailTrack({ + id: "thumb-1", + }); + const newThumbnailTrack2 = generateFakeThumbnailTrack({ + id: "thumb-2", + }); + const newPeriod = { + start: 500, + end: 520, + duration: 20, + thumbnailTracks: [newThumbnailTrack1, newThumbnailTrack2], + adaptations: { + video: [], + audio: [], + }, + getAdaptations() { + return []; + }, + }; + const mockLog = vi.spyOn(log, "warn"); + const res = updatePeriodInPlace( + oldPeriod as unknown as Period, + newPeriod as unknown as Period, + MANIFEST_UPDATE_TYPE.Partial, + ); + expect(res).toEqual({ + addedThumbnailTracks: [ + { + height: 100, + horizontalTiles: 5, + id: "thumb-2", + mimeType: "image/png", + verticalTiles: 3, + width: 200, + }, + ], + updatedThumbnailTracks: [ + { + height: 100, + horizontalTiles: 5, + id: "thumb-1", + mimeType: "image/png", + verticalTiles: 3, + width: 200, + }, + ], + removedThumbnailTracks: [], + addedAdaptations: [], + removedAdaptations: [], + updatedAdaptations: [], + }); + expect(mockLog).toHaveBeenCalled(); + expect(mockLog).toHaveBeenNthCalledWith( + 1, + "Manifest: 1 new Thumbnail tracks found when merging.", + ); + expect(oldPeriod.thumbnailTracks).toHaveLength(2); + mockLog.mockRestore(); + }); + + it("should remove unfound Thumbnail Tracks in Full mode", () => { + const oldThumbnailTrack1 = generateFakeThumbnailTrack({ + id: "thumb-1", + }); + const oldThumbnailTrack2 = generateFakeThumbnailTrack({ + id: "thumb-2", + }); + const oldPeriod = { + start: 5, + end: 15, + duration: 10, + thumbnailTracks: [oldThumbnailTrack1, oldThumbnailTrack2], + adaptations: { + video: [], + audio: [], + }, + getAdaptations() { + return []; + }, + }; + const newThumbnailTrack1 = generateFakeThumbnailTrack({ + id: "thumb-1", + }); + const newPeriod = { + start: 500, + end: 520, + duration: 20, + thumbnailTracks: [newThumbnailTrack1], + adaptations: { + video: [], + audio: [], + }, + getAdaptations() { + return []; + }, + }; + const mockLog = vi.spyOn(log, "warn"); + const res = updatePeriodInPlace( + oldPeriod as unknown as Period, + newPeriod as unknown as Period, + MANIFEST_UPDATE_TYPE.Full, + ); + expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [ + { + height: 100, + horizontalTiles: 5, + id: "thumb-1", + mimeType: "image/png", + verticalTiles: 3, + width: 200, + }, + ], + removedThumbnailTracks: [ + { + id: oldThumbnailTrack2.id, + }, + ], + addedAdaptations: [], + removedAdaptations: [], + updatedAdaptations: [], + }); + expect(mockLog).toHaveBeenCalled(); + expect(mockLog).toHaveBeenNthCalledWith( + 1, + 'Manifest: ThumbnailTrack "thumb-2" not found when merging.', + ); + expect(oldPeriod.thumbnailTracks).toHaveLength(1); + mockLog.mockRestore(); + }); + + it("should remove unfound ThumbnailTracks in Partial mode", () => { + const oldThumbnailTrack1 = generateFakeThumbnailTrack({ + id: "thumb-1", + }); + const oldThumbnailTrack2 = generateFakeThumbnailTrack({ + id: "thumb-2", + }); + const oldPeriod = { + start: 5, + end: 15, + duration: 10, + thumbnailTracks: [oldThumbnailTrack1, oldThumbnailTrack2], + adaptations: { + video: [], + audio: [], + }, + getAdaptations() { + return []; + }, + }; + const newThumbnailTrack1 = generateFakeThumbnailTrack({ + id: "thumb-1", + }); + const newPeriod = { + start: 500, + end: 520, + duration: 20, + thumbnailTracks: [newThumbnailTrack1], + adaptations: { + video: [], + audio: [], + }, + getAdaptations() { + return []; + }, + }; + const mockLog = vi.spyOn(log, "warn"); + const res = updatePeriodInPlace( + oldPeriod as unknown as Period, + newPeriod as unknown as Period, + MANIFEST_UPDATE_TYPE.Partial, + ); + expect(res).toEqual({ + addedThumbnailTracks: [], + updatedThumbnailTracks: [ + { + height: 100, + horizontalTiles: 5, + id: "thumb-1", + mimeType: "image/png", + verticalTiles: 3, + width: 200, + }, + ], + removedThumbnailTracks: [ + { + id: oldThumbnailTrack2.id, + }, + ], + addedAdaptations: [], + removedAdaptations: [], + updatedAdaptations: [], + }); + expect(mockLog).toHaveBeenCalled(); + expect(mockLog).toHaveBeenNthCalledWith( + 1, + 'Manifest: ThumbnailTrack "thumb-2" not found when merging.', + ); + expect(oldPeriod.thumbnailTracks).toHaveLength(1); + mockLog.mockRestore(); + }); }); diff --git a/src/manifest/classes/__tests__/update_periods.test.ts b/src/manifest/classes/__tests__/update_periods.test.ts index 96861cb99b..6b969015a2 100644 --- a/src/manifest/classes/__tests__/update_periods.test.ts +++ b/src/manifest/classes/__tests__/update_periods.test.ts @@ -32,6 +32,7 @@ function generateFakePeriod({ duration: end === undefined ? undefined : end - (start ?? 0), streamEvents: [], adaptations: {}, + thumbnailTracks: [], refreshCodecSupport() { // noop }, @@ -57,6 +58,7 @@ function generateFakePeriod({ id: id ?? String(start), streamEvents: [], adaptations: {}, + thumbnailTracks: [], }; }, }; @@ -393,7 +395,7 @@ describe("Manifest - updatePeriods", () => { removedPeriods: [], updatedPeriods: [ { - period: { id: "p2", start: 60, streamEvents: [] }, + period: { id: "p2", start: 60, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], @@ -434,7 +436,7 @@ describe("Manifest - updatePeriods", () => { removedPeriods: [], updatedPeriods: [ { - period: { id: "p2", start: 60, streamEvents: [] }, + period: { id: "p2", start: 60, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], @@ -527,11 +529,11 @@ describe("Manifest - updatePeriods", () => { removedPeriods: [{ id: "p1.5", start: 69, end: 70 }], updatedPeriods: [ { - period: { id: "p1", start: 60, end: 69, streamEvents: [] }, + period: { id: "p1", start: 60, end: 69, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, { - period: { id: "p2", start: 70, streamEvents: [] }, + period: { id: "p2", start: 70, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], @@ -805,11 +807,11 @@ describe("Manifest - updatePeriods", () => { removedPeriods: [], updatedPeriods: [ { - period: { id: "p1", start: 60, end: 70, streamEvents: [] }, + period: { id: "p1", start: 60, end: 70, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, { - period: { id: "p2", start: 70, streamEvents: [] }, + period: { id: "p2", start: 70, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], @@ -865,11 +867,11 @@ describe("Manifest - updatePeriods", () => { removedPeriods: [{ id: "p2", start: 70, end: 80 }], updatedPeriods: [ { - period: { id: "p1", start: 60, end: 70, streamEvents: [] }, + period: { id: "p1", start: 60, end: 70, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, { - period: { id: "p3", start: 80, streamEvents: [] }, + period: { id: "p3", start: 80, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], @@ -928,11 +930,11 @@ describe("Manifest - updatePeriods", () => { ], updatedPeriods: [ { - period: { id: "p1", start: 60, end: 70, streamEvents: [] }, + period: { id: "p1", start: 60, end: 70, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, { - period: { id: "p3", start: 80, end: 90, streamEvents: [] }, + period: { id: "p3", start: 80, end: 90, streamEvents: [], thumbnailTracks: [] }, result: fakeUpdatePeriodInPlaceRes, }, ], diff --git a/src/manifest/classes/index.ts b/src/manifest/classes/index.ts index 4db4d9de1a..c98d80f332 100644 --- a/src/manifest/classes/index.ts +++ b/src/manifest/classes/index.ts @@ -19,6 +19,7 @@ import type { ICodecSupportInfo } from "./codec_support_cache"; import type { IDecipherabilityUpdateElement, IManifestParsingOptions } from "./manifest"; import Manifest from "./manifest"; import Period from "./period"; +import type { IThumbnailTrack } from "./period"; import Representation from "./representation"; import type { IMetaPlaylistPrivateInfos, @@ -42,6 +43,7 @@ export type { IRepresentationIndex, IPrivateInfos, ISegment, + IThumbnailTrack, }; export { areSameContent, diff --git a/src/manifest/classes/period.ts b/src/manifest/classes/period.ts index e31996c27c..05a35b34c4 100644 --- a/src/manifest/classes/period.ts +++ b/src/manifest/classes/period.ts @@ -14,7 +14,11 @@ * limitations under the License. */ import { MediaError } from "../../errors"; -import type { IManifestStreamEvent, IParsedPeriod } from "../../parsers/manifest"; +import type { + ICdnMetadata, + IManifestStreamEvent, + IParsedPeriod, +} from "../../parsers/manifest"; import type { ITrackType, IRepresentationFilter } from "../../public_types"; import arrayFind from "../../utils/array_find"; import isNullOrUndefined from "../../utils/is_null_or_undefined"; @@ -22,6 +26,7 @@ import type { IAdaptationMetadata, IPeriodMetadata } from "../types"; import { getAdaptations, getSupportedAdaptations, periodContainsTime } from "../utils"; import Adaptation from "./adaptation"; import type CodecSupportCache from "./codec_support_cache"; +import type { IRepresentationIndex } from "./representation_index"; /** Structure listing every `Adaptation` in a Period. */ export type IManifestAdaptations = Partial>; @@ -56,6 +61,11 @@ export default class Period implements IPeriodMetadata { /** Array containing every stream event happening on the period */ public streamEvents: IManifestStreamEvent[]; + /** + * If set to an object, this Period has thumbnail tracks. + */ + public thumbnailTracks: IThumbnailTrack[]; + /** * @constructor * @param {Object} args @@ -126,6 +136,16 @@ export default class Period implements IPeriodMetadata { ); } + this.thumbnailTracks = args.thumbnailTracks.map((thumbnailTrack) => ({ + id: thumbnailTrack.id, + mimeType: thumbnailTrack.mimeType, + index: thumbnailTrack.index, + cdnMetadata: thumbnailTrack.cdnMetadata, + height: thumbnailTrack.height, + width: thumbnailTrack.width, + horizontalTiles: thumbnailTrack.horizontalTiles, + verticalTiles: thumbnailTrack.verticalTiles, + })); this.duration = args.duration; this.start = args.start; @@ -278,6 +298,50 @@ export default class Period implements IPeriodMetadata { id: this.id, streamEvents: this.streamEvents, adaptations, + thumbnailTracks: this.thumbnailTracks.map((thumbnailTrack) => ({ + id: thumbnailTrack.id, + mimeType: thumbnailTrack.mimeType, + height: thumbnailTrack.height, + width: thumbnailTrack.width, + horizontalTiles: thumbnailTrack.horizontalTiles, + verticalTiles: thumbnailTrack.verticalTiles, + })), }; } } + +/** + * Metadata on an image thumbnail track associated to a Period. + */ +export interface IThumbnailTrack { + /** Identifier for that thumbnail track. */ + id: string; + /** interface allowing to obtain information on the actual thumbnails. */ + index: IRepresentationIndex; + /** Mime-type for loaded thumbnails, allowing to know their format. */ + mimeType: string; + /** CDN(s) on which the thumbnails may be loaded. */ + cdnMetadata: ICdnMetadata[] | null; + /** + * A loaded thumbnail's height in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + height: number; + /** + * A loaded thumbnail's width in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + width: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained horizontally in a whole loaded thumbnail resource. + */ + horizontalTiles: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained vertically in a whole loaded thumbnail resource. + */ + verticalTiles: number; +} diff --git a/src/manifest/classes/update_period_in_place.ts b/src/manifest/classes/update_period_in_place.ts index 918ee888f6..ebbfa4b127 100644 --- a/src/manifest/classes/update_period_in_place.ts +++ b/src/manifest/classes/update_period_in_place.ts @@ -18,6 +18,7 @@ import log from "../../log"; import type { IAdaptationMetadata, IRepresentationMetadata } from "../../manifest"; import type { ITrackType } from "../../public_types"; import arrayFindIndex from "../../utils/array_find_index"; +import type { IThumbnailTrackMetadata } from "../types"; import type Period from "./period"; import { MANIFEST_UPDATE_TYPE } from "./types"; @@ -38,12 +39,77 @@ export default function updatePeriodInPlace( updatedAdaptations: [], removedAdaptations: [], addedAdaptations: [], + updatedThumbnailTracks: [], + removedThumbnailTracks: [], + addedThumbnailTracks: [], }; oldPeriod.start = newPeriod.start; oldPeriod.end = newPeriod.end; oldPeriod.duration = newPeriod.duration; oldPeriod.streamEvents = newPeriod.streamEvents; + const oldThumbnailTracks = oldPeriod.thumbnailTracks; + const newThumbnailTracks = newPeriod.thumbnailTracks; + for (let j = 0; j < oldThumbnailTracks.length; j++) { + const oldThumbnailTrack = oldThumbnailTracks[j]; + const newThumbnailTrackIdx = arrayFindIndex( + newThumbnailTracks, + (a) => a.id === oldThumbnailTrack.id, + ); + + if (newThumbnailTrackIdx === -1) { + log.warn( + 'Manifest: ThumbnailTrack "' + + oldThumbnailTracks[j].id + + '" not found when merging.', + ); + const [removed] = oldThumbnailTracks.splice(j, 1); + j--; + res.removedThumbnailTracks.push({ + id: removed.id, + }); + } else { + const [newThumbnailTrack] = newThumbnailTracks.splice(newThumbnailTrackIdx, 1); + oldThumbnailTrack.mimeType = newThumbnailTrack.mimeType; + oldThumbnailTrack.height = newThumbnailTrack.height; + oldThumbnailTrack.width = newThumbnailTrack.width; + oldThumbnailTrack.horizontalTiles = newThumbnailTrack.horizontalTiles; + oldThumbnailTrack.verticalTiles = newThumbnailTrack.verticalTiles; + oldThumbnailTrack.cdnMetadata = newThumbnailTrack.cdnMetadata; + if (updateType === MANIFEST_UPDATE_TYPE.Full) { + oldThumbnailTrack.index._replace(newThumbnailTrack.index); + } else { + oldThumbnailTrack.index._update(newThumbnailTrack.index); + } + res.updatedThumbnailTracks.push({ + id: oldThumbnailTrack.id, + mimeType: oldThumbnailTrack.mimeType, + height: oldThumbnailTrack.height, + width: oldThumbnailTrack.width, + horizontalTiles: oldThumbnailTrack.horizontalTiles, + verticalTiles: oldThumbnailTrack.verticalTiles, + }); + } + } + + if (newThumbnailTracks.length > 0) { + log.warn( + `Manifest: ${newThumbnailTracks.length} new Thumbnail tracks ` + + "found when merging.", + ); + res.addedThumbnailTracks.push( + ...newThumbnailTracks.map((t) => ({ + id: t.id, + mimeType: t.mimeType, + height: t.height, + width: t.width, + horizontalTiles: t.horizontalTiles, + verticalTiles: t.verticalTiles, + })), + ); + oldPeriod.thumbnailTracks.push(...newThumbnailTracks); + } + const oldAdaptations = oldPeriod.getAdaptations(); const newAdaptations = newPeriod.getAdaptations(); @@ -160,4 +226,13 @@ export interface IUpdatedPeriodResult { }>; /** Adaptation that have been added to the Period. */ addedAdaptations: IAdaptationMetadata[]; + + /** Information on Thumbnail Tracks that have been updated. */ + updatedThumbnailTracks: IThumbnailTrackMetadata[]; + /** Thumbnail tracks that have been removed from the Period. */ + removedThumbnailTracks: Array<{ + id: string; + }>; + /** Thumbnail tracks that have been added to the Period. */ + addedThumbnailTracks: IThumbnailTrackMetadata[]; } diff --git a/src/manifest/index.ts b/src/manifest/index.ts index b42643cefd..a366f1d008 100644 --- a/src/manifest/index.ts +++ b/src/manifest/index.ts @@ -9,6 +9,7 @@ import type { IRepresentationIndex, IMetaPlaylistPrivateInfos, IPrivateInfos, + IThumbnailTrack, } from "./classes"; import type Manifest from "./classes"; import { areSameContent, getLoggableSegmentId } from "./classes"; @@ -33,6 +34,7 @@ export type { ISegment, IMetaPlaylistPrivateInfos, IPrivateInfos, + IThumbnailTrack, }; export { areSameContent, getLoggableSegmentId }; export type { diff --git a/src/manifest/types.ts b/src/manifest/types.ts index 0edc8e4140..94497de68f 100644 --- a/src/manifest/types.ts +++ b/src/manifest/types.ts @@ -254,6 +254,47 @@ export interface IPeriodMetadata { adaptations: Partial>; /** Array containing every stream event happening on the period */ streamEvents: IManifestStreamEvent[]; + /** If set to an object, this Period has a thumbnail track. */ + thumbnailTracks: IThumbnailTrackMetadata[]; +} + +/** Describes metadata about a single image thumbnail track. */ +export interface IThumbnailTrackMetadata { + /** Identify that thumbnail track. */ + id: string; + /** Estimated mime-type for the loaded thumbnails (e.g. `"image/jpeg"`). */ + mimeType: string; + /** + * A loaded thumbnail's height in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + height: number; + /** + * A loaded thumbnail's width in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + width: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained horizontally in a whole loaded thumbnail resource. + */ + horizontalTiles: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained vertically in a whole loaded thumbnail resource. + */ + verticalTiles: number; +} + +export interface ILoadedThumbnailData { + data: BufferSource; + mimeType: string; + start: number; + end?: number | undefined; + height?: number | undefined; + width?: number | undefined; } /** diff --git a/src/manifest/utils.ts b/src/manifest/utils.ts index 47dd9f715d..5604794cfd 100644 --- a/src/manifest/utils.ts +++ b/src/manifest/utils.ts @@ -19,6 +19,7 @@ import type { IManifestMetadata, IPeriodMetadata, IRepresentationMetadata, + IThumbnailTrackMetadata, } from "./types"; /** List in an array every possible value for the Adaptation's `type` property. */ @@ -584,6 +585,41 @@ export function replicateUpdatesOnManifestMetadata( } } + for (const removedThumbnailTrack of updatedPeriod.result.removedThumbnailTracks) { + for ( + let thumbIdx = 0; + thumbIdx < basePeriod.thumbnailTracks.length; + thumbIdx++ + ) { + if (basePeriod.thumbnailTracks[thumbIdx].id === removedThumbnailTrack.id) { + basePeriod.thumbnailTracks.splice(thumbIdx, 1); + break; + } + } + } + for (const updatedThumbnailTrack of updatedPeriod.result.updatedThumbnailTracks) { + const newThumbnailTrack = updatedThumbnailTrack; + for ( + let thumbIdx = 0; + thumbIdx < basePeriod.thumbnailTracks.length; + thumbIdx++ + ) { + if (basePeriod.thumbnailTracks[thumbIdx].id === newThumbnailTrack.id) { + const baseThumbnailTrack = basePeriod.thumbnailTracks[thumbIdx]; + for (const prop of Object.keys(newThumbnailTrack) as Array< + keyof IThumbnailTrackMetadata + >) { + // eslint-disable-next-line + (baseThumbnailTrack as any)[prop] = newThumbnailTrack[prop]; + } + break; + } + } + } + for (const addedThumbnailTrack of updatedPeriod.result.addedThumbnailTracks) { + basePeriod.thumbnailTracks.push(addedThumbnailTrack); + } + for (const removedAdaptation of updatedPeriod.result.removedAdaptations) { const ttype = removedAdaptation.trackType; const adaptationsForType = basePeriod.adaptations[ttype] ?? []; diff --git a/src/multithread_types.ts b/src/multithread_types.ts index e7e4b926a7..87f4aaabd7 100644 --- a/src/multithread_types.ts +++ b/src/multithread_types.ts @@ -30,7 +30,7 @@ import type { } from "./mse"; import type { IFreezingStatus, IRebufferingStatus } from "./playback_observer"; import type { ICmcdOptions, ITrackType } from "./public_types"; -import type { ITransportOptions } from "./transports"; +import type { IThumbnailResponse, ITransportOptions } from "./transports"; import type { ILogFormat, ILoggerLevel } from "./utils/logger"; import type { IRange } from "./utils/ranges"; @@ -525,6 +525,18 @@ export interface IRemoveTextDataErrorMessage { }; } +/** Message sent from main thread when it wants to fetch thumbnail data. */ +export interface IThumbnailDataRequestMainMessage { + type: MainThreadMessageType.ThumbnailDataRequest; + contentId: string; + value: { + requestId: number; + periodId: string; + thumbnailTrackId: string; + time: number; + }; +} + /** * Template for a message originating from main thread to update * `SharedReference` objects (a common abstraction of the RxPlayer allowing for @@ -548,7 +560,7 @@ export type IReferenceUpdateMessage = export interface IPullSegmentSinkStoreInfos { type: MainThreadMessageType.PullSegmentSinkStoreInfos; - value: { messageId: number }; + value: { requestId: number }; } export const enum MainThreadMessageType { @@ -573,6 +585,7 @@ export const enum MainThreadMessageType { StopContent = "stop", TrackUpdate = "track-update", PullSegmentSinkStoreInfos = "pull-segment-sink-store-infos", + ThumbnailDataRequest = "thumbnail-request", } export type IMainThreadMessage = @@ -596,7 +609,8 @@ export type IMainThreadMessage = | IPushTextDataErrorMessage | IRemoveTextDataErrorMessage | IMediaSourceReadyStateChangeMainMessage - | IPullSegmentSinkStoreInfos; + | IPullSegmentSinkStoreInfos + | IThumbnailDataRequestMainMessage; export type ISentError = | ISerializedNetworkError @@ -947,10 +961,26 @@ export interface ISegmentSinkStoreUpdateMessage { contentId: string; value: { segmentSinkMetrics: ISegmentSinkMetrics; - messageId: number; + requestId: number; }; } +export interface IThumbnailDataResponseWorkerMessage { + type: WorkerMessageType.ThumbnailDataResponse; + contentId: string; + value: + | { + status: "error"; + requestId: number; + error: ISentError; + } + | { + status: "success"; + requestId: number; + data: IThumbnailResponse; + }; +} + export const enum WorkerMessageType { AbortSourceBuffer = "abort-source-buffer", ActivePeriodChanged = "active-period-changed", @@ -989,6 +1019,7 @@ export const enum WorkerMessageType { UpdatePlaybackRate = "update-playback-rate", Warning = "warning", SegmentSinkStoreUpdate = "segment-sink-store-update", + ThumbnailDataResponse = "thumbnail-response", } export type IWorkerMessage = @@ -1028,4 +1059,5 @@ export type IWorkerMessage = | IUpdateMediaSourceDurationWorkerMessage | IUpdatePlaybackRateWorkerMessage | IWarningWorkerMessage - | ISegmentSinkStoreUpdateMessage; + | ISegmentSinkStoreUpdateMessage + | IThumbnailDataResponseWorkerMessage; diff --git a/src/parsers/manifest/dash/common/__tests__/flatten_overlapping_period.test.ts b/src/parsers/manifest/dash/common/__tests__/flatten_overlapping_period.test.ts index 9098852e64..e2bf13fe8e 100644 --- a/src/parsers/manifest/dash/common/__tests__/flatten_overlapping_period.test.ts +++ b/src/parsers/manifest/dash/common/__tests__/flatten_overlapping_period.test.ts @@ -17,9 +17,9 @@ describe("flattenOverlappingPeriods", function () { const mockLog = vi.spyOn(log, "warn").mockImplementation(vi.fn()); const periods = [ - { id: "1", start: 0, duration: 60, adaptations: {} }, - { id: "2", start: 60, duration: 60, adaptations: {} }, - { id: "3", start: 60, duration: 60, adaptations: {} }, + { id: "1", start: 0, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "2", start: 60, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "3", start: 60, duration: 60, thumbnailTracks: [], adaptations: {} }, ]; const flattenPeriods = flattenOverlappingPeriods(periods); @@ -42,9 +42,9 @@ describe("flattenOverlappingPeriods", function () { const mockLog = vi.spyOn(log, "warn").mockImplementation(vi.fn()); const periods = [ - { id: "1", start: 0, duration: 60, adaptations: {} }, - { id: "2", start: 60, duration: 60, adaptations: {} }, - { id: "3", start: 90, duration: 60, adaptations: {} }, + { id: "1", start: 0, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "2", start: 60, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "3", start: 90, duration: 60, thumbnailTracks: [], adaptations: {} }, ]; const flattenPeriods = flattenOverlappingPeriods(periods); @@ -70,9 +70,9 @@ describe("flattenOverlappingPeriods", function () { const mockLog = vi.spyOn(log, "warn").mockImplementation(vi.fn()); const periods = [ - { id: "1", start: 0, duration: 60, adaptations: {} }, - { id: "2", start: 60, duration: 60, adaptations: {} }, - { id: "3", start: 50, duration: 120, adaptations: {} }, + { id: "1", start: 0, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "2", start: 60, duration: 60, thumbnailTracks: [], adaptations: {} }, + { id: "3", start: 50, duration: 120, thumbnailTracks: [], adaptations: {} }, ]; const flattenPeriods = flattenOverlappingPeriods(periods); @@ -97,13 +97,16 @@ describe("flattenOverlappingPeriods", function () { it("should keep last announced period from multiple periods with same start and end", function () { const mockLog = vi.spyOn(log, "warn").mockImplementation(vi.fn()); - const periods = [{ id: "1", start: 0, duration: 60, adaptations: {} }]; + const periods = [ + { id: "1", start: 0, duration: 60, thumbnailTracks: [], adaptations: {} }, + ]; for (let i = 1; i <= 100; i++) { periods.push({ id: i.toString(), start: 60, duration: 60, + thumbnailTracks: [], adaptations: {}, }); } @@ -127,9 +130,9 @@ describe("flattenOverlappingPeriods", function () { const mockLog = vi.spyOn(log, "warn").mockImplementation(vi.fn()); const periods = [ - { id: "1", start: 40, duration: 20, adaptations: {} }, - { id: "2", start: 60, duration: 20, adaptations: {} }, - { id: "3", start: 20, duration: 100, adaptations: {} }, + { id: "1", start: 40, duration: 20, thumbnailTracks: [], adaptations: {} }, + { id: "2", start: 60, duration: 20, thumbnailTracks: [], adaptations: {} }, + { id: "3", start: 20, duration: 100, thumbnailTracks: [], adaptations: {} }, ]; const flattenPeriods = flattenOverlappingPeriods(periods); diff --git a/src/parsers/manifest/dash/common/infer_adaptation_type.ts b/src/parsers/manifest/dash/common/infer_adaptation_type.ts index ea725105f2..a33065a48f 100644 --- a/src/parsers/manifest/dash/common/infer_adaptation_type.ts +++ b/src/parsers/manifest/dash/common/infer_adaptation_type.ts @@ -14,10 +14,16 @@ * limitations under the License. */ +import log from "../../../../log"; import { SUPPORTED_ADAPTATIONS_TYPE } from "../../../../manifest"; import arrayFind from "../../../../utils/array_find"; import arrayIncludes from "../../../../utils/array_includes"; -import type { IRepresentationIntermediateRepresentation } from "../node_parser_types"; +import isNonEmptyString from "../../../../utils/is_non_empty_string"; +import isNullOrUndefined from "../../../../utils/is_null_or_undefined"; +import type { + IAdaptationSetIntermediateRepresentation, + IRepresentationIntermediateRepresentation, +} from "../node_parser_types"; /** Different "type" a parsed Adaptation can be. */ type IAdaptationType = "audio" | "video" | "text"; @@ -31,6 +37,56 @@ interface IScheme { value?: string | undefined; } +/** + * From a thumbnail AdaptationSet, returns core information such as the number + * of tiles vertically and horizontally per image. + * + * Returns `null` if the information could not be parsed. + * @param {Object} adaptation + * @returns {Object|null} + */ +export function getThumbnailAdaptationSetInfo( + adaptation: IAdaptationSetIntermediateRepresentation, + representation?: IRepresentationIntermediateRepresentation | undefined, +): { + horizontalTiles: number; + verticalTiles: number; +} | null { + const thumbnailProp = + arrayFind( + adaptation.children.essentialProperties ?? [], + (p) => + p.schemeIdUri === "http://dashif.org/guidelines/thumbnail_tile" || + p.schemeIdUri === "http://dashif.org/thumbnail_tile", + ) ?? + arrayFind( + (representation ?? adaptation.children.representations[0])?.children + .essentialProperties ?? [], + (p) => + p.schemeIdUri === "http://dashif.org/guidelines/thumbnail_tile" || + p.schemeIdUri === "http://dashif.org/thumbnail_tile", + ); + if (thumbnailProp === undefined) { + return null; + } + const tilesRegex = /(\d+)x(\d+)/; + if ( + thumbnailProp === undefined || + thumbnailProp.value === undefined || + !tilesRegex.test(thumbnailProp.value) + ) { + log.warn("DASH: Invalid thumbnails Representation, no tile-related information"); + return null; + } + const match = thumbnailProp.value.match(tilesRegex) as RegExpMatchArray; + const horizontalTiles = parseInt(match[1], 10); + const verticalTiles = parseInt(match[2], 10); + return { + horizontalTiles, + verticalTiles, + }; +} + /** * Infers the type of adaptation from codec and mimetypes found in it. * @@ -42,18 +98,29 @@ interface IScheme { * 3. codec * * Note: This is based on DASH-IF-IOP-v4.0 with some more freedom. + * @param {Object} adaptation * @param {Array.} representations - * @param {string|null} adaptationMimeType - * @param {string|null} adaptationCodecs - * @param {Array.|null} adaptationRoles * @returns {string} - "audio"|"video"|"text"|"metadata"|"unknown" */ export default function inferAdaptationType( + adaptation: IAdaptationSetIntermediateRepresentation, representations: IRepresentationIntermediateRepresentation[], - adaptationMimeType: string | null, - adaptationCodecs: string | null, - adaptationRoles: IScheme[] | null, -): IAdaptationType | undefined { +): IAdaptationType | "thumbnails" | undefined { + if (adaptation.attributes.contentType === "image") { + if (getThumbnailAdaptationSetInfo(adaptation) !== null) { + return "thumbnails"; + } + return undefined; + } + const adaptationMimeType = isNonEmptyString(adaptation.attributes.mimeType) + ? adaptation.attributes.mimeType + : null; + const adaptationCodecs = isNonEmptyString(adaptation.attributes.codecs) + ? adaptation.attributes.codecs + : null; + const adaptationRoles = !isNullOrUndefined(adaptation.children.roles) + ? adaptation.children.roles + : null; function fromMimeType( mimeType: string, roles: IScheme[] | null, diff --git a/src/parsers/manifest/dash/common/parse_adaptation_sets.ts b/src/parsers/manifest/dash/common/parse_adaptation_sets.ts index f7ffeee88d..4409fe9436 100644 --- a/src/parsers/manifest/dash/common/parse_adaptation_sets.ts +++ b/src/parsers/manifest/dash/common/parse_adaptation_sets.ts @@ -23,14 +23,21 @@ import arrayFindIndex from "../../../../utils/array_find_index"; import arrayIncludes from "../../../../utils/array_includes"; import isNonEmptyString from "../../../../utils/is_non_empty_string"; import isNullOrUndefined from "../../../../utils/is_null_or_undefined"; -import type { IParsedAdaptation, IParsedAdaptations } from "../../types"; +import type { + IParsedAdaptation, + IParsedAdaptations, + IParsedRepresentation, + IParsedThumbnailTrack, +} from "../../types"; import type { IAdaptationSetIntermediateRepresentation, ISegmentTemplateIntermediateRepresentation, } from "../node_parser_types"; import attachTrickModeTrack from "./attach_trickmode_track"; import type ContentProtectionParser from "./content_protection_parser"; -import inferAdaptationType from "./infer_adaptation_type"; +import inferAdaptationType, { + getThumbnailAdaptationSetInfo, +} from "./infer_adaptation_type"; import type { IRepresentationContext } from "./parse_representations"; import parseRepresentations from "./parse_representations"; import resolveBaseURLs from "./resolve_base_urls"; @@ -259,11 +266,15 @@ function getAdaptationSetSwitchingIDs( export default function parseAdaptationSets( adaptationsIR: IAdaptationSetIntermediateRepresentation[], context: IAdaptationSetContext, -): IParsedAdaptations { +): { + adaptations: IParsedAdaptations; + thumbnailTracks: IParsedThumbnailTrack[]; +} { const parsedAdaptations: Record< ITrackType, Array<[IParsedAdaptation, IAdaptationSetOrderingData]> > = { video: [], audio: [], text: [] }; + const parsedThumbnailTracks: IParsedThumbnailTrack[] = []; const trickModeAdaptations: Array<{ adaptation: IParsedAdaptation; trickModeAttachedAdaptationIds: string[]; @@ -297,14 +308,7 @@ export default function parseAdaptationSets( (context.availabilityTimeOffset ?? 0); } - const adaptationMimeType = adaptation.attributes.mimeType; - const adaptationCodecs = adaptation.attributes.codecs; - const type = inferAdaptationType( - representationsIR, - isNonEmptyString(adaptationMimeType) ? adaptationMimeType : null, - isNonEmptyString(adaptationCodecs) ? adaptationCodecs : null, - !isNullOrUndefined(adaptationChildren.roles) ? adaptationChildren.roles : null, - ); + const type = inferAdaptationType(adaptation, representationsIR); if (type === undefined) { continue; } @@ -407,6 +411,15 @@ export default function parseAdaptationSets( context.unsafelyBaseOnPreviousPeriod?.getAdaptation(adaptationID) ?? null; const representations = parseRepresentations(representationsIR, adaptation, reprCtxt); + + if (type === "thumbnails") { + const track = createThumbnailTracks(adaptation, representations); + if (track !== null) { + parsedThumbnailTracks.push(...track); + } + continue; + } + const parsedAdaptationSet: IParsedAdaptation = { id: adaptationID, representations, @@ -505,7 +518,10 @@ export default function parseAdaptationSets( ); parsedAdaptations.video.sort(compareAdaptations); attachTrickModeTrack(adaptationsPerType, trickModeAdaptations); - return adaptationsPerType; + return { + adaptations: adaptationsPerType, + thumbnailTracks: parsedThumbnailTracks, + }; } /** Metadata allowing to order AdaptationSets between one another. */ @@ -524,6 +540,55 @@ interface IAdaptationSetOrderingData { indexInMpd: number; } +/** + * From the given attributes, returns a parsed thumbnail track, or null if it + * fails to do so. + * @param {Object} adaptation + * @param {Array.} representations + * @returns {Object|null} + */ +function createThumbnailTracks( + adaptation: IAdaptationSetIntermediateRepresentation, + representations: IParsedRepresentation[], +): IParsedThumbnailTrack[] { + const tracks = []; + for (let i = 0; i < representations.length; i++) { + const representation = representations[i]; + if (representation !== undefined) { + if (representation.mimeType === undefined) { + log.warn("DASH: Invalid thumbnails Representation, no mime-type"); + continue; + } + const tileInfo = getThumbnailAdaptationSetInfo( + adaptation, + adaptation.children.representations[i], + ); + if (tileInfo === null) { + continue; + } + if (representation.height === undefined) { + log.warn("DASH: Invalid thumbnails Representation, no height information"); + continue; + } + if (representation.width === undefined) { + log.warn("DASH: Invalid thumbnails Representation, no width information"); + continue; + } + tracks.push({ + id: representation.id, + cdnMetadata: representation.cdnMetadata, + index: representation.index, + mimeType: representation.mimeType, + height: representation.height, + width: representation.width, + horizontalTiles: tileInfo.horizontalTiles, + verticalTiles: tileInfo.verticalTiles, + }); + } + } + return tracks; +} + /** * Compare groups of parsed AdaptationSet, alongside some ordering metadata, * allowing to easily sort them through JavaScript's `Array.prototype.sort` diff --git a/src/parsers/manifest/dash/common/parse_periods.ts b/src/parsers/manifest/dash/common/parse_periods.ts index ffe26f737c..b5340a389b 100644 --- a/src/parsers/manifest/dash/common/parse_periods.ts +++ b/src/parsers/manifest/dash/common/parse_periods.ts @@ -126,7 +126,10 @@ export default function parsePeriods( start: periodStart, unsafelyBaseOnPreviousPeriod, }; - const adaptations = parseAdaptationSets(periodIR.children.adaptations, adapCtxt); + const { adaptations, thumbnailTracks } = parseAdaptationSets( + periodIR.children.adaptations, + adapCtxt, + ); const namespaces = (context.xmlNamespaces ?? []).concat( periodIR.attributes.namespaces ?? [], @@ -141,6 +144,7 @@ export default function parsePeriods( start: periodStart, end: periodEnd, duration: periodDuration, + thumbnailTracks, adaptations, streamEvents, }; diff --git a/src/parsers/manifest/dash/common/parse_representations.ts b/src/parsers/manifest/dash/common/parse_representations.ts index ce8faaceeb..9e138f0328 100644 --- a/src/parsers/manifest/dash/common/parse_representations.ts +++ b/src/parsers/manifest/dash/common/parse_representations.ts @@ -101,6 +101,7 @@ function getHDRInformation({ /** * Process intermediate representations to create final parsed representations. + * In the same order. * @param {Array.} representationsIR * @param {Object} context * @returns {Array.} diff --git a/src/parsers/manifest/dash/fast-js-parser/node_parsers/Representation.ts b/src/parsers/manifest/dash/fast-js-parser/node_parsers/Representation.ts index 6869a1f6fe..4f0a364ccf 100644 --- a/src/parsers/manifest/dash/fast-js-parser/node_parsers/Representation.ts +++ b/src/parsers/manifest/dash/fast-js-parser/node_parsers/Representation.ts @@ -103,6 +103,13 @@ function parseRepresentationChildren( } break; } + case "EssentialProperty": + if (isNullOrUndefined(children.essentialProperties)) { + children.essentialProperties = [parseScheme(currentElement)]; + } else { + children.essentialProperties.push(parseScheme(currentElement)); + } + break; case "SupplementalProperty": if (isNullOrUndefined(children.supplementalProperties)) { children.supplementalProperties = [parseScheme(currentElement)]; diff --git a/src/parsers/manifest/dash/native-parser/node_parsers/Representation.ts b/src/parsers/manifest/dash/native-parser/node_parsers/Representation.ts index 8a86659dd1..0fb76f5e78 100644 --- a/src/parsers/manifest/dash/native-parser/node_parsers/Representation.ts +++ b/src/parsers/manifest/dash/native-parser/node_parsers/Representation.ts @@ -100,6 +100,13 @@ function parseRepresentationChildren( } break; } + case "EssentialProperty": + if (isNullOrUndefined(children.essentialProperties)) { + children.essentialProperties = [parseScheme(currentElement)]; + } else { + children.essentialProperties.push(parseScheme(currentElement)); + } + break; case "SupplementalProperty": if (isNullOrUndefined(children.supplementalProperties)) { children.supplementalProperties = [parseScheme(currentElement)]; diff --git a/src/parsers/manifest/dash/node_parser_types.ts b/src/parsers/manifest/dash/node_parser_types.ts index 70083f2fff..80a65d62f8 100644 --- a/src/parsers/manifest/dash/node_parser_types.ts +++ b/src/parsers/manifest/dash/node_parser_types.ts @@ -261,6 +261,7 @@ export interface IRepresentationChildren { segmentList?: ISegmentListIntermediateRepresentation; segmentTemplate?: ISegmentTemplateIntermediateRepresentation; supplementalProperties?: IScheme[] | undefined; + essentialProperties?: IScheme[] | undefined; } /* Intermediate representation for A Representation node's attributes. */ diff --git a/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts b/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts index ff8c4ff117..411cd58cca 100644 --- a/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts +++ b/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts @@ -87,6 +87,17 @@ export function generateRepresentationChildrenParser( break; } + case TagName.EssentialProperty: { + const essentialProperty = {}; + if (childrenObj.essentialProperties === undefined) { + childrenObj.essentialProperties = []; + } + childrenObj.essentialProperties.push(essentialProperty); + const attributeParser = generateSchemeAttrParser(essentialProperty, linearMemory); + parsersStack.pushParsers(nodeId, noop, attributeParser); + break; + } + case TagName.SupplementalProperty: { const supplementalProperty = {}; if (childrenObj.supplementalProperties === undefined) { diff --git a/src/parsers/manifest/local/parse_local_manifest.ts b/src/parsers/manifest/local/parse_local_manifest.ts index 2ae6530f27..3e61a05b9f 100644 --- a/src/parsers/manifest/local/parse_local_manifest.ts +++ b/src/parsers/manifest/local/parse_local_manifest.ts @@ -93,6 +93,7 @@ function parsePeriod( start: period.start, end: period.end, duration: period.end - period.start, + thumbnailTracks: [], adaptations: period.adaptations.reduce>>( (acc, ada) => { const type = ada.type; diff --git a/src/parsers/manifest/metaplaylist/metaplaylist_parser.ts b/src/parsers/manifest/metaplaylist/metaplaylist_parser.ts index 35eefba8a1..956e5ef46e 100644 --- a/src/parsers/manifest/metaplaylist/metaplaylist_parser.ts +++ b/src/parsers/manifest/metaplaylist/metaplaylist_parser.ts @@ -308,6 +308,7 @@ function createManifest( adaptations, duration: currentPeriod.duration, start: contentOffset + currentPeriod.start, + thumbnailTracks: currentPeriod.thumbnailTracks, }; manifestPeriods.push(newPeriod); } diff --git a/src/parsers/manifest/smooth/create_parser.ts b/src/parsers/manifest/smooth/create_parser.ts index b8ba8c6085..9f66254252 100644 --- a/src/parsers/manifest/smooth/create_parser.ts +++ b/src/parsers/manifest/smooth/create_parser.ts @@ -681,6 +681,7 @@ function createSmoothStreamingParser( end: periodEnd, id: "gen-smooth-period-0", start: periodStart, + thumbnailTracks: [], }, ], suggestedPresentationDelay, diff --git a/src/parsers/manifest/types.ts b/src/parsers/manifest/types.ts index 629f9d4b20..b1171a932a 100644 --- a/src/parsers/manifest/types.ts +++ b/src/parsers/manifest/types.ts @@ -103,6 +103,52 @@ export interface ICdnMetadata { id?: string | undefined; } +/** Information linked to an image thumbnail track. */ +export interface IParsedThumbnailTrack { + /** Identifier for that thumbnail track. */ + id: string; + /** + * Information on the CDN(s) on which requests should be done to request + * thumbnails. + * + * `null` if there's no CDN involved here (e.g. resources are not + * requested through the network). + * + * An empty array means that no CDN are left to request the resource. As such, + * no resource can be loaded in that situation. + */ + cdnMetadata: ICdnMetadata[] | null; + /** Interface allowing to get timed thumbnail metadata to then be able to fetch them. */ + index: IRepresentationIndex; + /** + * Mimetype of the image thumbnails available here. + * Allows to know the image format (e.g. jpeg, png etc.) + */ + mimeType: string; + /** + * A loaded thumbnail's height in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + height: number; + /** + * A loaded thumbnail's width in pixels. Note that there can be multiple actual + * thumbnails per loaded thumbnail resource (see `horizontalTiles` and + * `verticalTiles` properties. + */ + width: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained horizontally in a whole loaded thumbnail resource. + */ + horizontalTiles: number; + /** + * Thumbnail tracks are usually grouped together. This is the number of + * images contained vertically in a whole loaded thumbnail resource. + */ + verticalTiles: number; +} + /** Representation of a "quality" available in an Adaptation. */ export interface IParsedRepresentation { /** Maximum bitrate the Representation is available in, in bits per seconds. */ @@ -269,6 +315,7 @@ export interface IParsedPeriod { * `undefined` if no parsed stream event in manifest. */ streamEvents?: IManifestStreamEvent[] | undefined; + thumbnailTracks: IParsedThumbnailTrack[]; } /** Information on the whole content */ diff --git a/src/public_types.ts b/src/public_types.ts index aa5e14a481..2ae3c3cafa 100644 --- a/src/public_types.ts +++ b/src/public_types.ts @@ -1284,3 +1284,65 @@ export interface IModeInformation { isDirectFile: boolean; useWorker: boolean; } + +/** Information returned by the `getAvailableThumbnailsTracks` method. */ +export interface IThumbnailTrackInfo { + /** Identifier identifying a particular thumbnail track. */ + id: string; + /** + * Width in pixels of the individual thumbnails available in that + * thumbnail track. + */ + width: number | undefined; + /** + * Height in pixels of the individual thumbnails available in that + * thumbnail track. + */ + height: number | undefined; + /** + * Expected mime-type of the images in that thumbnail track (e.g. + * `image/jpeg` or `image/png`. + */ + mimeType: string | undefined; +} + +/** + * Options that can be provided to the `renderThumbnail` method + */ +export interface IThumbnailRenderingOptions { + /** + * HTMLElement inside which the thumbnail should be displayed. + * + * The resulting thumbnail will fill that container if the thumbnail loading + * and rendering operations succeeds. + * + * If there was already a thumbnail rendering request on that container, the + * previous operation is cancelled. + */ + container: HTMLElement; + /** Position, in seconds, for which you want to provide an image thumbnail. */ + time: number; + /** + * If set to `true`, we'll keep the potential previous thumbnail found inside + * the container if the current `renderThumbnail` call fail on an error. + * We'll still replace it if the new `renderThumbnail` call succeeds (with the + * new thumbnail). + * + * If set to `false`, to `undefined`, or not set, the previous thumbnail + * potentially found inside the container will also be removed if the new + * new `renderThumbnail` call fails. + * + * The default behavior (equivalent to `false`) is generally more expected, as + * you usually don't want to provide an unrelated preview thumbnail for a + * completely different time and prefer to display no thumbnail at all. + */ + keepPreviousThumbnailOnError?: boolean | undefined; + /** + * If set, specify from which thumbnail track you want to display the + * thumbnail from. That identifier can be obtained from the + * `getThumbnailMetadata` call (the `id` property). + * + * This is mainly useful when encountering multiple thumbnail track qualities. + */ + thumbnailTrackId?: string | undefined; +} diff --git a/src/transports/dash/pipelines.ts b/src/transports/dash/pipelines.ts index edf71825bb..0321c5d0ba 100644 --- a/src/transports/dash/pipelines.ts +++ b/src/transports/dash/pipelines.ts @@ -23,6 +23,7 @@ import generateSegmentLoader from "./segment_loader"; import generateAudioVideoSegmentParser from "./segment_parser"; import generateTextTrackLoader from "./text_loader"; import generateTextTrackParser from "./text_parser"; +import { loadThumbnail, parseThumbnail } from "./thumbnails"; /** * Returns pipelines used for DASH streaming. @@ -55,6 +56,10 @@ export default function (options: ITransportOptions): ITransportPipelines { parseSegment: audioVideoSegmentParser, }, text: { loadSegment: textTrackLoader, parseSegment: textTrackParser }, + thumbnails: { + loadThumbnail, + parseThumbnail, + }, }; } diff --git a/src/transports/dash/thumbnails.ts b/src/transports/dash/thumbnails.ts new file mode 100644 index 0000000000..60d508cfdf --- /dev/null +++ b/src/transports/dash/thumbnails.ts @@ -0,0 +1,96 @@ +import type { ISegment } from "../../manifest"; +import type { ICdnMetadata } from "../../parsers/manifest"; +import request from "../../utils/request/xhr"; +import type { CancellationSignal } from "../../utils/task_canceller"; +import type { + IRequestedData, + IThumbnailContext, + IThumbnailLoaderOptions, + IThumbnailResponse, +} from "../types"; +import addQueryString from "../utils/add_query_string"; +import byteRange from "../utils/byte_range"; +import constructSegmentUrl from "./construct_segment_url"; + +/** + * Load thumbnails for DASH content. + * @param {Object|null} wantedCdn + * @param {Object} thumbnail + * @param {Object} options + * @param {Object} cancelSignal + * @returns {Promise} + */ +export async function loadThumbnail( + wantedCdn: ICdnMetadata | null, + thumbnail: ISegment, + options: IThumbnailLoaderOptions, + cancelSignal: CancellationSignal, +): Promise> { + const initialUrl = constructSegmentUrl(wantedCdn, thumbnail); + if (initialUrl === null) { + return Promise.reject(new Error("Cannot load thumbnail: no URL")); + } + const url = + options.cmcdPayload?.type === "query" + ? addQueryString(initialUrl, options.cmcdPayload.value) + : initialUrl; + + const cmcdHeaders = + options.cmcdPayload?.type === "headers" ? options.cmcdPayload.value : undefined; + + let headers; + if (thumbnail.range !== undefined) { + headers = { + ...cmcdHeaders, + Range: byteRange(thumbnail.range), + }; + } else if (cmcdHeaders !== undefined) { + headers = cmcdHeaders; + } + return request({ + url, + responseType: "arraybuffer", + headers, + timeout: options.timeout, + connectionTimeout: options.connectionTimeout, + cancelSignal, + }); +} + +/** + * Parse loaded thumbnail data into exploitable thumbnail data and metadata. + * @param {ArrayBuffer} data - The loaded thumbnail data + * @param {Object} context + * @returns {Object} + */ +export function parseThumbnail( + data: ArrayBuffer, + context: IThumbnailContext, +): IThumbnailResponse { + const { thumbnailTrack, thumbnail: wantedThumbnail } = context; + const height = thumbnailTrack.height / thumbnailTrack.verticalTiles; + const width = thumbnailTrack.width / thumbnailTrack.horizontalTiles; + const thumbnails = []; + const tileDuration = + (wantedThumbnail.end - wantedThumbnail.time) / + (thumbnailTrack.horizontalTiles * thumbnailTrack.verticalTiles); + let start = wantedThumbnail.time; + for (let row = 0; row < thumbnailTrack.verticalTiles; row++) { + for (let column = 0; column < thumbnailTrack.horizontalTiles; column++) { + thumbnails.push({ + start, + end: start + tileDuration, + offsetX: Math.round(column * width), + offsetY: Math.round(row * height), + height: Math.floor(height), + width: Math.floor(width), + }); + start += tileDuration; + } + } + return { + mimeType: thumbnailTrack.mimeType, + data, + thumbnails, + }; +} diff --git a/src/transports/local/pipelines.ts b/src/transports/local/pipelines.ts index d24733ee31..4b3668e7c7 100644 --- a/src/transports/local/pipelines.ts +++ b/src/transports/local/pipelines.ts @@ -93,5 +93,14 @@ export default function getLocalManifestPipelines( audio: segmentPipeline, video: segmentPipeline, text: textTrackPipeline, + thumbnails: { + loadThumbnail: () => + Promise.reject( + new Error("Thumbnail tracks aren't implemented with the local transport"), + ), + parseThumbnail: () => { + throw new Error("Thumbnail tracks aren't implemented with the local transport"); + }, + }, }; } diff --git a/src/transports/metaplaylist/pipelines.ts b/src/transports/metaplaylist/pipelines.ts index 722b21bcd2..5e380c634f 100644 --- a/src/transports/metaplaylist/pipelines.ts +++ b/src/transports/metaplaylist/pipelines.ts @@ -398,5 +398,14 @@ export default function (options: ITransportOptions): ITransportPipelines { audio: audioPipeline, video: videoPipeline, text: textTrackPipeline, + thumbnails: { + loadThumbnail: () => + Promise.reject( + new Error("Thumbnail tracks aren't implemented with MetaPlaylist"), + ), + parseThumbnail: () => { + throw new Error("Thumbnail tracks aren't implemented with MetaPlaylist"); + }, + }, }; } diff --git a/src/transports/smooth/pipelines.ts b/src/transports/smooth/pipelines.ts index 3378391d5e..07612de933 100644 --- a/src/transports/smooth/pipelines.ts +++ b/src/transports/smooth/pipelines.ts @@ -429,5 +429,12 @@ export default function (transportOptions: ITransportOptions): ITransportPipelin audio: audioVideoPipeline, video: audioVideoPipeline, text: textTrackPipeline, + thumbnails: { + loadThumbnail: () => + Promise.reject(new Error("Thumbnail tracks aren't implemented with smooth")), + parseThumbnail: () => { + throw new Error("Thumbnail tracks aren't implemented with smooth"); + }, + }, }; } diff --git a/src/transports/types.ts b/src/transports/types.ts index 2b7a137ca3..206ae03f28 100644 --- a/src/transports/types.ts +++ b/src/transports/types.ts @@ -16,6 +16,7 @@ import type { IInbandEvent } from "../core/types"; import type { IManifest, ISegment } from "../manifest"; +import type { IThumbnailTrackMetadata } from "../manifest/types"; import type { ICdnMetadata } from "../parsers/manifest"; import type { ITrackType, @@ -62,6 +63,8 @@ export interface ITransportPipelines { >; /** Functions allowing to load an parse text (e.g. subtitles) segments. */ text: ISegmentPipeline; + /** Functions allowing to load image thumbnails. */ + thumbnails: IThumbnailPipeline; } /** Name describing the transport pipeline. */ @@ -309,6 +312,46 @@ export interface IManifestParserOptions { unsafeMode: boolean; } +/** "Pipeline" for image thumbnails. */ +export interface IThumbnailPipeline { + loadThumbnail: IThumbnailLoader; + parseThumbnail: IThumbnailParser; +} + +export type IThumbnailLoader = ( + wantedCdn: ICdnMetadata | null, + thumbnail: ISegment, + options: IThumbnailLoaderOptions, + cancelSignal: CancellationSignal, +) => Promise>; + +export type IThumbnailParser = ( + loadedThumbnail: ArrayBuffer, + context: IThumbnailContext, +) => IThumbnailResponse; + +export interface IThumbnailContext { + /** Metadata about the wanted thumbnail. */ + thumbnail: ISegment; + /** Metadata on the thumbnail track linked to that thumbnail. */ + thumbnailTrack: IThumbnailTrackMetadata; +} + +export interface IThumbnailResponse { + mimeType: string; + data: ArrayBuffer; + thumbnails: Array<{ + height: number; + width: number; + offsetX: number; + offsetY: number; + start: number; + end: number; + }>; +} + +export type IThumbnailLoaderOptions = ISegmentLoaderOptions; + export interface IManifestParserCallbacks { onWarning: (warning: Error) => void;