From 19f9f03f94920fd19ef9ba03bda7c9473f2c57a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaquim=20St=C3=A4hli?= Date: Fri, 22 Nov 2024 17:16:56 +0100 Subject: [PATCH] Sprite sheet (#777) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Gaëtan Muller Co-authored-by: Gaëtan Muller --- gradle/libs.versions.toml | 1 + pillarbox-core-business/build.gradle.kts | 1 + .../business/integrationlayer/data/Chapter.kt | 2 + .../integrationlayer/data/SpriteSheet.kt | 29 +++ .../core/business/source/SRGAssetLoader.kt | 8 +- .../business/source/SRGAssetLoaderConfig.kt | 24 +++ .../core/business/source/SpriteSheetLoader.kt | 45 +++++ .../business/source/SpriteSheetMediaPeriod.kt | 186 ++++++++++++++++++ .../business/source/SpriteSheetMediaSource.kt | 52 +++++ .../CommandersActTrackerIntegrationTest.kt | 6 + .../ComScoreTrackerIntegrationTest.kt | 6 + .../demo/shared/ui/NavigationRoutes.kt | 3 + .../demo/ui/showcases/ShowcasesHome.kt | 8 + .../demo/ui/showcases/ShowcasesNavigation.kt | 5 +- .../layouts/thumbnail/ThumbnailView.kt | 50 +++++ .../layouts/thumbnail/ThumbnailViewModel.kt | 87 ++++++++ .../src/main/res/values/strings.xml | 1 + .../pillarbox/player/extension/Tracks.kt | 10 + pillarbox-ui/build.gradle.kts | 2 +- .../ui/SmoothProgressTrackerState.kt | 26 ++- 20 files changed, 541 insertions(+), 11 deletions(-) create mode 100644 pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/integrationlayer/data/SpriteSheet.kt create mode 100644 pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetLoader.kt create mode 100644 pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaPeriod.kt create mode 100644 pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaSource.kt create mode 100644 pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/layouts/thumbnail/ThumbnailView.kt create mode 100644 pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/layouts/thumbnail/ThumbnailViewModel.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7e93b0ac9..f94e1e707 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -102,6 +102,7 @@ androidx-media3-cast = { group = "androidx.media3", name = "media3-cast", versio androidx-media3-common = { group = "androidx.media3", name = "media3-common", version.ref = "androidx-media3" } androidx-media3-datasource = { group = "androidx.media3", name = "media3-datasource", version.ref = "androidx-media3" } androidx-media3-datasource-okhttp = { group = "androidx.media3", name = "media3-datasource-okhttp", version.ref = "androidx-media3" } +androidx-media3-decoder = { group = "androidx.media3", name = "media3-decoder", version.ref = "androidx-media3" } androidx-media3-exoplayer = { group = "androidx.media3", name = "media3-exoplayer", version.ref = "androidx-media3" } androidx-media3-ui = { group = "androidx.media3", name = "media3-ui", version.ref = "androidx-media3" } androidx-media3-dash = { group = "androidx.media3", name = "media3-exoplayer-dash", version.ref = "androidx-media3" } diff --git a/pillarbox-core-business/build.gradle.kts b/pillarbox-core-business/build.gradle.kts index 9355ed6b0..149643242 100644 --- a/pillarbox-core-business/build.gradle.kts +++ b/pillarbox-core-business/build.gradle.kts @@ -26,6 +26,7 @@ dependencies { api(libs.androidx.media3.common) api(libs.androidx.media3.datasource) implementation(libs.androidx.media3.datasource.okhttp) + implementation(libs.androidx.media3.decoder) api(libs.androidx.media3.exoplayer) implementation(libs.guava) runtimeOnly(libs.kotlinx.coroutines.android) diff --git a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/integrationlayer/data/Chapter.kt b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/integrationlayer/data/Chapter.kt index 703c5740e..4b7b4dddf 100644 --- a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/integrationlayer/data/Chapter.kt +++ b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/integrationlayer/data/Chapter.kt @@ -28,6 +28,7 @@ import kotlinx.serialization.Serializable * @property timeIntervalList List of time intervals relevant to the chapter. * @property validFrom The [Instant] when the [Chapter] becomes valid. * @property validTo The [Instant] until when the [Chapter] is valid. + * @property spriteSheet The [SpriteSheet] information if available. */ @Serializable data class Chapter( @@ -51,6 +52,7 @@ data class Chapter( val timeIntervalList: List? = null, val validFrom: Instant? = null, val validTo: Instant? = null, + val spriteSheet: SpriteSheet? = null, ) : DataWithAnalytics { /** * Indicates whether this represents a full-length chapter. diff --git a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/integrationlayer/data/SpriteSheet.kt b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/integrationlayer/data/SpriteSheet.kt new file mode 100644 index 000000000..9927319dc --- /dev/null +++ b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/integrationlayer/data/SpriteSheet.kt @@ -0,0 +1,29 @@ +/* + * Copyright (c) SRG SSR. All rights reserved. + * License information is available from the LICENSE file. + */ +package ch.srgssr.pillarbox.core.business.integrationlayer.data + +import kotlinx.serialization.Serializable + +/** + * Represents a sprite sheet containing multiple thumbnail images arranged in a grid. + * + * @property urn The URN of the media. + * @property rows The number of rows in the sprite sheet. + * @property columns The number of columns in the sprite sheet. + * @property thumbnailHeight The height of each thumbnail image, in pixels. + * @property thumbnailWidth The width of each thumbnail image, in pixels. + * @property interval The interval between two thumbnail images, in milliseconds. + * @property url The URL of the sprite sheet image. + */ +@Serializable +data class SpriteSheet( + val urn: String, + val rows: Int, + val columns: Int, + val thumbnailHeight: Int, + val thumbnailWidth: Int, + val interval: Long, + val url: String +) diff --git a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoader.kt b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoader.kt index a8f0b3684..0078b0234 100644 --- a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoader.kt +++ b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoader.kt @@ -12,6 +12,7 @@ import androidx.media3.common.MediaMetadata import androidx.media3.common.MimeTypes import androidx.media3.datasource.DataSource.Factory import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import androidx.media3.exoplayer.source.MergingMediaSource import ch.srgssr.pillarbox.core.business.HttpResultException import ch.srgssr.pillarbox.core.business.akamai.AkamaiTokenDataSource import ch.srgssr.pillarbox.core.business.akamai.AkamaiTokenProvider @@ -99,6 +100,7 @@ class SRGAssetLoader internal constructor( private val customTrackerData: (MutableMediaItemTrackerData.(Resource, Chapter, MediaComposition) -> Unit)?, private val customMediaMetadata: (suspend MediaMetadata.Builder.(MediaMetadata, Chapter, MediaComposition) -> Unit)?, private val resourceSelector: ResourceSelector, + private val spriteSheetLoader: SpriteSheetLoader, ) : AssetLoader( mediaSourceFactory = DefaultMediaSourceFactory(AkamaiTokenDataSource.Factory(akamaiTokenProvider, dataSourceFactory)) ) { @@ -157,8 +159,12 @@ class SRGAssetLoader internal constructor( .setDrmConfiguration(fillDrmConfiguration(resource)) .setUri(uri) .build() + val contentMediaSource = mediaSourceFactory.createMediaSource(loadingMediaItem) + val mediaSource = chapter.spriteSheet?.let { + MergingMediaSource(contentMediaSource, SpriteSheetMediaSource(it, loadingMediaItem, spriteSheetLoader)) + } ?: contentMediaSource return Asset( - mediaSource = mediaSourceFactory.createMediaSource(loadingMediaItem), + mediaSource = mediaSource, trackersData = trackerData.toMediaItemTrackerData(), mediaMetadata = mediaItem.mediaMetadata.buildUpon().apply { defaultMediaMetadata.invoke(this, mediaItem.mediaMetadata, chapter, result) diff --git a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoaderConfig.kt b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoaderConfig.kt index 6482f4377..5da310ef0 100644 --- a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoaderConfig.kt +++ b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoaderConfig.kt @@ -5,6 +5,7 @@ package ch.srgssr.pillarbox.core.business.source import android.content.Context +import android.graphics.Bitmap import androidx.annotation.VisibleForTesting import androidx.media3.common.MediaMetadata import androidx.media3.datasource.DataSource.Factory @@ -16,6 +17,7 @@ import ch.srgssr.pillarbox.core.business.integrationlayer.ResourceSelector import ch.srgssr.pillarbox.core.business.integrationlayer.data.Chapter import ch.srgssr.pillarbox.core.business.integrationlayer.data.MediaComposition import ch.srgssr.pillarbox.core.business.integrationlayer.data.Resource +import ch.srgssr.pillarbox.core.business.integrationlayer.data.SpriteSheet import ch.srgssr.pillarbox.core.business.integrationlayer.service.HttpMediaCompositionService import ch.srgssr.pillarbox.core.business.integrationlayer.service.MediaCompositionService import ch.srgssr.pillarbox.core.business.tracker.commandersact.CommandersActTracker @@ -37,6 +39,7 @@ import kotlinx.coroutines.Dispatchers * - Setting an HTTP client for network requests. * - Injecting custom data into media item tracker data. * - Overriding the default media metadata. + * - Providing a custom [Bitmap] loader for sprite sheet. * * @param context The Android [Context]. */ @@ -50,6 +53,7 @@ class SRGAssetLoaderConfig internal constructor(context: Context) { private var commanderActTrackerFactory: MediaItemTracker.Factory = CommandersActTracker.Factory(SRGAnalytics.commandersAct, Dispatchers.Default) private var comscoreTrackerFactory: MediaItemTracker.Factory = ComScoreTracker.Factory() + private var spriteSheetLoader: SpriteSheetLoader = SpriteSheetLoader.Default() @VisibleForTesting internal fun commanderActTrackerFactory(commanderActTrackerFactory: MediaItemTracker.Factory) { @@ -142,6 +146,25 @@ class SRGAssetLoaderConfig internal constructor(context: Context) { mediaMetadataOverride = block } + /** + * Sets the [SpriteSheetLoader] to be used to load a [Bitmap] from a [SpriteSheet]. + * + * **Example** + * + * ```kotlin + * val srgAssetLoader = SRGAssetLoader(context) { + * spriteSheetLoader { spriteSheet, onComplete -> + * onComplete(loadBitmap(spriteSheet.url)) + * } + * } + * ``` + * + * @param spriteSheetLoader The [SpriteSheetLoader] instance to use. + */ + fun spriteSheetLoader(spriteSheetLoader: SpriteSheetLoader) { + this.spriteSheetLoader = spriteSheetLoader + } + internal fun create(): SRGAssetLoader { return SRGAssetLoader( akamaiTokenProvider = akamaiTokenProvider, @@ -152,6 +175,7 @@ class SRGAssetLoaderConfig internal constructor(context: Context) { comscoreTrackerFactory = comscoreTrackerFactory, mediaCompositionService = mediaCompositionService, resourceSelector = ResourceSelector(), + spriteSheetLoader = spriteSheetLoader ) } } diff --git a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetLoader.kt b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetLoader.kt new file mode 100644 index 000000000..b70673d92 --- /dev/null +++ b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetLoader.kt @@ -0,0 +1,45 @@ +/* + * Copyright (c) SRG SSR. All rights reserved. + * License information is available from the LICENSE file. + */ +package ch.srgssr.pillarbox.core.business.source + +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import ch.srgssr.pillarbox.core.business.integrationlayer.data.SpriteSheet +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.launch +import java.net.URL + +/** + * Load a [Bitmap] from a [SpriteSheet]. + * + * This interface allows integrators to use their own implementation to load [Bitmap]s using an external library like + * [Glide](https://bumptech.github.io/glide/), [Coil](https://coil-kt.github.io/coil/), ... + */ +fun interface SpriteSheetLoader { + /** + * Load sprite sheet + * + * @param spriteSheet The [SpriteSheet] to load the [Bitmap] from. + * @param onComplete The callback to call when the [Bitmap] has been loaded. Passing `null` means that the [Bitmap] could not be loaded. + */ + fun loadSpriteSheet(spriteSheet: SpriteSheet, onComplete: (Bitmap?) -> Unit) + + /** + * Default + * + * @param dispatcher The [CoroutineDispatcher] to use for loading the sprite sheet. Should not be on the main thread. + */ + class Default(private val dispatcher: CoroutineDispatcher = Dispatchers.IO) : SpriteSheetLoader { + override fun loadSpriteSheet(spriteSheet: SpriteSheet, onComplete: (Bitmap?) -> Unit) { + MainScope().launch(dispatcher) { + URL(spriteSheet.url).openStream().use { + onComplete(BitmapFactory.decodeStream(it)) + } + } + } + } +} diff --git a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaPeriod.kt b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaPeriod.kt new file mode 100644 index 000000000..a3b877da8 --- /dev/null +++ b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaPeriod.kt @@ -0,0 +1,186 @@ +/* + * Copyright (c) SRG SSR. All rights reserved. + * License information is available from the LICENSE file. + */ +package ch.srgssr.pillarbox.core.business.source + +import android.graphics.Bitmap +import androidx.media3.common.C +import androidx.media3.common.Format +import androidx.media3.common.MimeTypes +import androidx.media3.common.TrackGroup +import androidx.media3.decoder.DecoderInputBuffer +import androidx.media3.exoplayer.FormatHolder +import androidx.media3.exoplayer.LoadingInfo +import androidx.media3.exoplayer.SeekParameters +import androidx.media3.exoplayer.source.MediaPeriod +import androidx.media3.exoplayer.source.SampleStream +import androidx.media3.exoplayer.source.TrackGroupArray +import androidx.media3.exoplayer.trackselection.ExoTrackSelection +import ch.srgssr.pillarbox.core.business.integrationlayer.data.SpriteSheet +import java.io.ByteArrayOutputStream +import java.io.IOException +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.math.max +import kotlin.time.Duration.Companion.milliseconds + +/** + * A [MediaPeriod] that loads a [Bitmap] and pass it to a [SampleStream]. + */ +internal class SpriteSheetMediaPeriod( + private val spriteSheet: SpriteSheet, + private val spriteSheetLoader: SpriteSheetLoader, +) : MediaPeriod { + private var bitmap: Bitmap? = null + private val isLoading = AtomicBoolean(true) + private val format = Format.Builder() + .setId("SpriteSheet") + .setFrameRate(1f / spriteSheet.interval.milliseconds.inWholeSeconds) + .setCustomData(spriteSheet) + .setRoleFlags(C.ROLE_FLAG_MAIN) + .setContainerMimeType(MimeTypes.IMAGE_JPEG) + .setSampleMimeType(MimeTypes.IMAGE_JPEG) + .build() + private val tracks = TrackGroupArray(TrackGroup("sprite-sheet-srg", format)) + private var positionUs = 0L + + override fun prepare(callback: MediaPeriod.Callback, positionUs: Long) { + callback.onPrepared(this) + this.positionUs = positionUs + isLoading.set(true) + spriteSheetLoader.loadSpriteSheet(spriteSheet) { bitmap -> + this.bitmap = bitmap + isLoading.set(false) + } + } + + fun releasePeriod() { + bitmap?.recycle() + bitmap = null + } + + override fun selectTracks( + selections: Array, + mayRetainStreamFlags: BooleanArray, + streams: Array, + streamResetFlags: BooleanArray, + positionUs: Long + ): Long { + this.positionUs = getAdjustedSeekPositionUs(positionUs, SeekParameters.DEFAULT) + for (i in selections.indices) { + if (streams[i] != null && (selections[i] == null || !mayRetainStreamFlags[i])) { + streams[i] = null + } + if (streams[i] == null && selections[i] != null) { + val stream = SpriteSheetSampleStream() + streams[i] = stream + streamResetFlags[i] = true + } + } + return positionUs + } + + override fun getTrackGroups(): TrackGroupArray { + return tracks + } + + override fun getBufferedPositionUs(): Long { + return C.TIME_UNSET + } + + override fun getNextLoadPositionUs(): Long { + return C.TIME_UNSET + } + + override fun continueLoading(loadingInfo: LoadingInfo): Boolean { + return isLoading() + } + + override fun isLoading(): Boolean { + return isLoading.get() + } + + override fun reevaluateBuffer(positionUs: Long) = Unit + + override fun maybeThrowPrepareError() = Unit + + override fun discardBuffer(positionUs: Long, toKeyframe: Boolean) = Unit + + override fun readDiscontinuity(): Long { + return C.TIME_UNSET + } + + override fun seekToUs(positionUs: Long): Long { + this.positionUs = positionUs + return positionUs + } + + override fun getAdjustedSeekPositionUs(positionUs: Long, seekParameters: SeekParameters): Long { + val intervalUs = spriteSheet.interval.milliseconds.inWholeMicroseconds + return (positionUs / intervalUs) * intervalUs + } + + internal inner class SpriteSheetSampleStream : SampleStream { + + override fun isReady(): Boolean { + return !isLoading() + } + + override fun maybeThrowError() { + if (bitmap == null && !isLoading.get()) { + throw IOException("Can't decode ${spriteSheet.url}") + } + } + + @Suppress("ReturnCount") + override fun readData(formatHolder: FormatHolder, buffer: DecoderInputBuffer, readFlags: Int): Int { + if ((readFlags and SampleStream.FLAG_REQUIRE_FORMAT) != 0) { + formatHolder.format = tracks[0].getFormat(0) + return C.RESULT_FORMAT_READ + } + + if (isLoading.get()) { + return C.RESULT_NOTHING_READ + } + + val intervalUs = spriteSheet.interval.milliseconds.inWholeMicroseconds + val tileIndex = positionUs / intervalUs + buffer.addFlag(C.BUFFER_FLAG_KEY_FRAME) + buffer.timeUs = positionUs + bitmap?.let { bitmap -> + val data = cropTileFromImageGrid(bitmap, max((tileIndex.toInt() - 1), 0)) + buffer.ensureSpaceForWrite(data.size) + buffer.data?.put(data, /* offset= */0, data.size) + } + return C.RESULT_BUFFER_READ + } + + override fun skipData(positionUs: Long): Int { + return 0 + } + + private fun cropTileFromImageGrid(bitmap: Bitmap, tileIndex: Int): ByteArray { + val tileWidth: Int = spriteSheet.thumbnailWidth + val tileHeight: Int = spriteSheet.thumbnailHeight + val tileStartXCoordinate: Int = tileWidth * (tileIndex % spriteSheet.columns) + val tileStartYCoordinate: Int = tileHeight * (tileIndex / spriteSheet.columns) + val tile = Bitmap.createBitmap(bitmap, tileStartXCoordinate, tileStartYCoordinate, tileWidth, tileHeight) + return bitmapToByteArray(tile) + } + } + + private companion object { + private const val MAX_QUALITY = 100 + + private fun bitmapToByteArray( + bitmap: Bitmap, + format: Bitmap.CompressFormat = Bitmap.CompressFormat.JPEG, + quality: Int = MAX_QUALITY + ): ByteArray { + return ByteArrayOutputStream().use { + bitmap.compress(format, quality, it) + it.toByteArray() + } + } + } +} diff --git a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaSource.kt b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaSource.kt new file mode 100644 index 000000000..7ad26d24b --- /dev/null +++ b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaSource.kt @@ -0,0 +1,52 @@ +/* + * Copyright (c) SRG SSR. All rights reserved. + * License information is available from the LICENSE file. + */ +package ch.srgssr.pillarbox.core.business.source + +import android.graphics.Bitmap +import androidx.media3.common.MediaItem +import androidx.media3.datasource.TransferListener +import androidx.media3.exoplayer.source.BaseMediaSource +import androidx.media3.exoplayer.source.MediaPeriod +import androidx.media3.exoplayer.source.MediaSource +import androidx.media3.exoplayer.source.SinglePeriodTimeline +import androidx.media3.exoplayer.upstream.Allocator +import ch.srgssr.pillarbox.core.business.integrationlayer.data.SpriteSheet +import kotlin.time.Duration.Companion.milliseconds + +/** + * An implementation of a [BaseMediaSource] that loads a [SpriteSheet]. + * + * @param spriteSheet The [SpriteSheet] to build thumbnails. + * @param mediaItem The [MediaItem]. + * @param spriteSheetLoader The [SpriteSheetLoader] to use to load a [Bitmap] from a [SpriteSheet]. + */ +internal class SpriteSheetMediaSource( + private val spriteSheet: SpriteSheet, + private val mediaItem: MediaItem, + private val spriteSheetLoader: SpriteSheetLoader, +) : BaseMediaSource() { + + override fun getMediaItem(): MediaItem { + return mediaItem + } + + override fun maybeThrowSourceInfoRefreshError() = Unit + + override fun createPeriod(id: MediaSource.MediaPeriodId, allocator: Allocator, startPositionUs: Long): MediaPeriod { + return SpriteSheetMediaPeriod(spriteSheet, spriteSheetLoader) + } + + override fun releasePeriod(mediaPeriod: MediaPeriod) { + (mediaPeriod as SpriteSheetMediaPeriod).releasePeriod() + } + + override fun prepareSourceInternal(mediaTransferListener: TransferListener?) { + val duration = (spriteSheet.rows * spriteSheet.columns * spriteSheet.interval).milliseconds + val timeline = SinglePeriodTimeline(duration.inWholeMicroseconds, true, false, false, null, mediaItem) + refreshSourceInfo(timeline) + } + + override fun releaseSourceInternal() = Unit +} diff --git a/pillarbox-core-business/src/test/java/ch/srgssr/pillarbox/core/business/tracker/commandersact/CommandersActTrackerIntegrationTest.kt b/pillarbox-core-business/src/test/java/ch/srgssr/pillarbox/core/business/tracker/commandersact/CommandersActTrackerIntegrationTest.kt index 210e2ff4c..4944b9696 100644 --- a/pillarbox-core-business/src/test/java/ch/srgssr/pillarbox/core/business/tracker/commandersact/CommandersActTrackerIntegrationTest.kt +++ b/pillarbox-core-business/src/test/java/ch/srgssr/pillarbox/core/business/tracker/commandersact/CommandersActTrackerIntegrationTest.kt @@ -6,6 +6,7 @@ package ch.srgssr.pillarbox.core.business.tracker.commandersact import android.content.Context import android.os.Looper +import androidx.media3.common.C import androidx.media3.common.MediaItem import androidx.media3.common.Player import androidx.media3.exoplayer.ExoPlayer @@ -79,6 +80,11 @@ class CommandersActTrackerIntegrationTest { clock(clock) // Use other CoroutineContext to avoid infinite loop because Heartbeat is also running in Pillarbox. coroutineContext(EmptyCoroutineContext) + }.apply { + // FIXME Investigate why we need to disable the image track in tests + trackSelectionParameters = trackSelectionParameters.buildUpon() + .setTrackTypeDisabled(C.TRACK_TYPE_IMAGE, true) + .build() } } diff --git a/pillarbox-core-business/src/test/java/ch/srgssr/pillarbox/core/business/tracker/comscore/ComScoreTrackerIntegrationTest.kt b/pillarbox-core-business/src/test/java/ch/srgssr/pillarbox/core/business/tracker/comscore/ComScoreTrackerIntegrationTest.kt index 726e38986..8709c23fd 100644 --- a/pillarbox-core-business/src/test/java/ch/srgssr/pillarbox/core/business/tracker/comscore/ComScoreTrackerIntegrationTest.kt +++ b/pillarbox-core-business/src/test/java/ch/srgssr/pillarbox/core/business/tracker/comscore/ComScoreTrackerIntegrationTest.kt @@ -9,6 +9,7 @@ import android.os.Looper import android.view.SurfaceView import android.view.ViewGroup import androidx.core.view.updateLayoutParams +import androidx.media3.common.C import androidx.media3.common.MediaItem import androidx.media3.common.Player import androidx.media3.test.utils.FakeClock @@ -65,6 +66,11 @@ class ComScoreTrackerIntegrationTest { comscoreTrackerFactory(comScoreFactory) commanderActTrackerFactory(mockk(relaxed = true)) } + }.apply { + // FIXME Investigate why we need to disable the image track in tests + trackSelectionParameters = trackSelectionParameters.buildUpon() + .setTrackTypeDisabled(C.TRACK_TYPE_IMAGE, true) + .build() } } diff --git a/pillarbox-demo-shared/src/main/java/ch/srgssr/pillarbox/demo/shared/ui/NavigationRoutes.kt b/pillarbox-demo-shared/src/main/java/ch/srgssr/pillarbox/demo/shared/ui/NavigationRoutes.kt index 1d70f3727..87870f358 100644 --- a/pillarbox-demo-shared/src/main/java/ch/srgssr/pillarbox/demo/shared/ui/NavigationRoutes.kt +++ b/pillarbox-demo-shared/src/main/java/ch/srgssr/pillarbox/demo/shared/ui/NavigationRoutes.kt @@ -77,4 +77,7 @@ sealed interface NavigationRoutes { @Serializable data object CountdownShowcase : NavigationRoutes + + @Serializable + data object ThumbnailShowcase : NavigationRoutes } diff --git a/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/ShowcasesHome.kt b/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/ShowcasesHome.kt index 9a9557870..4a6684066 100644 --- a/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/ShowcasesHome.kt +++ b/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/ShowcasesHome.kt @@ -85,6 +85,14 @@ fun ShowcasesHome(navController: NavController) { modifier = itemModifier, onClick = { navController.navigate(NavigationRoutes.Chapters) } ) + + HorizontalDivider() + + DemoListItemView( + title = stringResource(R.string.thumbnail), + modifier = itemModifier, + onClick = { navController.navigate(NavigationRoutes.ThumbnailShowcase) } + ) } DemoListHeaderView( diff --git a/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/ShowcasesNavigation.kt b/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/ShowcasesNavigation.kt index ba67df6ea..f0af5928d 100644 --- a/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/ShowcasesNavigation.kt +++ b/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/ShowcasesNavigation.kt @@ -14,6 +14,7 @@ import ch.srgssr.pillarbox.demo.ui.showcases.integrations.ExoPlayerShowcase import ch.srgssr.pillarbox.demo.ui.showcases.layouts.ChapterShowcase import ch.srgssr.pillarbox.demo.ui.showcases.layouts.SimpleLayoutShowcase import ch.srgssr.pillarbox.demo.ui.showcases.layouts.StoryLayoutShowcase +import ch.srgssr.pillarbox.demo.ui.showcases.layouts.thumbnail.ThumbnailView import ch.srgssr.pillarbox.demo.ui.showcases.misc.ContentNotYetAvailable import ch.srgssr.pillarbox.demo.ui.showcases.misc.MultiPlayerShowcase import ch.srgssr.pillarbox.demo.ui.showcases.misc.ResizablePlayerShowcase @@ -67,10 +68,12 @@ fun NavGraphBuilder.showcasesNavGraph(navController: NavController) { composable(DemoPageView("Chapters", Levels)) { ChapterShowcase() } - composable(DemoPageView("CountdownShowcase", Levels)) { ContentNotYetAvailable() } + composable(DemoPageView("ThumbnailShowcase", Levels)) { + ThumbnailView() + } } private val Levels = listOf("app", "pillarbox", "showcase") diff --git a/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/layouts/thumbnail/ThumbnailView.kt b/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/layouts/thumbnail/ThumbnailView.kt new file mode 100644 index 000000000..2dbe39baa --- /dev/null +++ b/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/layouts/thumbnail/ThumbnailView.kt @@ -0,0 +1,50 @@ +/* + * Copyright (c) SRG SSR. All rights reserved. + * License information is available from the LICENSE file. + */ +package ch.srgssr.pillarbox.demo.ui.showcases.layouts.thumbnail + +import android.graphics.Bitmap +import androidx.compose.foundation.Image +import androidx.compose.foundation.interaction.MutableInteractionSource +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.runtime.Composable +import androidx.compose.runtime.remember +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.asImageBitmap +import androidx.lifecycle.compose.LifecycleResumeEffect +import androidx.lifecycle.viewmodel.compose.viewModel +import ch.srgssr.pillarbox.demo.ui.player.controls.PlayerControls +import ch.srgssr.pillarbox.ui.widget.player.PlayerSurface + +/** + * Thumbnail view + */ +@Composable +fun ThumbnailView() { + val thumbnailViewModel = viewModel() + val player = thumbnailViewModel.player + LifecycleResumeEffect(player) { + player.play() + onPauseOrDispose { + player.pause() + } + } + + Box { + PlayerSurface(player) { + val thumbnail: Bitmap? = thumbnailViewModel.thumbnail + thumbnail?.let { + Image(bitmap = it.asImageBitmap(), contentDescription = null, modifier = Modifier.fillMaxSize()) + } + } + val interactionSource = remember { MutableInteractionSource() } + PlayerControls( + modifier = Modifier.matchParentSize(), + player = player, + progressTracker = thumbnailViewModel.progressTrackerState, + interactionSource = interactionSource + ) {} + } +} diff --git a/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/layouts/thumbnail/ThumbnailViewModel.kt b/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/layouts/thumbnail/ThumbnailViewModel.kt new file mode 100644 index 000000000..d775f3118 --- /dev/null +++ b/pillarbox-demo/src/main/java/ch/srgssr/pillarbox/demo/ui/showcases/layouts/thumbnail/ThumbnailViewModel.kt @@ -0,0 +1,87 @@ +/* + * Copyright (c) SRG SSR. All rights reserved. + * License information is available from the LICENSE file. + */ +package ch.srgssr.pillarbox.demo.ui.showcases.layouts.thumbnail + +import android.app.Application +import android.graphics.Bitmap +import android.graphics.drawable.BitmapDrawable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.setValue +import androidx.lifecycle.AndroidViewModel +import androidx.lifecycle.viewModelScope +import androidx.media3.exoplayer.image.ImageOutput +import ch.srgssr.pillarbox.core.business.PillarboxExoPlayer +import ch.srgssr.pillarbox.core.business.SRGMediaItem +import ch.srgssr.pillarbox.demo.shared.data.DemoItem +import ch.srgssr.pillarbox.player.network.PillarboxOkHttp +import ch.srgssr.pillarbox.ui.ProgressTrackerState +import ch.srgssr.pillarbox.ui.SmoothProgressTrackerState +import coil.imageLoader +import coil.request.ImageRequest +import coil.size.Scale + +/** + * A ViewModel to demonstrate how to work with Image track. + * + * @param application The [Application]. + */ +class ThumbnailViewModel(application: Application) : AndroidViewModel(application), ImageOutput { + private val imageLoader = application.imageLoader.newBuilder() + .okHttpClient(PillarboxOkHttp()) + .build() + + /** + * Player + */ + val player = PillarboxExoPlayer(application) { + srgAssetLoader(application) { + spriteSheetLoader { spriteSheet, onComplete -> + val request = ImageRequest.Builder(application) + .data(spriteSheet.url) + .scale(Scale.FILL) // FILL to have the source image size! + .allowConversionToBitmap(enable = true) + .target { result -> + val bitmap = (result as BitmapDrawable).bitmap + onComplete(bitmap) + } + .build() + imageLoader.enqueue(request) + } + } + } + + /** + * Thumbnail + */ + var thumbnail by mutableStateOf(null) + private set + + /** + * Progress tracker state + */ + val progressTrackerState: ProgressTrackerState = SmoothProgressTrackerState(player, viewModelScope, this) + + init { + player.prepare() + player.addMediaItem(SRGMediaItem("urn:srf:video:881be9c2-65ec-4fa9-ba4a-926d15d046ef")) + player.addMediaItem(DemoItem.OnDemandHorizontalVideo.toMediaItem()) + player.addMediaItem(SRGMediaItem("urn:rsi:video:2366175")) + player.addMediaItem(DemoItem.UnifiedStreamingOnDemand_Dash_TiledThumbnails.toMediaItem()) + player.addMediaItem(DemoItem.UnifiedStreamingOnDemand_Dash_TrickPlay.toMediaItem()) + } + + override fun onCleared() { + player.release() + } + + override fun onImageAvailable(presentationTimeUs: Long, bitmap: Bitmap) { + thumbnail = bitmap + } + + override fun onDisabled() { + thumbnail = null + } +} diff --git a/pillarbox-demo/src/main/res/values/strings.xml b/pillarbox-demo/src/main/res/values/strings.xml index 529112c83..d7afe8e43 100644 --- a/pillarbox-demo/src/main/res/values/strings.xml +++ b/pillarbox-demo/src/main/res/values/strings.xml @@ -35,6 +35,7 @@ all Pause at end of media items Chapters + Thumbnail Library version Choose text color Choose text size diff --git a/pillarbox-player/src/main/java/ch/srgssr/pillarbox/player/extension/Tracks.kt b/pillarbox-player/src/main/java/ch/srgssr/pillarbox/player/extension/Tracks.kt index 6d1daad38..8b3f93777 100644 --- a/pillarbox-player/src/main/java/ch/srgssr/pillarbox/player/extension/Tracks.kt +++ b/pillarbox-player/src/main/java/ch/srgssr/pillarbox/player/extension/Tracks.kt @@ -4,6 +4,7 @@ */ package ch.srgssr.pillarbox.player.extension +import androidx.media3.common.C import androidx.media3.common.Tracks import ch.srgssr.pillarbox.player.asset.timeRange.BlockedTimeRange import ch.srgssr.pillarbox.player.source.PillarboxMediaSource @@ -27,3 +28,12 @@ fun Tracks.getBlockedTimeRangeOrNull(): List? { it.type == PillarboxMediaSource.TRACK_TYPE_PILLARBOX_BLOCKED }?.getTrackFormat(0)?.customData as? List } + +/** + * Contains image track + * + * @return `true` if there is a track of type [C.TRACK_TYPE_IMAGE], `false` otherwise + */ +fun Tracks.containsImageTrack(): Boolean { + return containsType(C.TRACK_TYPE_IMAGE) +} diff --git a/pillarbox-ui/build.gradle.kts b/pillarbox-ui/build.gradle.kts index c294110e0..d6de4fffe 100644 --- a/pillarbox-ui/build.gradle.kts +++ b/pillarbox-ui/build.gradle.kts @@ -30,7 +30,7 @@ dependencies { implementation(libs.androidx.lifecycle.runtime.compose) implementation(libs.guava) api(libs.androidx.media3.common) - implementation(libs.androidx.media3.exoplayer) + api(libs.androidx.media3.exoplayer) api(libs.androidx.media3.ui) implementation(libs.kotlinx.coroutines.core) diff --git a/pillarbox-ui/src/main/java/ch/srgssr/pillarbox/ui/SmoothProgressTrackerState.kt b/pillarbox-ui/src/main/java/ch/srgssr/pillarbox/ui/SmoothProgressTrackerState.kt index c30c031cb..f4aeb0831 100644 --- a/pillarbox-ui/src/main/java/ch/srgssr/pillarbox/ui/SmoothProgressTrackerState.kt +++ b/pillarbox-ui/src/main/java/ch/srgssr/pillarbox/ui/SmoothProgressTrackerState.kt @@ -7,7 +7,9 @@ package ch.srgssr.pillarbox.ui import androidx.media3.common.C import androidx.media3.common.Player import androidx.media3.exoplayer.SeekParameters +import androidx.media3.exoplayer.image.ImageOutput import ch.srgssr.pillarbox.player.PillarboxExoPlayer +import ch.srgssr.pillarbox.player.extension.containsImageTrack import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.StateFlow import kotlin.time.Duration @@ -17,10 +19,12 @@ import kotlin.time.Duration * * @param player The [Player] whose progress needs to be tracked. * @param coroutineScope The [CoroutineScope] used for managing [StateFlow]s. + * @param imageOutput The [ImageOutput] to render the image track. */ class SmoothProgressTrackerState( private val player: PillarboxExoPlayer, - coroutineScope: CoroutineScope + coroutineScope: CoroutineScope, + private val imageOutput: ImageOutput = ImageOutput.NO_OP, ) : ProgressTrackerState { private var storedSeekParameters = player.seekParameters private var storedPlayWhenReady = player.playWhenReady @@ -41,13 +45,18 @@ class SmoothProgressTrackerState( player.setSeekParameters(SeekParameters.CLOSEST_SYNC) player.smoothSeekingEnabled = true player.playWhenReady = false - player.trackSelectionParameters = player.trackSelectionParameters.buildUpon() - .setPreferredVideoRoleFlags(C.ROLE_FLAG_TRICK_PLAY) - .setTrackTypeDisabled(C.TRACK_TYPE_TEXT, true) - .setTrackTypeDisabled(C.TRACK_TYPE_AUDIO, true) - .setTrackTypeDisabled(C.TRACK_TYPE_METADATA, true) - .setTrackTypeDisabled(C.TRACK_TYPE_IMAGE, true) - .build() + player.trackSelectionParameters = player.trackSelectionParameters.buildUpon().apply { + setPreferredVideoRoleFlags(C.ROLE_FLAG_TRICK_PLAY) + setTrackTypeDisabled(C.TRACK_TYPE_TEXT, true) + setTrackTypeDisabled(C.TRACK_TYPE_AUDIO, true) + setTrackTypeDisabled(C.TRACK_TYPE_METADATA, true) + if (player.currentTracks.containsImageTrack() && imageOutput != ImageOutput.NO_OP) { + setPrioritizeImageOverVideoEnabled(true) + } else { + setTrackTypeDisabled(C.TRACK_TYPE_IMAGE, true) + } + }.build() + player.setImageOutput(imageOutput) } player.seekTo(progress.inWholeMilliseconds) } @@ -59,5 +68,6 @@ class SmoothProgressTrackerState( player.smoothSeekingEnabled = storedSmoothSeeking player.setSeekParameters(storedSeekParameters) player.playWhenReady = storedPlayWhenReady + player.setImageOutput(null) } }