diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt index 96bd54bbb..9d0b312b1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt @@ -28,7 +28,8 @@ import io.github.thibaultbee.streampack.data.VideoConfig import io.github.thibaultbee.streampack.internal.gl.EglWindowSurface import io.github.thibaultbee.streampack.internal.gl.FullFrameRect import io.github.thibaultbee.streampack.internal.gl.Texture2DProgram -import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider +import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationListener +import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.utils.av.video.DynamicRangeProfile import io.github.thibaultbee.streampack.listeners.OnErrorListener import java.util.concurrent.Executors @@ -125,7 +126,7 @@ class VideoMediaCodecEncoder( class CodecSurface( private val orientationProvider: ISourceOrientationProvider? ) : - SurfaceTexture.OnFrameAvailableListener { + SurfaceTexture.OnFrameAvailableListener, ISourceOrientationListener { private var eglSurface: EglWindowSurface? = null private var fullFrameRect: FullFrameRect? = null private var textureId = -1 @@ -134,8 +135,9 @@ class VideoMediaCodecEncoder( private var surfaceTexture: SurfaceTexture? = null private val stMatrix = FloatArray(16) + private var _inputSurface: Surface? = null val inputSurface: Surface? - get() = surfaceTexture?.let { Surface(surfaceTexture) } + get() = _inputSurface /** * If true, the encoder will use high bit depth (10 bits) for encoding. @@ -162,6 +164,10 @@ class VideoMediaCodecEncoder( field = value } + init { + orientationProvider?.addListener(this) + } + private fun initOrUpdateSurfaceTexture(surface: Surface) { eglSurface = ensureGlContext(EglWindowSurface(surface, useHighBitDepth)) { val width = it.getWidth() @@ -173,7 +179,8 @@ class VideoMediaCodecEncoder( textureId = createTextureObject() setMVPMatrixAndViewPort( orientation.toFloat(), - size + size, + orientationProvider?.mirroredVertically ?: false ) } @@ -189,7 +196,9 @@ class VideoMediaCodecEncoder( @SuppressLint("Recycle") private fun attachOrBuildSurfaceTexture(surfaceTexture: SurfaceTexture?): SurfaceTexture { return if (surfaceTexture == null) { - SurfaceTexture(textureId) + SurfaceTexture(textureId).apply { + _inputSurface = Surface(this) + } } else { surfaceTexture.attachToGLContext(textureId) surfaceTexture @@ -208,12 +217,40 @@ class VideoMediaCodecEncoder( return surface } - override fun onFrameAvailable(surfaceTexture: SurfaceTexture) { - synchronized(this) { - if (!isRunning) { - return + override fun onOrientationChanged() { + executor.execute { + synchronized(this) { + ensureGlContext(eglSurface) { + val width = it.getWidth() + val height = it.getHeight() + + fullFrameRect?.setMVPMatrixAndViewPort( + (orientationProvider?.orientation ?: 0).toFloat(), + orientationProvider?.getOrientedSize(Size(width, height)) ?: Size( + width, + height + ), + orientationProvider?.mirroredVertically ?: false + ) + + /** + * Flushing spurious latest camera frames that block SurfaceTexture buffer + * to avoid having a misoriented frame. + */ + surfaceTexture?.updateTexImage() + surfaceTexture?.releaseTexImage() + } } - executor.execute { + } + } + + override fun onFrameAvailable(surfaceTexture: SurfaceTexture) { + if (!isRunning) { + return + } + + executor.execute { + synchronized(this) { eglSurface?.let { it.makeCurrent() surfaceTexture.updateTexImage() @@ -256,8 +293,10 @@ class VideoMediaCodecEncoder( }.get() } - fun dispose() { + fun release() { + orientationProvider?.removeListener(this) stopStream() + surfaceTexture?.setOnFrameAvailableListener(null) surfaceTexture?.release() surfaceTexture = null } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/FullFrameRect.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/FullFrameRect.kt index 00bbca5c6..aa59342fd 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/FullFrameRect.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/FullFrameRect.kt @@ -103,8 +103,9 @@ class FullFrameRect(var program: Texture2DProgram) { return program.createTextureObject() } - fun setMVPMatrixAndViewPort(rotation: Float, resolution: Size) { + fun setMVPMatrixAndViewPort(rotation: Float, resolution: Size, mirroredVertically: Boolean) { Matrix.setIdentityM(mvpMatrix, 0) + Matrix.scaleM(mvpMatrix, 0, if (mirroredVertically) -1f else 1f, 1f, 0f) Matrix.rotateM( mvpMatrix, 0, rotation, 0f, 0f, -1f diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/IMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/IMuxer.kt index 5d5f87acf..09c420423 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/IMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/IMuxer.kt @@ -17,9 +17,9 @@ package io.github.thibaultbee.streampack.internal.muxers import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.internal.data.Frame -import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.interfaces.Releaseable import io.github.thibaultbee.streampack.internal.interfaces.Streamable +import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider interface IMuxer : Streamable, Releaseable { val helper: IMuxerHelper diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/FlvMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/FlvMuxer.kt index 62fc60ba5..bb15e7c06 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/FlvMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/FlvMuxer.kt @@ -19,7 +19,7 @@ import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.internal.data.Frame import io.github.thibaultbee.streampack.internal.data.Packet import io.github.thibaultbee.streampack.internal.data.PacketType -import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider +import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.IMuxer import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener import io.github.thibaultbee.streampack.internal.muxers.flv.tags.AVTagsFactory diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadata.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadata.kt index e20f2daa4..80d95f97e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadata.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadata.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.internal.muxers.flv.tags import io.github.thibaultbee.streampack.data.AudioConfig import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.data.VideoConfig -import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider +import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.flv.amf.containers.AmfContainer import io.github.thibaultbee.streampack.internal.muxers.flv.amf.containers.AmfEcmaArray import io.github.thibaultbee.streampack.internal.muxers.flv.tags.video.CodecID diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/MP4Muxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/MP4Muxer.kt index a1bbfeddd..95b89cfb0 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/MP4Muxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/MP4Muxer.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.internal.muxers.mp4 import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.internal.data.Frame import io.github.thibaultbee.streampack.internal.data.Packet -import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider +import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.IMuxer import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.FileTypeBox diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/ts/TSMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/ts/TSMuxer.kt index a3e4a2179..5de15ce41 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/ts/TSMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/ts/TSMuxer.kt @@ -20,7 +20,7 @@ import android.media.MediaFormat import io.github.thibaultbee.streampack.data.AudioConfig import io.github.thibaultbee.streampack.data.Config import io.github.thibaultbee.streampack.internal.data.Frame -import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider +import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.IMuxer import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener import io.github.thibaultbee.streampack.internal.muxers.ts.data.Service diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/orientation/AbstractSourceOrientationProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/orientation/AbstractSourceOrientationProvider.kt new file mode 100644 index 000000000..1a0e31695 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/orientation/AbstractSourceOrientationProvider.kt @@ -0,0 +1,18 @@ +package io.github.thibaultbee.streampack.internal.orientation + +abstract class AbstractSourceOrientationProvider : ISourceOrientationProvider { + protected val listeners = mutableSetOf() + + override val mirroredVertically = false + override fun addListener(listener: ISourceOrientationListener) { + listeners.add(listener) + } + + override fun removeListener(listener: ISourceOrientationListener) { + listeners.remove(listener) + } + + override fun removeAllListeners() { + listeners.clear() + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/interfaces/ISourceOrientationProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/orientation/ISourceOrientationProvider.kt similarity index 57% rename from core/src/main/java/io/github/thibaultbee/streampack/internal/interfaces/ISourceOrientationProvider.kt rename to core/src/main/java/io/github/thibaultbee/streampack/internal/orientation/ISourceOrientationProvider.kt index 4b2239084..96ebaf5dc 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/interfaces/ISourceOrientationProvider.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/orientation/ISourceOrientationProvider.kt @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.github.thibaultbee.streampack.internal.interfaces +package io.github.thibaultbee.streampack.internal.orientation import android.graphics.SurfaceTexture import android.util.Size @@ -31,7 +31,13 @@ interface ISourceOrientationProvider { val orientation: Int /** - * Return the size with the correct orientation. + * If true, the source is mirrored vertically. + * Example: should be true for a front camera. + */ + val mirroredVertically: Boolean + + /** + * Returns the size with the correct orientation. * If orientation is portrait, it returns a portrait size. * Example: * - Size = 1920x1080, if orientation is portrait, it returns 1080x1920. @@ -39,8 +45,38 @@ interface ISourceOrientationProvider { fun getOrientedSize(size: Size): Size /** - * Return the size for [SurfaceTexture.setDefaultBufferSize]. + * Returns the size for [SurfaceTexture.setDefaultBufferSize]. * Override this method if the image is stretched. */ fun getDefaultBufferSize(size: Size) = size + + /** + * Adds a listener to be notified when the orientation changes. + * + * @param listener to add. + */ + fun addListener(listener: ISourceOrientationListener) + + /** + * Removes a listener. + * + * @param listener to remove. + */ + fun removeListener(listener: ISourceOrientationListener) + + /** + * Removes all registered listeners. + */ + fun removeAllListeners() +} + +/** + * Interface to be notified when the orientation changes. + */ +interface ISourceOrientationListener { + /** + * Called when the orientation changes. + * Only called if [ISourceOrientationProvider.mirroredVertically] changes for now. + */ + fun onOrientationChanged() } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoSource.kt index 393d7f4b7..a215f35c7 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoSource.kt @@ -16,7 +16,7 @@ package io.github.thibaultbee.streampack.internal.sources import io.github.thibaultbee.streampack.data.VideoConfig -import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider +import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider interface IVideoSource : IFrameSource, ISurfaceSource { /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt index f1b57ec65..974eb8304 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt @@ -17,12 +17,13 @@ package io.github.thibaultbee.streampack.internal.sources.camera import android.Manifest import android.content.Context +import android.hardware.camera2.CameraCharacteristics import android.util.Size import android.view.Surface import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.data.VideoConfig import io.github.thibaultbee.streampack.internal.data.Frame -import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider +import io.github.thibaultbee.streampack.internal.orientation.AbstractSourceOrientationProvider import io.github.thibaultbee.streampack.internal.sources.IVideoSource import io.github.thibaultbee.streampack.internal.utils.av.video.DynamicRangeProfile import io.github.thibaultbee.streampack.internal.utils.extensions.deviceOrientation @@ -30,7 +31,9 @@ import io.github.thibaultbee.streampack.internal.utils.extensions.isDevicePortra import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize import io.github.thibaultbee.streampack.internal.utils.extensions.portraitize import io.github.thibaultbee.streampack.utils.CameraSettings +import io.github.thibaultbee.streampack.utils.cameraList import io.github.thibaultbee.streampack.utils.defaultCameraId +import io.github.thibaultbee.streampack.utils.getFacingDirection import io.github.thibaultbee.streampack.utils.isFrameRateSupported import kotlinx.coroutines.runBlocking import java.nio.ByteBuffer @@ -66,7 +69,7 @@ class CameraSource( override val timestampOffset = CameraHelper.getTimeOffsetToMonoClock(context, cameraId) override val hasSurface = true override val hasFrames = false - override val orientationProvider = CameraOrientationProvider(context) + override val orientationProvider = CameraOrientationProvider(context, cameraId) override fun getFrame(buffer: ByteBuffer): Frame { throw UnsupportedOperationException("Camera expects to run in Surface mode") @@ -96,6 +99,7 @@ class CameraSource( } cameraController.startRequestSession(fps, targets) isPreviewing = true + orientationProvider.cameraId = cameraId } fun stopPreview() { @@ -130,8 +134,22 @@ class CameraSource( } - class CameraOrientationProvider(private val context: Context) : - ISourceOrientationProvider { + class CameraOrientationProvider(private val context: Context, initialCameraId: String) : + AbstractSourceOrientationProvider() { + private val isFrontFacingMap = + context.cameraList.associateWith { (context.getFacingDirection(it) == CameraCharacteristics.LENS_FACING_FRONT) } + + var cameraId: String = initialCameraId + set(value) { + if (field == value) { + return + } + val orientationChanged = mirroredVertically != isFrontFacing(value) + field = value + if (orientationChanged) { + listeners.forEach { it.onOrientationChanged() } + } + } override val orientation: Int get() = when (context.deviceOrientation) { @@ -142,6 +160,13 @@ class CameraSource( else -> 0 } + private fun isFrontFacing(cameraId: String): Boolean { + return isFrontFacingMap[cameraId] ?: false + } + + override val mirroredVertically: Boolean + get() = isFrontFacing(cameraId) + override fun getOrientedSize(size: Size): Size { return if (context.isDevicePortrait) { size.portraitize() diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenSource.kt index e0fe3dddb..ecb7fcf19 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenSource.kt @@ -28,7 +28,7 @@ import androidx.activity.result.ActivityResult import io.github.thibaultbee.streampack.data.VideoConfig import io.github.thibaultbee.streampack.error.StreamPackError import io.github.thibaultbee.streampack.internal.data.Frame -import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider +import io.github.thibaultbee.streampack.internal.orientation.AbstractSourceOrientationProvider import io.github.thibaultbee.streampack.internal.sources.IVideoSource import io.github.thibaultbee.streampack.internal.utils.extensions.isDevicePortrait import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize @@ -147,7 +147,7 @@ class ScreenSource( } class ScreenSourceOrientationProvider(private val context: Context) : - ISourceOrientationProvider { + AbstractSourceOrientationProvider() { override val orientation = 0 override fun getOrientedSize(size: Size): Size { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseStreamer.kt index 914bfce49..d32022bcc 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseStreamer.kt @@ -29,7 +29,7 @@ import io.github.thibaultbee.streampack.internal.encoders.IEncoderListener import io.github.thibaultbee.streampack.internal.encoders.VideoMediaCodecEncoder import io.github.thibaultbee.streampack.internal.endpoints.IEndpoint import io.github.thibaultbee.streampack.internal.events.EventHandler -import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider +import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider import io.github.thibaultbee.streampack.internal.muxers.IMuxer import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener import io.github.thibaultbee.streampack.internal.sources.IAudioSource @@ -375,7 +375,7 @@ abstract class BaseStreamer( */ override fun release() { audioEncoder?.release() - videoEncoder?.codecSurface?.dispose() + videoEncoder?.codecSurface?.release() videoEncoder?.release() audioSource?.release() videoSource?.release()