From a2a88c6faa7937bbb5a2a182505b67dfc513b351 Mon Sep 17 00:00:00 2001 From: Holden Date: Thu, 15 Feb 2024 22:00:49 -0500 Subject: [PATCH] Update GStreamer versions to 1.24.2 --- .github/workflows/android.yml | 10 +- .github/workflows/linux.yml | 34 + .github/workflows/macos_debug.yml | 13 +- .github/workflows/macos_release.yml | 6 +- .github/workflows/windows.yml | 3 +- QGCCommon.pri | 3 +- cmake/DownloadGstreamer.cmake | 11 + libs/qmlglsink/CMakeLists.txt | 133 ++- libs/qmlglsink/qt6-linux/gstplugin.cc | 62 ++ libs/qmlglsink/qt6-linux/gstqml6glsink.cc | 576 +++++++++++++ libs/qmlglsink/qt6-linux/gstqml6glsink.h | 62 ++ .../{qt6 => qt6-linux}/gstqsg6glnode.cc | 0 .../{qt6 => qt6-linux}/gstqsg6glnode.h | 0 libs/qmlglsink/qt6-linux/gstqt6element.cc | 38 + libs/qmlglsink/qt6-linux/gstqt6elements.h | 37 + libs/qmlglsink/qt6-linux/gstqt6gl.h | 56 ++ libs/qmlglsink/qt6-linux/gstqt6glutility.cc | 359 ++++++++ libs/qmlglsink/qt6-linux/gstqt6glutility.h | 55 ++ libs/qmlglsink/qt6-linux/meson.build | 145 ++++ libs/qmlglsink/qt6-linux/qt6glitem.cc | 755 +++++++++++++++++ libs/qmlglsink/qt6-linux/qt6glitem.h | 128 +++ libs/qmlglsink/qt6/RGBA.frag | 24 + libs/qmlglsink/qt6/YUV_TRIPLANAR.frag | 37 + libs/qmlglsink/qt6/gstplugin.cc | 14 +- libs/qmlglsink/qt6/gstqml6glmixer.cc | 628 ++++++++++++++ libs/qmlglsink/qt6/gstqml6glmixer.h | 40 + libs/qmlglsink/qt6/gstqml6gloverlay.cc | 571 +++++++++++++ libs/qmlglsink/qt6/gstqml6gloverlay.h | 76 ++ libs/qmlglsink/qt6/gstqml6glsink.cc | 79 +- libs/qmlglsink/qt6/gstqml6glsrc.cc | 599 ++++++++++++++ libs/qmlglsink/qt6/gstqml6glsrc.h | 62 ++ libs/qmlglsink/qt6/gstqsg6material.cc | 641 +++++++++++++++ libs/qmlglsink/qt6/gstqsg6material.h | 75 ++ libs/qmlglsink/qt6/gstqt6elements.h | 9 +- libs/qmlglsink/qt6/gstqt6glutility.cc | 127 +-- libs/qmlglsink/qt6/gstqt6glutility.h | 7 +- libs/qmlglsink/qt6/meson.build | 158 ++-- libs/qmlglsink/qt6/qt6glitem.cc | 213 ++++- libs/qmlglsink/qt6/qt6glitem.h | 2 +- libs/qmlglsink/qt6/qt6glrenderer.cc | 766 ++++++++++++++++++ libs/qmlglsink/qt6/qt6glrenderer.h | 124 +++ libs/qmlglsink/qt6/qt6glwindow.cc | 470 +++++++++++ libs/qmlglsink/qt6/qt6glwindow.h | 69 ++ libs/qmlglsink/qt6/resources.qrc | 7 + libs/qmlglsink/qt6/vertex.vert | 23 + qmlglsink.pri | 72 +- src/VideoReceiver/GStreamer.cc | 12 +- src/VideoReceiver/VideoReceiver.pri | 15 +- tools/setup/ubuntu.sh | 35 +- 49 files changed, 7188 insertions(+), 253 deletions(-) create mode 100644 cmake/DownloadGstreamer.cmake create mode 100644 libs/qmlglsink/qt6-linux/gstplugin.cc create mode 100644 libs/qmlglsink/qt6-linux/gstqml6glsink.cc create mode 100644 libs/qmlglsink/qt6-linux/gstqml6glsink.h rename libs/qmlglsink/{qt6 => qt6-linux}/gstqsg6glnode.cc (100%) rename libs/qmlglsink/{qt6 => qt6-linux}/gstqsg6glnode.h (100%) create mode 100644 libs/qmlglsink/qt6-linux/gstqt6element.cc create mode 100644 libs/qmlglsink/qt6-linux/gstqt6elements.h create mode 100644 libs/qmlglsink/qt6-linux/gstqt6gl.h create mode 100644 libs/qmlglsink/qt6-linux/gstqt6glutility.cc create mode 100644 libs/qmlglsink/qt6-linux/gstqt6glutility.h create mode 100644 libs/qmlglsink/qt6-linux/meson.build create mode 100644 libs/qmlglsink/qt6-linux/qt6glitem.cc create mode 100644 libs/qmlglsink/qt6-linux/qt6glitem.h create mode 100644 libs/qmlglsink/qt6/RGBA.frag create mode 100644 libs/qmlglsink/qt6/YUV_TRIPLANAR.frag create mode 100644 libs/qmlglsink/qt6/gstqml6glmixer.cc create mode 100644 libs/qmlglsink/qt6/gstqml6glmixer.h create mode 100644 libs/qmlglsink/qt6/gstqml6gloverlay.cc create mode 100644 libs/qmlglsink/qt6/gstqml6gloverlay.h create mode 100644 libs/qmlglsink/qt6/gstqml6glsrc.cc create mode 100644 libs/qmlglsink/qt6/gstqml6glsrc.h create mode 100644 libs/qmlglsink/qt6/gstqsg6material.cc create mode 100644 libs/qmlglsink/qt6/gstqsg6material.h create mode 100644 libs/qmlglsink/qt6/qt6glrenderer.cc create mode 100644 libs/qmlglsink/qt6/qt6glrenderer.h create mode 100644 libs/qmlglsink/qt6/qt6glwindow.cc create mode 100644 libs/qmlglsink/qt6/qt6glwindow.h create mode 100644 libs/qmlglsink/qt6/resources.qrc create mode 100644 libs/qmlglsink/qt6/vertex.vert diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index c2e67bb78cdb..2bbe01220a71 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -85,6 +85,7 @@ jobs: dir: ${{ runner.temp }} modules: qtcharts qtlocation qtpositioning qtspeech qt5compat qtmultimedia qtserialport qtimageformats qtshadertools qtconnectivity qtquick3d cache: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' }} + tools: 'tools_cmake tools_ninja' - name: Install Qt6 for Android (armv7) uses: ./install-qt-action/action/ @@ -141,11 +142,12 @@ jobs: - name: Install gstreamer working-directory: ${{ github.workspace }} env: - GST_VERSION: 1.18.5 + GST_VERSION: 1.24.2 run: | - wget --quiet https://gstreamer.freedesktop.org/data/pkg/android/${GST_VERSION}/gstreamer-1.0-android-universal-${GST_VERSION}.tar.xz - mkdir gstreamer-1.0-android-universal-${GST_VERSION} - tar xf gstreamer-1.0-android-universal-${GST_VERSION}.tar.xz -C gstreamer-1.0-android-universal-${GST_VERSION} + wget --quiet https://gstreamer.freedesktop.org/data/pkg/android/${{ env.GST_VERSION }}/gstreamer-1.0-android-universal-${{ env.GST_VERSION }}.tar.xz + mkdir gstreamer-1.0-android-universal-${{ env.GST_VERSION }} + tar xf gstreamer-1.0-android-universal-${{ env.GST_VERSION }}.tar.xz -C gstreamer-1.0-android-universal-${{ env.GST_VERSION }} + rm gstreamer-1.0-android-universal-${{ env.GST_VERSION }}.tar.xz - name: Install dependencies run: sudo apt-get install -y ninja-build diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 622fee7b6782..b89aa451a6bc 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -59,11 +59,45 @@ jobs: modules: qtcharts qtlocation qtpositioning qtspeech qt5compat qtmultimedia qtserialport qtimageformats qtshadertools qtconnectivity qtquick3d setup-python: true cache: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' }} + tools: 'tools_cmake tools_ninja' - name: Install Dependencies run: | chmod a+x ./tools/setup/ubuntu.sh ./tools/setup/ubuntu.sh + python3 -m pip install meson ninja + + - uses: actions/checkout@v4 + with: + repository: GStreamer/gstreamer + ref: 1.24.2 + path: gstreamer + + # - name: Clone GStreamer + # working-directory: ${{ runner.temp }} + # run: git clone --depth 1 --branch 1.24.2 https://github.com/GStreamer/gstreamer.git + + - name: Configure GStreamer + working-directory: ${{ runner.temp }}/gstreamer + run: meson setup + --prefix=${{ runner.temp }}/gst + -Dgst-full-target-type=static_library + -Dauto_features=disabled + build + # --prefix=${{ runner.temp }}/gst + # --buildtype=${{ matrix.BuildType }} + # -Dauto_features=disabled -Dgood=enabled -Dgst-plugins-good:qml6=enabled -Dgst-plugins-good:qt-egl=enabled -Dgst-plugins-good:qt-wayland=enabled -Dgst-plugins-good:qt-x11=enabled -Dbase=enabled -Dgst-plugins-base:gl=enabled -Dgst-plugins-base:gl_platform=glx,egl -Dgst-plugins-base:gl_winsys=x11,wayland -Dgst-plugins-base:x11=enabled build + # -Dgst-full-target-type=static_library + # --default-library=static -Dgst-full-libraries=app,video + # -Dges=disabled -Dtests=disabled -Dexamples=disabled -Dgst-examples=disabled -Ddoc=disabled -Dgtk_doc=disabled -Dgpl=enabled build + + - name: Compile GStreamer + working-directory: ${{ runner.temp }}/gstreamer + run: meson compile -C build + + - name: Install GStreamer + working-directory: ${{ runner.temp }}/gstreamer + run: meson install -C build - name: Create build directory run: mkdir ${{ runner.temp }}/shadow_build_dir diff --git a/.github/workflows/macos_debug.yml b/.github/workflows/macos_debug.yml index c6afebba2e22..bab3ef039b34 100644 --- a/.github/workflows/macos_debug.yml +++ b/.github/workflows/macos_debug.yml @@ -64,10 +64,15 @@ jobs: - name: Install Dependencies run: brew install ninja SDL2 - - name: Setup GStreamer - uses: blinemedical/setup-gstreamer@v1 - with: - version: 1.18.6 + - name: Install Gstreamer + env: + GST_VERSION: 1.24.2 + run: | + wget https://gstreamer.freedesktop.org/data/pkg/osx/${{ env.GST_VERSION }}/gstreamer-1.0-devel-${{ env.GST_VERSION }}-universal.pkg + wget https://gstreamer.freedesktop.org/data/pkg/osx/${{ env.GST_VERSION }}/gstreamer-1.0-${{ env.GST_VERSION }}-universal.pkg + for package in *.pkg ; + do sudo installer -verbose -pkg "$package" -target / + done - name: Create build directory run: mkdir ${{ runner.temp }}/shadow_build_dir diff --git a/.github/workflows/macos_release.yml b/.github/workflows/macos_release.yml index bf9d1b669ea4..d8f53877cf3e 100644 --- a/.github/workflows/macos_release.yml +++ b/.github/workflows/macos_release.yml @@ -63,9 +63,11 @@ jobs: cache: true - name: Install Gstreamer + env: + GST_VERSION: 1.24.2 run: | - wget https://gstreamer.freedesktop.org/data/pkg/osx/1.18.6/gstreamer-1.0-devel-1.18.6-x86_64.pkg - wget https://gstreamer.freedesktop.org/data/pkg/osx/1.18.6/gstreamer-1.0-1.18.6-x86_64.pkg + wget https://gstreamer.freedesktop.org/data/pkg/osx/${{ env.GST_VERSION }}/gstreamer-1.0-devel-${{ env.GST_VERSION }}-universal.pkg + wget https://gstreamer.freedesktop.org/data/pkg/osx/${{ env.GST_VERSION }}/gstreamer-1.0-${{ env.GST_VERSION }}-universal.pkg for package in *.pkg ; do sudo installer -verbose -pkg "$package" -target / done diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index fceedb7233b9..b58b8d2915f5 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -54,8 +54,7 @@ jobs: - name: Install Gstreamer uses: blinemedical/setup-gstreamer@v1 with: - version: 1.18.6 - # run: choco install --no-progress gstreamer gstreamer-devel --version=1.18.6 + version: 1.24.2 - name: Install Dependencies run: choco install --no-progress ninja -y diff --git a/QGCCommon.pri b/QGCCommon.pri index 5041e7a889fe..0472318f0d5d 100644 --- a/QGCCommon.pri +++ b/QGCCommon.pri @@ -99,7 +99,8 @@ linux { QMAKE_CXXFLAGS_WARN_ON += -Werror \ -Wno-unused-parameter \ # gst-plugins-good -Wno-unused-but-set-variable \ # QGCTileCacheWorker.cpp - -Wno-deprecated-declarations + -Wno-deprecated-declarations \ + -Wno-unused-private-field } else { error("Unsupported Mac toolchain, only 64-bit LLVM+clang is supported") } diff --git a/cmake/DownloadGstreamer.cmake b/cmake/DownloadGstreamer.cmake new file mode 100644 index 000000000000..1c8c7273bb2a --- /dev/null +++ b/cmake/DownloadGstreamer.cmake @@ -0,0 +1,11 @@ +set(GST_VERSION "1.24.2") +if(ANDROID) + set(GST_URL https://gstreamer.freedesktop.org/data/pkg/android/${GST_VERSION}/gstreamer-1.0-android-universal-${GST_VERSION}.tar.xz) + set(GST_ZIP_PATH ${CMAKE_SOURCE_DIR}/gstreamer-1.0-android-universal.tar.xz) + set(GST_TARGET_PATH ${CMAKE_SOURCE_DIR}/gstreamer-1.0-android-universal) + if(NOT EXISTS "${GST_TARGET_PATH}") + file(DOWNLOAD ${GST_URL} ${GST_ZIP_PATH}) + execute_process(COMMAND mkdir gstreamer-1.0-android-universal) + execute_process(COMMAND tar xf ${GST_ZIP_PATH} -C ${GST_TARGET_PATH}) + endif() +endif() diff --git a/libs/qmlglsink/CMakeLists.txt b/libs/qmlglsink/CMakeLists.txt index 4fb6af35b2ed..64ed39d8eb15 100644 --- a/libs/qmlglsink/CMakeLists.txt +++ b/libs/qmlglsink/CMakeLists.txt @@ -2,6 +2,8 @@ option(QGC_ENABLE_VIDEOSTREAMING "Enable video streaming" ON) if(QGC_ENABLE_VIDEOSTREAMING) message(STATUS "Enabling video streaming support") + # include(DownloadGstreamer) + if(ANDROID) set(GST_STATIC_BUILD ON) else() @@ -9,11 +11,11 @@ if(QGC_ENABLE_VIDEOSTREAMING) endif() if(LINUX) - set(GST_TARGET_VERSION 1.16) + set(GST_TARGET_VERSION 1.16.0) elseif(ANDROID) - set(GST_TARGET_VERSION 1.18.5) + set(GST_TARGET_VERSION 1.24.2) else() - set(GST_TARGET_VERSION 1.18) + set(GST_TARGET_VERSION 1.24.0) endif() set(GST_TARGET_PLUGINS @@ -46,8 +48,39 @@ if(QGC_ENABLE_VIDEOSTREAMING) ) if(LINUX) list(APPEND GST_TARGET_MODULES egl) + # gstreamer-gl-x11-1.0 + # gstreamer-gl-wayland-1.0 + # gstreamer-gl-egl-1.0 endif() +# gstreamer-rtsp-1.0 +# gstreamer-audio-1.0 +# # gstreamer-plugins-base-1.0 +# gstreamer-codecparsers-1.0 +# gstreamer-rtp-1.0 +# gstreamer-pbutils-1.0 +# gstreamer-net-1.0 +# gstreamer-tag-1.0 +# gstreamer-mpegts-1.0 +# gstreamer-riff-1.0 +# gstreamer-photography-1.0 +# gstreamer-controller-1.0 +# gmodule-2.0 +# graphene-1.0 +# orc-0.4 +# bzip2 +# libffi +# libpcre2-8 +# zlib +# # libpng +# # libjpeg +# # libavcodec +# # libavfilter +# # libavformat +# # libavutil +# # x264 +# ) + # find_library(GSTREAMER_${_upper_name}_LIBRARY # NAMES # gst${_lower_name}-${_abi_version} @@ -76,14 +109,15 @@ if(QGC_ENABLE_VIDEOSTREAMING) elseif(MACOS) set(GSTREAMER_ROOT "/Library/Frameworks/GStreamer.framework") elseif(ANDROID) + set(GSTREAMER_ROOT ${CMAKE_SOURCE_DIR}/gstreamer-1.0-android-universal-${GST_TARGET_VERSION}) if(${ANDROID_ABI} STREQUAL armeabi-v7a) - set(GSTREAMER_ROOT ${CMAKE_SOURCE_DIR}/gstreamer-1.0-android-universal-${GST_TARGET_VERSION}/armv7) + set(GSTREAMER_ROOT ${GSTREAMER_ROOT}/armv7) elseif(${ANDROID_ABI} STREQUAL arm64-v8a) - set(GSTREAMER_ROOT ${CMAKE_SOURCE_DIR}/gstreamer-1.0-android-universal-${GST_TARGET_VERSION}/arm64) + set(GSTREAMER_ROOT ${GSTREAMER_ROOT}/arm64) elseif(${ANDROID_ABI} STREQUAL x86) - set(GSTREAMER_ROOT ${CMAKE_SOURCE_DIR}/gstreamer-1.0-android-universal-${GST_TARGET_VERSION}/x86) + set(GSTREAMER_ROOT ${GSTREAMER_ROOT}/x86) elseif(${ANDROID_ABI} STREQUAL x86_64) - set(GSTREAMER_ROOT ${CMAKE_SOURCE_DIR}/gstreamer-1.0-android-universal-${GST_TARGET_VERSION}/x86_64) + set(GSTREAMER_ROOT ${GSTREAMER_ROOT}/x86_64) endif() endif() @@ -159,6 +193,15 @@ if(QGC_ENABLE_VIDEOSTREAMING) endif() endif() # file(GET_RUNTIME_DEPENDENCIES) + # target_link_libraries(qmlglsink + # PUBLIC + # iconv + # # x264 + # z + # png16 + # jpeg + # android + # ) endif() else() message(WARNING "Gstreamer Not Found") @@ -166,21 +209,49 @@ if(QGC_ENABLE_VIDEOSTREAMING) if(GST_FOUND) message(STATUS "GST Modules Found") - target_sources(qmlglsink - PRIVATE - qt6/gstplugin.cc - qt6/gstqml6glsink.cc - qt6/gstqml6glsink.h - qt6/gstqsg6glnode.cc - qt6/gstqsg6glnode.h - qt6/gstqt6element.cc - qt6/gstqt6elements.h - qt6/gstqt6gl.h - qt6/gstqt6glutility.cc - qt6/gstqt6glutility.h - qt6/qt6glitem.cc - qt6/qt6glitem.h - ) + if(LINUX) + target_sources(qmlglsink + PRIVATE + qt6-linux/gstplugin.cc + qt6-linux/gstqml6glsink.cc + qt6-linux/gstqml6glsink.h + qt6-linux/gstqsg6glnode.cc + qt6-linux/gstqsg6glnode.h + qt6-linux/gstqt6element.cc + qt6-linux/gstqt6elements.h + qt6-linux/gstqt6gl.h + qt6-linux/gstqt6glutility.cc + qt6-linux/gstqt6glutility.h + qt6-linux/qt6glitem.cc + qt6-linux/qt6glitem.h + ) + else() + target_sources(qmlglsink + PRIVATE + qt6/gstplugin.cc + qt6/gstqml6glmixer.cc + qt6/gstqml6glmixer.h + qt6/gstqml6gloverlay.cc + qt6/gstqml6gloverlay.h + qt6/gstqml6glsink.cc + qt6/gstqml6glsink.h + qt6/gstqml6glsrc.cc + qt6/gstqml6glsrc.h + qt6/gstqsg6material.cc + qt6/gstqsg6material.h + qt6/gstqt6element.cc + qt6/gstqt6elements.h + qt6/gstqt6gl.h + qt6/gstqt6glutility.cc + qt6/gstqt6glutility.h + qt6/qt6glitem.cc + qt6/qt6glitem.h + qt6/qt6glrenderer.cc + qt6/qt6glrenderer.h + qt6/qt6glwindow.cc + qt6/qt6glwindow.h + ) + endif() find_package(Qt6 REQUIRED COMPONENTS Core Gui OpenGL Qml Quick) target_link_libraries(qmlglsink @@ -203,6 +274,9 @@ if(QGC_ENABLE_VIDEOSTREAMING) endif() target_include_directories(qmlglsink PUBLIC qt6) + if(WIN32) + # target_include_directories(qmlglsink PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/windows) + endif() target_compile_definitions(qmlglsink PRIVATE @@ -240,6 +314,19 @@ if(QGC_ENABLE_VIDEOSTREAMING) ) endif() + qt_add_resources(qmlglsink "gst" + PREFIX "/org/freedesktop/gstreamer/qml6" + FILES + qt6/RGBA.frag + qt6/vertex.vert + qt6/YUV_TRIPLANAR.frag + ) + # qt_add_qml_module(qmlglsink + # URI org.freedesktop.gstreamer.qml6 + # VERSION 1.0 + # IMPORT_PATH ${QT_QML_OUTPUT_DIRECTORY} + # ) + message(STATUS "GStreamer version: ${GST_gstreamer-1.0_VERSION}") message(STATUS "GStreamer prefix: ${GST_gstreamer-1.0_PREFIX}") message(STATUS "GStreamer include dir: ${GST_gstreamer-1.0_INCLUDEDIR}") @@ -283,5 +370,5 @@ if(QGC_ENABLE_VIDEOSTREAMING) message(WARNING "GST Modules Not Found") endif() else() - message(STATUS "Video streaming disabled") + message(STATUS "Video streaming disabled") endif() diff --git a/libs/qmlglsink/qt6-linux/gstplugin.cc b/libs/qmlglsink/qt6-linux/gstplugin.cc new file mode 100644 index 000000000000..eb24046dc5ee --- /dev/null +++ b/libs/qmlglsink/qt6-linux/gstplugin.cc @@ -0,0 +1,62 @@ +/* + * GStreamer + * Copyright (C) 2015 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstqt6elements.h" +#include "qt6glitem.h" + +#include + +static gboolean +plugin_init (GstPlugin * plugin) +{ + gboolean ret = FALSE; +// TODO(zdanek) fix after switching to gstreamer 1.20.0+ +// original code from 1.20.0 +// ret |= GST_ELEMENT_REGISTER (qml6glsink, plugin); + ret |= gst_element_register_qml6glsink (plugin); + + return ret; +} + +static void registerMetatypes() +{ + qmlRegisterType ("org.freedesktop.gstreamer.GLVideoItem", 1, 0, "GstGLVideoItem"); +} + +Q_CONSTRUCTOR_FUNCTION(registerMetatypes) + +#ifndef GST_PACKAGE_NAME +#define GST_PACKAGE_NAME "GStreamer Bad Plug-ins (qmake)" +#define GST_PACKAGE_ORIGIN "Unknown package origin" +#define GST_LICENSE "LGPL" +#define PACKAGE "gst-plugins-bad (qmake)" +#define PACKAGE_VERSION "1.21.0.1" +#endif + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + qml6, + "Qt6 Qml plugin", + plugin_init, PACKAGE_VERSION, GST_LICENSE, GST_PACKAGE_NAME, + GST_PACKAGE_ORIGIN) diff --git a/libs/qmlglsink/qt6-linux/gstqml6glsink.cc b/libs/qmlglsink/qt6-linux/gstqml6glsink.cc new file mode 100644 index 000000000000..9c9ad386ef09 --- /dev/null +++ b/libs/qmlglsink/qt6-linux/gstqml6glsink.cc @@ -0,0 +1,576 @@ +/* + * GStreamer + * Copyright (C) 2015 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:gstqml6glsink + * + * qml6glsink provides a way to render a video stream as a Qml object inside + * the Qml scene graph. This is achieved by providing the incoming OpenGL + * textures to Qt as a scene graph object. + * + * qml6glsink will attempt to retrieve the windowing system display connection + * that Qt is using (#GstGLDisplay). This may be different to any already + * existing window system display connection already in use in the pipeline for + * a number of reasons. A couple of examples of this are: + * + * 1. Adding qml6glsink to an already running pipeline + * 2. Not having any qml6glsink element start up before any + * other OpenGL-based element in the pipeline. + * + * If one of these scenarios occurs, then there will be multiple OpenGL contexts + * in use in the pipeline. This means that either the pipeline will fail to + * start up correctly, a downstream element may reject buffers, or a complete + * GPU->System memory->GPU transfer is performed for every buffer. + * + * The requirement to avoid this is that all elements share the same + * #GstGLDisplay object and as Qt cannot currently share an existing window + * system display connection, GStreamer must use the window system display + * connection provided by Qt. This window system display connection can be + * retrieved by either a qmlglsink element or a qmlgloverlay element. The + * recommended usage is to have either element (qmlglsink or qmlgloverlay) + * be the first to propagate the #GstGLDisplay for the entire pipeline to use by + * setting either element to the READY element state before any other OpenGL + * element in the pipeline. + * + * In a dynamically adding qmlglsink (or qmlgloverlay) to a pipeline case, + * there are some considerations for ensuring that the window system display + * and OpenGL contexts are compatible with Qt. When the qmlgloverlay (or + * qmlglsink) element is added and brought up to READY, it will propagate it's + * own #GstGLDisplay using the #GstContext mechanism regardless of any existing + * #GstGLDisplay used by the pipeline previously. In order for the new + * #GstGLDisplay to be used, the application must then set the provided + * #GstGLDisplay containing #GstContext on the pipeline. This may effectively + * cause each OpenGL element to replace the window system display and also the + * OpenGL context it is using. As such this process may take a significant + * amount of time and resources as objects are recreated in the new OpenGL + * context. + * + * All instances of qmlglsink and qmlgloverlay will return the exact same + * #GstGLDisplay object while the pipeline is running regardless of whether + * any qmlglsink or qmlgloverlay elements are added or removed from the + * pipeline. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstqt6elements.h" +#include "gstqml6glsink.h" +#include + +#include + +#define GST_CAT_DEFAULT gst_debug_qml6_gl_sink +GST_DEBUG_CATEGORY (GST_CAT_DEFAULT); + +static void gst_qml6_gl_sink_navigation_interface_init (GstNavigationInterface * iface); +static void gst_qml6_gl_sink_finalize (GObject * object); +static void gst_qml6_gl_sink_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * param_spec); +static void gst_qml6_gl_sink_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * param_spec); + +static gboolean gst_qml6_gl_sink_stop (GstBaseSink * bsink); + +static gboolean gst_qml6_gl_sink_query (GstBaseSink * bsink, GstQuery * query); + +static GstStateChangeReturn +gst_qml6_gl_sink_change_state (GstElement * element, GstStateChange transition); + +static void gst_qml6_gl_sink_get_times (GstBaseSink * bsink, GstBuffer * buf, + GstClockTime * start, GstClockTime * end); +static gboolean gst_qml6_gl_sink_set_caps (GstBaseSink * bsink, GstCaps * caps); +static GstFlowReturn gst_qml6_gl_sink_show_frame (GstVideoSink * bsink, + GstBuffer * buf); +static gboolean gst_qml6_gl_sink_propose_allocation (GstBaseSink * bsink, + GstQuery * query); + +static GstStaticPadTemplate gst_qt_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " + "format = (string) { RGB, RGBA }, " + "width = " GST_VIDEO_SIZE_RANGE ", " + "height = " GST_VIDEO_SIZE_RANGE ", " + "framerate = " GST_VIDEO_FPS_RANGE ", " + "texture-target = (string) 2D")); + +#define DEFAULT_FORCE_ASPECT_RATIO TRUE +#define DEFAULT_PAR_N 0 +#define DEFAULT_PAR_D 1 + +enum +{ + ARG_0, + PROP_WIDGET, + PROP_FORCE_ASPECT_RATIO, + PROP_PIXEL_ASPECT_RATIO, +}; + +enum +{ + SIGNAL_0, + LAST_SIGNAL +}; + +#define gst_qml6_gl_sink_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstQml6GLSink, gst_qml6_gl_sink, + GST_TYPE_VIDEO_SINK, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, + "qtsink", 0, "Qt Video Sink"); + G_IMPLEMENT_INTERFACE (GST_TYPE_NAVIGATION, + gst_qml6_gl_sink_navigation_interface_init)); +G_BEGIN_DECLS gboolean G_PASTE(gst_element_register_, qml6glsink)(GstPlugin *plugin) +{ + { + { + qt6_element_init(plugin); + } + } + return gst_element_register(plugin, "qml6glsink", GST_RANK_NONE, (gst_qml6_gl_sink_get_type())); +} +G_END_DECLS; + +static void +gst_qml6_gl_sink_class_init (GstQml6GLSinkClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstBaseSinkClass *gstbasesink_class; + GstVideoSinkClass *gstvideosink_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + gstbasesink_class = (GstBaseSinkClass *) klass; + gstvideosink_class = (GstVideoSinkClass *) klass; + + gobject_class->set_property = gst_qml6_gl_sink_set_property; + gobject_class->get_property = gst_qml6_gl_sink_get_property; + + gst_element_class_set_metadata (gstelement_class, "Qt6 Video Sink", + "Sink/Video", "A video sink that renders to a QQuickItem for Qt6", + "Matthew Waters "); + + g_object_class_install_property (gobject_class, PROP_WIDGET, + g_param_spec_pointer ("widget", "QQuickItem", + "The QQuickItem to place in the object hierarchy", + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_FORCE_ASPECT_RATIO, + g_param_spec_boolean ("force-aspect-ratio", + "Force aspect ratio", + "When enabled, scaling will respect original aspect ratio", + DEFAULT_FORCE_ASPECT_RATIO, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_PIXEL_ASPECT_RATIO, + gst_param_spec_fraction ("pixel-aspect-ratio", "Pixel Aspect Ratio", + "The pixel aspect ratio of the device", DEFAULT_PAR_N, DEFAULT_PAR_D, + G_MAXINT, 1, 1, 1, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + gst_element_class_add_static_pad_template (gstelement_class, &gst_qt_sink_template); + + gobject_class->finalize = gst_qml6_gl_sink_finalize; + + gstelement_class->change_state = gst_qml6_gl_sink_change_state; + gstbasesink_class->query = gst_qml6_gl_sink_query; + gstbasesink_class->set_caps = gst_qml6_gl_sink_set_caps; + gstbasesink_class->get_times = gst_qml6_gl_sink_get_times; + gstbasesink_class->propose_allocation = gst_qml6_gl_sink_propose_allocation; + gstbasesink_class->stop = gst_qml6_gl_sink_stop; + + gstvideosink_class->show_frame = gst_qml6_gl_sink_show_frame; +} + +static void +gst_qml6_gl_sink_init (GstQml6GLSink * qt_sink) +{ + qt_sink->widget = QSharedPointer(); + if (qt_sink->widget) + qt_sink->widget->setSink (GST_ELEMENT_CAST (qt_sink)); +} + +static void +gst_qml6_gl_sink_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstQml6GLSink *qt_sink = GST_QML6_GL_SINK (object); + + switch (prop_id) { + case PROP_WIDGET: { + Qt6GLVideoItem *qt_item = static_cast (g_value_get_pointer (value)); + if (qt_item) { + qt_sink->widget = qt_item->getInterface(); + if (qt_sink->widget) { + qt_sink->widget->setSink (GST_ELEMENT_CAST (qt_sink)); + } + } else { + qt_sink->widget.clear(); + } + break; + } + case PROP_FORCE_ASPECT_RATIO: + g_return_if_fail (qt_sink->widget); + qt_sink->widget->setForceAspectRatio (g_value_get_boolean (value)); + break; + case PROP_PIXEL_ASPECT_RATIO: + g_return_if_fail (qt_sink->widget); + qt_sink->widget->setDAR (gst_value_get_fraction_numerator (value), + gst_value_get_fraction_denominator (value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_reset (GstQml6GLSink * qt_sink) +{ + if (qt_sink->display) { + gst_object_unref (qt_sink->display); + qt_sink->display = NULL; + } + + if (qt_sink->context) { + gst_object_unref (qt_sink->context); + qt_sink->context = NULL; + } + + if (qt_sink->qt_context) { + gst_object_unref (qt_sink->qt_context); + qt_sink->qt_context = NULL; + } +} + +static void +gst_qml6_gl_sink_finalize (GObject * object) +{ + GstQml6GLSink *qt_sink = GST_QML6_GL_SINK (object); + + _reset (qt_sink); + + qt_sink->widget.clear(); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_qml6_gl_sink_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstQml6GLSink *qt_sink = GST_QML6_GL_SINK (object); + + switch (prop_id) { + case PROP_WIDGET: + /* This is not really safe - the app needs to be + * sure the widget is going to be kept alive or + * this can crash */ + if (qt_sink->widget) + g_value_set_pointer (value, qt_sink->widget->videoItem()); + else + g_value_set_pointer (value, NULL); + break; + case PROP_FORCE_ASPECT_RATIO: + if (qt_sink->widget) + g_value_set_boolean (value, qt_sink->widget->getForceAspectRatio ()); + else + g_value_set_boolean (value, DEFAULT_FORCE_ASPECT_RATIO); + break; + case PROP_PIXEL_ASPECT_RATIO: + if (qt_sink->widget) { + gint num, den; + qt_sink->widget->getDAR (&num, &den); + gst_value_set_fraction (value, num, den); + } else { + gst_value_set_fraction (value, DEFAULT_PAR_N, DEFAULT_PAR_D); + } + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_qml6_gl_sink_query (GstBaseSink * bsink, GstQuery * query) +{ + GstQml6GLSink *qt_sink = GST_QML6_GL_SINK (bsink); + gboolean res = FALSE; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + { + if (gst_gl_handle_context_query ((GstElement *) qt_sink, query, + qt_sink->display, qt_sink->context, qt_sink->qt_context)) + return TRUE; + + /* fallthrough */ + } + default: + res = GST_BASE_SINK_CLASS (parent_class)->query (bsink, query); + break; + } + + return res; +} + +static gboolean +gst_qml6_gl_sink_stop (GstBaseSink * bsink) +{ + return TRUE; +} + +static GstStateChangeReturn +gst_qml6_gl_sink_change_state (GstElement * element, GstStateChange transition) +{ + GstQml6GLSink *qt_sink = GST_QML6_GL_SINK (element); + GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + QGuiApplication *app; + + GST_DEBUG ("changing state: %s => %s", + gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)), + gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition))); + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: + app = static_cast (QCoreApplication::instance ()); + if (!app) { + GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND, + ("%s", "Failed to connect to Qt"), + ("%s", "Could not retrieve QGuiApplication instance")); + return GST_STATE_CHANGE_FAILURE; + } + + if (!qt_sink->widget) { + GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND, + ("%s", "Required property \'widget\' not set"), + (NULL)); + return GST_STATE_CHANGE_FAILURE; + } + + if (!qt_sink->widget->initWinSys()) { + GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND, + ("%s", "Could not initialize window system"), + (NULL)); + return GST_STATE_CHANGE_FAILURE; + } + + qt_sink->display = qt_sink->widget->getDisplay(); + qt_sink->context = qt_sink->widget->getContext(); + qt_sink->qt_context = qt_sink->widget->getQtContext(); + + if (!qt_sink->display || !qt_sink->context || !qt_sink->qt_context) { + GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND, + ("%s", "Could not retrieve window system OpenGL configuration"), + (NULL)); + return GST_STATE_CHANGE_FAILURE; + } + + GST_OBJECT_LOCK (qt_sink->display); + gst_gl_display_add_context (qt_sink->display, qt_sink->context); + GST_OBJECT_UNLOCK (qt_sink->display); + + gst_gl_element_propagate_display_context (GST_ELEMENT (qt_sink), qt_sink->display); + + break; + case GST_STATE_CHANGE_READY_TO_PAUSED: + break; + case GST_STATE_CHANGE_PAUSED_TO_PLAYING: + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + case GST_STATE_CHANGE_PLAYING_TO_PAUSED: + break; + case GST_STATE_CHANGE_PAUSED_TO_READY: + if (qt_sink->widget) + qt_sink->widget->setBuffer(NULL); + break; + case GST_STATE_CHANGE_READY_TO_NULL: + break; + default: + break; + } + + return ret; +} + +static void +gst_qml6_gl_sink_get_times (GstBaseSink * bsink, GstBuffer * buf, + GstClockTime * start, GstClockTime * end) +{ + GstQml6GLSink *qt_sink = GST_QML6_GL_SINK (bsink); + + if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) { + *start = GST_BUFFER_TIMESTAMP (buf); + if (GST_BUFFER_DURATION_IS_VALID (buf)) + *end = *start + GST_BUFFER_DURATION (buf); + else { + if (GST_VIDEO_INFO_FPS_N (&qt_sink->v_info) > 0) { + *end = *start + + gst_util_uint64_scale_int (GST_SECOND, + GST_VIDEO_INFO_FPS_D (&qt_sink->v_info), + GST_VIDEO_INFO_FPS_N (&qt_sink->v_info)); + } + } + } +} + +gboolean +gst_qml6_gl_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) +{ + GstQml6GLSink *qt_sink = GST_QML6_GL_SINK (bsink); + + GST_DEBUG ("set caps with %" GST_PTR_FORMAT, caps); + + if (!gst_video_info_from_caps (&qt_sink->v_info, caps)) + return FALSE; + + if (!qt_sink->widget) + return FALSE; + + return qt_sink->widget->setCaps(caps); +} + +static GstFlowReturn +gst_qml6_gl_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf) +{ + GstQml6GLSink *qt_sink = GST_QML6_GL_SINK (vsink); + + GST_TRACE ("rendering buffer:%p", buf); + + if (qt_sink->widget) + qt_sink->widget->setBuffer(buf); + + return GST_FLOW_OK; +} + +static gboolean +gst_qml6_gl_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) +{ + GstQml6GLSink *qt_sink = GST_QML6_GL_SINK (bsink); + GstBufferPool *pool; + GstStructure *config; + GstCaps *caps; + guint size; + gboolean need_pool; + + if (!qt_sink->display || !qt_sink->context) + return FALSE; + + gst_query_parse_allocation (query, &caps, &need_pool); + + if (caps == NULL) + goto no_caps; + + /* FIXME re-using buffer pool breaks renegotiation */ + if ((pool = qt_sink->pool)) + gst_object_ref (pool); + + if (pool != NULL) { + GstCaps *pcaps; + + /* we had a pool, check caps */ + GST_DEBUG_OBJECT (qt_sink, "check existing pool caps"); + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_get_params (config, &pcaps, &size, NULL, NULL); + + if (!gst_caps_is_equal (caps, pcaps)) { + GST_DEBUG_OBJECT (qt_sink, "pool has different caps"); + /* different caps, we can't use this pool */ + gst_object_unref (pool); + pool = NULL; + } + gst_structure_free (config); + } else { + GstVideoInfo info; + + if (!gst_video_info_from_caps (&info, caps)) + goto invalid_caps; + + /* the normal size of a frame */ + size = info.size; + } + + if (pool == NULL && need_pool) { + + GST_DEBUG_OBJECT (qt_sink, "create new pool"); + pool = gst_gl_buffer_pool_new (qt_sink->context); + + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (config, caps, size, 0, 0); + if (!gst_buffer_pool_set_config (pool, config)) + goto config_failed; + } + + /* we need at least 2 buffer because we hold on to the last one */ + gst_query_add_allocation_pool (query, pool, size, 2, 0); + if (pool) + gst_object_unref (pool); + + /* we also support various metadata */ + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0); + + if (qt_sink->context->gl_vtable->FenceSync) + gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0); + + return TRUE; + + /* ERRORS */ +no_caps: + { + GST_DEBUG_OBJECT (bsink, "no caps specified"); + return FALSE; + } +invalid_caps: + { + GST_DEBUG_OBJECT (bsink, "invalid caps specified"); + return FALSE; + } +config_failed: + { + GST_DEBUG_OBJECT (bsink, "failed setting config"); + return FALSE; + } +} + +static void +gst_qml6_gl_sink_navigation_send_event (GstNavigation * navigation, + GstStructure *structure) +{ + GstQml6GLSink *qt_sink = GST_QML6_GL_SINK (navigation); + GST_TRACE_OBJECT (qt_sink, "navigation event %" GST_PTR_FORMAT, + structure); + // TODO(zdanek) this will be available from gstreamer 1.22 +} + +static void gst_qml6_gl_sink_navigation_interface_init (GstNavigationInterface * iface) +{ + iface->send_event = gst_qml6_gl_sink_navigation_send_event; +} diff --git a/libs/qmlglsink/qt6-linux/gstqml6glsink.h b/libs/qmlglsink/qt6-linux/gstqml6glsink.h new file mode 100644 index 000000000000..4eeeddbced1a --- /dev/null +++ b/libs/qmlglsink/qt6-linux/gstqml6glsink.h @@ -0,0 +1,62 @@ +/* + * GStreamer + * Copyright (C) 2015 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_QT6_SINK_H__ +#define __GST_QT6_SINK_H__ + +#include +#include +#include +#include +#include "qt6glitem.h" + +typedef struct _GstQml6GLSinkPrivate GstQml6GLSinkPrivate; + +G_BEGIN_DECLS + +#define GST_TYPE_QML6_GL_SINK (gst_qml6_gl_sink_get_type()) +G_DECLARE_FINAL_TYPE (GstQml6GLSink, gst_qml6_gl_sink, GST, QML6_GL_SINK, GstVideoSink) +#define GST_QML6_GL_SINK_CAST(obj) ((GstQml6GLSink*)(obj)) + +/** + * GstQml6GLSink: + * + * Opaque #GstQml6GLSink object + */ +struct _GstQml6GLSink +{ + /* */ + GstVideoSink parent; + + GstVideoInfo v_info; + GstBufferPool *pool; + + GstGLDisplay *display; + GstGLContext *context; + GstGLContext *qt_context; + + QSharedPointer widget; +}; + +GstQml6GLSink * gst_qml6_gl_sink_new (void); + +G_END_DECLS + +#endif /* __GST_QT6_SINK_H__ */ diff --git a/libs/qmlglsink/qt6/gstqsg6glnode.cc b/libs/qmlglsink/qt6-linux/gstqsg6glnode.cc similarity index 100% rename from libs/qmlglsink/qt6/gstqsg6glnode.cc rename to libs/qmlglsink/qt6-linux/gstqsg6glnode.cc diff --git a/libs/qmlglsink/qt6/gstqsg6glnode.h b/libs/qmlglsink/qt6-linux/gstqsg6glnode.h similarity index 100% rename from libs/qmlglsink/qt6/gstqsg6glnode.h rename to libs/qmlglsink/qt6-linux/gstqsg6glnode.h diff --git a/libs/qmlglsink/qt6-linux/gstqt6element.cc b/libs/qmlglsink/qt6-linux/gstqt6element.cc new file mode 100644 index 000000000000..aa9347103126 --- /dev/null +++ b/libs/qmlglsink/qt6-linux/gstqt6element.cc @@ -0,0 +1,38 @@ +/* + * GStreamer + * Copyright (C) 2015 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstqt6elements.h" +#include "qt6glitem.h" +#include + +void +qt6_element_init (GstPlugin * plugin) +{ + static gsize res = FALSE; + if (g_once_init_enter (&res)) { + /* this means the plugin must be loaded before the qml engine is loaded */ + qmlRegisterType ("org.freedesktop.gstreamer.Qt6GLVideoItem", 1, 0, "GstGLQt6VideoItem"); + g_once_init_leave (&res, TRUE); + } +} diff --git a/libs/qmlglsink/qt6-linux/gstqt6elements.h b/libs/qmlglsink/qt6-linux/gstqt6elements.h new file mode 100644 index 000000000000..c22984216f96 --- /dev/null +++ b/libs/qmlglsink/qt6-linux/gstqt6elements.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2020 Huawei Technologies Co., Ltd. + * @Author: Julian Bouzas + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_QT6_ELEMENTS_H__ +#define __GST_QT6_ELEMENTS_H__ + +#include + +G_BEGIN_DECLS + +void qt6_element_init (GstPlugin * plugin); + +// TODO(zdanek) fix after switching to gstreamer 1.20.0+ +// original code from 1.20.0 +// GST_ELEMENT_REGISTER_DECLARE (qml6glsink); +// backported to: +extern "C" { gboolean gst_element_register_qml6glsink (GstPlugin * plugin); }; + +G_END_DECLS + +#endif /* __GST_QT6_ELEMENTS_H__ */ diff --git a/libs/qmlglsink/qt6-linux/gstqt6gl.h b/libs/qmlglsink/qt6-linux/gstqt6gl.h new file mode 100644 index 000000000000..a1babaad18cf --- /dev/null +++ b/libs/qmlglsink/qt6-linux/gstqt6gl.h @@ -0,0 +1,56 @@ +/* + * GStreamer + * Copyright (C) 2015 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#include +#if (QT_VERSION >= QT_VERSION_CHECK(5, 9, 0)) +#include +#endif + +#include + +/* The glext.h guard was renamed in 2018, but some software which + * includes their own copy of the GL headers (such as qt) might have + * older version which use the old guard. This would result in the + * header being included again (and symbols redefined). + * + * To avoid this, we define the "old" guard if the "new" guard is + * defined.*/ +#if GST_GL_HAVE_OPENGL +#ifdef __gl_glext_h_ +#ifndef __glext_h_ +#define __glext_h_ 1 +#endif +#endif +#endif + +/* pulls in GLsync, see below */ +#include + +/* qt uses the same trick as us to typedef GLsync on GLES2 but to a different + * type which confuses the preprocessor. Instead of trying to reconcile the + * two, we instead use the GLsync definition from Qt from above, and ensure + * that we don't typedef GLsync in gstglfuncs.h */ +#undef GST_GL_HAVE_GLSYNC +#define GST_GL_HAVE_GLSYNC 1 +#include + +#if defined(QT_OPENGL_ES_2) +#include +#include +#endif /* defined(QT_OPENGL_ES_2) */ diff --git a/libs/qmlglsink/qt6-linux/gstqt6glutility.cc b/libs/qmlglsink/qt6-linux/gstqt6glutility.cc new file mode 100644 index 000000000000..013ded4bfef3 --- /dev/null +++ b/libs/qmlglsink/qt6-linux/gstqt6glutility.cc @@ -0,0 +1,359 @@ +/* + * GStreamer + * Copyright (C) 2016 Freescale Semiconductor, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstqt6glutility.h" +#include +#if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11) +#include +//#include +#endif +#if GST_GL_HAVE_PLATFORM_EGL && (defined (HAVE_QT_WAYLAND) || defined (HAVE_QT_EGLFS) || defined (HAVE_QT_ANDROID)) +#include +#ifdef HAVE_QT_QPA_HEADER +#include +#endif +//#include +#include +#endif + +#if GST_GL_HAVE_WINDOW_WAYLAND && defined (HAVE_QT_WAYLAND) +#include +#endif +#if 0 +#if GST_GL_HAVE_WINDOW_VIV_FB +#include +#endif + +#if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32) +#include +#include +#endif +#endif +#include + +#define GST_CAT_DEFAULT qml6_gl_utils_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +G_LOCK_DEFINE_STATIC (display_lock); +static GWeakRef qt_display; +static gboolean sink_retrieved = FALSE; + +GstGLDisplay * +gst_qml6_get_gl_display (gboolean sink) +{ + GstGLDisplay *display = NULL; + QGuiApplication *app = static_cast (QCoreApplication::instance ()); + static gsize _debug; + + g_assert (app != NULL); + + if (g_once_init_enter (&_debug)) { + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtglutility", 0, + "Qt gl utility functions"); + g_once_init_leave (&_debug, 1); + } + + G_LOCK (display_lock); + /* XXX: this assumes that only one display will ever be created by Qt */ + display = static_cast(g_weak_ref_get (&qt_display)); + if (display) { + if (sink_retrieved) { + GST_INFO ("returning previously created display"); + G_UNLOCK (display_lock); + return display; + } + gst_clear_object (&display); + } + if (sink) + sink_retrieved = sink; + + GST_INFO ("QGuiApplication::instance()->platformName() %s", app->platformName().toUtf8().data()); +#if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11) + if (QString::fromUtf8 ("xcb") == app->platformName()) { + auto x11_native = app->nativeInterface(); + if (x11_native) { + display = (GstGLDisplay *) + gst_gl_display_x11_new_with_display (x11_native->display()); + } + } +#endif +#if GST_GL_HAVE_WINDOW_WAYLAND && GST_GL_HAVE_PLATFORM_EGL && defined (HAVE_QT_WAYLAND) + if (QString::fromUtf8 ("wayland") == app->platformName() + || QString::fromUtf8 ("wayland-egl") == app->platformName()){ + struct wl_display * wayland_display; + QPlatformNativeInterface *native = + QGuiApplication::platformNativeInterface(); + wayland_display = (struct wl_display *) + native->nativeResourceForWindow("display", NULL); + display = (GstGLDisplay *) + gst_gl_display_wayland_new_with_display (wayland_display); + } +#endif +#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_WINDOW_ANDROID + if (QString::fromUtf8 ("android") == app->platformName()) { + EGLDisplay egl_display = (EGLDisplay) gst_gl_display_egl_get_from_native (GST_GL_DISPLAY_TYPE_ANY, 0); + display = (GstGLDisplay *) gst_gl_display_egl_new_with_egl_display (egl_display); + } +#elif GST_GL_HAVE_PLATFORM_EGL && defined (HAVE_QT_EGLFS) + if (QString::fromUtf8("eglfs") == app->platformName()) { +#if GST_GL_HAVE_WINDOW_VIV_FB + /* FIXME: Could get the display directly from Qt like this + * QPlatformNativeInterface *native = + * QGuiApplication::platformNativeInterface(); + * EGLDisplay egl_display = (EGLDisplay) + * native->nativeResourceForWindow("egldisplay", NULL); + * + * However we seem to have no way for getting the EGLNativeDisplayType, aka + * native_display, via public API. As such we have to assume that display 0 + * is always used. Only way around that is parsing the index the same way as + * Qt does in QEGLDeviceIntegration::fbDeviceName(), so let's do that. + */ + const gchar *fb_dev; + gint disp_idx = 0; + + fb_dev = g_getenv ("QT_QPA_EGLFS_FB"); + if (fb_dev) { + if (sscanf (fb_dev, "/dev/fb%d", &disp_idx) != 1) + disp_idx = 0; + } + + display = (GstGLDisplay *) gst_gl_display_viv_fb_new (disp_idx); +#elif defined(HAVE_QT_QPA_HEADER) + QPlatformNativeInterface *native = + QGuiApplication::platformNativeInterface(); + EGLDisplay egl_display = (EGLDisplay) + native->nativeResourceForWindow("egldisplay", NULL); + if (egl_display != EGL_NO_DISPLAY) + display = (GstGLDisplay *) gst_gl_display_egl_new_with_egl_display (egl_display); +#else + EGLDisplay egl_display = (EGLDisplay) gst_gl_display_egl_get_from_native (GST_GL_DISPLAY_TYPE_ANY, 0); + display = (GstGLDisplay *) gst_gl_display_egl_new_with_egl_display (egl_display); +#endif + } +#endif +#if GST_GL_HAVE_WINDOW_COCOA && GST_GL_HAVE_PLATFORM_CGL && defined (HAVE_QT_MAC) + if (QString::fromUtf8 ("cocoa") == app->platformName()) + display = (GstGLDisplay *) gst_gl_display_new (); +#endif +#if GST_GL_HAVE_WINDOW_EAGL && GST_GL_HAVE_PLATFORM_EAGL && defined (HAVE_QT_IOS) + if (QString::fromUtf8 ("ios") == app->platformName()) + display = gst_gl_display_new (); +#endif +#if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32) + if (QString::fromUtf8 ("windows") == app->platformName()) + display = gst_gl_display_new (); +#endif + + if (!display) + display = gst_gl_display_new (); + + g_weak_ref_set (&qt_display, display); + G_UNLOCK (display_lock); + + return display; +} + +gboolean +gst_qml6_get_gl_wrapcontext (GstGLDisplay * display, + GstGLContext **wrap_glcontext, GstGLContext **context) +{ + GstGLPlatform G_GNUC_UNUSED platform = (GstGLPlatform) 0; + GstGLAPI G_GNUC_UNUSED gl_api; + guintptr G_GNUC_UNUSED gl_handle; + GstGLContext *current; + GError *error = NULL; + + g_return_val_if_fail (display != NULL && wrap_glcontext != NULL, FALSE); +#if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11) + if (GST_IS_GL_DISPLAY_X11 (display)) { +#if GST_GL_HAVE_PLATFORM_GLX + platform = GST_GL_PLATFORM_GLX; +#elif GST_GL_HAVE_PLATFORM_EGL + platform = GST_GL_PLATFORM_EGL; +#endif + } +#endif +#if GST_GL_HAVE_WINDOW_WAYLAND && defined (HAVE_QT_WAYLAND) + if (GST_IS_GL_DISPLAY_WAYLAND (display)) { + platform = GST_GL_PLATFORM_EGL; + } +#endif +#if GST_GL_HAVE_PLATFORM_EGL && defined (HAVE_QT_EGLFS) +#if GST_GL_HAVE_WINDOW_VIV_FB + if (GST_IS_GL_DISPLAY_VIV_FB (display)) { +#else + if (GST_IS_GL_DISPLAY_EGL (display)) { +#endif + platform = GST_GL_PLATFORM_EGL; + } +#endif + if (platform == 0) { +#if GST_GL_HAVE_WINDOW_COCOA && GST_GL_HAVE_PLATFORM_CGL && defined (HAVE_QT_MAC) + platform = GST_GL_PLATFORM_CGL; +#elif GST_GL_HAVE_WINDOW_EAGL && GST_GL_HAVE_PLATFORM_EAGL && defined (HAVE_QT_IOS) + platform = GST_GL_PLATFORM_EAGL; +#elif GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32) + platform = GST_GL_PLATFORM_WGL; +#elif GST_GL_HAVE_WINDOW_ANDROID && GST_GL_HAVE_PLATFORM_EGL && defined (HAVE_QT_ANDROID) + platform = GST_GL_PLATFORM_EGL; +#else + GST_ERROR ("Unknown platform"); + return FALSE; +#endif + } + + gl_api = gst_gl_context_get_current_gl_api (platform, NULL, NULL); + gl_handle = gst_gl_context_get_current_gl_context (platform); + + /* see if we already have a current GL context in GStreamer for this thread */ + current = gst_gl_context_get_current (); + if (current && current->display == display) { + /* just use current context we found */ + *wrap_glcontext = static_cast (gst_object_ref (current)); + } + else { + if (gl_handle) + *wrap_glcontext = + gst_gl_context_new_wrapped (display, gl_handle, + platform, gl_api); + + if (!*wrap_glcontext) { + GST_ERROR ("cannot wrap qt OpenGL context"); + return FALSE; + } + + gst_gl_context_activate(*wrap_glcontext, TRUE); + if (!gst_gl_context_fill_info (*wrap_glcontext, &error)) { + GST_ERROR ("failed to retrieve qt context info: %s", error->message); + gst_gl_context_activate(*wrap_glcontext, FALSE); + gst_clear_object (wrap_glcontext); + return FALSE; + } + + gst_gl_display_filter_gl_api (display, gst_gl_context_get_gl_api (*wrap_glcontext)); + gst_gl_context_activate (*wrap_glcontext, FALSE); + } +#if 0 +#if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32) + g_return_val_if_fail (context != NULL, FALSE); + + G_STMT_START { + /* If there's no wglCreateContextAttribsARB() support, then we would fallback to + * wglShareLists() which will fail with ERROR_BUSY (0xaa) if either of the GL + * contexts are current in any other thread. + * + * The workaround here is to temporarily disable Qt's GL context while we + * set up our own. + * + * Sometimes wglCreateContextAttribsARB() + * exists, but isn't functional (some Intel drivers), so it's easiest to do this + * unconditionally. + */ + + /* retrieve Qt's GL device context as current device context */ + HDC device = wglGetCurrentDC (); + + *context = gst_gl_context_new (display); + + wglMakeCurrent (NULL, NULL); + if (!gst_gl_context_create (*context, *wrap_glcontext, &error)) { + GST_ERROR ("failed to create shared GL context: %s", error->message); + gst_clear_object (wrap_glcontext); + gst_clear_object (context); + } + wglMakeCurrent (device, (HGLRC) gl_handle); + + if (!*context) + return FALSE; + + } G_STMT_END; +#endif +#endif + return TRUE; +} +#if 0 +QVariant +qt_opengl_native_context_from_gst_gl_context (GstGLContext * context) +{ + guintptr handle; + GstGLPlatform platform; + + handle = gst_gl_context_get_gl_context (context); + platform = gst_gl_context_get_gl_platform (context); + +#if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11) + if (platform == GST_GL_PLATFORM_GLX) { + GstGLDisplay *display = gst_gl_context_get_display (context); + GstGLWindow *window = gst_gl_context_get_window (context); + Display *xdisplay = (Display *) gst_gl_display_get_handle (display); + Window win = gst_gl_window_get_window_handle (window); + gst_object_unref (window); + gst_object_unref (display); + return QVariant::fromValue(QGLXNativeContext((GLXContext) handle, xdisplay, win)); + } +#endif +#if GST_GL_HAVE_PLATFORM_EGL && (defined (HAVE_QT_WAYLAND) || defined (HAVE_QT_EGLFS) || defined (HAVE_QT_ANDROID)) + if (platform == GST_GL_PLATFORM_EGL) { + EGLDisplay egl_display = EGL_DEFAULT_DISPLAY; + GstGLDisplay *display = gst_gl_context_get_display (context); + GstGLDisplayEGL *display_egl = gst_gl_display_egl_from_gl_display (display); +#if GST_GL_HAVE_WINDOW_WAYLAND && defined (HAVE_QT_WAYLAND) + if (gst_gl_display_get_handle_type (display) == GST_GL_DISPLAY_TYPE_WAYLAND) { +#if 1 + g_warning ("Qt does not support wrapping native OpenGL contexts " + "on wayland. See https://bugreports.qt.io/browse/QTBUG-82528"); + gst_object_unref (display_egl); + gst_object_unref (display); + return QVariant::fromValue(nullptr); +#else + if (display_egl) + egl_display = (EGLDisplay) gst_gl_display_get_handle ((GstGLDisplay *) display_egl); +#endif + } +#endif + gst_object_unref (display_egl); + gst_object_unref (display); + return QVariant::fromValue(QEGLNativeContext((EGLContext) handle, egl_display)); + } +#endif +#if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32) + if (platform == GST_GL_PLATFORM_WGL) { + GstGLWindow *window = gst_gl_context_get_window (context); + guintptr hwnd = gst_gl_window_get_window_handle (window); + gst_object_unref (window); + return QVariant::fromValue(QWGLNativeContext((HGLRC) handle, (HWND) hwnd)); + } +#endif + { + gchar *platform_s = gst_gl_platform_to_string (platform); + g_warning ("Unimplemented configuration! This means either:\n" + "1. The qmlgl plugin was built without support for your platform.\n" + "2. The necessary code to convert from a GstGLContext to Qt's " + "native context type for \'%s\' currently does not exist.", + platform_s); + g_free (platform_s); + } + return QVariant::fromValue(nullptr); +} +#endif diff --git a/libs/qmlglsink/qt6-linux/gstqt6glutility.h b/libs/qmlglsink/qt6-linux/gstqt6glutility.h new file mode 100644 index 000000000000..ba436230da85 --- /dev/null +++ b/libs/qmlglsink/qt6-linux/gstqt6glutility.h @@ -0,0 +1,55 @@ +/* + * GStreamer + * Copyright (C) 2016 Freescale Semiconductor, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __QML6_GL_UTILS_H__ +#define __QML6_GL_UTILS_H__ + +#include +#include + +#include +#include + +G_BEGIN_DECLS + +struct RenderJob : public QRunnable { + using Callable = std::function; + + explicit RenderJob(Callable c) : _c(c) { } + + void run() { _c(); } + +private: + Callable _c; +}; + +GstGLDisplay * gst_qml6_get_gl_display (gboolean sink); +gboolean gst_qml6_get_gl_wrapcontext (GstGLDisplay * display, + GstGLContext **wrap_glcontext, GstGLContext **context); + +G_END_DECLS + +#if 0 +#if defined(__cplusplus) +QVariant qt_opengl_native_context_from_gst_gl_context (GstGLContext * context); +#endif +#endif + +#endif /* __QML6_GL_UTILS_H__ */ diff --git a/libs/qmlglsink/qt6-linux/meson.build b/libs/qmlglsink/qt6-linux/meson.build new file mode 100644 index 000000000000..150b0b9f222f --- /dev/null +++ b/libs/qmlglsink/qt6-linux/meson.build @@ -0,0 +1,145 @@ +sources = [ + 'gstplugin.cc', + 'gstqt6element.cc', + 'gstqsg6glnode.cc', + 'gstqt6glutility.cc', + 'gstqml6glsink.cc', + 'qt6glitem.cc', +] + +moc_headers = [ + 'qt6glitem.h', + 'gstqsg6glnode.h', +] + +qt6qml_dep = dependency('', required: false) +qt6_option = get_option('qt6') +qt6_method = get_option('qt-method') + +if qt6_option.disabled() + subdir_done() +endif + +if not have_gstgl + if qt6_option.enabled() + error('qt6 qmlglsink plugin is enabled, but gstreamer-gl-1.0 was not found') + endif + subdir_done() +endif + +if not add_languages('cpp', native: false, required: qt6_option) + subdir_done() +endif + +qt6_mod = import('qt6') +if not qt6_mod.has_tools() + if qt6_option.enabled() + error('qt6 qmlglsink plugin is enabled, but qt specific tools were not found') + endif + subdir_done() +endif + +qt6qml_dep = dependency('qt6', modules : ['Core', 'Gui', 'Qml', 'Quick'], + method: qt6_method, required: qt6_option, static: host_system == 'ios') +if not qt6qml_dep.found() + subdir_done() +endif + +optional_deps = [] +qt_defines = [] +have_qpa_include = false +have_qt_windowing = false + +# Look for the QPA platform native interface header +qpa_header_path = join_paths(qt6qml_dep.version(), 'QtGui') +qpa_header = join_paths(qpa_header_path, 'qpa/qplatformnativeinterface.h') +if cxx.has_header(qpa_header, dependencies : qt6qml_dep) + qt_defines += '-DHAVE_QT_QPA_HEADER' + qt_defines += '-DQT_QPA_HEADER=' + '<@0@>'.format(qpa_header) + have_qpa_include = true + message('Found QtGui QPA header in ' + qpa_header_path) +endif + +# Try to come up with all the platform/winsys combinations that will work + +if gst_gl_have_window_x11 and gst_gl_have_platform_glx + # FIXME: automagic + qt_defines += ['-DHAVE_QT_X11'] + have_qt_windowing = true +endif + +if gst_gl_have_platform_egl + # Embedded linux (e.g. i.MX6) with or without windowing support + qt_defines += ['-DHAVE_QT_EGLFS'] + optional_deps += gstglegl_dep + have_qt_windowing = true + if have_qpa_include + # Wayland windowing + if gst_gl_have_window_wayland + # FIXME: automagic + qt6waylandextras = dependency('qt6', modules : ['WaylandClient'], method: qt6_method, required : false) + if qt6waylandextras.found() + optional_deps += [qt6waylandextras, gstglwayland_dep] + qt_defines += ['-DHAVE_QT_WAYLAND'] + have_qt_windowing = true + endif + endif + # Android windowing +# if gst_gl_have_window_android + # FIXME: automagic +# qt5androidextras = dependency('qt5', modules : ['AndroidExtras'], method: qt6_method, required : false) + # for gl functions in QtGui/qopenglfunctions.h + # FIXME: automagic +# glesv2_dep = cc.find_library('GLESv2', required : false) +# if glesv2_dep.found() and qt5androidextras.found() +# optional_deps += [qt5androidextras, glesv2_dep] +# qt_defines += ['-DHAVE_QT_ANDROID'] +# have_qt_windowing = true + # Needed for C++11 support in Cerbero. People building with Android + # in some other way need to add the necessary bits themselves. +# optional_deps += dependency('gnustl', required : false) +# endif +# endif + endif +endif + +#if gst_gl_have_platform_wgl and gst_gl_have_window_win32 + # for wglMakeCurrent() + # FIXME: automagic +# opengl32_dep = cc.find_library('opengl32', required : false) +# if opengl32_dep.found() +# qt_defines += ['-DHAVE_QT_WIN32'] +# optional_deps += opengl32_dep +# have_qt_windowing = true +# endif +#endif + +if gst_gl_have_window_cocoa and gst_gl_have_platform_cgl + # FIXME: automagic + if host_machine.system() == 'darwin' + qt_defines += ['-DHAVE_QT_MAC'] + have_qt_windowing = true + endif +endif + +if gst_gl_have_window_eagl and gst_gl_have_platform_eagl + if host_machine.system() == 'ios' + qt_defines += ['-DHAVE_QT_IOS'] + have_qt_windowing = true + endif +endif + +if have_qt_windowing + # Build it! + moc_files = qt6_mod.preprocess(moc_headers : moc_headers) + gstqml6gl = library('gstqml6', sources, moc_files, + cpp_args : gst_plugins_good_args + qt_defines, + link_args : noseh_link_args, + include_directories: [configinc, libsinc], + dependencies : [gst_dep, gstvideo_dep, gstgl_dep, gstglproto_dep, qt6qml_dep, optional_deps], + override_options : ['cpp_std=c++17'], + install: true, + install_dir : plugins_install_dir) + pkgconfig.generate(gstqml6gl, install_dir : plugins_pkgconfig_install_dir) + plugins += [gstqml6gl] +endif diff --git a/libs/qmlglsink/qt6-linux/qt6glitem.cc b/libs/qmlglsink/qt6-linux/qt6glitem.cc new file mode 100644 index 000000000000..dfb012a05cd7 --- /dev/null +++ b/libs/qmlglsink/qt6-linux/qt6glitem.cc @@ -0,0 +1,755 @@ +/* + * GStreamer + * Copyright (C) 2015 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include +#include "qt6glitem.h" +#include "gstqsg6glnode.h" +#include "gstqt6glutility.h" + +#include +#include +#include +#include +#include + +/** + * SECTION:Qt6GLVideoItem + * @short_description: a Qt5 QtQuick item that renders GStreamer video #GstBuffers + * + * #QtGLVideoItem is an #QQuickItem that renders GStreamer video buffers. + */ + +#define GST_CAT_DEFAULT qt_item_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +#define DEFAULT_FORCE_ASPECT_RATIO TRUE +#define DEFAULT_PAR_N 0 +#define DEFAULT_PAR_D 1 + +enum +{ + PROP_0, + PROP_FORCE_ASPECT_RATIO, + PROP_PIXEL_ASPECT_RATIO, +}; + +struct _Qt6GLVideoItemPrivate +{ + GMutex lock; + + /* properties */ + gboolean force_aspect_ratio; + gint par_n, par_d; + + GWeakRef sink; + + gint display_width; + gint display_height; + + GstBuffer *buffer; + GstCaps *new_caps; + GstCaps *caps; + GstVideoInfo new_v_info; + GstVideoInfo v_info; + + gboolean initted; + GstGLDisplay *display; + QOpenGLContext *qt_context; + GstGLContext *other_context; + GstGLContext *context; + + /* buffers with textures that were bound by QML */ + GQueue bound_buffers; + /* buffers that were previously bound but in the meantime a new one was + * bound so this one is most likely not used anymore + * FIXME: Ideally we would use fences for this but there seems to be no + * way to reliably "try wait" on a fence */ + GQueue potentially_unbound_buffers; + + GstQSG6OpenGLNode *m_node; +}; + +Qt6GLVideoItem::Qt6GLVideoItem() +{ + static gsize _debug; + + if (g_once_init_enter (&_debug)) { + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtglwidget", 0, "Qt GL Widget"); + g_once_init_leave (&_debug, 1); + } + + this->setFlag (QQuickItem::ItemHasContents, true); + + this->priv = g_new0 (Qt6GLVideoItemPrivate, 1); + + this->priv->force_aspect_ratio = DEFAULT_FORCE_ASPECT_RATIO; + this->priv->par_n = DEFAULT_PAR_N; + this->priv->par_d = DEFAULT_PAR_D; + + this->priv->initted = FALSE; + + g_mutex_init (&this->priv->lock); + + g_weak_ref_init (&priv->sink, NULL); + + this->priv->display = gst_qml6_get_gl_display(TRUE); + + connect(this, SIGNAL(windowChanged(QQuickWindow*)), this, + SLOT(handleWindowChanged(QQuickWindow*))); + + this->proxy = QSharedPointer(new Qt6GLVideoItemInterface(this)); + + setFlag(ItemHasContents, true); + setAcceptedMouseButtons(Qt::AllButtons); + setAcceptHoverEvents(true); + + setAcceptTouchEvents(true); + + GST_DEBUG ("%p init Qt6 Video Item", this); +} + +Qt6GLVideoItem::~Qt6GLVideoItem() +{ + GstBuffer *tmp_buffer; + + /* Before destroying the priv info, make sure + * no qmlglsink's will call in again, and that + * any ongoing calls are done by invalidating the proxy + * pointer */ + GST_INFO ("%p Destroying QtGLVideoItem and invalidating the proxy %p", this, proxy.data()); + proxy->invalidateRef(); + proxy.clear(); + + g_mutex_clear (&this->priv->lock); + if (this->priv->context) + gst_object_unref(this->priv->context); + if (this->priv->other_context) + gst_object_unref(this->priv->other_context); + if (this->priv->display) + gst_object_unref(this->priv->display); + + while ((tmp_buffer = (GstBuffer*) g_queue_pop_head (&this->priv->potentially_unbound_buffers))) { + GST_TRACE ("old buffer %p should be unbound now, unreffing", tmp_buffer); + gst_buffer_unref (tmp_buffer); + } + while ((tmp_buffer = (GstBuffer*) g_queue_pop_head (&this->priv->bound_buffers))) { + GST_TRACE ("old buffer %p should be unbound now, unreffing", tmp_buffer); + gst_buffer_unref (tmp_buffer); + } + + gst_buffer_replace (&this->priv->buffer, NULL); + + gst_caps_replace (&this->priv->caps, NULL); + gst_caps_replace (&this->priv->new_caps, NULL); + + g_weak_ref_clear (&this->priv->sink); + + g_free (this->priv); + this->priv = NULL; +} + +void +Qt6GLVideoItem::setDAR(gint num, gint den) +{ + this->priv->par_n = num; + this->priv->par_d = den; +} + +void +Qt6GLVideoItem::getDAR(gint * num, gint * den) +{ + if (num) + *num = this->priv->par_n; + if (den) + *den = this->priv->par_d; +} + +void +Qt6GLVideoItem::setForceAspectRatio(bool force_aspect_ratio) +{ + this->priv->force_aspect_ratio = !!force_aspect_ratio; + + emit forceAspectRatioChanged(force_aspect_ratio); +} + +bool +Qt6GLVideoItem::getForceAspectRatio() +{ + return this->priv->force_aspect_ratio; +} + +bool +Qt6GLVideoItem::itemInitialized() +{ + return this->priv->initted; +} + +static gboolean +_calculate_par (Qt6GLVideoItem * widget, GstVideoInfo * info) +{ + gboolean ok; + gint width, height; + gint par_n, par_d; + gint display_par_n, display_par_d; + guint display_ratio_num, display_ratio_den; + + width = GST_VIDEO_INFO_WIDTH (info); + height = GST_VIDEO_INFO_HEIGHT (info); + + par_n = GST_VIDEO_INFO_PAR_N (info); + par_d = GST_VIDEO_INFO_PAR_D (info); + + if (!par_n) + par_n = 1; + + /* get display's PAR */ + if (widget->priv->par_n != 0 && widget->priv->par_d != 0) { + display_par_n = widget->priv->par_n; + display_par_d = widget->priv->par_d; + } else { + display_par_n = 1; + display_par_d = 1; + } + + ok = gst_video_calculate_display_ratio (&display_ratio_num, + &display_ratio_den, width, height, par_n, par_d, display_par_n, + display_par_d); + + if (!ok) + return FALSE; + + widget->setImplicitWidth (width); + widget->setImplicitHeight (height); + + GST_LOG ("%p PAR: %u/%u DAR:%u/%u", widget, par_n, par_d, display_par_n, + display_par_d); + + if (height % display_ratio_den == 0) { + GST_DEBUG ("%p keeping video height", widget); + widget->priv->display_width = (guint) + gst_util_uint64_scale_int (height, display_ratio_num, + display_ratio_den); + widget->priv->display_height = height; + } else if (width % display_ratio_num == 0) { + GST_DEBUG ("%p keeping video width", widget); + widget->priv->display_width = width; + widget->priv->display_height = (guint) + gst_util_uint64_scale_int (width, display_ratio_den, display_ratio_num); + } else { + GST_DEBUG ("%p approximating while keeping video height", widget); + widget->priv->display_width = (guint) + gst_util_uint64_scale_int (height, display_ratio_num, + display_ratio_den); + widget->priv->display_height = height; + } + GST_DEBUG ("%p scaling to %dx%d", widget, widget->priv->display_width, + widget->priv->display_height); + + return TRUE; +} + +QSGNode * +Qt6GLVideoItem::updatePaintNode(QSGNode * oldNode, + UpdatePaintNodeData * updatePaintNodeData) +{ + GstBuffer *old_buffer; + + if (!this->priv->initted) + return oldNode; + + GstQSG6OpenGLNode *texNode = static_cast (oldNode); + GstVideoRectangle src, dst, result; + + g_mutex_lock (&this->priv->lock); + + GST_TRACE ("%p updatePaintNode", this); + + if (gst_gl_context_get_current() == NULL) + gst_gl_context_activate (this->priv->other_context, TRUE); + + if (!texNode) { + texNode = new GstQSG6OpenGLNode (this); + this->priv->m_node = texNode; + } + + if ((old_buffer = texNode->getBuffer())) { + if (old_buffer == this->priv->buffer) { + /* same buffer */ + gst_buffer_unref (old_buffer); + } else { + GstBuffer *tmp_buffer; + + GST_TRACE ("old buffer %p was bound, queueing up for later", old_buffer); + /* Unref all buffers that were previously not bound anymore. At least + * one more buffer was bound in the meantime so this one is most likely + * not in use anymore. */ + while ((tmp_buffer = (GstBuffer*) g_queue_pop_head (&this->priv->potentially_unbound_buffers))) { + GST_TRACE ("old buffer %p should be unbound now, unreffing", tmp_buffer); + gst_buffer_unref (tmp_buffer); + } + + /* Move previous bound buffers to the next queue. We now know that + * another buffer was bound in the meantime and will free them on + * the next iteration above. */ + while ((tmp_buffer = (GstBuffer*) g_queue_pop_head (&this->priv->bound_buffers))) { + GST_TRACE ("old buffer %p is potentially unbound now", tmp_buffer); + g_queue_push_tail (&this->priv->potentially_unbound_buffers, tmp_buffer); + } + g_queue_push_tail (&this->priv->bound_buffers, old_buffer); + } + old_buffer = NULL; + } + + texNode->setCaps (this->priv->caps); + texNode->setBuffer (this->priv->buffer); + + if (this->priv->force_aspect_ratio && this->priv->caps) { + src.w = this->priv->display_width; + src.h = this->priv->display_height; + + dst.x = boundingRect().x(); + dst.y = boundingRect().y(); + dst.w = boundingRect().width(); + dst.h = boundingRect().height(); + + gst_video_sink_center_rect (src, dst, &result, TRUE); + } else { + result.x = boundingRect().x(); + result.y = boundingRect().y(); + result.w = boundingRect().width(); + result.h = boundingRect().height(); + } + + texNode->setRect (QRectF (result.x, result.y, result.w, result.h)); + + g_mutex_unlock (&this->priv->lock); + + return texNode; +} + +/* This method has to be invoked with the the priv->lock taken */ +void +Qt6GLVideoItem::fitStreamToAllocatedSize(GstVideoRectangle * result) +{ + if (this->priv->force_aspect_ratio) { + GstVideoRectangle src, dst; + + src.x = 0; + src.y = 0; + src.w = this->priv->display_width; + src.h = this->priv->display_height; + + dst.x = 0; + dst.y = 0; + dst.w = width(); + dst.h = height(); + + gst_video_sink_center_rect (src, dst, result, TRUE); + } else { + result->x = 0; + result->y = 0; + result->w = width(); + result->h = height(); + } +} + +/* This method has to be invoked with the the priv->lock taken */ +QPointF +Qt6GLVideoItem::mapPointToStreamSize(QPointF pos) +{ + gdouble stream_width, stream_height; + GstVideoRectangle result; + double stream_x, stream_y; + double x, y; + + fitStreamToAllocatedSize(&result); + + stream_width = (gdouble) GST_VIDEO_INFO_WIDTH (&this->priv->v_info); + stream_height = (gdouble) GST_VIDEO_INFO_HEIGHT (&this->priv->v_info); + x = pos.x(); + y = pos.y(); + + /* from display coordinates to stream coordinates */ + if (result.w > 0) + stream_x = (x - result.x) / result.w * stream_width; + else + stream_x = 0.; + + /* clip to stream size */ + stream_x = CLAMP(stream_x, 0., stream_width); + + /* same for y-axis */ + if (result.h > 0) + stream_y = (y - result.y) / result.h * stream_height; + else + stream_y = 0.; + + stream_y = CLAMP(stream_y, 0., stream_height); + GST_TRACE ("transform %fx%f into %fx%f", x, y, stream_x, stream_y); + + return QPointF(stream_x, stream_y); +} + +void +Qt6GLVideoItem::wheelEvent(QWheelEvent * event) +{ + // noop +} + +void +Qt6GLVideoItem::hoverEnterEvent(QHoverEvent *) +{ + mouseHovering = true; +} + +void +Qt6GLVideoItem::hoverLeaveEvent(QHoverEvent *) +{ + mouseHovering = false; +} + +void +Qt6GLVideoItem::hoverMoveEvent(QHoverEvent * event) +{ + // noop +} + +void +Qt6GLVideoItem::touchEvent(QTouchEvent * event) +{ + // noop +} + +void +Qt6GLVideoItem::sendMouseEvent(QMouseEvent * event, gboolean is_press) +{ + // noop +} + +void +Qt6GLVideoItem::mousePressEvent(QMouseEvent * event) +{ + forceActiveFocus(); + sendMouseEvent(event, TRUE); +} + +void +Qt6GLVideoItem::mouseReleaseEvent(QMouseEvent * event) +{ + sendMouseEvent(event, FALSE); +} + +void +Qt6GLVideoItemInterface::setSink (GstElement * sink) +{ + QMutexLocker locker(&lock); + if (qt_item == NULL) + return; + + g_mutex_lock (&qt_item->priv->lock); + g_weak_ref_set (&qt_item->priv->sink, sink); + g_mutex_unlock (&qt_item->priv->lock); +} + +void +Qt6GLVideoItemInterface::setBuffer (GstBuffer * buffer) +{ + QMutexLocker locker(&lock); + + if (qt_item == NULL) { + GST_WARNING ("%p actual item is NULL. setBuffer call ignored", this); + return; + } + + if (!qt_item->priv->caps && !qt_item->priv->new_caps) { + GST_WARNING ("%p Got buffer on unnegotiated QtGLVideoItem. Dropping", this); + return; + } + + g_mutex_lock (&qt_item->priv->lock); + + if (qt_item->priv->new_caps) { + GST_DEBUG ("%p caps change from %" GST_PTR_FORMAT " to %" GST_PTR_FORMAT, + this, qt_item->priv->caps, qt_item->priv->new_caps); + gst_caps_take (&qt_item->priv->caps, qt_item->priv->new_caps); + qt_item->priv->new_caps = NULL; + qt_item->priv->v_info = qt_item->priv->new_v_info; + + if (!_calculate_par (qt_item, &qt_item->priv->v_info)) { + g_mutex_unlock (&qt_item->priv->lock); + return; + } + } + + gst_buffer_replace (&qt_item->priv->buffer, buffer); + + QMetaObject::invokeMethod(qt_item, "update", Qt::QueuedConnection); + + g_mutex_unlock (&qt_item->priv->lock); +} + +void +Qt6GLVideoItem::onSceneGraphInitialized () +{ + QSGRendererInterface *renderer; + QOpenGLContext *gl_context; + + if (this->window() == NULL) + return; + + renderer = this->window()->rendererInterface(); + if (!renderer) + return; + + if (renderer->graphicsApi() != QSGRendererInterface::GraphicsApi::OpenGL) { + GST_WARNING ("%p scene graph initialized with a non-OpenGL renderer interface", this); + return; + } + + gl_context = + static_cast ( + renderer->getResource( + this->window(), + QSGRendererInterface::Resource::OpenGLContextResource)); + + GST_DEBUG ("%p scene graph initialization with Qt GL context %p", this, + gl_context); + + if (this->priv->qt_context == gl_context) + return; + + this->priv->qt_context = gl_context; + if (this->priv->qt_context == NULL) { + GST_ERROR ("%p failed to retrieve Qt GL context", this); + g_assert_not_reached (); + return; + } + + this->priv->initted = gst_qml6_get_gl_wrapcontext (this->priv->display, + &this->priv->other_context, &this->priv->context); + + GST_DEBUG ("%p created wrapped GL context %" GST_PTR_FORMAT, this, + this->priv->other_context); + + emit itemInitializedChanged(); +} + +void +Qt6GLVideoItem::onSceneGraphInvalidated () +{ + this->priv->m_node = nullptr; + GST_FIXME ("%p scene graph invalidated", this); +} + +/** + * Retrieve and populate the GL context information from the current + * OpenGL context. + */ +gboolean +Qt6GLVideoItemInterface::initWinSys () +{ + QMutexLocker locker(&lock); + + GError *error = NULL; + + if (qt_item == NULL) + return FALSE; + + g_mutex_lock (&qt_item->priv->lock); + + if (qt_item->priv->display && qt_item->priv->qt_context + && qt_item->priv->other_context && qt_item->priv->context) { + /* already have the necessary state */ + g_mutex_unlock (&qt_item->priv->lock); + return TRUE; + } + + if (!GST_IS_GL_DISPLAY (qt_item->priv->display)) { + GST_ERROR ("%p failed to retrieve display connection %" GST_PTR_FORMAT, + qt_item, qt_item->priv->display); + g_mutex_unlock (&qt_item->priv->lock); + return FALSE; + } + + if (!GST_IS_GL_CONTEXT (qt_item->priv->other_context)) { + GST_ERROR ("%p failed to retrieve wrapped context %" GST_PTR_FORMAT, qt_item, + qt_item->priv->other_context); + g_mutex_unlock (&qt_item->priv->lock); + return FALSE; + } + + qt_item->priv->context = gst_gl_context_new (qt_item->priv->display); + + if (!qt_item->priv->context) { + g_mutex_unlock (&qt_item->priv->lock); + return FALSE; + } + + if (!gst_gl_context_create (qt_item->priv->context, qt_item->priv->other_context, + &error)) { + GST_ERROR ("%s", error->message); + g_mutex_unlock (&qt_item->priv->lock); + return FALSE; + } + + g_mutex_unlock (&qt_item->priv->lock); + return TRUE; +} + +void +Qt6GLVideoItem::handleWindowChanged (QQuickWindow * win) +{ + if (win) { + if (win->isSceneGraphInitialized ()) + win->scheduleRenderJob (new RenderJob (std:: + bind (&Qt6GLVideoItem::onSceneGraphInitialized, this)), + QQuickWindow::BeforeSynchronizingStage); + else + connect (win, SIGNAL (sceneGraphInitialized ()), this, + SLOT (onSceneGraphInitialized ()), Qt::DirectConnection); + + connect (win, SIGNAL (sceneGraphInvalidated ()), this, + SLOT (onSceneGraphInvalidated ()), Qt::DirectConnection); + } else { + this->priv->qt_context = NULL; + this->priv->initted = FALSE; + } + this->priv->m_node = nullptr; +} + +void +Qt6GLVideoItem::releaseResources() +{ + this->priv->m_node = nullptr; +} + +gboolean +Qt6GLVideoItemInterface::setCaps (GstCaps * caps) +{ + QMutexLocker locker(&lock); + GstVideoInfo v_info; + + g_return_val_if_fail (GST_IS_CAPS (caps), FALSE); + g_return_val_if_fail (gst_caps_is_fixed (caps), FALSE); + + if (qt_item == NULL) + return FALSE; + + if (qt_item->priv->caps && gst_caps_is_equal_fixed (qt_item->priv->caps, caps)) + return TRUE; + + if (!gst_video_info_from_caps (&v_info, caps)) + return FALSE; + + g_mutex_lock (&qt_item->priv->lock); + + GST_DEBUG ("%p set caps %" GST_PTR_FORMAT, qt_item, caps); + + gst_caps_replace (&qt_item->priv->new_caps, caps); + + qt_item->priv->new_v_info = v_info; + + g_mutex_unlock (&qt_item->priv->lock); + + return TRUE; +} + +GstGLContext * +Qt6GLVideoItemInterface::getQtContext () +{ + QMutexLocker locker(&lock); + + if (!qt_item || !qt_item->priv->other_context) + return NULL; + + return (GstGLContext *) gst_object_ref (qt_item->priv->other_context); +} + +GstGLContext * +Qt6GLVideoItemInterface::getContext () +{ + QMutexLocker locker(&lock); + + if (!qt_item || !qt_item->priv->context) + return NULL; + + return (GstGLContext *) gst_object_ref (qt_item->priv->context); +} + +GstGLDisplay * +Qt6GLVideoItemInterface::getDisplay() +{ + QMutexLocker locker(&lock); + + if (!qt_item || !qt_item->priv->display) + return NULL; + + return (GstGLDisplay *) gst_object_ref (qt_item->priv->display); +} + +void +Qt6GLVideoItemInterface::setDAR(gint num, gint den) +{ + QMutexLocker locker(&lock); + if (!qt_item) + return; + qt_item->setDAR(num, den); +} + +void +Qt6GLVideoItemInterface::getDAR(gint * num, gint * den) +{ + QMutexLocker locker(&lock); + if (!qt_item) + return; + qt_item->getDAR (num, den); +} + +void +Qt6GLVideoItemInterface::setForceAspectRatio(bool force_aspect_ratio) +{ + QMutexLocker locker(&lock); + if (!qt_item) + return; + qt_item->setForceAspectRatio(force_aspect_ratio); +} + +bool +Qt6GLVideoItemInterface::getForceAspectRatio() +{ + QMutexLocker locker(&lock); + if (!qt_item) + return FALSE; + return qt_item->getForceAspectRatio(); +} + +void +Qt6GLVideoItemInterface::invalidateRef() +{ + QMutexLocker locker(&lock); + qt_item = NULL; +} + diff --git a/libs/qmlglsink/qt6-linux/qt6glitem.h b/libs/qmlglsink/qt6-linux/qt6glitem.h new file mode 100644 index 000000000000..c5c5efcfcfd9 --- /dev/null +++ b/libs/qmlglsink/qt6-linux/qt6glitem.h @@ -0,0 +1,128 @@ +/* + * GStreamer + * Copyright (C) 2015 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __QT6_GL_ITEM_H__ +#define __QT6_GL_ITEM_H__ + +#include +#include + +#include "gstqt6gl.h" +#include +#include +#include +#include +#include + +typedef struct _Qt6GLVideoItemPrivate Qt6GLVideoItemPrivate; + +class Qt6GLVideoItem; + +class Qt6GLVideoItemInterface : public QObject +{ + Q_OBJECT + QML_ELEMENT +public: + Qt6GLVideoItemInterface (Qt6GLVideoItem *w) : qt_item (w), lock() {}; + + void invalidateRef(); + + void setSink (GstElement * sink); + void setBuffer (GstBuffer * buffer); + gboolean setCaps (GstCaps *caps); + gboolean initWinSys (); + GstGLContext *getQtContext(); + GstGLContext *getContext(); + GstGLDisplay *getDisplay(); + Qt6GLVideoItem *videoItem () { return qt_item; }; + + void setDAR(gint, gint); + void getDAR(gint *, gint *); + void setForceAspectRatio(bool); + bool getForceAspectRatio(); +private: + Qt6GLVideoItem *qt_item; + QMutex lock; +}; + +class Qt6GLVideoItem : public QQuickItem, protected QOpenGLFunctions +{ + Q_OBJECT + QML_ELEMENT + + Q_PROPERTY(bool itemInitialized + READ itemInitialized + NOTIFY itemInitializedChanged) + Q_PROPERTY(bool forceAspectRatio + READ getForceAspectRatio + WRITE setForceAspectRatio + NOTIFY forceAspectRatioChanged) + +public: + Qt6GLVideoItem(); + ~Qt6GLVideoItem(); + + void setDAR(gint, gint); + void getDAR(gint *, gint *); + void setForceAspectRatio(bool); + bool getForceAspectRatio(); + bool itemInitialized(); + + QSharedPointer getInterface() { return proxy; }; + /* private for C interface ... */ + Qt6GLVideoItemPrivate *priv; + +Q_SIGNALS: + void itemInitializedChanged(); + void forceAspectRatioChanged(bool); + +private Q_SLOTS: + void handleWindowChanged(QQuickWindow * win); + void onSceneGraphInitialized(); + void onSceneGraphInvalidated(); + +protected: + QSGNode * updatePaintNode (QSGNode * oldNode, UpdatePaintNodeData * updatePaintNodeData) override; + void releaseResources() override; + void wheelEvent(QWheelEvent *) override; + void hoverEnterEvent(QHoverEvent *) override; + void hoverLeaveEvent (QHoverEvent *) override; + void hoverMoveEvent (QHoverEvent *) override; + void mousePressEvent(QMouseEvent*) override; + void mouseReleaseEvent(QMouseEvent*) override; + void touchEvent(QTouchEvent*) override; + +private: + + void setViewportSize(const QSize &size); + void shareContext(); + + void fitStreamToAllocatedSize(GstVideoRectangle * result); + QPointF mapPointToStreamSize(QPointF); + + void sendMouseEvent(QMouseEvent * event, gboolean is_press); + + quint32 mousePressedButton; + bool mouseHovering; + + QSharedPointer proxy; +}; + +#endif /* __QT_GL_ITEM_H__ */ diff --git a/libs/qmlglsink/qt6/RGBA.frag b/libs/qmlglsink/qt6/RGBA.frag new file mode 100644 index 000000000000..c1ab12010700 --- /dev/null +++ b/libs/qmlglsink/qt6/RGBA.frag @@ -0,0 +1,24 @@ +#version 440 + +layout(location = 0) in vec2 vTexCoord; + +layout(location = 0) out vec4 fragColor; + +layout(std140, binding = 0) uniform buf { + mat4 qt_Matrix; + ivec4 swizzle; + mat4 color_matrix; + float qt_Opacity; +} ubuf; + +layout(binding = 1) uniform sampler2D tex; + +vec4 swizzle(in vec4 texel, in ivec4 swizzle) { + return vec4(texel[swizzle[0]], texel[swizzle[1]], texel[swizzle[2]], texel[swizzle[3]]); +} + +void main() +{ + vec4 texel = swizzle(texture(tex, vTexCoord), ubuf.swizzle); + fragColor = texel * ubuf.qt_Opacity; +} diff --git a/libs/qmlglsink/qt6/YUV_TRIPLANAR.frag b/libs/qmlglsink/qt6/YUV_TRIPLANAR.frag new file mode 100644 index 000000000000..6c096848933a --- /dev/null +++ b/libs/qmlglsink/qt6/YUV_TRIPLANAR.frag @@ -0,0 +1,37 @@ +#version 440 + +layout(location = 0) in vec2 vTexCoord; + +layout(location = 0) out vec4 fragColor; + +layout(std140, binding = 0) uniform buf { + mat4 qt_Matrix; + ivec4 swizzle; + mat4 color_matrix; + float qt_Opacity; +} ubuf; + +layout(binding = 1) uniform sampler2D Ytex; +layout(binding = 2) uniform sampler2D Utex; +layout(binding = 3) uniform sampler2D Vtex; + +vec4 swizzle(in vec4 texel, in ivec4 swizzle) { + return vec4(texel[swizzle[0]], texel[swizzle[1]], texel[swizzle[2]], texel[swizzle[3]]); +} + +vec4 yuva_to_rgba(in vec4 yuva, in mat4 color_matrix) { + return yuva * color_matrix; +} + +void main() +{ + vec4 yuva; + yuva.x = texture(Ytex, vTexCoord).r; + yuva.y = texture(Utex, vTexCoord).r; + yuva.z = texture(Vtex, vTexCoord).r; + yuva.a = 1.0; + yuva = swizzle(yuva, ubuf.swizzle); + vec4 rgba = yuva_to_rgba (yuva, ubuf.color_matrix); + fragColor = rgba * ubuf.qt_Opacity; +} + diff --git a/libs/qmlglsink/qt6/gstplugin.cc b/libs/qmlglsink/qt6/gstplugin.cc index eb24046dc5ee..3a36df6575aa 100644 --- a/libs/qmlglsink/qt6/gstplugin.cc +++ b/libs/qmlglsink/qt6/gstplugin.cc @@ -23,28 +23,32 @@ #endif #include "gstqt6elements.h" +/////////////////////////////////// #include "qt6glitem.h" - #include +/////////////////////////////////// static gboolean plugin_init (GstPlugin * plugin) { gboolean ret = FALSE; -// TODO(zdanek) fix after switching to gstreamer 1.20.0+ -// original code from 1.20.0 -// ret |= GST_ELEMENT_REGISTER (qml6glsink, plugin); - ret |= gst_element_register_qml6glsink (plugin); + + ret |= GST_ELEMENT_REGISTER (qml6glsink, plugin); + ret |= GST_ELEMENT_REGISTER (qml6glsrc, plugin); + ret |= GST_ELEMENT_REGISTER (qml6glmixer, plugin); + ret |= GST_ELEMENT_REGISTER (qml6gloverlay, plugin); return ret; } +/////////////////////////////////// static void registerMetatypes() { qmlRegisterType ("org.freedesktop.gstreamer.GLVideoItem", 1, 0, "GstGLVideoItem"); } Q_CONSTRUCTOR_FUNCTION(registerMetatypes) +/////////////////////////////////// #ifndef GST_PACKAGE_NAME #define GST_PACKAGE_NAME "GStreamer Bad Plug-ins (qmake)" diff --git a/libs/qmlglsink/qt6/gstqml6glmixer.cc b/libs/qmlglsink/qt6/gstqml6glmixer.cc new file mode 100644 index 000000000000..b616cd4ad06c --- /dev/null +++ b/libs/qmlglsink/qt6/gstqml6glmixer.cc @@ -0,0 +1,628 @@ +/* + * GStreamer + * Copyright (C) 2023 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:gstqml6gmixer + * + * `qml6glmixer` provides a way to render an almost-arbitrary QML scene within + * GStreamer pipeline using the same OpenGL context that GStreamer uses + * internally. This avoids attempting to share multiple OpenGL contexts + * avoiding increased synchronisation points and attempting to share an OpenGL + * context at runtime which some drivers do not like. The Intel driver on + * Windows is a notable example of the last point. + * + * `qml6glmixer` will attempt to retrieve the windowing system display connection + * that Qt is using (#GstGLDisplay). This may be different to any already + * existing window system display connection already in use in the pipeline for + * a number of reasons. A couple of examples of this are: + * + * 1. Adding `qml6glmixer` to an already running pipeline + * 2. Not having any `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`) element + * start up before any other OpenGL-based element in the pipeline. + * + * If one of these scenarios occurs, then there will be multiple OpenGL contexts + * in use in the pipeline. This means that either the pipeline will fail to + * start up correctly, a downstream element may reject buffers, or a complete + * GPU->System memory->GPU transfer is performed for every buffer. + * + * The requirement to avoid this is that all elements share the same + * #GstGLDisplay object and as Qt cannot currently share an existing window + * system display connection, GStreamer must use the window system display + * connection provided by Qt. This window system display connection can be + * retrieved by either a `qml6glsink` element, a `qml6gloverlay` element or a + * `qmlglmixer element. The recommended usage is to have either elements + * (`qml6glsink` or `qml6gloverlay` or `qml6glmixer) be the first to propagate + * the #GstGLDisplay for the entire pipeline to use by setting either element + * to the READY element state before any other OpenGL element in the pipeline. + * + * In the dynamically adding `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`) + * to a pipeline case, there are some considerations for ensuring that the + * window system display and OpenGL contexts are compatible with Qt. When the + * `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`) element is added and + * brought up to READY, it will propagate it's own #GstGLDisplay using the + * #GstContext mechanism regardless of any existing #GstGLDisplay used by the + * pipeline previously. In order for the new #GstGLDisplay to be used, the + * application must then set the provided #GstGLDisplay containing #GstContext + * on the pipeline. This may effectively cause each OpenGL element to replace + * the window system display and also the OpenGL context it is using. As such + * this process may take a significant amount of time and resources as objects + * are recreated in the new OpenGL context. + * + * All instances of `qml6glmixer`, `qml6glsink`, and `qml6gloverlay` will return + * the exact same #GstGLDisplay object while the pipeline is running regardless + * of whether any `qml6glmixer`, `qml6glsink`, or `qml6gloverlay` elements are + * added or removed from the pipeline. + * + * The Qml scene will run at configured output framerate. The timestamps on the + * output buffers are used to drive the animation time. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstqt6elements.h" +#include "gstqml6glmixer.h" +#include "qt6glrenderer.h" +#include "gstqt6glutility.h" + +#include + +#include +#include + +#define GST_CAT_DEFAULT gst_debug_qml6_gl_mixer +GST_DEBUG_CATEGORY (GST_CAT_DEFAULT); + +enum +{ + PROP_PAD_0, + PROP_PAD_WIDGET, +}; + +struct _GstQml6GLMixerPad +{ + GstGLMixerPad parent; + + QSharedPointer widget; +}; + +G_DEFINE_FINAL_TYPE (GstQml6GLMixerPad, gst_qml6_gl_mixer_pad, GST_TYPE_GL_MIXER_PAD); + +static gboolean +gst_qml6_gl_mixer_pad_prepare_frame (GstVideoAggregatorPad *vagg_pad, GstVideoAggregator * vagg, + GstBuffer *buffer, GstVideoFrame * prepared_frame) +{ + GstQml6GLMixerPad *pad = GST_QML6_GL_MIXER_PAD (vagg_pad); + + if (!GST_VIDEO_AGGREGATOR_PAD_CLASS (gst_qml6_gl_mixer_pad_parent_class)->prepare_frame (vagg_pad, vagg, buffer, prepared_frame)) + return FALSE; + + if (pad->widget) { + GstMemory *mem; + GstGLMemory *gl_mem; + GstCaps *in_caps; + GstGLContext *context; + + in_caps = gst_video_info_to_caps (&vagg_pad->info); + gst_caps_set_features_simple (in_caps, gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + pad->widget->setCaps (in_caps); + gst_clear_caps (&in_caps); + + mem = gst_buffer_peek_memory (buffer, 0); + if (!gst_is_gl_memory (mem)) { + GST_ELEMENT_ERROR (vagg_pad, RESOURCE, NOT_FOUND, + (NULL), ("Input memory must be a GstGLMemory")); + return GST_FLOW_ERROR; + } + gl_mem = (GstGLMemory *) mem; + context = gst_gl_base_mixer_get_gl_context (GST_GL_BASE_MIXER (vagg)); + if (!gst_gl_context_can_share (gl_mem->mem.context, context)) { + GST_WARNING_OBJECT (vagg_pad, "Cannot use the current input texture " + "(input buffer GL context %" GST_PTR_FORMAT " cannot share " + "resources with the configured OpenGL context %" GST_PTR_FORMAT ")", + gl_mem->mem.context, context); + } else { + pad->widget->setBuffer (buffer); + } + } + + return TRUE; +} + +static void +gst_qml6_gl_mixer_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstQml6GLMixerPad *qml6_gl_mixer_pad = GST_QML6_GL_MIXER_PAD (object); + + switch (prop_id) { + case PROP_PAD_WIDGET: { + Qt6GLVideoItem *qt_item = static_cast (g_value_get_pointer (value)); + if (qt_item) + qml6_gl_mixer_pad->widget = qt_item->getInterface(); + else + qml6_gl_mixer_pad->widget.clear(); + break; + } + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_qml6_gl_mixer_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstQml6GLMixerPad *qml6_gl_mixer_pad = GST_QML6_GL_MIXER_PAD (object); + + switch (prop_id) { + case PROP_PAD_WIDGET: + /* This is not really safe - the app needs to be + * sure the widget is going to be kept alive or + * this can crash */ + if (qml6_gl_mixer_pad->widget) + g_value_set_pointer (value, qml6_gl_mixer_pad->widget->videoItem()); + else + g_value_set_pointer (value, NULL); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_qml6_gl_mixer_pad_finalize (GObject * object) +{ + GstQml6GLMixerPad *pad = GST_QML6_GL_MIXER_PAD (object); + + pad->widget.clear(); + + G_OBJECT_CLASS (gst_qml6_gl_mixer_pad_parent_class)->finalize (object); +} + +static void +gst_qml6_gl_mixer_pad_class_init (GstQml6GLMixerPadClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstVideoAggregatorPadClass *vagg_pad_class = (GstVideoAggregatorPadClass *) klass; + + gobject_class->set_property = gst_qml6_gl_mixer_pad_set_property; + gobject_class->get_property = gst_qml6_gl_mixer_pad_get_property; + gobject_class->finalize = gst_qml6_gl_mixer_pad_finalize; + + g_object_class_install_property (gobject_class, PROP_PAD_WIDGET, + g_param_spec_pointer ("widget", "QQuickItem", + "The QQuickItem to place the input video in the object hierarchy", + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + vagg_pad_class->prepare_frame = gst_qml6_gl_mixer_pad_prepare_frame; +} + +static void +gst_qml6_gl_mixer_pad_init (GstQml6GLMixerPad * pad) +{ + pad->widget = QSharedPointer(); +} + +static void gst_qml6_gl_mixer_finalize (GObject * object); +static void gst_qml6_gl_mixer_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * param_spec); +static void gst_qml6_gl_mixer_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * param_spec); + +static gboolean gst_qml6_gl_mixer_process_buffers (GstGLMixer * btrans, + GstBuffer * outbuf); + +static gboolean gst_qml6_gl_mixer_gl_start (GstGLBaseMixer * bmixer); +static void gst_qml6_gl_mixer_gl_stop (GstGLBaseMixer * bmixer); + +static GstFlowReturn gst_qml6_gl_mixer_create_output_buffer (GstVideoAggregator * vagg, GstBuffer ** outbuf); + +static gboolean gst_qml6_gl_mixer_negotiated_src_caps (GstAggregator * aggregator, GstCaps * out_caps); + +static GstStateChangeReturn gst_qml6_gl_mixer_change_state (GstElement * element, + GstStateChange transition); + +enum +{ + PROP_0, + PROP_QML_SCENE, + PROP_ROOT_ITEM, +}; + +enum +{ + SIGNAL_0, + SIGNAL_QML_SCENE_INITIALIZED, + SIGNAL_QML_SCENE_DESTROYED, + LAST_SIGNAL +}; + +static guint gst_qml6_gl_mixer_signals[LAST_SIGNAL] = { 0 }; + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "RGBA")) + ); + +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "{ RGBA, BGRA, YV12 }")) + ); + +struct _GstQml6GLMixer { + GstGLMixer parent; + + gchar *qml_scene; + + GstQt6QuickRenderer *renderer; + GstBuffer *outbuf; +}; + +#define gst_qml6_gl_mixer_parent_class parent_class +G_DEFINE_FINAL_TYPE_WITH_CODE (GstQml6GLMixer, gst_qml6_gl_mixer, + GST_TYPE_GL_MIXER, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, + "qml6glmixer", 0, "Qt6 Video Mixer")); +GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (qml6glmixer, "qml6glmixer", + GST_RANK_NONE, GST_TYPE_QML6_GL_MIXER, qt6_element_init (plugin)); + +static void +gst_qml6_gl_mixer_class_init (GstQml6GLMixerClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstAggregatorClass *agg_class; + GstVideoAggregatorClass *vagg_class; + GstGLBaseMixerClass *glbasemixer_class; + GstGLMixerClass *glmixer_class; + GstElementClass *element_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + glbasemixer_class = (GstGLBaseMixerClass *) klass; + glmixer_class = (GstGLMixerClass *) klass; + vagg_class = (GstVideoAggregatorClass *) klass; + agg_class = (GstAggregatorClass *) klass; + element_class = (GstElementClass *) klass; + + gobject_class->set_property = gst_qml6_gl_mixer_set_property; + gobject_class->get_property = gst_qml6_gl_mixer_get_property; + gobject_class->finalize = gst_qml6_gl_mixer_finalize; + + gst_element_class_set_metadata (gstelement_class, "Qt6 Video Mixer", + "Video/QML/Mixer", "A mixer that renders a QML scene", + "Matthew Waters "); + + g_object_class_install_property (gobject_class, PROP_QML_SCENE, + g_param_spec_string ("qml-scene", "QML Scene", + "The contents of the QML scene", NULL, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property (gobject_class, PROP_ROOT_ITEM, + g_param_spec_pointer ("root-item", "QQuickItem", + "The root QQuickItem from the qml-scene used to render", + (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + + /** + * GstQmlGLMixer::qml-scene-initialized + * @element: the #GstQmlGLMixer + * @user_data: user provided data + */ + gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_INITIALIZED] = + g_signal_new ("qml-scene-initialized", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0); + + /** + * GstQmlGLMixer::qml-scene-destroyed + * @element: the #GstQmlGLMixer + * @user_data: user provided data + */ + gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_DESTROYED] = + g_signal_new ("qml-scene-destroyed", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0); + + glbasemixer_class->gl_start = gst_qml6_gl_mixer_gl_start; + glbasemixer_class->gl_stop = gst_qml6_gl_mixer_gl_stop; + + glmixer_class->process_buffers = gst_qml6_gl_mixer_process_buffers; + + vagg_class->create_output_buffer = gst_qml6_gl_mixer_create_output_buffer; + + agg_class->negotiated_src_caps = gst_qml6_gl_mixer_negotiated_src_caps; + + element_class->change_state = gst_qml6_gl_mixer_change_state; + + gst_element_class_add_static_pad_template_with_gtype (element_class, + &src_factory, GST_TYPE_AGGREGATOR_PAD); + gst_element_class_add_static_pad_template_with_gtype (element_class, + &sink_factory, GST_TYPE_QML6_GL_MIXER_PAD); +} + +static void +gst_qml6_gl_mixer_init (GstQml6GLMixer * qml6_gl_mixer) +{ + qml6_gl_mixer->qml_scene = NULL; +} + +static void +gst_qml6_gl_mixer_finalize (GObject * object) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object); + + g_free (qml6_gl_mixer->qml_scene); + qml6_gl_mixer->qml_scene = NULL; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_qml6_gl_mixer_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object); + + switch (prop_id) { + case PROP_QML_SCENE: + g_free (qml6_gl_mixer->qml_scene); + qml6_gl_mixer->qml_scene = g_value_dup_string (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_qml6_gl_mixer_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object); + + switch (prop_id) { + case PROP_QML_SCENE: + g_value_set_string (value, qml6_gl_mixer->qml_scene); + break; + case PROP_ROOT_ITEM: + GST_OBJECT_LOCK (qml6_gl_mixer); + if (qml6_gl_mixer->renderer) { + QQuickItem *root = qml6_gl_mixer->renderer->rootItem(); + if (root) + g_value_set_pointer (value, root); + else + g_value_set_pointer (value, NULL); + } else { + g_value_set_pointer (value, NULL); + } + GST_OBJECT_UNLOCK (qml6_gl_mixer); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_qml6_gl_mixer_negotiated_src_caps (GstAggregator * aggregator, GstCaps * out_caps) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (aggregator); + GstVideoInfo out_info; + + if (!gst_video_info_from_caps (&out_info, out_caps)) + return FALSE; + + qml6_gl_mixer->renderer->setSize (GST_VIDEO_INFO_WIDTH (&out_info), + GST_VIDEO_INFO_HEIGHT (&out_info)); + + return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (aggregator, out_caps); +} + +static gboolean +gst_qml6_gl_mixer_gl_start (GstGLBaseMixer * bmixer) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (bmixer); + + QQuickItem *root; + GError *error = NULL; + + GST_TRACE_OBJECT (bmixer, "using scene:\n%s", qml6_gl_mixer->qml_scene); + + if (!qml6_gl_mixer->qml_scene || g_strcmp0 (qml6_gl_mixer->qml_scene, "") == 0) { + GST_ELEMENT_ERROR (bmixer, RESOURCE, NOT_FOUND, ("qml-scene property not set"), (NULL)); + return FALSE; + } + + if (!GST_GL_BASE_MIXER_CLASS (parent_class)->gl_start (bmixer)) + return FALSE; + + GST_OBJECT_LOCK (bmixer); + qml6_gl_mixer->renderer = new GstQt6QuickRenderer; + if (!qml6_gl_mixer->renderer->init (bmixer->context, &error)) { + GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND, + ("%s", error->message), (NULL)); + delete qml6_gl_mixer->renderer; + qml6_gl_mixer->renderer = NULL; + GST_OBJECT_UNLOCK (bmixer); + return FALSE; + } + + /* FIXME: Qml may do async loading and we need to propagate qml errors in that case as well */ + if (!qml6_gl_mixer->renderer->setQmlScene (qml6_gl_mixer->qml_scene, &error)) { + GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND, + ("%s", error->message), (NULL)); + goto fail_renderer; + } + + root = qml6_gl_mixer->renderer->rootItem(); + if (!root) { + GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND, + ("Qml scene does not have a root item"), (NULL)); + goto fail_renderer; + } + GST_OBJECT_UNLOCK (bmixer); + + g_object_notify (G_OBJECT (qml6_gl_mixer), "root-item"); + g_signal_emit (qml6_gl_mixer, gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_INITIALIZED], 0); + + return TRUE; + +fail_renderer: + { + qml6_gl_mixer->renderer->cleanup(); + delete qml6_gl_mixer->renderer; + qml6_gl_mixer->renderer = NULL; + GST_OBJECT_UNLOCK (bmixer); + return FALSE; + } +} + +static void +gst_qml6_gl_mixer_gl_stop (GstGLBaseMixer * bmixer) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (bmixer); + GstQt6QuickRenderer *renderer = NULL; + + /* notify before actually destroying anything */ + GST_OBJECT_LOCK (qml6_gl_mixer); + if (qml6_gl_mixer->renderer) + renderer = qml6_gl_mixer->renderer; + qml6_gl_mixer->renderer = NULL; + GST_OBJECT_UNLOCK (qml6_gl_mixer); + + g_signal_emit (qml6_gl_mixer, gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_DESTROYED], 0); + g_object_notify (G_OBJECT (qml6_gl_mixer), "root-item"); + + /* TODO: clear all pad buffers in the items? + if (qml6_gl_mixer->widget) + qml6_gl_mixer->widget->setBuffer (NULL); +*/ + if (renderer) { + renderer->cleanup(); + delete renderer; + } + + GST_GL_BASE_MIXER_CLASS (parent_class)->gl_stop (bmixer); +} + +static GstFlowReturn +gst_qml6_gl_mixer_create_output_buffer (GstVideoAggregator * vagg, GstBuffer ** outbuf) +{ + *outbuf = gst_buffer_new(); + + return GST_FLOW_OK; +} + +static gboolean +qml6_gl_mixer_gl_callback (GstGLContext *context, GstQml6GLMixer * qml6_gl_mixer) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (qml6_gl_mixer); + GstGLMemory *out_mem; + + /* XXX: is this the correct ts to drive the animation */ + out_mem = qml6_gl_mixer->renderer->generateOutput (GST_BUFFER_PTS (qml6_gl_mixer->outbuf)); + if (!out_mem) { + GST_ERROR_OBJECT (qml6_gl_mixer, "Failed to generate output"); + return FALSE; + } + + gst_buffer_append_memory (qml6_gl_mixer->outbuf, (GstMemory *) out_mem); + gst_buffer_add_video_meta (qml6_gl_mixer->outbuf, (GstVideoFrameFlags) 0, + GST_VIDEO_INFO_FORMAT (&vagg->info), + GST_VIDEO_INFO_WIDTH (&vagg->info), + GST_VIDEO_INFO_HEIGHT (&vagg->info)); + + return TRUE; +} + +static gboolean +gst_qml6_gl_mixer_process_buffers (GstGLMixer * mix, + GstBuffer * outbuf) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (mix); + GstGLBaseMixer *bmix = GST_GL_BASE_MIXER (mix); + GstGLContext *context = gst_gl_base_mixer_get_gl_context (bmix); + + qml6_gl_mixer->outbuf = outbuf; + gst_gl_context_thread_add (context, + (GstGLContextThreadFunc) qml6_gl_mixer_gl_callback, qml6_gl_mixer); + qml6_gl_mixer->outbuf = NULL; + + gst_clear_object (&context); + + return TRUE; +} + +static GstStateChangeReturn +gst_qml6_gl_mixer_change_state (GstElement * element, + GstStateChange transition) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (element); + GstGLBaseMixer *bmixer = GST_GL_BASE_MIXER (element); + GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + + GST_DEBUG_OBJECT (element, "changing state: %s => %s", + gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)), + gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition))); + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: { + QGuiApplication *app; + GstGLDisplay *display = NULL; + + app = static_cast (QCoreApplication::instance ()); + if (!app) { + GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND, + ("%s", "Failed to connect to Qt"), + ("%s", "Could not retrieve QGuiApplication instance")); + return GST_STATE_CHANGE_FAILURE; + } + + display = gst_qml6_get_gl_display (FALSE); + + if (display != bmixer->display) + /* always propagate. The application may need to choose between window + * system display connections */ + gst_gl_element_propagate_display_context (GST_ELEMENT (qml6_gl_mixer), display); + gst_object_unref (display); + break; + } + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + default: + break; + } + + return ret; + +} diff --git a/libs/qmlglsink/qt6/gstqml6glmixer.h b/libs/qmlglsink/qt6/gstqml6glmixer.h new file mode 100644 index 000000000000..e58cb5e16b06 --- /dev/null +++ b/libs/qmlglsink/qt6/gstqml6glmixer.h @@ -0,0 +1,40 @@ +/* + * GStreamer + * Copyright (C) 2020 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_QML6_GL_MIXER_H__ +#define __GST_QML6_GL_MIXER_H__ + +#include +#include +#include +#include "qt6glrenderer.h" +#include "qt6glitem.h" + +G_BEGIN_DECLS + +#define GST_TYPE_QML6_GL_MIXER_PAD (gst_qml6_gl_mixer_pad_get_type()) +G_DECLARE_FINAL_TYPE(GstQml6GLMixerPad, gst_qml6_gl_mixer_pad, GST, QML6_GL_MIXER_PAD, GstGLMixerPad); + +#define GST_TYPE_QML6_GL_MIXER (gst_qml6_gl_mixer_get_type()) +G_DECLARE_FINAL_TYPE(GstQml6GLMixer, gst_qml6_gl_mixer, GST, QML6_GL_MIXER, GstGLMixer); + +G_END_DECLS + +#endif /* __GST_QML6_GL_MIXER_H__ */ diff --git a/libs/qmlglsink/qt6/gstqml6gloverlay.cc b/libs/qmlglsink/qt6/gstqml6gloverlay.cc new file mode 100644 index 000000000000..85f5a380f6f7 --- /dev/null +++ b/libs/qmlglsink/qt6/gstqml6gloverlay.cc @@ -0,0 +1,571 @@ +/* + * GStreamer + * Copyright (C) 2022 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:gstqml6gloverlay + * + * `qml6gloverlay` provides a way to render an almost-arbitrary QML scene within + * GStreamer pipeline using the same OpenGL context that GStreamer uses + * internally. This avoids attempting to share multiple OpenGL contexts + * avoiding increased synchronisation points and attempting to share an OpenGL + * context at runtime which some drivers do not like. The Intel driver on + * Windows is a notable example of the last point. + * + * `qml6gloverlay` will attempt to retrieve the windowing system display connection + * that Qt is using (#GstGLDisplay). This may be different to any already + * existing window system display connection already in use in the pipeline for + * a number of reasons. A couple of examples of this are: + * + * 1. Adding `qml6gloverlay` to an already running pipeline + * 2. Not having any `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element + * start up before any other OpenGL-based element in the pipeline. + * + * If one of these scenarios occurs, then there will be multiple OpenGL contexts + * in use in the pipeline. This means that either the pipeline will fail to + * start up correctly, a downstream element may reject buffers, or a complete + * GPU->System memory->GPU transfer is performed for every buffer. + * + * The requirement to avoid this is that all elements share the same + * #GstGLDisplay object and as Qt cannot currently share an existing window + * system display connection, GStreamer must use the window system display + * connection provided by Qt. This window system display connection can be + * retrieved by either a `qml6glsink` element, a `qml6gloverlay`, or a + * `qml6glmixer` element. The recommended usage is to have either element + * (`qml6glsink` or `qml6gloverlay` or `qml6glmixer`) be the first to propagate + * the #GstGLDisplay for the entire pipeline to use by setting either element + * to the READY element state before any other OpenGL element in the pipeline. + * + * In the dynamically adding `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) + * to a pipeline case, there are some considerations for ensuring that the + * window system display and OpenGL contexts are compatible with Qt. When the + * `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element is added and + * brought up to READY, it will propagate it's own #GstGLDisplay using the + * #GstContext mechanism regardless of any existing #GstGLDisplay used by the + * pipeline previously. In order for the new #GstGLDisplay to be used, the + * application must then set the provided #GstGLDisplay containing #GstContext + * on the pipeline. This may effectively cause each OpenGL element to replace + * the window system display and also the OpenGL context it is using. As such + * this process may take a significant amount of time and resources as objects + * are recreated in the new OpenGL context. + * + * All instances of `qml6gloverlay`, `qml6glsink`, and `qml6glmixer` will + * return the exact same #GstGLDisplay object while the pipeline is running + * regardless of whether any `qml6gloverlay` or `qml6glsink` elements are + * added or removed from the pipeline. + * + * The Qml scene will run at the pace of incoming buffers. One input buffer + * will cause a render of one output buffer. The timestamps on the input + * buffers are used to drive the animation time. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstqt6elements.h" +#include "gstqml6gloverlay.h" +#include "qt6glrenderer.h" +#include "gstqt6glutility.h" + +#include + +#include + +#define GST_CAT_DEFAULT gst_debug_qml6_gl_overlay +GST_DEBUG_CATEGORY (GST_CAT_DEFAULT); + +/* *INDENT-OFF* */ +static GstStaticPadTemplate qml6_overlay_src_pad_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " + "format = (string) RGBA, " + "width = " GST_VIDEO_SIZE_RANGE ", " + "height = " GST_VIDEO_SIZE_RANGE ", " + "framerate = " GST_VIDEO_FPS_RANGE "," + "texture-target = (string) 2D" + )); + +static GstStaticPadTemplate qml6_overlay_sink_pad_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("video/x-raw(ANY), " + "format = (string) { RGBA, BGRA, YV12 }, " + "width = " GST_VIDEO_SIZE_RANGE ", " + "height = " GST_VIDEO_SIZE_RANGE ", " + "framerate = " GST_VIDEO_FPS_RANGE "," + "texture-target = (string) 2D" + )); +/* *INDENT-ON* */ + +static void gst_qml6_gl_overlay_finalize (GObject * object); +static void gst_qml6_gl_overlay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * param_spec); +static void gst_qml6_gl_overlay_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * param_spec); + +static GstCaps * gst_qml6_overlay_transform_internal_caps (GstGLFilter * filter, + GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps); + +static gboolean gst_qml6_gl_overlay_gl_start (GstGLBaseFilter * bfilter); +static void gst_qml6_gl_overlay_gl_stop (GstGLBaseFilter * bfilter); +static gboolean gst_qml6_gl_overlay_gl_set_caps (GstGLBaseFilter * bfilter, + GstCaps * in_caps, GstCaps * out_caps); + +static GstFlowReturn gst_qml6_gl_overlay_prepare_output_buffer (GstBaseTransform * btrans, + GstBuffer * buffer, GstBuffer ** outbuf); +static GstFlowReturn gst_qml6_gl_overlay_transform (GstBaseTransform * btrans, + GstBuffer * inbuf, GstBuffer * outbuf); + +static GstStateChangeReturn gst_qml6_gl_overlay_change_state (GstElement * element, + GstStateChange transition); + +enum +{ + PROP_0, + PROP_WIDGET, + PROP_QML_SCENE, + PROP_ROOT_ITEM, +}; + +enum +{ + SIGNAL_0, + SIGNAL_QML_SCENE_INITIALIZED, + SIGNAL_QML_SCENE_DESTROYED, + LAST_SIGNAL +}; + +static guint gst_qml6_gl_overlay_signals[LAST_SIGNAL] = { 0 }; + +#define gst_qml6_gl_overlay_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstQml6GLOverlay, gst_qml6_gl_overlay, + GST_TYPE_GL_FILTER, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, + "qml6gloverlay", 0, "Qt6 Video Overlay")); +GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (qml6gloverlay, "qml6gloverlay", + GST_RANK_NONE, GST_TYPE_QML6_GL_OVERLAY, qt6_element_init (plugin)); + +static void +gst_qml6_gl_overlay_class_init (GstQml6GLOverlayClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstBaseTransformClass *btrans_class; + GstGLBaseFilterClass *glbasefilter_class; + GstGLFilterClass *glfilter_class; + GstElementClass *element_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + glbasefilter_class = (GstGLBaseFilterClass *) klass; + glfilter_class = (GstGLFilterClass *) klass; + btrans_class = (GstBaseTransformClass *) klass; + element_class = (GstElementClass *) klass; + + gobject_class->set_property = gst_qml6_gl_overlay_set_property; + gobject_class->get_property = gst_qml6_gl_overlay_get_property; + gobject_class->finalize = gst_qml6_gl_overlay_finalize; + + gst_element_class_set_metadata (gstelement_class, "Qt Video Overlay", + "Filter/QML/Overlay", "A filter that renders a QML scene onto a video stream", + "Matthew Waters "); + + g_object_class_install_property (gobject_class, PROP_QML_SCENE, + g_param_spec_string ("qml-scene", "QML Scene", + "The contents of the QML scene", NULL, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_WIDGET, + g_param_spec_pointer ("widget", "QQuickItem", + "The QQuickItem to place the input video in the object hierarchy", + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_ROOT_ITEM, + g_param_spec_pointer ("root-item", "QQuickItem", + "The root QQuickItem from the qml-scene used to render", + (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + + /** + * GstQmlGLOverlay::qml-scene-initialized + * @element: the #GstQmlGLOverlay + * @user_data: user provided data + */ + gst_qml6_gl_overlay_signals[SIGNAL_QML_SCENE_INITIALIZED] = + g_signal_new ("qml-scene-initialized", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0); + + /** + * GstQmlGLOverlay::qml-scene-destroyed + * @element: the #GstQmlGLOverlay + * @user_data: user provided data + */ + gst_qml6_gl_overlay_signals[SIGNAL_QML_SCENE_DESTROYED] = + g_signal_new ("qml-scene-destroyed", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0); + + gst_element_class_add_static_pad_template (element_class, + &qml6_overlay_src_pad_template); + gst_element_class_add_static_pad_template (element_class, + &qml6_overlay_sink_pad_template); + + btrans_class->prepare_output_buffer = gst_qml6_gl_overlay_prepare_output_buffer; + btrans_class->transform = gst_qml6_gl_overlay_transform; + + glfilter_class->transform_internal_caps = gst_qml6_overlay_transform_internal_caps; + + glbasefilter_class->gl_start = gst_qml6_gl_overlay_gl_start; + glbasefilter_class->gl_stop = gst_qml6_gl_overlay_gl_stop; + glbasefilter_class->gl_set_caps = gst_qml6_gl_overlay_gl_set_caps; + + element_class->change_state = gst_qml6_gl_overlay_change_state; +} + +static void +gst_qml6_gl_overlay_init (GstQml6GLOverlay * qml6_gl_overlay) +{ + qml6_gl_overlay->widget = QSharedPointer(); + qml6_gl_overlay->qml_scene = NULL; +} + +static void +gst_qml6_gl_overlay_finalize (GObject * object) +{ + GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (object); + + g_free (qml6_gl_overlay->qml_scene); + qml6_gl_overlay->qml_scene = NULL; + + qml6_gl_overlay->widget.clear(); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_qml6_gl_overlay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (object); + + switch (prop_id) { + case PROP_WIDGET: { + Qt6GLVideoItem *qt_item = static_cast (g_value_get_pointer (value)); + if (qt_item) + qml6_gl_overlay->widget = qt_item->getInterface(); + else + qml6_gl_overlay->widget.clear(); + break; + } + case PROP_QML_SCENE: + g_free (qml6_gl_overlay->qml_scene); + qml6_gl_overlay->qml_scene = g_value_dup_string (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_qml6_gl_overlay_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (object); + + switch (prop_id) { + case PROP_WIDGET: + /* This is not really safe - the app needs to be + * sure the widget is going to be kept alive or + * this can crash */ + if (qml6_gl_overlay->widget) + g_value_set_pointer (value, qml6_gl_overlay->widget->videoItem()); + else + g_value_set_pointer (value, NULL); + break; + case PROP_QML_SCENE: + g_value_set_string (value, qml6_gl_overlay->qml_scene); + break; + case PROP_ROOT_ITEM: + GST_OBJECT_LOCK (qml6_gl_overlay); + if (qml6_gl_overlay->renderer) { + QQuickItem *root = qml6_gl_overlay->renderer->rootItem(); + if (root) + g_value_set_pointer (value, root); + else + g_value_set_pointer (value, NULL); + } else { + g_value_set_pointer (value, NULL); + } + GST_OBJECT_UNLOCK (qml6_gl_overlay); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_qml6_gl_overlay_gl_start (GstGLBaseFilter * bfilter) +{ + GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (bfilter); + QQuickItem *root; + GError *error = NULL; + + GST_TRACE_OBJECT (bfilter, "using scene:\n%s", qml6_gl_overlay->qml_scene); + + if (!qml6_gl_overlay->qml_scene || g_strcmp0 (qml6_gl_overlay->qml_scene, "") == 0) { + GST_ELEMENT_ERROR (bfilter, RESOURCE, NOT_FOUND, ("qml-scene property not set"), (NULL)); + return FALSE; + } + + if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (bfilter)) + return FALSE; + + GST_OBJECT_LOCK (bfilter); + qml6_gl_overlay->renderer = new GstQt6QuickRenderer; + if (!qml6_gl_overlay->renderer->init (bfilter->context, &error)) { + GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND, + ("%s", error->message), (NULL)); + delete qml6_gl_overlay->renderer; + qml6_gl_overlay->renderer = NULL; + GST_OBJECT_UNLOCK (bfilter); + return FALSE; + } + + /* FIXME: Qml may do async loading and we need to propagate qml errors in that case as well */ + if (!qml6_gl_overlay->renderer->setQmlScene (qml6_gl_overlay->qml_scene, &error)) { + GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND, + ("%s", error->message), (NULL)); + goto fail_renderer; + return FALSE; + } + + root = qml6_gl_overlay->renderer->rootItem(); + if (!root) { + GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND, + ("Qml scene does not have a root item"), (NULL)); + goto fail_renderer; + } + GST_OBJECT_UNLOCK (bfilter); + + g_object_notify (G_OBJECT (qml6_gl_overlay), "root-item"); + g_signal_emit (qml6_gl_overlay, gst_qml6_gl_overlay_signals[SIGNAL_QML_SCENE_INITIALIZED], 0); + + GST_OBJECT_LOCK (bfilter); + if (!qml6_gl_overlay->widget) { + Qt6GLVideoItem *qt_item = static_cast(root->findChild ()); + if (qt_item) + qml6_gl_overlay->widget = qt_item->getInterface(); + } + GST_OBJECT_UNLOCK (bfilter); + + return TRUE; + +fail_renderer: + { + qml6_gl_overlay->renderer->cleanup(); + delete qml6_gl_overlay->renderer; + qml6_gl_overlay->renderer = NULL; + GST_OBJECT_UNLOCK (bfilter); + return FALSE; + } +} + +static void +gst_qml6_gl_overlay_gl_stop (GstGLBaseFilter * bfilter) +{ + GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (bfilter); + GstQt6QuickRenderer *renderer = NULL; + + /* notify before actually destroying anything */ + GST_OBJECT_LOCK (qml6_gl_overlay); + if (qml6_gl_overlay->renderer) + renderer = qml6_gl_overlay->renderer; + qml6_gl_overlay->renderer = NULL; + GST_OBJECT_UNLOCK (qml6_gl_overlay); + + g_signal_emit (qml6_gl_overlay, gst_qml6_gl_overlay_signals[SIGNAL_QML_SCENE_DESTROYED], 0); + g_object_notify (G_OBJECT (qml6_gl_overlay), "root-item"); + + if (qml6_gl_overlay->widget) + qml6_gl_overlay->widget->setBuffer (NULL); + + if (renderer) { + renderer->cleanup(); + delete renderer; + } + + GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (bfilter); +} + +static gboolean +gst_qml6_gl_overlay_gl_set_caps (GstGLBaseFilter * bfilter, GstCaps * in_caps, + GstCaps * out_caps) +{ + GstGLFilter *filter = GST_GL_FILTER (bfilter); + GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (bfilter); + + if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_set_caps (bfilter, in_caps, out_caps)) + return FALSE; + + qml6_gl_overlay->renderer->setSize (GST_VIDEO_INFO_WIDTH (&filter->out_info), + GST_VIDEO_INFO_HEIGHT (&filter->out_info)); + + return TRUE; +} + +static GstCaps * +gst_qml6_overlay_transform_internal_caps (GstGLFilter * filter, + GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps) +{ + GstCaps *tmp = GST_GL_FILTER_CLASS (parent_class)->transform_internal_caps (filter, direction, caps, filter_caps); + int i, n; + + n = gst_caps_get_size (tmp); + for (i = 0; i < n; i++) { + GstStructure *s = gst_caps_get_structure (tmp, i); + + gst_structure_remove_fields (s, "format", "colorimetry", "chroma-site", + "texture-target", NULL); + } + + return tmp; +} + +static GstFlowReturn +gst_qml6_gl_overlay_prepare_output_buffer (GstBaseTransform * btrans, + GstBuffer * buffer, GstBuffer ** outbuf) +{ + GstBaseTransformClass *bclass = GST_BASE_TRANSFORM_GET_CLASS (btrans); + GstGLBaseFilter *bfilter = GST_GL_BASE_FILTER (btrans); + GstGLFilter *filter = GST_GL_FILTER (btrans); + GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (btrans); + GstGLMemory *out_mem; + GstGLSyncMeta *sync_meta; + + if (gst_buffer_n_memory (buffer) <= 0) { + GST_ELEMENT_ERROR (btrans, RESOURCE, NOT_FOUND, + (NULL), ("Buffer must have a memory object")); + return GST_FLOW_ERROR; + } + + if (qml6_gl_overlay->widget) { + GstMemory *mem; + GstGLMemory *gl_mem; + + qml6_gl_overlay->widget->setCaps (bfilter->in_caps); + + mem = gst_buffer_peek_memory (buffer, 0); + if (!gst_is_gl_memory (mem)) { + GST_ELEMENT_ERROR (btrans, RESOURCE, NOT_FOUND, + (NULL), ("Input memory must be a GstGLMemory")); + return GST_FLOW_ERROR; + } + gl_mem = (GstGLMemory *) mem; + if (!gst_gl_context_can_share (gl_mem->mem.context, bfilter->context)) { + GST_WARNING_OBJECT (bfilter, "Cannot use the current input texture " + "(input buffer GL context %" GST_PTR_FORMAT " cannot share " + "resources with the configured OpenGL context %" GST_PTR_FORMAT ")", + gl_mem->mem.context, bfilter->context); + } else { + qml6_gl_overlay->widget->setBuffer (buffer); + } + } + + /* XXX: is this the correct ts to drive the animation */ + out_mem = qml6_gl_overlay->renderer->generateOutput (GST_BUFFER_PTS (buffer)); + if (!out_mem) { + GST_ERROR_OBJECT (qml6_gl_overlay, "Failed to generate output"); + return GST_FLOW_ERROR; + } + + *outbuf = gst_buffer_new (); + gst_buffer_append_memory (*outbuf, (GstMemory *) out_mem); + gst_buffer_add_video_meta (*outbuf, (GstVideoFrameFlags) 0, + GST_VIDEO_INFO_FORMAT (&filter->out_info), + GST_VIDEO_INFO_WIDTH (&filter->in_info), + GST_VIDEO_INFO_HEIGHT (&filter->out_info)); + + sync_meta = gst_buffer_add_gl_sync_meta (bfilter->context, *outbuf); + gst_gl_sync_meta_set_sync_point (sync_meta, bfilter->context); + + bclass->copy_metadata (btrans, buffer, *outbuf); + + return GST_FLOW_OK; +} + +static GstFlowReturn +gst_qml6_gl_overlay_transform (GstBaseTransform * btrans, GstBuffer * inbuf, + GstBuffer * outbuf) +{ + return GST_FLOW_OK; +} + +static GstStateChangeReturn +gst_qml6_gl_overlay_change_state (GstElement * element, + GstStateChange transition) +{ + GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (element); + GstGLBaseFilter *filter = GST_GL_BASE_FILTER (element); + GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + + GST_DEBUG_OBJECT (filter, "changing state: %s => %s", + gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)), + gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition))); + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: { + QGuiApplication *app; + GstGLDisplay *display = NULL; + + app = static_cast (QCoreApplication::instance ()); + if (!app) { + GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND, + ("%s", "Failed to connect to Qt"), + ("%s", "Could not retrieve QGuiApplication instance")); + return GST_STATE_CHANGE_FAILURE; + } + + display = gst_qml6_get_gl_display (FALSE); + + if (display != filter->display) + /* always propagate. The application may need to choose between window + * system display connections */ + gst_gl_element_propagate_display_context (GST_ELEMENT (qml6_gl_overlay), display); + gst_object_unref (display); + break; + } + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + default: + break; + } + + return ret; + +} diff --git a/libs/qmlglsink/qt6/gstqml6gloverlay.h b/libs/qmlglsink/qt6/gstqml6gloverlay.h new file mode 100644 index 000000000000..704706775052 --- /dev/null +++ b/libs/qmlglsink/qt6/gstqml6gloverlay.h @@ -0,0 +1,76 @@ +/* + * GStreamer + * Copyright (C) 2020 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_QML6_GL_OVERLAY_H__ +#define __GST_QML6_GL_OVERLAY_H__ + +#include +#include +#include +#include "qt6glrenderer.h" +#include "qt6glitem.h" + +typedef struct _GstQml6GLOverlay GstQml6GLOverlay; +typedef struct _GstQml6GLOverlayClass GstQml6GLOverlayClass; +typedef struct _GstQml6GLOverlayPrivate GstQml6GLOverlayPrivate; + +G_BEGIN_DECLS + +GType gst_qml6_gl_overlay_get_type (void); +#define GST_TYPE_QML6_GL_OVERLAY (gst_qml6_gl_overlay_get_type()) +#define GST_QML6_GL_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_QML6_GL_OVERLAY,GstQml6GLOverlay)) +#define GST_QML6_GL_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_QML6_GL_OVERLAY,GstQml6GLOverlayClass)) +#define GST_IS_QML6_GL_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_QML6_GL_OVERLAY)) +#define GST_IS_QML6_GL_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_QML6_GL_OVERLAY)) +#define GST_QML6_GL_OVERLAY_CAST(obj) ((GstQml6GLOverlay*)(obj)) + +/** + * GstQml6GLOverlay: + * + * Opaque #GstQml6GLOverlay object + */ +struct _GstQml6GLOverlay +{ + /* */ + GstGLFilter parent; + + gchar *qml_scene; + + GstQt6QuickRenderer *renderer; + + QSharedPointer widget; +}; + +/** + * GstQml6GLOverlayClass: + * + * The #GstQml6GLOverlayClass struct only contains private data + */ +struct _GstQml6GLOverlayClass +{ + /* */ + GstGLFilterClass parent_class; +}; + +GstQml6GLOverlay * gst_qml6_gl_overlay_new (void); + +G_END_DECLS + +#endif /* __GST_QML6_GL_OVERLAY_H__ */ diff --git a/libs/qmlglsink/qt6/gstqml6glsink.cc b/libs/qmlglsink/qt6/gstqml6glsink.cc index 9c9ad386ef09..068d343a3387 100644 --- a/libs/qmlglsink/qt6/gstqml6glsink.cc +++ b/libs/qmlglsink/qt6/gstqml6glsink.cc @@ -43,29 +43,30 @@ * #GstGLDisplay object and as Qt cannot currently share an existing window * system display connection, GStreamer must use the window system display * connection provided by Qt. This window system display connection can be - * retrieved by either a qmlglsink element or a qmlgloverlay element. The - * recommended usage is to have either element (qmlglsink or qmlgloverlay) - * be the first to propagate the #GstGLDisplay for the entire pipeline to use by - * setting either element to the READY element state before any other OpenGL - * element in the pipeline. + * retrieved by either a `qml6glsink` element, a `qml6gloverlay` element, or a + * `qml6glmixer` element. The recommended usage is to have either element + * (`qml6glsink`, or `qml6gloverlay`, or `qml6glmixer`) be the first to + * propagate the #GstGLDisplay for the entire pipeline to use by setting either + * element to the READY element state before any other OpenGL element in the + * pipeline. * - * In a dynamically adding qmlglsink (or qmlgloverlay) to a pipeline case, - * there are some considerations for ensuring that the window system display - * and OpenGL contexts are compatible with Qt. When the qmlgloverlay (or - * qmlglsink) element is added and brought up to READY, it will propagate it's - * own #GstGLDisplay using the #GstContext mechanism regardless of any existing - * #GstGLDisplay used by the pipeline previously. In order for the new - * #GstGLDisplay to be used, the application must then set the provided - * #GstGLDisplay containing #GstContext on the pipeline. This may effectively - * cause each OpenGL element to replace the window system display and also the - * OpenGL context it is using. As such this process may take a significant - * amount of time and resources as objects are recreated in the new OpenGL - * context. + * In the dynamically adding `qml6glsink` (or `qml6gloverlay`, or `qml6glmixer`) + * to a pipeline case, there are some considerations for ensuring that the + * window system display and OpenGL contexts are compatible with Qt. When the + * `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element is added and + * brought up to READY, it will propagate it's own #GstGLDisplay using the + * #GstContext mechanism regardless of any existing #GstGLDisplay used by the + * pipeline previously. In order for the new #GstGLDisplay to be used, the + * application must then set the provided #GstGLDisplay containing #GstContext + * on the pipeline. This may effectively cause each OpenGL element to replace + * the window system display and also the OpenGL context it is using. As such + * this process may take a significant amount of time and resources as objects + * are recreated in the new OpenGL context. * - * All instances of qmlglsink and qmlgloverlay will return the exact same - * #GstGLDisplay object while the pipeline is running regardless of whether - * any qmlglsink or qmlgloverlay elements are added or removed from the - * pipeline. + * All instances of `qml6glsink`, `qml6gloverlay`, and `qml6glmixer` will + * return the exact same #GstGLDisplay object while the pipeline is running + * regardless of whether any `qml6glsink` or `qml6gloverlay` elements are + * added or removed from the pipeline. */ #ifdef HAVE_CONFIG_H @@ -108,7 +109,7 @@ GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " - "format = (string) { RGB, RGBA }, " + "format = (string) { RGBA, BGRA, RGB, YV12 }, " "width = " GST_VIDEO_SIZE_RANGE ", " "height = " GST_VIDEO_SIZE_RANGE ", " "framerate = " GST_VIDEO_FPS_RANGE ", " @@ -138,16 +139,8 @@ G_DEFINE_TYPE_WITH_CODE (GstQml6GLSink, gst_qml6_gl_sink, "qtsink", 0, "Qt Video Sink"); G_IMPLEMENT_INTERFACE (GST_TYPE_NAVIGATION, gst_qml6_gl_sink_navigation_interface_init)); -G_BEGIN_DECLS gboolean G_PASTE(gst_element_register_, qml6glsink)(GstPlugin *plugin) -{ - { - { - qt6_element_init(plugin); - } - } - return gst_element_register(plugin, "qml6glsink", GST_RANK_NONE, (gst_qml6_gl_sink_get_type())); -} -G_END_DECLS; +GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (qml6glsink, "qml6glsink", + GST_RANK_NONE, GST_TYPE_QML6_GL_SINK, qt6_element_init (plugin)); static void gst_qml6_gl_sink_class_init (GstQml6GLSinkClass * klass) @@ -562,15 +555,29 @@ gst_qml6_gl_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) static void gst_qml6_gl_sink_navigation_send_event (GstNavigation * navigation, - GstStructure *structure) + GstEvent * event) { GstQml6GLSink *qt_sink = GST_QML6_GL_SINK (navigation); + GstPad *pad; + + pad = gst_pad_get_peer (GST_VIDEO_SINK_PAD (qt_sink)); + GST_TRACE_OBJECT (qt_sink, "navigation event %" GST_PTR_FORMAT, - structure); - // TODO(zdanek) this will be available from gstreamer 1.22 + gst_event_get_structure(event)); + + if (GST_IS_PAD (pad) && GST_IS_EVENT (event)) { + if (!gst_pad_send_event (pad, gst_event_ref (event))) { + /* If upstream didn't handle the event we'll post a message with it + * for the application in case it wants to do something with it */ + gst_element_post_message (GST_ELEMENT_CAST (qt_sink), + gst_navigation_message_new_event (GST_OBJECT_CAST (qt_sink), event)); + } + gst_event_unref (event); + gst_object_unref (pad); + } } static void gst_qml6_gl_sink_navigation_interface_init (GstNavigationInterface * iface) { - iface->send_event = gst_qml6_gl_sink_navigation_send_event; + iface->send_event_simple = gst_qml6_gl_sink_navigation_send_event; } diff --git a/libs/qmlglsink/qt6/gstqml6glsrc.cc b/libs/qmlglsink/qt6/gstqml6glsrc.cc new file mode 100644 index 000000000000..dbd76844bf2a --- /dev/null +++ b/libs/qmlglsink/qt6/gstqml6glsrc.cc @@ -0,0 +1,599 @@ +/* + * GStreamer + * Copyright (C) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright (C) 2022 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:qml6glsrc + * + * Since: 1.24 + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstqt6elements.h" +#include "gstqml6glsrc.h" +#include + +#define GST_CAT_DEFAULT gst_debug_qml6_gl_src +GST_DEBUG_CATEGORY (GST_CAT_DEFAULT); + +#define DEFAULT_IS_LIVE TRUE + +static void gst_qml6_gl_src_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_qml6_gl_src_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static void gst_qml6_gl_src_finalize (GObject * object); + +static gboolean gst_qml6_gl_src_setcaps (GstBaseSrc * bsrc, GstCaps * caps); +static GstCaps *gst_qml6_gl_src_get_caps (GstBaseSrc * bsrc, GstCaps * filter); +static gboolean gst_qml6_gl_src_query (GstBaseSrc * bsrc, GstQuery * query); + +static gboolean gst_qml6_gl_src_decide_allocation (GstBaseSrc * bsrc, + GstQuery * query); +static GstFlowReturn gst_qml6_gl_src_create (GstPushSrc * psrc, GstBuffer ** buffer); +static gboolean gst_qml6_gl_src_unlock(GstBaseSrc * bsrc); +static gboolean gst_qml6_gl_src_unlock_stop (GstBaseSrc * bsrc); +static GstStateChangeReturn gst_qml6_gl_src_change_state (GstElement * element, + GstStateChange transition); +static gboolean gst_qml6_gl_src_start (GstBaseSrc * basesrc); +static gboolean gst_qml6_gl_src_stop (GstBaseSrc * basesrc); + +static GstStaticPadTemplate gst_qml6_gl_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " + "format = (string) RGBA, " + "width = " GST_VIDEO_SIZE_RANGE ", " + "height = " GST_VIDEO_SIZE_RANGE ", " + "framerate = " GST_VIDEO_FPS_RANGE ", " + "texture-target = (string) 2D")); + +enum +{ + ARG_0, + PROP_WINDOW, + PROP_DEFAULT_FBO +}; + +#define gst_qml6_gl_src_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstQml6GLSrc, gst_qml6_gl_src, + GST_TYPE_PUSH_SRC, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, + "qml6glsrc", 0, "Qt6 Qml Video Src")); +GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (qml6glsrc, "qml6glsrc", + GST_RANK_NONE, GST_TYPE_QML6_GL_SRC, qt6_element_init (plugin)); + +static const gfloat vertical_flip_matrix[] = { + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 1.0f, +}; + +static void +gst_qml6_gl_src_class_init (GstQml6GLSrcClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstElementClass *gstelement_class = (GstElementClass *) klass; + GstBaseSrcClass *gstbasesrc_class = (GstBaseSrcClass *) klass; + GstPushSrcClass *gstpushsrc_class = (GstPushSrcClass *) klass; + + gobject_class->set_property = gst_qml6_gl_src_set_property; + gobject_class->get_property = gst_qml6_gl_src_get_property; + gobject_class->finalize = gst_qml6_gl_src_finalize; + + gst_element_class_set_metadata (gstelement_class, "Qt Video Source", + "Source/Video", "A video src that captures a window from a QML view", + "Multimedia Team "); + + g_object_class_install_property (gobject_class, PROP_WINDOW, + g_param_spec_pointer ("window", "QQuickWindow", + "The QQuickWindow to place in the object hierarchy", + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_DEFAULT_FBO, + g_param_spec_boolean ("use-default-fbo", + "Whether to use default FBO", + "When set it will not create a new FBO for the QML render thread", + FALSE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&gst_qml6_gl_src_template)); + + gstelement_class->change_state = gst_qml6_gl_src_change_state; + gstbasesrc_class->set_caps = gst_qml6_gl_src_setcaps; + gstbasesrc_class->get_caps = gst_qml6_gl_src_get_caps; + gstbasesrc_class->query = gst_qml6_gl_src_query; + gstbasesrc_class->start = gst_qml6_gl_src_start; + gstbasesrc_class->stop = gst_qml6_gl_src_stop; + gstbasesrc_class->decide_allocation = gst_qml6_gl_src_decide_allocation; + gstbasesrc_class->unlock = gst_qml6_gl_src_unlock; + gstbasesrc_class->unlock_stop = gst_qml6_gl_src_unlock_stop; + + gstpushsrc_class->create = gst_qml6_gl_src_create; +} + +static void +gst_qml6_gl_src_init (GstQml6GLSrc * src) +{ + gst_base_src_set_format (GST_BASE_SRC (src), GST_FORMAT_TIME); + gst_base_src_set_live (GST_BASE_SRC (src), DEFAULT_IS_LIVE); + src->default_fbo = FALSE; + src->pending_image_orientation = TRUE; +} + +static void +gst_qml6_gl_src_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (object); + + switch (prop_id) { + case PROP_WINDOW:{ + qt_src->qwindow = + static_cast < QQuickWindow * >(g_value_get_pointer (value)); + + if (qt_src->window) { + delete qt_src->window; + qt_src->window = NULL; + } + + if (qt_src->qwindow) + qt_src->window = new Qt6GLWindow (NULL, qt_src->qwindow); + + break; + } + case PROP_DEFAULT_FBO: + qt_src->default_fbo = g_value_get_boolean (value); + if (qt_src->window) + qt6_gl_window_use_default_fbo (qt_src->window, qt_src->default_fbo); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_qml6_gl_src_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (object); + + switch (prop_id) { + case PROP_WINDOW: + g_value_set_pointer (value, qt_src->qwindow); + break; + case PROP_DEFAULT_FBO: + g_value_set_boolean (value, qt_src->default_fbo); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_qml6_gl_src_finalize (GObject * object) +{ + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (object); + + GST_DEBUG ("qmlglsrc finalize"); + if (qt_src->context) + gst_object_unref (qt_src->context); + qt_src->context = NULL; + + if (qt_src->qt_context) + gst_object_unref (qt_src->qt_context); + qt_src->qt_context = NULL; + + if (qt_src->display) + gst_object_unref (qt_src->display); + qt_src->display = NULL; + + if (qt_src->window) + delete qt_src->window; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static gboolean +gst_qml6_gl_src_setcaps (GstBaseSrc * bsrc, GstCaps * caps) +{ + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (bsrc); + + GST_DEBUG ("set caps with %" GST_PTR_FORMAT, caps); + + if (!gst_video_info_from_caps (&qt_src->v_info, caps)) + return FALSE; + + return TRUE; +} + +static GstCaps * +gst_qml6_gl_src_get_caps (GstBaseSrc * bsrc, GstCaps * filter) +{ + GstCaps *caps = NULL, *temp = NULL; + GstPadTemplate *pad_template; + GstBaseSrcClass *bclass = GST_BASE_SRC_GET_CLASS (bsrc); + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (bsrc); + guint i; + gint width, height; + + if (qt_src->window) { + qt_src->window->getGeometry (&width, &height); + } + + pad_template = + gst_element_class_get_pad_template (GST_ELEMENT_CLASS (bclass), "src"); + if (pad_template != NULL) + caps = gst_pad_template_get_caps (pad_template); + + if (qt_src->window) { + temp = gst_caps_copy (caps); + guint n_caps = gst_caps_get_size (caps); + + for (i = 0; i < n_caps; i++) { + GstStructure *s = gst_caps_get_structure (temp, i); + gst_structure_set (s, "width", G_TYPE_INT, width, NULL); + gst_structure_set (s, "height", G_TYPE_INT, height, NULL); + /* because the framerate is unknown */ + gst_structure_set (s, "framerate", GST_TYPE_FRACTION, 0, 1, NULL); + gst_structure_set (s, "pixel-aspect-ratio", + GST_TYPE_FRACTION, 1, 1, NULL); + } + + gst_caps_unref (caps); + caps = temp; + } + + if (filter) { + GstCaps *intersection; + + intersection = + gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (caps); + caps = intersection; + } + + return caps; +} + +static gboolean +gst_qml6_gl_src_query (GstBaseSrc * bsrc, GstQuery * query) +{ + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (bsrc); + gboolean res = FALSE; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + { + if (!qt6_gl_window_is_scenegraph_initialized (qt_src->window)) + return FALSE; + + if (!qt_src->display && !qt_src->qt_context) { + if (!qt_src->display) + qt_src->display = qt6_gl_window_get_display (qt_src->window); + if (!qt_src->qt_context) + qt_src->qt_context = qt6_gl_window_get_qt_context (qt_src->window); + if (!qt_src->context) + qt_src->context = qt6_gl_window_get_context (qt_src->window); + } + + if (gst_gl_handle_context_query ((GstElement *) qt_src, query, + qt_src->display, qt_src->context, qt_src->qt_context)) + return TRUE; + + /* fallthrough */ + } + default: + res = GST_BASE_SRC_CLASS (parent_class)->query (bsrc, query); + break; + } + + return res; +} + +static gboolean +_find_local_gl_context (GstQml6GLSrc * qt_src) +{ + if (gst_gl_query_local_gl_context (GST_ELEMENT (qt_src), GST_PAD_SRC, + &qt_src->context)) + return TRUE; + return FALSE; +} + +static gboolean +gst_qml6_gl_src_decide_allocation (GstBaseSrc * bsrc, GstQuery * query) +{ + GstBufferPool *pool = NULL; + GstStructure *config; + GstCaps *caps; + guint min, max, size, n, i; + gboolean update_pool, update_allocator; + GstAllocator *allocator; + GstAllocationParams params; + GstGLVideoAllocationParams *glparams; + GstVideoInfo vinfo; + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (bsrc); + + if (gst_query_find_allocation_meta (query, + GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, NULL)) { + qt_src->downstream_supports_affine_meta = TRUE; + } else { + qt_src->downstream_supports_affine_meta = FALSE; + } + + gst_query_parse_allocation (query, &caps, NULL); + if (!caps) + return FALSE; + + gst_video_info_from_caps (&vinfo, caps); + + n = gst_query_get_n_allocation_pools (query); + if (n > 0) { + update_pool = TRUE; + for (i = 0; i < n; i++) { + gst_query_parse_nth_allocation_pool (query, i, &pool, &size, &min, &max); + + if (!pool || !GST_IS_GL_BUFFER_POOL (pool)) { + if (pool) + gst_object_unref (pool); + pool = NULL; + } + } + } + + if (!pool) { + size = vinfo.size; + min = max = 0; + update_pool = FALSE; + } + + if (!qt_src->context && !_find_local_gl_context (qt_src)) + return FALSE; + + if (!qt6_gl_window_set_context (qt_src->window, qt_src->context)) + return FALSE; + + if (!pool) { + if (!qt_src->context || !GST_IS_GL_CONTEXT (qt_src->context)) + return FALSE; + + pool = gst_gl_buffer_pool_new (qt_src->context); + GST_INFO_OBJECT (qt_src, "No pool, create one ourself %p", pool); + } + + config = gst_buffer_pool_get_config (pool); + + gst_buffer_pool_config_set_params (config, caps, size, min, max); + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + if (gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, NULL)) + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_GL_SYNC_META); + + if (gst_query_get_n_allocation_params (query) > 0) { + gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms); + gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); + GST_INFO_OBJECT (qt_src, "got allocator %p", allocator); + update_allocator = TRUE; + } else { + allocator = NULL; + gst_allocation_params_init (¶ms); + update_allocator = FALSE; + } + + glparams = + gst_gl_video_allocation_params_new (qt_src->context, ¶ms, &vinfo, 0, + NULL, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGBA); + gst_buffer_pool_config_set_gl_allocation_params (config, + (GstGLAllocationParams *) glparams); + gst_gl_allocation_params_free ((GstGLAllocationParams *) glparams); + + if (!gst_buffer_pool_set_config (pool, config)) + GST_WARNING_OBJECT (qt_src, "Failed to set buffer pool config"); + + if (update_allocator) + gst_query_set_nth_allocation_param (query, 0, allocator, ¶ms); + else + gst_query_add_allocation_param (query, allocator, ¶ms); + if (allocator) + gst_object_unref (allocator); + + if (update_pool) + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + else + gst_query_add_allocation_pool (query, pool, size, min, max); + gst_object_unref (pool); + + GST_INFO_OBJECT (qt_src, "successfully decide_allocation"); + return TRUE; +} + +static GstFlowReturn +gst_qml6_gl_src_create (GstPushSrc * psrc, GstBuffer ** buffer) +{ + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (psrc); + GstCaps *updated_caps = NULL; + GstGLContext* context = qt_src->context; + GstGLSyncMeta *sync_meta; + + *buffer = qt6_gl_window_take_buffer (qt_src->window, &updated_caps); + GST_DEBUG_OBJECT (qt_src, "produced buffer %p", *buffer); + if (!*buffer) + return GST_FLOW_FLUSHING; + + if (updated_caps) { + GST_DEBUG_OBJECT (qt_src, "new_caps %" GST_PTR_FORMAT, updated_caps); + gst_base_src_set_caps (GST_BASE_SRC (qt_src), updated_caps); + } + gst_clear_caps (&updated_caps); + + sync_meta = gst_buffer_get_gl_sync_meta(*buffer); + if (sync_meta) + gst_gl_sync_meta_wait(sync_meta, context); + + if (!qt_src->downstream_supports_affine_meta) { + if (qt_src->pending_image_orientation) { + /* let downstream know the image orientation is vertical filp */ + GstTagList *image_orientation_tag = + gst_tag_list_new (GST_TAG_IMAGE_ORIENTATION, "flip-rotate-180", NULL); + + gst_pad_push_event (GST_BASE_SRC_PAD (psrc), + gst_event_new_tag (image_orientation_tag)); + + qt_src->pending_image_orientation = FALSE; + } + } else { + GstVideoAffineTransformationMeta *trans_meta; + trans_meta = gst_buffer_add_video_affine_transformation_meta (*buffer); + gst_video_affine_transformation_meta_apply_matrix (trans_meta, + vertical_flip_matrix); + } + + GST_DEBUG_OBJECT (qt_src, "buffer create done %p", *buffer); + + return GST_FLOW_OK; +} + +static gboolean +gst_qml6_gl_src_unlock (GstBaseSrc * bsrc) +{ + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (bsrc); + + if (!qt_src->window) + return TRUE; + + qt6_gl_window_unlock (qt_src->window); + + return TRUE; +} + +static gboolean +gst_qml6_gl_src_unlock_stop (GstBaseSrc * bsrc) +{ + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (bsrc); + + if (!qt_src->window) + return TRUE; + + qt6_gl_window_unlock_stop (qt_src->window); + + return TRUE; +} + +static GstStateChangeReturn +gst_qml6_gl_src_change_state (GstElement * element, GstStateChange transition) +{ + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (element); + GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + QGuiApplication *app; + + GST_DEBUG ("changing state: %s => %s", + gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)), + gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition))); + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: + app = static_cast < QGuiApplication * >(QCoreApplication::instance ()); + if (!app) { + GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND, + ("%s", "Failed to connect to Qt"), + ("%s", "Could not retrieve QGuiApplication instance")); + return GST_STATE_CHANGE_FAILURE; + } + + if (!qt_src->window) { + GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND, + ("%s", "Required property \'window\' not set"), (NULL)); + return GST_STATE_CHANGE_FAILURE; + } + + if (!qt6_gl_window_is_scenegraph_initialized (qt_src->window)) { + GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND, + ("%s", "Could not initialize window system"), (NULL)); + return GST_STATE_CHANGE_FAILURE; + } + + qt6_gl_window_use_default_fbo (qt_src->window, qt_src->default_fbo); + + break; + case GST_STATE_CHANGE_READY_TO_PAUSED: + break; + case GST_STATE_CHANGE_PAUSED_TO_PLAYING: + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + case GST_STATE_CHANGE_PLAYING_TO_PAUSED: + break; + case GST_STATE_CHANGE_PAUSED_TO_READY: + break; + case GST_STATE_CHANGE_READY_TO_NULL: + break; + default: + break; + } + + return ret; +} + +static gboolean +gst_qml6_gl_src_start (GstBaseSrc * basesrc) +{ + GstQml6GLSrc *qt_src = GST_QML6_GL_SRC (basesrc); + + /* already has get OpenGL configuration from qt */ + if (qt_src->display && qt_src->qt_context) + return TRUE; + + if (!qt6_gl_window_is_scenegraph_initialized (qt_src->window)) + return FALSE; + + qt_src->display = qt6_gl_window_get_display (qt_src->window); + qt_src->qt_context = qt6_gl_window_get_qt_context (qt_src->window); + qt_src->context = qt6_gl_window_get_context (qt_src->window); + + if (!qt_src->display || !qt_src->qt_context) { + GST_ERROR_OBJECT (qt_src, + "Could not retrieve window system OpenGL configuration"); + return FALSE; + } + + GST_DEBUG_OBJECT (qt_src, "Got qt display %p and qt gl context %p", + qt_src->display, qt_src->qt_context); + return TRUE; +} + +static gboolean +gst_qml6_gl_src_stop (GstBaseSrc * basesrc) +{ + return TRUE; +} diff --git a/libs/qmlglsink/qt6/gstqml6glsrc.h b/libs/qmlglsink/qt6/gstqml6glsrc.h new file mode 100644 index 000000000000..96ad503f179c --- /dev/null +++ b/libs/qmlglsink/qt6/gstqml6glsrc.h @@ -0,0 +1,62 @@ +/* + * GStreamer + * Copyright (C) 2016 Freescale Semiconductor, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_QML6_GL_SRC_H__ +#define __GST_QML6_GL_SRC_H__ + +#include +#include +#include +#include +#include "qt6glwindow.h" + +G_BEGIN_DECLS + +#define GST_TYPE_QML6_GL_SRC (gst_qml6_gl_src_get_type()) +G_DECLARE_FINAL_TYPE (GstQml6GLSrc, gst_qml6_gl_src, GST, QML6_GL_SRC, GstPushSrc) +#define GST_QML6_GL_SRC_CAST(obj) ((GstQml6GLSrc*)(obj)) + +/** + * GstQml6GLSrc: + * + * Opaque #GstQml6GLSrc object + */ +struct _GstQml6GLSrc +{ + /* */ + GstPushSrc parent; + + QQuickWindow *qwindow; + Qt6GLWindow *window; + + GstVideoInfo v_info; + + GstGLDisplay *display; + GstGLContext *context; + GstGLContext *qt_context; + + gboolean default_fbo; + gboolean downstream_supports_affine_meta; + gboolean pending_image_orientation; +}; + +G_END_DECLS + +#endif /* __GST_QML6_GL_SRC_H__ */ diff --git a/libs/qmlglsink/qt6/gstqsg6material.cc b/libs/qmlglsink/qt6/gstqsg6material.cc new file mode 100644 index 000000000000..34587d69e1bd --- /dev/null +++ b/libs/qmlglsink/qt6/gstqsg6material.cc @@ -0,0 +1,641 @@ +/* + * GStreamer + * Copyright (C) 2023 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include +#include +#include +#include "gstqsg6material.h" +#include + +#define GST_CAT_DEFAULT gst_qsg_texture_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +/* matrix colour conversion code from vkvideoconvert.c */ +typedef struct +{ + double dm[4][4]; +} Matrix4; + +static void +matrix_debug (const Matrix4 * s) +{ + GST_DEBUG ("[%f %f %f %f]", s->dm[0][0], s->dm[0][1], s->dm[0][2], + s->dm[0][3]); + GST_DEBUG ("[%f %f %f %f]", s->dm[1][0], s->dm[1][1], s->dm[1][2], + s->dm[1][3]); + GST_DEBUG ("[%f %f %f %f]", s->dm[2][0], s->dm[2][1], s->dm[2][2], + s->dm[2][3]); + GST_DEBUG ("[%f %f %f %f]", s->dm[3][0], s->dm[3][1], s->dm[3][2], + s->dm[3][3]); +} + +static void +matrix_to_float (const Matrix4 * m, float *ret) +{ + int i, j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + ret[j * 4 + i] = m->dm[i][j]; + } + } +} + +static void +matrix_set_identity (Matrix4 * m) +{ + int i, j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + m->dm[i][j] = (i == j); + } + } +} + +static void +matrix_copy (Matrix4 * d, const Matrix4 * s) +{ + gint i, j; + + for (i = 0; i < 4; i++) + for (j = 0; j < 4; j++) + d->dm[i][j] = s->dm[i][j]; +} + +/* Perform 4x4 matrix multiplication: + * - @dst@ = @a@ * @b@ + * - @dst@ may be a pointer to @a@ andor @b@ + */ +static void +matrix_multiply (Matrix4 * dst, Matrix4 * a, Matrix4 * b) +{ + Matrix4 tmp; + int i, j, k; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + double x = 0; + for (k = 0; k < 4; k++) { + x += a->dm[i][k] * b->dm[k][j]; + } + tmp.dm[i][j] = x; + } + } + matrix_copy (dst, &tmp); +} + +static void +matrix_offset_components (Matrix4 * m, double a1, double a2, double a3) +{ + Matrix4 a; + + matrix_set_identity (&a); + a.dm[0][3] = a1; + a.dm[1][3] = a2; + a.dm[2][3] = a3; + matrix_debug (&a); + matrix_multiply (m, &a, m); +} + +static void +matrix_scale_components (Matrix4 * m, double a1, double a2, double a3) +{ + Matrix4 a; + + matrix_set_identity (&a); + a.dm[0][0] = a1; + a.dm[1][1] = a2; + a.dm[2][2] = a3; + matrix_multiply (m, &a, m); +} + +static void +matrix_YCbCr_to_RGB (Matrix4 * m, double Kr, double Kb) +{ + double Kg = 1.0 - Kr - Kb; + Matrix4 k = { + { + {1., 0., 2 * (1 - Kr), 0.}, + {1., -2 * Kb * (1 - Kb) / Kg, -2 * Kr * (1 - Kr) / Kg, 0.}, + {1., 2 * (1 - Kb), 0., 0.}, + {0., 0., 0., 1.}, + } + }; + + matrix_multiply (m, &k, m); +} + +static void +convert_to_RGB (GstVideoInfo *info, Matrix4 * m) +{ + { + const GstVideoFormatInfo *uinfo; + gint offset[4], scale[4], depth[4]; + guint i; + + uinfo = gst_video_format_get_info (GST_VIDEO_INFO_FORMAT (info)); + + /* bring color components to [0..1.0] range */ + gst_video_color_range_offsets (info->colorimetry.range, uinfo, offset, + scale); + + for (i = 0; i < uinfo->n_components; i++) + depth[i] = (1 << uinfo->depth[i]) - 1; + + matrix_offset_components (m, -offset[0] / (float) depth[0], + -offset[1] / (float) depth[1], -offset[2] / (float) depth[2]); + matrix_scale_components (m, depth[0] / ((float) scale[0]), + depth[1] / ((float) scale[1]), depth[2] / ((float) scale[2])); + GST_DEBUG ("to RGB scale/offset matrix"); + matrix_debug (m); + } + + if (GST_VIDEO_INFO_IS_YUV (info)) { + gdouble Kr, Kb; + + if (gst_video_color_matrix_get_Kr_Kb (info->colorimetry.matrix, &Kr, &Kb)) + matrix_YCbCr_to_RGB (m, Kr, Kb); + GST_DEBUG ("to RGB matrix"); + matrix_debug (m); + } +} + +class GstQSGTexture : public QSGTexture { +public: + GstQSGTexture(QRhiTexture *); + ~GstQSGTexture(); + + qint64 comparisonKey() const override; + bool hasAlphaChannel() const override; + bool hasMipmaps() const override { return false; }; + bool isAtlasTexture() const override { return false; }; + QSize textureSize() const override; + + QRhiTexture *rhiTexture() const override; + +private: + QRhiTexture *m_texture; + bool m_has_alpha; +}; + +GstQSGTexture::GstQSGTexture(QRhiTexture * texture) + : m_texture(texture) +{ + switch (texture->format()) { + case QRhiTexture::RGBA8: +#if (QT_VERSION >= QT_VERSION_CHECK(6, 4, 0)) + case QRhiTexture::RGB10A2: +#endif + case QRhiTexture::RGBA16F: + case QRhiTexture::RGBA32F: + this->m_has_alpha = true; + break; + default: + this->m_has_alpha = false; + } +} + +GstQSGTexture::~GstQSGTexture() +{ + if (m_texture) { + m_texture->deleteLater(); + m_texture = nullptr; + } +} + +qint64 +GstQSGTexture::comparisonKey() const +{ + if (this->m_texture) + return qint64(qintptr(this->m_texture)); + + return qint64(qintptr(this)); +} + +bool +GstQSGTexture::hasAlphaChannel() const +{ + return m_has_alpha; +} + +QSize +GstQSGTexture::textureSize() const +{ + // XXX: currently unused + return QSize(0, 0); +} + +QRhiTexture * +GstQSGTexture::rhiTexture() const +{ + return m_texture; +} + +class GstQSGMaterialShader : public QSGMaterialShader { +public: + GstQSGMaterialShader(GstVideoFormat v_format); + ~GstQSGMaterialShader(); + + bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; + void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, QSGMaterial *newMaterial, QSGMaterial *) override; + +private: + GstVideoFormat v_format; + QSGTexture *m_textures[GST_VIDEO_MAX_PLANES]; +}; + +GstQSGMaterialShader::GstQSGMaterialShader(GstVideoFormat v_format) + : v_format(v_format) +{ + setShaderFileName(VertexStage, ":/org/freedesktop/gstreamer/qml6/vertex.vert.qsb"); + + switch (v_format) { + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_BGRA: + case GST_VIDEO_FORMAT_RGB: + setShaderFileName(FragmentStage, ":/org/freedesktop/gstreamer/qml6/RGBA.frag.qsb"); + break; + case GST_VIDEO_FORMAT_YV12: + setShaderFileName(FragmentStage, ":/org/freedesktop/gstreamer/qml6/YUV_TRIPLANAR.frag.qsb"); + break; + default: + g_assert_not_reached (); + } + + m_textures[0] = nullptr; + m_textures[1] = nullptr; + m_textures[2] = nullptr; + m_textures[3] = nullptr; +} + +GstQSGMaterialShader::~GstQSGMaterialShader() +{ + for (int i = 0; i < 4; i++) { + if (m_textures[i]) { + delete m_textures[i]; + m_textures[i] = nullptr; + } + } +} + +bool +GstQSGMaterialShader::updateUniformData(RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) +{ + const GstVideoFormatInfo *finfo = gst_video_format_get_info (v_format); + bool changed = false; + QByteArray *buf = state.uniformData(); + Q_ASSERT(buf->size() >= 84); + + GST_TRACE ("%p new material %p old material %p", this, newMaterial, oldMaterial); + + if (state.isMatrixDirty()) { + const QMatrix4x4 m = state.combinedMatrix(); + memcpy(buf->data(), m.constData(), 64); + changed = true; + } + + if (state.isOpacityDirty()) { + const float opacity = state.opacity(); + memcpy(buf->data() + 144, &opacity, 4); + changed = true; + } + + auto *mat = static_cast(newMaterial); + if (oldMaterial != newMaterial || mat->uniforms.dirty) { + memcpy(buf->data() + 64, &mat->uniforms.input_swizzle, 4 * sizeof (int)); + memcpy(buf->data() + 80, mat->uniforms.color_matrix.constData(), 64); + mat->uniforms.dirty = false; + changed = true; + } + + for (guint i = 0; i < GST_VIDEO_MAX_PLANES; i++) { + if (this->m_textures[i]) { + delete this->m_textures[i]; + this->m_textures[i] = nullptr; + } + if (i < finfo->n_planes) + this->m_textures[i] = mat->bind(this, state.rhi(), state.resourceUpdateBatch(), i, v_format); + } + + return changed; +} + +void +GstQSGMaterialShader::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, + QSGMaterial *newMaterial, QSGMaterial *) +{ + *texture = this->m_textures[binding - 1]; + GST_TRACE ("%p binding:%d texture %p", this, binding, *texture); +} + +#define DEFINE_MATERIAL(format) \ +class G_PASTE(GstQSGMaterial_,format) : public GstQSGMaterial { \ +public: \ + G_PASTE(GstQSGMaterial_,format)(); \ + ~G_PASTE(GstQSGMaterial_,format)(); \ + QSGMaterialType *type() const override { static QSGMaterialType type; return &type; }; \ +}; \ +G_PASTE(GstQSGMaterial_,format)::G_PASTE(GstQSGMaterial_,format)() {} \ +G_PASTE(GstQSGMaterial_,format)::~G_PASTE(GstQSGMaterial_,format)() {} + +DEFINE_MATERIAL(RGBA_SWIZZLE); +DEFINE_MATERIAL(YUV_TRIPLANAR); + +GstQSGMaterial * +GstQSGMaterial::new_for_format(GstVideoFormat format) +{ + const GstVideoFormatInfo *finfo = gst_video_format_get_info (format); + + if (GST_VIDEO_FORMAT_INFO_IS_RGB (finfo) && finfo->n_planes == 1) { + return static_cast(new GstQSGMaterial_RGBA_SWIZZLE()); + } + + switch (format) { + case GST_VIDEO_FORMAT_YV12: + return static_cast(new GstQSGMaterial_YUV_TRIPLANAR()); + default: + g_assert_not_reached (); + } +} + +GstQSGMaterial::GstQSGMaterial () +{ + static gsize _debug; + + if (g_once_init_enter (&_debug)) { + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtqsg6material", 0, + "Qt6 Scenegraph Material"); + g_once_init_leave (&_debug, 1); + } + + g_weak_ref_init (&this->qt_context_ref_, NULL); + gst_video_info_init (&this->v_info); + memset (&this->v_frame, 0, sizeof (this->v_frame)); + + this->buffer_ = NULL; + this->buffer_was_bound = false; + this->sync_buffer_ = gst_buffer_new (); + + this->uniforms.dirty = true; +} + +GstQSGMaterial::~GstQSGMaterial () +{ + g_weak_ref_clear (&this->qt_context_ref_); + gst_buffer_replace (&this->buffer_, NULL); + gst_buffer_replace (&this->sync_buffer_, NULL); + this->buffer_was_bound = false; + + if (this->v_frame.buffer) { + gst_video_frame_unmap (&this->v_frame); + memset (&this->v_frame, 0, sizeof (this->v_frame)); + } +} + +bool +GstQSGMaterial::compatibleWith(GstVideoInfo * v_info) +{ + if (GST_VIDEO_INFO_FORMAT (&this->v_info) != GST_VIDEO_INFO_FORMAT (v_info)) + return false; + + return true; +} + +QSGMaterialShader * +GstQSGMaterial::createShader(QSGRendererInterface::RenderMode renderMode) const +{ + GstVideoFormat v_format = GST_VIDEO_INFO_FORMAT (&this->v_info); + + return new GstQSGMaterialShader(v_format); +} + +/* only called from the streaming thread with scene graph thread blocked */ +void +GstQSGMaterial::setCaps (GstCaps * caps) +{ + GST_LOG ("%p setCaps %" GST_PTR_FORMAT, this, caps); + + gst_video_info_from_caps (&this->v_info, caps); +} + +/* only called from the streaming thread with scene graph thread blocked */ +gboolean +GstQSGMaterial::setBuffer (GstBuffer * buffer) +{ + GST_LOG ("%p setBuffer %" GST_PTR_FORMAT, this, buffer); + /* FIXME: update more state here */ + if (!gst_buffer_replace (&this->buffer_, buffer)) + return FALSE; + + this->buffer_was_bound = false; + + g_weak_ref_set (&this->qt_context_ref_, gst_gl_context_get_current ()); + + if (this->v_frame.buffer) { + gst_video_frame_unmap (&this->v_frame); + memset (&this->v_frame, 0, sizeof (this->v_frame)); + } + + if (this->buffer_) { + if (!gst_video_frame_map (&this->v_frame, &this->v_info, this->buffer_, + (GstMapFlags) (GST_MAP_READ | GST_MAP_GL))) { + g_assert_not_reached (); + return FALSE; + } + gst_gl_video_format_swizzle(GST_VIDEO_INFO_FORMAT (&this->v_info), this->uniforms.input_swizzle); + + Matrix4 m; + float matrix_data[16] = { 0.0, }; + + matrix_set_identity (&m); + convert_to_RGB (&this->v_info, &m); + matrix_debug (&m); + matrix_to_float (&m, matrix_data); + + this->uniforms.color_matrix = QMatrix4x4(matrix_data); + this->uniforms.dirty = true; + } + + return TRUE; +} + +/* only called from the streaming thread with scene graph thread blocked */ +GstBuffer * +GstQSGMaterial::getBuffer (bool * was_bound) +{ + GstBuffer *buffer = NULL; + + if (this->buffer_) + buffer = gst_buffer_ref (this->buffer_); + if (was_bound) + *was_bound = this->buffer_was_bound; + + return buffer; +} + +void +GstQSGMaterial::setFiltering(QSGTexture::Filtering filtering) +{ + m_filtering = filtering; +} + +static QRhiTexture::Format +video_format_to_rhi_format (GstVideoFormat format, guint plane) +{ + switch (format) { + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_BGRA: + return QRhiTexture::RGBA8; + case GST_VIDEO_FORMAT_YV12: + return QRhiTexture::R8; + default: + g_assert_not_reached (); + } +} + +static int +video_format_to_texel_size (GstVideoFormat format, guint plane) +{ + switch (format) { + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_BGRA: + return 4; + case GST_VIDEO_FORMAT_YV12: + return 1; + default: + g_assert_not_reached (); + } +} + +QSGTexture * +GstQSGMaterial::bind(GstQSGMaterialShader *shader, QRhi * rhi, QRhiResourceUpdateBatch *res_updates, guint plane, GstVideoFormat v_format) +{ + GstGLContext *qt_context, *context; + GstMemory *mem; + GstGLMemory *gl_mem; + GstGLSyncMeta *sync_meta; + gboolean use_dummy_tex = TRUE; + guint tex_id; + GstQSGTexture *ret; + QRhiTexture *rhi_tex; + QSize tex_size; + + qt_context = GST_GL_CONTEXT (g_weak_ref_get (&this->qt_context_ref_)); + if (!qt_context) + goto out; + + if (!this->buffer_) + goto out; + if (GST_VIDEO_INFO_FORMAT (&this->v_info) == GST_VIDEO_FORMAT_UNKNOWN) + goto out; + + mem = gst_buffer_peek_memory (this->buffer_, plane); + g_assert (gst_is_gl_memory (mem)); + gl_mem = (GstGLMemory *) mem; + context = ((GstGLBaseMemory *)mem)->context; + + /* Texture was successfully bound, so we do not need + * to use the dummy texture */ + use_dummy_tex = FALSE; + + this->buffer_was_bound = true; + tex_id = *(guint *) this->v_frame.data[plane]; + + tex_size = QSize(gst_gl_memory_get_texture_width(gl_mem), gst_gl_memory_get_texture_height (gl_mem)); + + rhi_tex = rhi->newTexture (video_format_to_rhi_format (v_format, plane), tex_size, 1, {}); + rhi_tex->createFrom({(guint64) tex_id, 0}); + + sync_meta = gst_buffer_get_gl_sync_meta (this->sync_buffer_); + if (!sync_meta) + sync_meta = gst_buffer_add_gl_sync_meta (context, this->sync_buffer_); + + gst_gl_sync_meta_set_sync_point (sync_meta, context); + + gst_gl_sync_meta_wait (sync_meta, qt_context); + + GST_LOG ("%p binding GL texture %u for plane %d", this, tex_id, plane); + +out: + if (G_UNLIKELY (use_dummy_tex)) { + /* Create dummy texture if not already present. + * Use the Qt RHI functions instead of the GstGL ones. + */ + + /* Make this a black 64x64 pixel RGBA texture. + * This size and format is supported pretty much everywhere, so these + * are a safe pick. (64 pixel sidelength must be supported according + * to the GLES2 spec, table 6.18.) */ + const int tex_sidelength = 64; + + rhi_tex = rhi->newTexture (video_format_to_rhi_format (v_format, plane), QSize(tex_sidelength, tex_sidelength), 1, {}); + g_assert (rhi_tex->create()); + + int ts = video_format_to_texel_size (v_format, plane); + QByteArray dummy_data (tex_sidelength * tex_sidelength * ts, 0); + char *data = dummy_data.data(); + + switch (v_format) { + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_BGRA: + case GST_VIDEO_FORMAT_RGB: + for (gsize j = 0; j < tex_sidelength; j++) { + for (gsize k = 0; k < tex_sidelength; k++) { + data[(j * tex_sidelength + k) * ts + 3] = 0xFF; // opaque + } + } + break; + case GST_VIDEO_FORMAT_YV12: + if (plane == 1 || plane == 2) { + for (gsize j = 0; j < tex_sidelength; j++) { + for (gsize k = 0; k < tex_sidelength; k++) { + data[(j * tex_sidelength + k) * ts + 0] = 0x7F; + } + } + } + break; + default: + g_assert_not_reached (); + break; + } + + QRhiTextureSubresourceUploadDescription sub_desc(dummy_data); + QRhiTextureUploadEntry entry(0, 0, sub_desc); + QRhiTextureUploadDescription desc(entry); + res_updates->uploadTexture(rhi_tex, desc); + + GST_LOG ("%p binding for plane %d fallback dummy Qt texture", this, plane); + } + + ret = new GstQSGTexture(rhi_tex); + ret->setFiltering(m_filtering); + + gst_clear_object (&qt_context); + + return static_cast(ret); +} diff --git a/libs/qmlglsink/qt6/gstqsg6material.h b/libs/qmlglsink/qt6/gstqsg6material.h new file mode 100644 index 000000000000..42a2a87768d5 --- /dev/null +++ b/libs/qmlglsink/qt6/gstqsg6material.h @@ -0,0 +1,75 @@ +/* + * GStreamer + * Copyright (C) 2023 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_QSG6_MATERIAL_H__ +#define __GST_QSG6_MATERIAL_H__ + +#include +#include +#include + +#include "gstqt6gl.h" +#include +#include +#include +#include + +class QRhi; +class QRhiResourceUpdateBatch; +class GstQSGMaterialShader; + +class GstQSGMaterial : public QSGMaterial +{ +protected: + GstQSGMaterial(); + ~GstQSGMaterial(); +public: + static GstQSGMaterial *new_for_format (GstVideoFormat format); + + void setCaps (GstCaps * caps); + gboolean setBuffer (GstBuffer * buffer); + GstBuffer * getBuffer (bool * was_bound); + bool compatibleWith(GstVideoInfo *v_info); + + void setFiltering(QSGTexture::Filtering); + + QSGTexture * bind(GstQSGMaterialShader *, QRhi *, QRhiResourceUpdateBatch *, guint binding, GstVideoFormat); + + /* QSGMaterial */ + QSGMaterialShader *createShader(QSGRendererInterface::RenderMode renderMode) const override; + + struct { + int input_swizzle[4]; + QMatrix4x4 color_matrix; + bool dirty; + } uniforms; + +private: + GstBuffer * buffer_; + bool buffer_was_bound; + GWeakRef qt_context_ref_; + GstBuffer * sync_buffer_; + GstMemory * mem_; + GstVideoInfo v_info; + GstVideoFrame v_frame; + QSGTexture::Filtering m_filtering; +}; + +#endif /* __GST_QSG6_MATERIAL_H__ */ diff --git a/libs/qmlglsink/qt6/gstqt6elements.h b/libs/qmlglsink/qt6/gstqt6elements.h index c22984216f96..c1a7955a1adb 100644 --- a/libs/qmlglsink/qt6/gstqt6elements.h +++ b/libs/qmlglsink/qt6/gstqt6elements.h @@ -26,11 +26,10 @@ G_BEGIN_DECLS void qt6_element_init (GstPlugin * plugin); -// TODO(zdanek) fix after switching to gstreamer 1.20.0+ -// original code from 1.20.0 -// GST_ELEMENT_REGISTER_DECLARE (qml6glsink); -// backported to: -extern "C" { gboolean gst_element_register_qml6glsink (GstPlugin * plugin); }; +GST_ELEMENT_REGISTER_DECLARE (qml6glsink); +GST_ELEMENT_REGISTER_DECLARE (qml6glsrc); +GST_ELEMENT_REGISTER_DECLARE (qml6glmixer); +GST_ELEMENT_REGISTER_DECLARE (qml6gloverlay); G_END_DECLS diff --git a/libs/qmlglsink/qt6/gstqt6glutility.cc b/libs/qmlglsink/qt6/gstqt6glutility.cc index 013ded4bfef3..6802cf16d287 100644 --- a/libs/qmlglsink/qt6/gstqt6glutility.cc +++ b/libs/qmlglsink/qt6/gstqt6glutility.cc @@ -31,7 +31,7 @@ #if GST_GL_HAVE_PLATFORM_EGL && (defined (HAVE_QT_WAYLAND) || defined (HAVE_QT_EGLFS) || defined (HAVE_QT_ANDROID)) #include #ifdef HAVE_QT_QPA_HEADER -#include +#include QT_QPA_HEADER #endif //#include #include @@ -254,7 +254,6 @@ gst_qml6_get_gl_wrapcontext (GstGLDisplay * display, gst_gl_display_filter_gl_api (display, gst_gl_context_get_gl_api (*wrap_glcontext)); gst_gl_context_activate (*wrap_glcontext, FALSE); } -#if 0 #if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32) g_return_val_if_fail (context != NULL, FALSE); @@ -288,72 +287,100 @@ gst_qml6_get_gl_wrapcontext (GstGLDisplay * display, return FALSE; } G_STMT_END; -#endif #endif return TRUE; } -#if 0 -QVariant + +QOpenGLContext * qt_opengl_native_context_from_gst_gl_context (GstGLContext * context) { - guintptr handle; - GstGLPlatform platform; + guintptr handle; + GstGLPlatform platform; + QOpenGLContext *ret = NULL; + + handle = gst_gl_context_get_gl_context (context); + platform = gst_gl_context_get_gl_platform (context); - handle = gst_gl_context_get_gl_context (context); - platform = gst_gl_context_get_gl_platform (context); + /* this is required as Qt doesn't allow retrieving the relevant native + * interface unless the underlying context has been created */ + QOpenGLContext *qt_gl_context = new QOpenGLContext(); + qt_gl_context->create(); #if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11) - if (platform == GST_GL_PLATFORM_GLX) { - GstGLDisplay *display = gst_gl_context_get_display (context); - GstGLWindow *window = gst_gl_context_get_window (context); - Display *xdisplay = (Display *) gst_gl_display_get_handle (display); - Window win = gst_gl_window_get_window_handle (window); - gst_object_unref (window); - gst_object_unref (display); - return QVariant::fromValue(QGLXNativeContext((GLXContext) handle, xdisplay, win)); + if (!ret && platform == GST_GL_PLATFORM_GLX) { + auto glx = qt_gl_context->nativeInterface(); + if (!glx) { + GST_WARNING ("Retriving GLX context interface from Qt failed"); + } else { + GstGLDisplay *display = gst_gl_context_get_display (context); + GstGLWindow *window = gst_gl_context_get_window (context); + gst_object_unref (window); + gst_object_unref (display); + ret = glx->fromNative((GLXContext) handle); } + } #endif #if GST_GL_HAVE_PLATFORM_EGL && (defined (HAVE_QT_WAYLAND) || defined (HAVE_QT_EGLFS) || defined (HAVE_QT_ANDROID)) - if (platform == GST_GL_PLATFORM_EGL) { - EGLDisplay egl_display = EGL_DEFAULT_DISPLAY; - GstGLDisplay *display = gst_gl_context_get_display (context); - GstGLDisplayEGL *display_egl = gst_gl_display_egl_from_gl_display (display); + if (!ret && platform == GST_GL_PLATFORM_EGL) { + auto egl = qt_gl_context->nativeInterface(); + if (!egl) { + GST_WARNING ("Retriving EGL context interface from Qt failed"); + } else { + EGLDisplay egl_display = EGL_DEFAULT_DISPLAY; + GstGLDisplay *display = gst_gl_context_get_display (context); + GstGLDisplayEGL *display_egl = gst_gl_display_egl_from_gl_display (display); #if GST_GL_HAVE_WINDOW_WAYLAND && defined (HAVE_QT_WAYLAND) - if (gst_gl_display_get_handle_type (display) == GST_GL_DISPLAY_TYPE_WAYLAND) { -#if 1 - g_warning ("Qt does not support wrapping native OpenGL contexts " - "on wayland. See https://bugreports.qt.io/browse/QTBUG-82528"); - gst_object_unref (display_egl); - gst_object_unref (display); - return QVariant::fromValue(nullptr); + if (gst_gl_display_get_handle_type (display) == GST_GL_DISPLAY_TYPE_WAYLAND) { +#if 0 + g_warning ("Qt does not support wrapping native OpenGL contexts " + "on wayland. See https://bugreports.qt.io/browse/QTBUG-82528"); + gst_object_unref (display_egl); + gst_object_unref (display); + return NULL; #else - if (display_egl) - egl_display = (EGLDisplay) gst_gl_display_get_handle ((GstGLDisplay *) display_egl); + if (display_egl) + egl_display = (EGLDisplay) gst_gl_display_get_handle ((GstGLDisplay *) display_egl); #endif - } + } #endif - gst_object_unref (display_egl); - gst_object_unref (display); - return QVariant::fromValue(QEGLNativeContext((EGLContext) handle, egl_display)); + gst_object_unref (display_egl); + gst_object_unref (display); + GST_ERROR ("creating native context from context %p and display %p", (void *) handle, egl_display); + ret = egl->fromNative((EGLContext) handle, egl_display); + GST_ERROR ("created native context %p", ret); } + } #endif #if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32) - if (platform == GST_GL_PLATFORM_WGL) { - GstGLWindow *window = gst_gl_context_get_window (context); - guintptr hwnd = gst_gl_window_get_window_handle (window); - gst_object_unref (window); - return QVariant::fromValue(QWGLNativeContext((HGLRC) handle, (HWND) hwnd)); + if (!ret && platform == GST_GL_PLATFORM_WGL) { + auto wgl = qt_gl_context->nativeInterface(); + if (!wgl) { + GST_WARNING ("Retriving WGL context interface from Qt failed"); + } else { + GstGLWindow *window = gst_gl_context_get_window (context); + guintptr hwnd = gst_gl_window_get_window_handle (window); + gst_object_unref (window); + ret = wgl->fromNative((HGLRC) handle, (HWND) hwnd); } + } #endif - { - gchar *platform_s = gst_gl_platform_to_string (platform); - g_warning ("Unimplemented configuration! This means either:\n" - "1. The qmlgl plugin was built without support for your platform.\n" - "2. The necessary code to convert from a GstGLContext to Qt's " - "native context type for \'%s\' currently does not exist.", - platform_s); - g_free (platform_s); - } - return QVariant::fromValue(nullptr); + if (!ret) { + gchar *platform_s = gst_gl_platform_to_string (platform); + g_warning ("Unimplemented configuration! This means either:\n" + "1. Qt6 wasn't built with support for \'%s\'\n" + "2. The qmlgl plugin was built without support for your platform.\n" + "3. The necessary code to convert from a GstGLContext to Qt's " + "native context type for \'%s\' currently does not exist." + "4. Qt failed to wrap an existing native context.", + platform_s, platform_s); + g_free (platform_s); + } + + qt_gl_context->doneCurrent(); + delete qt_gl_context; + + gst_gl_context_activate (context, FALSE); + gst_gl_context_activate (context, TRUE); + + return ret; } -#endif diff --git a/libs/qmlglsink/qt6/gstqt6glutility.h b/libs/qmlglsink/qt6/gstqt6glutility.h index ba436230da85..a3a5561306b6 100644 --- a/libs/qmlglsink/qt6/gstqt6glutility.h +++ b/libs/qmlglsink/qt6/gstqt6glutility.h @@ -26,6 +26,7 @@ #include #include +#include G_BEGIN_DECLS @@ -46,10 +47,6 @@ gboolean gst_qml6_get_gl_wrapcontext (GstGLDisplay * display, G_END_DECLS -#if 0 -#if defined(__cplusplus) -QVariant qt_opengl_native_context_from_gst_gl_context (GstGLContext * context); -#endif -#endif +QOpenGLContext * qt_opengl_native_context_from_gst_gl_context (GstGLContext * context); #endif /* __QML6_GL_UTILS_H__ */ diff --git a/libs/qmlglsink/qt6/meson.build b/libs/qmlglsink/qt6/meson.build index 150b0b9f222f..2b1ab1d87112 100644 --- a/libs/qmlglsink/qt6/meson.build +++ b/libs/qmlglsink/qt6/meson.build @@ -1,19 +1,34 @@ sources = [ 'gstplugin.cc', 'gstqt6element.cc', - 'gstqsg6glnode.cc', + 'gstqsg6material.cc', 'gstqt6glutility.cc', 'gstqml6glsink.cc', + 'gstqml6glsrc.cc', + 'gstqml6glmixer.cc', + 'gstqml6gloverlay.cc', 'qt6glitem.cc', + 'qt6glwindow.cc', + 'qt6glrenderer.cc', ] moc_headers = [ 'qt6glitem.h', - 'gstqsg6glnode.h', + 'qt6glwindow.h', + 'qt6glrenderer.h', +] + +shader_sources = [ + 'vertex.vert', + 'RGBA.frag', + 'YUV_TRIPLANAR.frag', ] qt6qml_dep = dependency('', required: false) qt6_option = get_option('qt6') +qt6_egl = get_option('qt-egl') +qt6_wayland = get_option('qt-wayland') +qt6_x11 = get_option('qt-x11') qt6_method = get_option('qt-method') if qt6_option.disabled() @@ -32,7 +47,7 @@ if not add_languages('cpp', native: false, required: qt6_option) endif qt6_mod = import('qt6') -if not qt6_mod.has_tools() +if not qt6_mod.has_tools(method: qt6_method) if qt6_option.enabled() error('qt6 qmlglsink plugin is enabled, but qt specific tools were not found') endif @@ -40,11 +55,17 @@ if not qt6_mod.has_tools() endif qt6qml_dep = dependency('qt6', modules : ['Core', 'Gui', 'Qml', 'Quick'], - method: qt6_method, required: qt6_option, static: host_system == 'ios') + method: qt6_method, required: qt6_option, static: host_system == 'ios', private_headers: true) if not qt6qml_dep.found() subdir_done() endif +qt6_bindir = qt6qml_dep.get_variable('bindir', configtool: 'QT_HOST_BINS') +qsb = find_program('qsb-qt6', 'qsb', dirs: [qt6_bindir], required: qt6_option) +if not qsb.found() + subdir_done() +endif + optional_deps = [] qt_defines = [] have_qpa_include = false @@ -53,86 +74,113 @@ have_qt_windowing = false # Look for the QPA platform native interface header qpa_header_path = join_paths(qt6qml_dep.version(), 'QtGui') qpa_header = join_paths(qpa_header_path, 'qpa/qplatformnativeinterface.h') -if cxx.has_header(qpa_header, dependencies : qt6qml_dep) +need_qpa_include = qt6_option.enabled() and (host_system == 'android' or qt6_wayland.enabled()) +if cxx.has_header(qpa_header, dependencies : qt6qml_dep, required: need_qpa_include) qt_defines += '-DHAVE_QT_QPA_HEADER' qt_defines += '-DQT_QPA_HEADER=' + '<@0@>'.format(qpa_header) have_qpa_include = true message('Found QtGui QPA header in ' + qpa_header_path) endif -# Try to come up with all the platform/winsys combinations that will work +## Try to come up with all the platform/winsys combinations that will work -if gst_gl_have_window_x11 and gst_gl_have_platform_glx - # FIXME: automagic +# X11 windowing +qt6_x11 = qt6_x11 \ + .require(gstglx11_dep.found(), error_message: 'gstreamer-gl-x11-1.0 is required') \ + .require(gst_gl_have_window_x11, error_message: 'x11 windowing support in gstreamer-gl is required') \ + .require(gst_gl_have_platform_glx, error_message: 'glx platform support in gstreamer-gl is required') +if qt6_x11.allowed() qt_defines += ['-DHAVE_QT_X11'] have_qt_windowing = true endif -if gst_gl_have_platform_egl - # Embedded linux (e.g. i.MX6) with or without windowing support +# Wayland windowing +qt6_wayland = qt6_wayland \ + .require(gstglwayland_dep.found(), error_message: 'gstreamer-gl-wayland-1.0 is required') \ + .require(gst_gl_have_window_wayland, error_message: 'wayland windowing support in gstreamer-gl is required') \ + .require(gst_gl_have_platform_egl, error_message: 'egl platform support in gstreamer-gl is required') \ + .require(have_qpa_include, error_message: 'QPA platform native interface header is required') +if qt6_wayland.allowed() + qt6waylandextras = dependency('qt6', modules : ['WaylandClient'], method: qt6_method, required: qt6_wayland) + if qt6waylandextras.found() + optional_deps += [qt6waylandextras, gstglwayland_dep] + qt_defines += ['-DHAVE_QT_WAYLAND'] + have_qt_windowing = true + endif +endif + +# EGL windowing for Embedded linux (e.g. i.MX6) with or without windowing +# support +qt6_egl = qt6_egl \ + .require(host_system == 'linux') \ + .require(gstglegl_dep.found(), error_message: 'gstreamer-gl-egl-1.0 is required') \ + .require(gst_gl_have_platform_egl, error_message: 'egl platform support in gstreamer-gl is required') +if qt6_egl.allowed() qt_defines += ['-DHAVE_QT_EGLFS'] optional_deps += gstglegl_dep have_qt_windowing = true - if have_qpa_include - # Wayland windowing - if gst_gl_have_window_wayland - # FIXME: automagic - qt6waylandextras = dependency('qt6', modules : ['WaylandClient'], method: qt6_method, required : false) - if qt6waylandextras.found() - optional_deps += [qt6waylandextras, gstglwayland_dep] - qt_defines += ['-DHAVE_QT_WAYLAND'] - have_qt_windowing = true - endif +endif + +# TODO: Android windowing + +# Win32 windowing +if host_system == 'windows' + qt6_win32 = qt6_option \ + .require(gst_gl_have_window_win32, error_message: 'win32 windowing support in gstreamer-gl is required') \ + .require(gst_gl_have_platform_wgl, error_message: 'wgl platform support in gstreamer-gl is required') + if qt6_win32.allowed() + # for wglMakeCurrent() + opengl32_dep = cc.find_library('opengl32', required : qt6_win32) + if opengl32_dep.found() + qt_defines += ['-DHAVE_QT_WIN32'] + optional_deps += opengl32_dep + have_qt_windowing = true endif - # Android windowing -# if gst_gl_have_window_android - # FIXME: automagic -# qt5androidextras = dependency('qt5', modules : ['AndroidExtras'], method: qt6_method, required : false) - # for gl functions in QtGui/qopenglfunctions.h - # FIXME: automagic -# glesv2_dep = cc.find_library('GLESv2', required : false) -# if glesv2_dep.found() and qt5androidextras.found() -# optional_deps += [qt5androidextras, glesv2_dep] -# qt_defines += ['-DHAVE_QT_ANDROID'] -# have_qt_windowing = true - # Needed for C++11 support in Cerbero. People building with Android - # in some other way need to add the necessary bits themselves. -# optional_deps += dependency('gnustl', required : false) -# endif -# endif endif endif -#if gst_gl_have_platform_wgl and gst_gl_have_window_win32 - # for wglMakeCurrent() - # FIXME: automagic -# opengl32_dep = cc.find_library('opengl32', required : false) -# if opengl32_dep.found() -# qt_defines += ['-DHAVE_QT_WIN32'] -# optional_deps += opengl32_dep -# have_qt_windowing = true -# endif -#endif - -if gst_gl_have_window_cocoa and gst_gl_have_platform_cgl - # FIXME: automagic - if host_machine.system() == 'darwin' +# macOS windowing +if host_system == 'darwin' + qt6_macos = qt6_option \ + .require(gst_gl_have_window_cocoa, error_message: 'cocoa windowing support in gstreamer-gl is required') \ + .require(gst_gl_have_platform_cgl, error_message: 'cgl platform support in gstreamer-gl is required') + if qt6_macos.allowed() qt_defines += ['-DHAVE_QT_MAC'] have_qt_windowing = true endif endif -if gst_gl_have_window_eagl and gst_gl_have_platform_eagl - if host_machine.system() == 'ios' +# iOS windowing +if host_system == 'ios' + qt6_ios = qt6_option \ + .require(gst_gl_have_window_eagl, error_message: 'eagl windowing support in gstreamer-gl is required') \ + .require(gst_gl_have_platform_eagl, error_message: 'eagl platform support in gstreamer-gl is required') + if qt6_ios.allowed() qt_defines += ['-DHAVE_QT_IOS'] have_qt_windowing = true endif endif -if have_qt_windowing +if qt6_option.require(have_qt_windowing, error_message: 'No windowing, enable one of the qt-* windowing options').allowed() # Build it! - moc_files = qt6_mod.preprocess(moc_headers : moc_headers) - gstqml6gl = library('gstqml6', sources, moc_files, + moc_files = qt6_mod.preprocess(moc_headers : moc_headers, method: qt6_method) + # TODO: dist backup qsb shaders? + shaders = [] + foreach shader: shader_sources + qsb_shader = shader + '.qsb' + dist_shader = shader + '-dist.qsb' + + compiled_shader = custom_target(qsb_shader, + input: shader, + output: qsb_shader, + command: [qsb, '--glsl=100 es,120,330', '--batchable', '--output', '@OUTPUT@', '@INPUT@'] + ) + shaders += [compiled_shader] + endforeach + resource_file = configure_file(input: 'resources.qrc', output: 'resources.qrc', copy: true) + qresources = qt6_mod.compile_resources(sources: resource_file, method: qt6_method) + + gstqml6gl = library('gstqml6', sources, moc_files, qresources, cpp_args : gst_plugins_good_args + qt_defines, link_args : noseh_link_args, include_directories: [configinc, libsinc], diff --git a/libs/qmlglsink/qt6/qt6glitem.cc b/libs/qmlglsink/qt6/qt6glitem.cc index dfb012a05cd7..99c256708e83 100644 --- a/libs/qmlglsink/qt6/qt6glitem.cc +++ b/libs/qmlglsink/qt6/qt6glitem.cc @@ -26,14 +26,13 @@ #include #include "qt6glitem.h" -#include "gstqsg6glnode.h" #include "gstqt6glutility.h" +#include "gstqsg6material.h" #include #include #include #include -#include /** * SECTION:Qt6GLVideoItem @@ -88,8 +87,6 @@ struct _Qt6GLVideoItemPrivate * FIXME: Ideally we would use fences for this but there seems to be no * way to reliably "try wait" on a fence */ GQueue potentially_unbound_buffers; - - GstQSG6OpenGLNode *m_node; }; Qt6GLVideoItem::Qt6GLVideoItem() @@ -276,29 +273,57 @@ Qt6GLVideoItem::updatePaintNode(QSGNode * oldNode, UpdatePaintNodeData * updatePaintNodeData) { GstBuffer *old_buffer; + GstQSGMaterial *tex = nullptr; + QSGGeometry *geometry = nullptr; + bool was_bound = false; if (!this->priv->initted) return oldNode; - GstQSG6OpenGLNode *texNode = static_cast (oldNode); + QSGGeometryNode *texNode = static_cast (oldNode); GstVideoRectangle src, dst, result; g_mutex_lock (&this->priv->lock); GST_TRACE ("%p updatePaintNode", this); + if (!this->priv->caps) { + GST_LOG ("%p no caps yet", this); + g_mutex_unlock (&this->priv->lock); + return NULL; + } + if (gst_gl_context_get_current() == NULL) gst_gl_context_activate (this->priv->other_context, TRUE); + if (texNode) { + tex = static_cast(texNode->material()); + if (tex && !tex->compatibleWith(&this->priv->v_info)) { + delete texNode; + texNode = nullptr; + } + } + if (!texNode) { - texNode = new GstQSG6OpenGLNode (this); - this->priv->m_node = texNode; + bool is_smooth = this->smooth (); + texNode = new QSGGeometryNode(); + geometry = new QSGGeometry(QSGGeometry::defaultAttributes_TexturedPoint2D(), 4); + texNode->setGeometry(geometry); + texNode->setFlag(QSGGeometryNode::OwnsGeometry); + tex = GstQSGMaterial::new_for_format(GST_VIDEO_INFO_FORMAT (&this->priv->v_info)); + tex->setFiltering(is_smooth ? QSGTexture::Filtering::Linear : + QSGTexture::Filtering::Nearest); + texNode->setMaterial(tex); + texNode->setFlag(QSGGeometryNode::OwnsMaterial); } - if ((old_buffer = texNode->getBuffer())) { + if ((old_buffer = tex->getBuffer(&was_bound))) { if (old_buffer == this->priv->buffer) { /* same buffer */ gst_buffer_unref (old_buffer); + } else if (!was_bound) { + GST_TRACE ("old buffer %p was not bound yet, unreffing", old_buffer); + gst_buffer_unref (old_buffer); } else { GstBuffer *tmp_buffer; @@ -323,8 +348,8 @@ Qt6GLVideoItem::updatePaintNode(QSGNode * oldNode, old_buffer = NULL; } - texNode->setCaps (this->priv->caps); - texNode->setBuffer (this->priv->buffer); + tex->setCaps (this->priv->caps); + tex->setBuffer (this->priv->buffer); if (this->priv->force_aspect_ratio && this->priv->caps) { src.w = this->priv->display_width; @@ -343,7 +368,10 @@ Qt6GLVideoItem::updatePaintNode(QSGNode * oldNode, result.h = boundingRect().height(); } - texNode->setRect (QRectF (result.x, result.y, result.w, result.h)); + geometry = texNode->geometry(); + QRectF rect(result.x, result.y, result.w, result.h); + QRectF sourceRect(0, 0, 1, 1); + QSGGeometry::updateTexturedRectGeometry(geometry, rect, sourceRect); g_mutex_unlock (&this->priv->lock); @@ -413,10 +441,44 @@ Qt6GLVideoItem::mapPointToStreamSize(QPointF pos) return QPointF(stream_x, stream_y); } +static GstNavigationModifierType +translateModifiers(Qt::KeyboardModifiers modifiers) +{ + return (GstNavigationModifierType)( + ((modifiers & Qt::KeyboardModifier::ShiftModifier) ? GST_NAVIGATION_MODIFIER_SHIFT_MASK : 0) | + ((modifiers & Qt::KeyboardModifier::ControlModifier) ? GST_NAVIGATION_MODIFIER_CONTROL_MASK : 0) | + ((modifiers & Qt::KeyboardModifier::AltModifier) ? GST_NAVIGATION_MODIFIER_MOD1_MASK : 0) | + ((modifiers & Qt::KeyboardModifier::MetaModifier) ? GST_NAVIGATION_MODIFIER_META_MASK : 0)); +} + +static GstNavigationModifierType +translateMouseButtons(Qt::MouseButtons buttons) +{ + return (GstNavigationModifierType)( + ((buttons & Qt::LeftButton) ? GST_NAVIGATION_MODIFIER_BUTTON1_MASK : 0) | + ((buttons & Qt::RightButton) ? GST_NAVIGATION_MODIFIER_BUTTON2_MASK : 0) | + ((buttons & Qt::MiddleButton) ? GST_NAVIGATION_MODIFIER_BUTTON3_MASK : 0) | + ((buttons & Qt::BackButton) ? GST_NAVIGATION_MODIFIER_BUTTON4_MASK : 0) | + ((buttons & Qt::ForwardButton) ? GST_NAVIGATION_MODIFIER_BUTTON5_MASK : 0)); +} + void Qt6GLVideoItem::wheelEvent(QWheelEvent * event) { - // noop + g_mutex_lock (&this->priv->lock); + QPoint delta = event->angleDelta(); + GstElement *element = GST_ELEMENT_CAST (g_weak_ref_get (&this->priv->sink)); + + if (element != NULL) { + auto position = event->position(); + gst_navigation_send_event_simple (GST_NAVIGATION (element), + gst_navigation_event_new_mouse_scroll (position.x(), position.y(), + delta.x(), delta.y(), + (GstNavigationModifierType) ( + translateModifiers(event->modifiers()) | translateMouseButtons(event->buttons())))); + g_object_unref (element); + } + g_mutex_unlock (&this->priv->lock); } void @@ -434,19 +496,136 @@ Qt6GLVideoItem::hoverLeaveEvent(QHoverEvent *) void Qt6GLVideoItem::hoverMoveEvent(QHoverEvent * event) { - // noop + if (!mouseHovering) + return; + + g_mutex_lock (&this->priv->lock); + + /* can't do anything when we don't have input format */ + if (!this->priv->caps) { + g_mutex_unlock (&this->priv->lock); + return; + } + + if (event->position() != event->oldPos()) { + QPointF pos = mapPointToStreamSize(event->position()); + GstElement *element = GST_ELEMENT_CAST (g_weak_ref_get (&this->priv->sink)); + + if (element != NULL) { + gst_navigation_send_event_simple (GST_NAVIGATION (element), + gst_navigation_event_new_mouse_move (pos.x(), pos.y(), + translateModifiers(event->modifiers()))); + g_object_unref (element); + } + } + g_mutex_unlock (&this->priv->lock); } void Qt6GLVideoItem::touchEvent(QTouchEvent * event) { - // noop + g_mutex_lock (&this->priv->lock); + + /* can't do anything when we don't have input format */ + if (!this->priv->caps) { + g_mutex_unlock (&this->priv->lock); + return; + } + + GstElement *element = GST_ELEMENT_CAST (g_weak_ref_get (&this->priv->sink)); + if (element == NULL) + return; + + if (event->type() == QEvent::TouchCancel) { + gst_navigation_send_event_simple (GST_NAVIGATION (element), + gst_navigation_event_new_touch_cancel (translateModifiers(event->modifiers()))); + } else { + const QList points = event->points(); + gboolean sent_event = FALSE; + + for (int i = 0; i < points.count(); i++) { + GstEvent *nav_event; + QPointF pos = mapPointToStreamSize(points[i].position()); + + switch (points[i].state()) { + case QEventPoint::Pressed: + nav_event = gst_navigation_event_new_touch_down ((guint) points[i].id(), + pos.x(), pos.y(), (gdouble) points[i].pressure(), translateModifiers(event->modifiers())); + break; + case QEventPoint::Updated: + nav_event = gst_navigation_event_new_touch_motion ((guint) points[i].id(), + pos.x(), pos.y(), (gdouble) points[i].pressure(), translateModifiers(event->modifiers())); + break; + case QEventPoint::Released: + nav_event = gst_navigation_event_new_touch_up ((guint) points[i].id(), + pos.x(), pos.y(), translateModifiers(event->modifiers())); + break; + /* Don't send an event if the point did not change */ + default: + nav_event = NULL; + break; + } + + if (nav_event) { + gst_navigation_send_event_simple (GST_NAVIGATION (element), nav_event); + sent_event = TRUE; + } + } + + /* Group simultaneos touch events with a frame event */ + if (sent_event) { + gst_navigation_send_event_simple (GST_NAVIGATION (element), + gst_navigation_event_new_touch_frame (translateModifiers(event->modifiers()))); + } + } + + g_object_unref (element); + g_mutex_unlock (&this->priv->lock); } void Qt6GLVideoItem::sendMouseEvent(QMouseEvent * event, gboolean is_press) { - // noop + quint32 button = 0; + + switch (event->button()) { + case Qt::LeftButton: + button = 1; + break; + case Qt::RightButton: + button = 2; + break; + default: + break; + } + + mousePressedButton = button; + + g_mutex_lock (&this->priv->lock); + + /* can't do anything when we don't have input format */ + if (!this->priv->caps) { + g_mutex_unlock (&this->priv->lock); + return; + } + + QPointF pos = mapPointToStreamSize(event->pos()); + GstElement *element = GST_ELEMENT_CAST (g_weak_ref_get (&this->priv->sink)); + + if (element != NULL) { + gst_navigation_send_event_simple (GST_NAVIGATION (element), + (is_press) ? gst_navigation_event_new_mouse_button_press (button, + pos.x(), pos.y(), + (GstNavigationModifierType) ( + translateModifiers(event->modifiers()) | translateMouseButtons(event->buttons()))) : + gst_navigation_event_new_mouse_button_release (button, pos.x(), + pos.y(), + (GstNavigationModifierType) ( + translateModifiers(event->modifiers()) | translateMouseButtons(event->buttons())))); + g_object_unref (element); + } + + g_mutex_unlock (&this->priv->lock); } void @@ -543,7 +722,6 @@ Qt6GLVideoItem::onSceneGraphInitialized () this->priv->qt_context = gl_context; if (this->priv->qt_context == NULL) { - GST_ERROR ("%p failed to retrieve Qt GL context", this); g_assert_not_reached (); return; } @@ -560,7 +738,6 @@ Qt6GLVideoItem::onSceneGraphInitialized () void Qt6GLVideoItem::onSceneGraphInvalidated () { - this->priv->m_node = nullptr; GST_FIXME ("%p scene graph invalidated", this); } @@ -637,13 +814,11 @@ Qt6GLVideoItem::handleWindowChanged (QQuickWindow * win) this->priv->qt_context = NULL; this->priv->initted = FALSE; } - this->priv->m_node = nullptr; } void Qt6GLVideoItem::releaseResources() { - this->priv->m_node = nullptr; } gboolean diff --git a/libs/qmlglsink/qt6/qt6glitem.h b/libs/qmlglsink/qt6/qt6glitem.h index c5c5efcfcfd9..6f0b7f0d2a94 100644 --- a/libs/qmlglsink/qt6/qt6glitem.h +++ b/libs/qmlglsink/qt6/qt6glitem.h @@ -27,9 +27,9 @@ #include "gstqt6gl.h" #include #include +#include // QGC #include #include -#include typedef struct _Qt6GLVideoItemPrivate Qt6GLVideoItemPrivate; diff --git a/libs/qmlglsink/qt6/qt6glrenderer.cc b/libs/qmlglsink/qt6/qt6glrenderer.cc new file mode 100644 index 000000000000..20b803cdb7c7 --- /dev/null +++ b/libs/qmlglsink/qt6/qt6glrenderer.cc @@ -0,0 +1,766 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "gstqt6gl.h" + +#include "qt6glrenderer.h" +#include "gstqt6glutility.h" + +#define GST_CAT_DEFAULT gst_qt6_gl_renderer_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +static void +init_debug (void) +{ + static gsize _debug; + + if (g_once_init_enter (&_debug)) { + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qt6glrenderer", 0, + "Qt6 OpenGL Renderer"); + g_once_init_leave (&_debug, 1); + } +} + +/* Needs to be based on QWindow otherwise (at least) windows and nvidia + * proprietary on linux does not work + * We also need to override the size handling to get the correct output size + */ +class GstQt6BackingSurface : public QWindow +{ +public: + GstQt6BackingSurface(); + ~GstQt6BackingSurface(); + + void setSize (int width, int height); + QSize size() const override; + +private: + QSize m_size; +}; + +GstQt6BackingSurface::GstQt6BackingSurface() + : m_size(QSize()) +{ + /* we do OpenGL things so need an OpenGL surface */ + setSurfaceType(QSurface::OpenGLSurface); +} + +GstQt6BackingSurface::~GstQt6BackingSurface() +{ +} + +QSize GstQt6BackingSurface::size () const +{ + return m_size; +} + +void GstQt6BackingSurface::setSize (int width, int height) +{ + m_size = QSize (width, height); +} + +class GstQt6AnimationDriver : public QAnimationDriver +{ +public: + GstQt6AnimationDriver(); + + void setNextTime(qint64 ms); + void advance() override; + qint64 elapsed() const override; +private: + qint64 m_elapsed; + qint64 m_next; +}; + +GstQt6AnimationDriver::GstQt6AnimationDriver() + : m_elapsed(0), + m_next(0) +{ +} + +void GstQt6AnimationDriver::advance() +{ + m_elapsed = m_next; + advanceAnimation(); +} + +qint64 GstQt6AnimationDriver::elapsed() const +{ + return m_elapsed; +} + +void GstQt6AnimationDriver::setNextTime(qint64 ms) +{ + m_next = ms; +} + +typedef enum +{ + STATE_ERROR = -1, + STATE_NEW = 0, + STATE_WAITING_FOR_WINDOW, + STATE_WINDOW_CREATED, + STATE_READY, +} SharedRenderDataState; + +struct SharedRenderData +{ + int refcount; + SharedRenderDataState state; + GMutex lock; + GCond cond; + GstQt6AnimationDriver *m_animationDriver; + QOpenGLContext *m_context; + GstQt6BackingSurface *m_surface; + QThread *m_renderThread; +}; + +static struct SharedRenderData * +shared_render_data_new (void) +{ + struct SharedRenderData *ret = g_new0 (struct SharedRenderData, 1); + + g_atomic_int_set (&ret->refcount, 1); + g_mutex_init (&ret->lock); + + return ret; +} + +static void +shared_render_data_free (struct SharedRenderData * data) +{ + GST_DEBUG ("%p freeing shared render data", data); + + g_mutex_clear (&data->lock); + + if (data->m_animationDriver) { + data->m_animationDriver->uninstall(); + delete data->m_animationDriver; + } + data->m_animationDriver = nullptr; + if (data->m_context) { + if (QOpenGLContext::currentContext() == data->m_context) + data->m_context->doneCurrent(); + delete data->m_context; + } + data->m_context = nullptr; + if (data->m_surface) + data->m_surface->deleteLater(); + data->m_surface = nullptr; +} + +static struct SharedRenderData * +shared_render_data_ref (struct SharedRenderData * data) +{ + GST_TRACE ("%p reffing shared render data", data); + g_atomic_int_inc (&data->refcount); + return data; +} + +static void +shared_render_data_unref (struct SharedRenderData * data) +{ + GST_TRACE ("%p unreffing shared render data", data); + if (g_atomic_int_dec_and_test (&data->refcount)) + shared_render_data_free (data); +} + +void +GstQt6QuickRenderer::deactivateContext () +{ +} + +void +GstQt6QuickRenderer::activateContext () +{ +} + +struct FBOUserData +{ + GstGLContext * context; + QOpenGLFramebufferObject * fbo; +}; + +GstQt6QuickRenderer::GstQt6QuickRenderer() + : gl_context(NULL), + m_quickWindow(nullptr), + m_renderControl(nullptr), + m_qmlEngine(nullptr), + m_qmlComponent(nullptr), + m_rootItem(nullptr), + gl_allocator(NULL), + gl_params(NULL), + gl_mem(NULL), + m_sharedRenderData(NULL) +{ + init_debug (); +} + +static gpointer +dup_shared_render_data (gpointer data, gpointer user_data) +{ + struct SharedRenderData *render_data = (struct SharedRenderData *) data; + + if (render_data) + return shared_render_data_ref (render_data); + + return NULL; +} + +class CreateSurfaceEvent : public QEvent +{ +public: + CreateSurfaceEvent (CreateSurfaceWorker * worker) + : QEvent(CreateSurfaceEvent::type()) + { + m_worker = worker; + } + + ~CreateSurfaceEvent() + { + GST_TRACE ("%p destroying create surface event", this); + delete m_worker; + } + + static QEvent::Type type() + { + if (customEventType == QEvent::None) { + int generatedType = QEvent::registerEventType(); + customEventType = static_cast(generatedType); + } + return customEventType; + } + +private: + static QEvent::Type customEventType; + CreateSurfaceWorker *m_worker; +}; + +QEvent::Type CreateSurfaceEvent::customEventType = QEvent::None; + + +CreateSurfaceWorker::CreateSurfaceWorker (struct SharedRenderData * rdata) +{ + m_sharedRenderData = shared_render_data_ref (rdata); +} + +CreateSurfaceWorker::~CreateSurfaceWorker () +{ + shared_render_data_unref (m_sharedRenderData); +} + +bool CreateSurfaceWorker::event(QEvent * ev) +{ + if (ev->type() == CreateSurfaceEvent::type()) { + GST_TRACE ("%p creating surface", m_sharedRenderData); + /* create the window surface in the main thread */ + g_mutex_lock (&m_sharedRenderData->lock); + m_sharedRenderData->m_surface = new GstQt6BackingSurface; + m_sharedRenderData->m_surface->create(); + GST_TRACE ("%p created surface %p", m_sharedRenderData, + m_sharedRenderData->m_surface); + g_cond_broadcast (&m_sharedRenderData->cond); + g_mutex_unlock (&m_sharedRenderData->lock); + } + + return QObject::event(ev); +} + +bool GstQt6QuickRenderer::init (GstGLContext * context, GError ** error) +{ + g_return_val_if_fail (GST_IS_GL_CONTEXT (context), false); + g_return_val_if_fail (gst_gl_context_get_current () == context, false); + + QOpenGLContext *qt_native_context = qt_opengl_native_context_from_gst_gl_context (context); + + if (!qt_native_context) { + g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND, + "Could not convert from the provided GstGLContext to a Qt " + "native context"); + return false; + } + + struct SharedRenderData *render_data = NULL, *old_render_data; + do { + if (render_data) + shared_render_data_unref (render_data); + + old_render_data = render_data = (struct SharedRenderData *) + g_object_dup_data (G_OBJECT (context), + "qt.gl.render.shared.data", dup_shared_render_data, NULL); + if (!render_data) + render_data = shared_render_data_new (); + } while (old_render_data != render_data + && !g_object_replace_data (G_OBJECT (context), + "qt.gl.render.shared.data", old_render_data, render_data, + NULL, NULL)); + m_sharedRenderData = render_data; + GST_TRACE ("%p retrieved shared render data %p", this, m_sharedRenderData); + + g_mutex_lock (&m_sharedRenderData->lock); + if (m_sharedRenderData->state == STATE_ERROR) { + g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND, + "In an error state from a previous attempt"); + g_mutex_unlock (&m_sharedRenderData->lock); + return false; + } + + if (m_sharedRenderData->state != STATE_READY) { + /* this state handling and locking is so that two qtglrenderer's will + * not attempt to create an OpenGL context without freeing the previous + * OpenGL context and cause a leak. It also only allows one + * CreateSurfaceEvent() to be posted to the main thread + * (QCoreApplication::instance()->thread()) while still allowing + * multiple waiters to wait for the window to be created */ + if (m_sharedRenderData->state == STATE_NEW) { + QCoreApplication *app = QCoreApplication::instance (); + + if (!app) { + g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND, + "Could not retrieve QCoreApplication instance"); + m_sharedRenderData->state = STATE_ERROR; + g_mutex_unlock (&m_sharedRenderData->lock); + return false; + } + + m_sharedRenderData->m_renderThread = QThread::currentThread(); + m_sharedRenderData->m_context = qt_native_context; + GST_TRACE ("%p new QOpenGLContext %p", this, m_sharedRenderData->m_context); + + CreateSurfaceWorker *w = new CreateSurfaceWorker (m_sharedRenderData); + GST_TRACE ("%p posting create surface event to main thread with " + "worker %p", this, w); + w->moveToThread (app->thread()); + app->postEvent (w, new CreateSurfaceEvent (w)); + m_sharedRenderData->state = STATE_WAITING_FOR_WINDOW; + } + + if (m_sharedRenderData->state == STATE_WAITING_FOR_WINDOW) { + gint64 end_time = g_get_monotonic_time () + 5 * G_TIME_SPAN_SECOND; + while (!m_sharedRenderData->m_surface) { + /* XXX: This might deadlock with the main thread if the + * QCoreApplication is not running and will not be able to + * execute. We only wait for 5 seconds until a better + * approach can be found here */ + if (!g_cond_wait_until (&m_sharedRenderData->cond, + &m_sharedRenderData->lock, end_time)) { + g_set_error (error, GST_RESOURCE_ERROR, + GST_RESOURCE_ERROR_NOT_FOUND, + "Could not create Qt window within 5 seconds"); + m_sharedRenderData->state = STATE_ERROR; + g_mutex_unlock (&m_sharedRenderData->lock); + return false; + } + } + + GST_TRACE ("%p surface successfully created", this); + m_sharedRenderData->state = STATE_WINDOW_CREATED; + } + + if (m_sharedRenderData->state == STATE_WINDOW_CREATED) { + /* Qt does some things that may require the OpenGL context current + * in ->create() so that it has the necessry information to create + * the QOpenGLContext from the native handle. This may fail if the + * OpenGL context is already current in another thread so we need + * to deactivate the context from GStreamer's thread before asking + * Qt to create the QOpenGLContext with ->create(). + */ + gst_gl_context_activate (context, FALSE); + //m_sharedRenderData->m_context->create(); + //m_sharedRenderData->m_context->doneCurrent(); + + if (!m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface)) { + g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND, + "Could not make Qt OpenGL context current"); + /* try to keep the same OpenGL context state */ + gst_gl_context_activate (context, TRUE); + m_sharedRenderData->state = STATE_ERROR; + g_mutex_unlock (&m_sharedRenderData->lock); + return false; + } + + if (!gst_gl_context_activate (context, TRUE)) { + g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND, + "Could not make GStreamer OpenGL context current again"); + m_sharedRenderData->state = STATE_ERROR; + g_mutex_unlock (&m_sharedRenderData->lock); + return false; + } + m_sharedRenderData->state = STATE_READY; + } + } + + m_renderControl = new QQuickRenderControl(); + /* Create a QQuickWindow that is associated with our render control. Note that this + * window never gets created or shown, meaning that it will never get an underlying + * native (platform) window. + */ + m_quickWindow = new QQuickWindow(m_renderControl); + m_quickWindow->setGraphicsDevice(QQuickGraphicsDevice::fromOpenGLContext(qt_native_context)); + /* after QQuickWindow creation as QQuickRenderControl requires it */ + m_renderControl->prepareThread (m_sharedRenderData->m_renderThread); + g_mutex_unlock (&m_sharedRenderData->lock); + + /* Create a QML engine. */ + m_qmlEngine = new QQmlEngine; + if (!m_qmlEngine->incubationController()) + m_qmlEngine->setIncubationController(m_quickWindow->incubationController()); + + /* TODO: use buffer pool */ + gl_context = static_cast(gst_object_ref (context)); + gl_allocator = (GstGLBaseMemoryAllocator *) gst_gl_memory_allocator_get_default (gl_context); + gl_params = (GstGLAllocationParams *) + gst_gl_video_allocation_params_new (gl_context, + NULL, &this->v_info, 0, NULL, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGBA8); + + /* This is a gross hack relying on the internals of Qt and GStreamer + * however it's the only way to remove this warning on shutdown of all + * resources. + * + * GLib-CRITICAL **: 17:35:24.988: g_main_context_pop_thread_default: assertion 'g_queue_peek_head (stack) == context' failed + * + * The reason is that libgstgl has a GMainContext that it pushes as the + * thread default context. Then later, Qt pushes a thread default main + * context. The detruction order of the GMainContext's is reversed as + * GStreamer will explicitly pop the thread default main context however + * Qt pops when the thread is about to be destroyed. GMainContext is + * unhappy with the ordering of the pops. + */ + GMainContext *gst_main_context = g_main_context_ref_thread_default (); + + /* make Qt allocate and push a thread-default GMainContext if it is + * going to */ + QEventLoop loop; + if (loop.processEvents()) + GST_LOG ("pending QEvents processed"); + + GMainContext *qt_main_context = g_main_context_ref_thread_default (); + + if (qt_main_context == gst_main_context) { + g_main_context_unref (qt_main_context); + g_main_context_unref (gst_main_context); + } else { + /* We flip the order of the GMainContext's so that the destruction + * order can be preserved. */ + g_main_context_pop_thread_default (qt_main_context); + g_main_context_pop_thread_default (gst_main_context); + g_main_context_push_thread_default (qt_main_context); + g_main_context_push_thread_default (gst_main_context); + g_main_context_unref (qt_main_context); + g_main_context_unref (gst_main_context); + } + + return true; +} + +GstQt6QuickRenderer::~GstQt6QuickRenderer() +{ + gst_gl_allocation_params_free (gl_params); + gst_clear_object (&gl_allocator); +} + +void GstQt6QuickRenderer::stopGL () +{ + QOpenGLContext *current_qt_context = QOpenGLContext::currentContext(); + + GST_DEBUG ("%p stop QOpenGLContext current: %p stored: %p", this, + current_qt_context, m_sharedRenderData->m_context); + /* Invalidating the renderer will cause Qt6 to clear the current qt-tracked OpenGL context. + * We however may be using the QOpenGLContext for multiple qml6gloverlay + * elements so need to recurrent it */ + if (current_qt_context) + g_assert (current_qt_context == m_sharedRenderData->m_context); + else + m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface); + + if (m_renderControl) + m_renderControl->invalidate(); + + GST_ERROR ("%p %p", this, QOpenGLContext::currentContext()); + + QEventLoop loop; + if (loop.processEvents()) + GST_LOG ("%p pending QEvents processed", this); +} + +void GstQt6QuickRenderer::stopAfterGL () +{ + GST_DEBUG ("%p stop QOpenGLContext curent: %p stored: %p", this, + QOpenGLContext::currentContext(), m_sharedRenderData->m_context); + g_assert (QOpenGLContext::currentContext() == nullptr); + + if (!m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface)) + g_warn_if_reached(); + + if (m_sharedRenderData) + shared_render_data_unref (m_sharedRenderData); + m_sharedRenderData = NULL; + + /* XXX: reset the OpenGL context and drawable as Qt may have clobbered it. + * Fixes any attempt to access OpenGL after shutting down qmlgloverlay. */ + gst_gl_context_activate (gl_context, FALSE); + gst_gl_context_activate (gl_context, TRUE); +} + +void GstQt6QuickRenderer::cleanup() +{ + if (gl_context) + gst_gl_context_thread_add (gl_context, + (GstGLContextThreadFunc) GstQt6QuickRenderer::stop_c, this); + + /* Delete the render control first since it will free the scenegraph resources. + * Destroy the QQuickWindow only afterwards. */ + if (m_renderControl) + delete m_renderControl; + m_renderControl = nullptr; + + if (m_qmlComponent) + delete m_qmlComponent; + m_qmlComponent = nullptr; + if (m_quickWindow) + delete m_quickWindow; + m_quickWindow = nullptr; + if (m_qmlEngine) + delete m_qmlEngine; + m_qmlEngine = nullptr; + if (m_rootItem) + delete m_rootItem; + m_rootItem = nullptr; + + if (gl_context) + gst_gl_context_thread_add (gl_context, + (GstGLContextThreadFunc) GstQt6QuickRenderer::stop_after_c, this); + + gst_clear_object (&gl_context); +} + +static QSize +gl_params_get_QSize(GstGLAllocationParams * gl_params) +{ + GstGLVideoAllocationParams * gl_vid_params = (GstGLVideoAllocationParams *) gl_params; + + if (!gl_vid_params) + return QSize (0, 0); + + return QSize(GST_VIDEO_INFO_WIDTH (gl_vid_params->v_info), GST_VIDEO_INFO_HEIGHT(gl_vid_params->v_info)); +} + +void +GstQt6QuickRenderer::renderGstGL () +{ +// const GstGLFuncs *gl = gl_context->gl_vtable; + + GST_TRACE ("%p current QOpenGLContext %p", this, + QOpenGLContext::currentContext()); + + m_sharedRenderData->m_animationDriver->advance(); + + QEventLoop loop; + if (loop.processEvents()) + GST_LOG ("pending QEvents processed"); + + loop.exit(); + + if (gl_params && gl_params_get_QSize(gl_params) != m_sharedRenderData->m_surface->size()) { + gst_gl_allocation_params_free(gl_params); + gl_params = NULL; + } + + if (!gl_params) + gl_params = (GstGLAllocationParams *) + gst_gl_video_allocation_params_new (gl_context, + NULL, &this->v_info, 0, NULL, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGBA8); + + + gl_mem = (GstGLMemory *) gst_gl_base_memory_alloc (gl_allocator, gl_params); + m_quickWindow->setRenderTarget(QQuickRenderTarget::fromOpenGLTexture(gst_gl_memory_get_texture_id (gl_mem), gl_params_get_QSize(gl_params))); + + m_renderControl->beginFrame(); + if (m_renderControl->sync()) + GST_LOG ("sync successful"); + + m_renderControl->render(); + m_renderControl->endFrame(); + + /* Qt doesn't seem to reset this, breaking glimagesink */ +// if (gl->DrawBuffer) +// gl->DrawBuffer (GL_BACK); +} + +GstGLMemory *GstQt6QuickRenderer::generateOutput(GstClockTime input_ns) +{ + m_sharedRenderData->m_animationDriver->setNextTime(input_ns / GST_MSECOND); + + /* run an event loop to update any changed values for rendering */ + QEventLoop loop; + if (loop.processEvents()) + GST_LOG ("pending QEvents processed"); + + GST_LOG ("generating output for time %" GST_TIME_FORMAT " ms: %" + G_GUINT64_FORMAT, GST_TIME_ARGS (input_ns), input_ns / GST_MSECOND); + + m_quickWindow->update(); + + /* Polishing happens on the gui thread. */ + m_renderControl->polishItems(); + + /* TODO: an async version could be used instead */ + gst_gl_context_thread_add (gl_context, + (GstGLContextThreadFunc) GstQt6QuickRenderer::render_gst_gl_c, this); + + GstGLMemory *tmp = gl_mem; + gl_mem = NULL; + + return tmp; +} + +void GstQt6QuickRenderer::initializeGstGL () +{ + GST_TRACE ("current QOpenGLContext %p", QOpenGLContext::currentContext()); + if (!m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface)) { + m_errorString = "Failed to make Qt's wrapped OpenGL context current"; + return; + } + GST_INFO ("current QOpenGLContext %p", QOpenGLContext::currentContext()); + + /* XXX: Avoid an assertion inside QSGDefaultRenderContext::initialize() + * from an unused (in this scenario) property when using multiple + * QQuickRenderControl's with the same QOpenGLContext. + * + * First noticed with Qt 5.15. Idea from: + * https://forum.qt.io/topic/55888/is-it-impossible-that-2-qquickrendercontrol-use-same-qopenglcontext/2 + * + * ASSERT: "!m_gl->property(QSG_RENDERCONTEXT_PROPERTY).isValid()" in file /path/to/qt5/qtdeclarative/src/quick/scenegraph/qsgdefaultrendercontext.cpp, line 121 + */ + //m_sharedRenderData->m_context->setProperty("_q_sgrendercontext", QVariant()); + + m_renderControl->initialize(); + + /* 1. QAnimationDriver's are thread-specific + * 2. QAnimationDriver controls the 'animation time' that the Qml scene is + * rendered at + */ + /* FIXME: what happens with multiple qmlgloverlay elements? Do we need a + * shared animation driver? */ + g_mutex_lock (&m_sharedRenderData->lock); + if (m_sharedRenderData->m_animationDriver == nullptr) { + m_sharedRenderData->m_animationDriver = new GstQt6AnimationDriver; + m_sharedRenderData->m_animationDriver->install(); + } + g_mutex_unlock (&m_sharedRenderData->lock); + /* XXX: reset the OpenGL context drawable as Qt may have clobbered it. + * Fixes glimagesink output where Qt replaces the Surface to use in its + * own MakeCurrent call. Qt does this on it's OpenGL initialisation + * the the rendering engine. */ + gst_gl_context_activate (gl_context, FALSE); + gst_gl_context_activate (gl_context, TRUE); +} + +void GstQt6QuickRenderer::initializeQml() +{ + disconnect(m_qmlComponent, &QQmlComponent::statusChanged, this, + &GstQt6QuickRenderer::initializeQml); + + if (m_qmlComponent->isError()) { + const QList errorList = m_qmlComponent->errors(); + for (const QQmlError &error : errorList) + m_errorString += error.toString(); + return; + } + + QObject *rootObject = m_qmlComponent->create(); + if (m_qmlComponent->isError()) { + const QList errorList = m_qmlComponent->errors(); + for (const QQmlError &error : errorList) + m_errorString += error.toString(); + delete rootObject; + return; + } + + m_rootItem = qobject_cast(rootObject); + if (!m_rootItem) { + m_errorString += "root QML item is not a QQuickItem"; + delete rootObject; + return; + } + + /* The root item is ready. Associate it with the window. */ + m_rootItem->setParentItem(m_quickWindow->contentItem()); + + /* Update item and rendering related geometries. */ + updateSizes(); + + /* Initialize the render control and our OpenGL resources. */ + gst_gl_context_thread_add (gl_context, + (GstGLContextThreadFunc) GstQt6QuickRenderer::initialize_gst_gl_c, this); +} + +void GstQt6QuickRenderer::updateSizes() +{ + GstQt6BackingSurface *surface = + static_cast(m_sharedRenderData->m_surface); + /* Behave like SizeRootObjectToView. */ + QSize size = surface->size(); + + m_rootItem->setWidth(size.width()); + m_rootItem->setHeight(size.height()); + + m_quickWindow->setGeometry(0, 0, size.width(), size.height()); + + gst_video_info_set_format (&v_info, GST_VIDEO_FORMAT_RGBA, size.width(), + size.height()); + GstGLVideoAllocationParams *params = (GstGLVideoAllocationParams *) (gl_params); + gst_video_info_set_format (params->v_info, GST_VIDEO_FORMAT_RGBA, size.width(), + size.height()); +} + +void GstQt6QuickRenderer::setSize(int w, int h) +{ + static_cast(m_sharedRenderData->m_surface)->setSize(w, h); + updateSizes(); +} + +bool GstQt6QuickRenderer::setQmlScene (const gchar * scene, GError ** error) +{ + /* replacing the scene is not supported */ + g_return_val_if_fail (m_qmlComponent == NULL, false); + + m_errorString = ""; + + m_qmlComponent = new QQmlComponent(m_qmlEngine); + /* XXX: do we need to provide a propper base name? */ + m_qmlComponent->setData(QByteArray (scene), QUrl("")); + if (m_qmlComponent->isLoading()) + /* TODO: handle async properly */ + connect(m_qmlComponent, &QQmlComponent::statusChanged, this, + &GstQt6QuickRenderer::initializeQml); + else + initializeQml(); + + if (m_errorString != "") { + QByteArray string = m_errorString.toUtf8(); + g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_SETTINGS, + "%s", string.constData()); + return FALSE; + } + + return TRUE; +} + +QQuickItem * GstQt6QuickRenderer::rootItem() const +{ + return m_rootItem; +} diff --git a/libs/qmlglsink/qt6/qt6glrenderer.h b/libs/qmlglsink/qt6/qt6glrenderer.h new file mode 100644 index 000000000000..0d36a6611737 --- /dev/null +++ b/libs/qmlglsink/qt6/qt6glrenderer.h @@ -0,0 +1,124 @@ +/* + * GStreamer + * Copyright (C) 2022 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_QT6_GL_RENDER_H__ +#define __GST_QT6_GL_RENDER_H__ + +#include +#include + +#include + +QT_FORWARD_DECLARE_CLASS(QOpenGLContext) +QT_FORWARD_DECLARE_CLASS(QOpenGLFramebufferObject) +QT_FORWARD_DECLARE_CLASS(QQuickRenderControl) +QT_FORWARD_DECLARE_CLASS(QQuickWindow) +QT_FORWARD_DECLARE_CLASS(QQmlEngine) +QT_FORWARD_DECLARE_CLASS(QQmlComponent) +QT_FORWARD_DECLARE_CLASS(QQuickItem) +QT_FORWARD_DECLARE_CLASS(GstAnimationDriver) +QT_FORWARD_DECLARE_CLASS(GstBackingSurface) + +class GstQt6QuickRenderer : public QObject +{ + Q_OBJECT + +public: + GstQt6QuickRenderer(); + ~GstQt6QuickRenderer(); + + /* initialize the GStreamer/Qt integration. On failure returns false + * and fills @error. + * Must be called with @context not wrapped and current in the current + * thread */ + bool init (GstGLContext * context, GError ** error); + + /* set the qml scene. returns false and fills @error on failure */ + bool setQmlScene (const gchar * scene, GError ** error); + + void setSize(int w, int h); + + GstGLMemory *generateOutput(GstClockTime input_ns); + + /* cleanup any resources. Any use of this object after calling this + * function may result in undefined behaviour */ + void cleanup(); + + /* retrieve the rootItem from the qml scene. Only valid after + * setQmlScene() has been successfully called */ + QQuickItem *rootItem() const; + +private slots: + void initializeQml(); + +private: + void init(); + void ensureFbo(); + + void updateSizes(); + + static void render_gst_gl_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->renderGstGL (); } + void renderGstGL (); + + static void initialize_gst_gl_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->initializeGstGL (); } + void initializeGstGL (); + + static void stop_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->stopGL (); } + void stopGL (); + static void stop_after_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->stopAfterGL (); } + void stopAfterGL (); + + static void activate_context_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->activateContext (); } + void activateContext (); + + static void deactivate_context_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->deactivateContext (); } + void deactivateContext (); + + GstGLContext *gl_context; + QQuickWindow *m_quickWindow; + QQuickRenderControl *m_renderControl; + QQmlEngine *m_qmlEngine; + QQmlComponent *m_qmlComponent; + QQuickItem *m_rootItem; + + GstGLBaseMemoryAllocator *gl_allocator; + GstGLAllocationParams *gl_params; + GstVideoInfo v_info; + GstGLMemory *gl_mem; + + QString m_errorString; + struct SharedRenderData *m_sharedRenderData; +}; + +class CreateSurfaceWorker : public QObject +{ + Q_OBJECT + +public: + CreateSurfaceWorker (struct SharedRenderData * rdata); + ~CreateSurfaceWorker (); + + bool event(QEvent *ev) override; + +private: + struct SharedRenderData *m_sharedRenderData; +}; + +#endif /* __GST_QT6_GL_RENDER_H__ */ diff --git a/libs/qmlglsink/qt6/qt6glwindow.cc b/libs/qmlglsink/qt6/qt6glwindow.cc new file mode 100644 index 000000000000..7c7cdcff3cbb --- /dev/null +++ b/libs/qmlglsink/qt6/qt6glwindow.cc @@ -0,0 +1,470 @@ +/* + * GStreamer + * Copyright (C) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright (C) 2022 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include +#include +#include "qt6glwindow.h" +#include "gstqt6glutility.h" + +#include +#include +#include +#include + +/* compatibility definitions... */ +#ifndef GL_READ_FRAMEBUFFER +#define GL_READ_FRAMEBUFFER 0x8CA8 +#endif +#ifndef GL_DRAW_FRAMEBUFFER +#define GL_DRAW_FRAMEBUFFER 0x8CA9 +#endif + +/** + * SECTION: + * + * #Qt6GLWindow is an #QQuickWindow that grab QtQuick view to GStreamer OpenGL video buffers. + */ + +GST_DEBUG_CATEGORY_STATIC (qt6_gl_window_debug); +#define GST_CAT_DEFAULT qt6_gl_window_debug + +struct _Qt6GLWindowPrivate +{ + GMutex lock; + GCond update_cond; + + GstBuffer *buffer; + GstVideoInfo v_info; + GstVideoFrame mapped_frame; + GstGLBaseMemoryAllocator *gl_allocator; + GstGLAllocationParams *gl_params; + + gboolean initted; + gboolean updated; + gboolean quit; + gboolean result; + gboolean useDefaultFbo; + + GstGLDisplay *display; + GstGLContext *other_context; + GstGLContext *context; + + guint fbo; + + gboolean new_caps; + GstBuffer *produced_buffer; +}; + +Qt6GLWindow::Qt6GLWindow (QWindow * parent, QQuickWindow *src) + : QQuickWindow( parent ), source (src) +{ + QGuiApplication *app = static_cast (QCoreApplication::instance ()); + static gsize _debug; + + g_assert (app != NULL); + + if (g_once_init_enter (&_debug)) { + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qt6glwindow", 0, "Qt6 GL QuickWindow"); + g_once_init_leave (&_debug, 1); + } + + this->priv = g_new0 (Qt6GLWindowPrivate, 1); + + g_mutex_init (&this->priv->lock); + g_cond_init (&this->priv->update_cond); + + this->priv->display = gst_qml6_get_gl_display(FALSE); + this->priv->result = TRUE; + + connect (source, SIGNAL(beforeRendering()), this, SLOT(beforeRendering()), Qt::DirectConnection); + connect (source, SIGNAL(afterRendering()), this, SLOT(afterRendering()), Qt::DirectConnection); + if (source->isSceneGraphInitialized()) + source->scheduleRenderJob(new RenderJob(std::bind(&Qt6GLWindow::onSceneGraphInitialized, this)), QQuickWindow::BeforeSynchronizingStage); + else + connect (source, SIGNAL(sceneGraphInitialized()), this, SLOT(onSceneGraphInitialized()), Qt::DirectConnection); + + connect (source, SIGNAL(sceneGraphInvalidated()), this, SLOT(onSceneGraphInvalidated()), Qt::DirectConnection); + + GST_DEBUG ("%p init Qt Window", this->priv->display); +} + +Qt6GLWindow::~Qt6GLWindow() +{ + GST_DEBUG ("deinit Qt Window"); + g_mutex_clear (&this->priv->lock); + g_cond_clear (&this->priv->update_cond); + gst_clear_object (&this->priv->other_context); + gst_clear_buffer (&this->priv->buffer); + gst_clear_buffer (&this->priv->produced_buffer); + gst_clear_object (&this->priv->display); + gst_clear_object (&this->priv->context); + gst_clear_object (&this->priv->gl_allocator); + + if (this->priv->gl_params) + gst_gl_allocation_params_free (this->priv->gl_params); + this->priv->gl_params = NULL; + + g_free (this->priv); + this->priv = NULL; +} + +void +Qt6GLWindow::beforeRendering() +{ + g_mutex_lock (&this->priv->lock); + + if (!this->priv->context) { + GST_LOG ("no GStreamer GL context set yet, skipping frame"); + g_mutex_unlock (&this->priv->lock); + return; + } + + QSize size = source->size(); + + if (!this->priv->gl_allocator) + this->priv->gl_allocator = + (GstGLBaseMemoryAllocator *) gst_gl_memory_allocator_get_default (this->priv->context); + + if (GST_VIDEO_INFO_WIDTH (&this->priv->v_info) != size.width() + || GST_VIDEO_INFO_HEIGHT (&this->priv->v_info) != size.height()) { + this->priv->new_caps = TRUE; + + gst_video_info_set_format (&this->priv->v_info, GST_VIDEO_FORMAT_RGBA, + size.width(), size.height()); + + if (this->priv->gl_params) { + GstGLVideoAllocationParams *gl_vid_params = (GstGLVideoAllocationParams *) this->priv->gl_params; + if (GST_VIDEO_INFO_WIDTH (gl_vid_params->v_info) != source->width() + || GST_VIDEO_INFO_HEIGHT (gl_vid_params->v_info) != source->height()) + this->priv->gl_params = NULL; + gst_clear_buffer (&this->priv->buffer); + } + } + + if (!this->priv->gl_params) { + this->priv->gl_params = (GstGLAllocationParams *) + gst_gl_video_allocation_params_new (this->priv->context, NULL, + &this->priv->v_info, 0, NULL, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGBA); + } + + if (!this->priv->buffer) { + GstGLMemory *gl_mem = + (GstGLMemory *) gst_gl_base_memory_alloc (this->priv->gl_allocator, + this->priv->gl_params); + this->priv->buffer = gst_buffer_new (); + gst_buffer_append_memory (this->priv->buffer, (GstMemory *) gl_mem); + } + + if (!gst_video_frame_map (&this->priv->mapped_frame, &this->priv->v_info, + this->priv->buffer, (GstMapFlags) (GST_MAP_WRITE | GST_MAP_GL))) { + GST_WARNING ("failed map video frame"); + gst_clear_buffer (&this->priv->buffer); + return; + } + + if (!this->priv->useDefaultFbo) { + guint tex_id = *(guint *) this->priv->mapped_frame.data[0]; + + source->setRenderTarget(QQuickRenderTarget::fromOpenGLTexture(tex_id, source->size())); + } else if (this->priv->useDefaultFbo) { + GST_DEBUG ("use default fbo for render target"); + source->setRenderTarget(QQuickRenderTarget()); + } + + g_mutex_unlock (&this->priv->lock); +} + +void +Qt6GLWindow::afterRendering() +{ + gboolean ret; + guint width, height; + const GstGLFuncs *gl; + GstGLSyncMeta *sync_meta; + + g_mutex_lock (&this->priv->lock); + + if (!this->priv->buffer) { + GST_LOG ("no buffer created in beforeRendering(), skipping"); + g_mutex_unlock (&this->priv->lock); + return; + } + + width = GST_VIDEO_INFO_WIDTH (&this->priv->v_info); + height = GST_VIDEO_INFO_HEIGHT (&this->priv->v_info); + + gst_gl_context_activate (this->priv->other_context, TRUE); + gl = this->priv->other_context->gl_vtable; + + if (!this->priv->useDefaultFbo) { + gst_video_frame_unmap (&this->priv->mapped_frame); + ret = TRUE; + } else { + gl->BindFramebuffer (GL_READ_FRAMEBUFFER, 0); + + ret = gst_gl_context_check_framebuffer_status (this->priv->other_context, GL_READ_FRAMEBUFFER); + if (!ret) { + GST_ERROR ("FBO errors"); + goto errors; + } + + guint dst_tex = *(guint *) this->priv->mapped_frame.data[0]; + gl->BindTexture (GL_TEXTURE_2D, dst_tex); + if (gl->BlitFramebuffer) { + gl->BindFramebuffer (GL_DRAW_FRAMEBUFFER, this->priv->fbo); + gl->FramebufferTexture2D (GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, dst_tex, 0); + + ret = gst_gl_context_check_framebuffer_status (this->priv->other_context, GL_DRAW_FRAMEBUFFER); + if (!ret) { + GST_ERROR ("FBO errors"); + goto errors; + } + gl->ReadBuffer (GL_BACK); + gl->BlitFramebuffer (0, 0, width, height, + 0, 0, width, height, + GL_COLOR_BUFFER_BIT, GL_LINEAR); + } else { + gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA, 0, 0, width, height, 0); + } + } + + gst_video_frame_unmap (&this->priv->mapped_frame); + gl->BindFramebuffer (GL_FRAMEBUFFER, 0); + + if (this->priv->context) { + sync_meta = gst_buffer_get_gl_sync_meta (this->priv->buffer); + if (!sync_meta) { + sync_meta = gst_buffer_add_gl_sync_meta (this->priv->context, this->priv->buffer); + } + gst_gl_sync_meta_set_sync_point (sync_meta, this->priv->other_context); + } + + GST_DEBUG ("rendering finished"); + +done: + gst_gl_context_activate (this->priv->other_context, FALSE); + + this->priv->result = ret; + this->priv->produced_buffer = this->priv->buffer; + this->priv->buffer = NULL; + this->priv->updated = TRUE; + g_cond_signal (&this->priv->update_cond); + g_mutex_unlock (&this->priv->lock); + return; + +errors: + gl->BindFramebuffer (GL_FRAMEBUFFER, 0); + gst_video_frame_unmap (&this->priv->mapped_frame); + goto done; +} + +void +Qt6GLWindow::onSceneGraphInitialized() +{ + QSGRendererInterface *renderer = source->rendererInterface(); + if (!renderer) + return; + + if (renderer->graphicsApi() != QSGRendererInterface::GraphicsApi::OpenGL) { + GST_WARNING ("%p scene graph initialized with a non-OpenGL renderer interface", this); + return; + } + + this->priv->initted = gst_qml6_get_gl_wrapcontext (this->priv->display, + &this->priv->other_context, &this->priv->context); + + if (this->priv->initted && this->priv->other_context) { + const GstGLFuncs *gl; + + gst_gl_context_activate (this->priv->other_context, TRUE); + gl = this->priv->other_context->gl_vtable; + + gl->GenFramebuffers (1, &this->priv->fbo); + + gst_gl_context_activate (this->priv->other_context, FALSE); + } + + GST_DEBUG ("%p created wrapped GL context %" GST_PTR_FORMAT, this, + this->priv->other_context); +} + +void +Qt6GLWindow::onSceneGraphInvalidated() +{ + GST_DEBUG ("scene graph invalidated"); + + if (this->priv->fbo && this->priv->other_context) { + const GstGLFuncs *gl; + + gst_gl_context_activate (this->priv->other_context, TRUE); + gl = this->priv->other_context->gl_vtable; + + gl->DeleteFramebuffers (1, &this->priv->fbo); + + gst_gl_context_activate (this->priv->other_context, FALSE); + } + + gst_clear_buffer (&this->priv->buffer); + gst_clear_buffer (&this->priv->produced_buffer); +} + +bool +Qt6GLWindow::getGeometry(int * width, int * height) +{ + if (width == NULL || height == NULL) + return FALSE; + + *width = this->source->width(); + *height = this->source->height(); + + return TRUE; +} + +GstGLContext * +qt6_gl_window_get_qt_context (Qt6GLWindow * qt6_gl_window) +{ + g_return_val_if_fail (qt6_gl_window != NULL, NULL); + + if (!qt6_gl_window->priv->other_context) + return NULL; + + return (GstGLContext *) gst_object_ref (qt6_gl_window->priv->other_context); +} + +GstGLDisplay * +qt6_gl_window_get_display (Qt6GLWindow * qt6_gl_window) +{ + g_return_val_if_fail (qt6_gl_window != NULL, NULL); + + if (!qt6_gl_window->priv->display) + return NULL; + + return (GstGLDisplay *) gst_object_ref (qt6_gl_window->priv->display); +} + +GstGLContext * +qt6_gl_window_get_context (Qt6GLWindow * qt6_gl_window) +{ + g_return_val_if_fail (qt6_gl_window != NULL, NULL); + + if (!qt6_gl_window->priv->context) + return NULL; + + return (GstGLContext *) gst_object_ref (qt6_gl_window->priv->context); +} + +gboolean +qt6_gl_window_set_context (Qt6GLWindow * qt6_gl_window, GstGLContext * context) +{ + g_return_val_if_fail (qt6_gl_window != NULL, FALSE); + + if (qt6_gl_window->priv->context && qt6_gl_window->priv->context != context) + return FALSE; + + gst_object_replace ((GstObject **) &qt6_gl_window->priv->context, (GstObject *) context); + + return TRUE; +} + +gboolean +qt6_gl_window_is_scenegraph_initialized (Qt6GLWindow * qt6_gl_window) +{ + g_return_val_if_fail (qt6_gl_window != NULL, FALSE); + + return qt6_gl_window->priv->initted; +} + +GstBuffer * +qt6_gl_window_take_buffer (Qt6GLWindow * qt6_gl_window, GstCaps ** updated_caps) +{ + g_return_val_if_fail (qt6_gl_window != NULL, FALSE); + g_return_val_if_fail (qt6_gl_window->priv->initted, FALSE); + GstBuffer *ret; + + g_mutex_lock (&qt6_gl_window->priv->lock); + + if (qt6_gl_window->priv->quit){ + GST_DEBUG("about to quit, drop this buffer"); + g_mutex_unlock (&qt6_gl_window->priv->lock); + return NULL; + } + + while (!qt6_gl_window->priv->produced_buffer && qt6_gl_window->priv->result) + g_cond_wait (&qt6_gl_window->priv->update_cond, &qt6_gl_window->priv->lock); + + ret = qt6_gl_window->priv->produced_buffer; + qt6_gl_window->priv->produced_buffer = NULL; + + if (qt6_gl_window->priv->new_caps) { + *updated_caps = gst_video_info_to_caps (&qt6_gl_window->priv->v_info); + gst_caps_set_features (*updated_caps, 0, + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + qt6_gl_window->priv->new_caps = FALSE; + } + + g_mutex_unlock (&qt6_gl_window->priv->lock); + + return ret; +} + +void +qt6_gl_window_use_default_fbo (Qt6GLWindow * qt6_gl_window, gboolean useDefaultFbo) +{ + g_return_if_fail (qt6_gl_window != NULL); + + g_mutex_lock (&qt6_gl_window->priv->lock); + + GST_DEBUG ("set to use default fbo %d", useDefaultFbo); + qt6_gl_window->priv->useDefaultFbo = useDefaultFbo; + + g_mutex_unlock (&qt6_gl_window->priv->lock); +} + +void +qt6_gl_window_unlock(Qt6GLWindow* qt6_gl_window) +{ + g_mutex_lock(&qt6_gl_window->priv->lock); + + GST_DEBUG("unlock window"); + qt6_gl_window->priv->result = FALSE; + g_cond_signal(&qt6_gl_window->priv->update_cond); + + g_mutex_unlock(&qt6_gl_window->priv->lock); +} + +void +qt6_gl_window_unlock_stop(Qt6GLWindow* qt6_gl_window) +{ + g_mutex_lock(&qt6_gl_window->priv->lock); + + GST_DEBUG("unlock stop window"); + qt6_gl_window->priv->result = TRUE; + g_cond_signal(&qt6_gl_window->priv->update_cond); + + g_mutex_unlock(&qt6_gl_window->priv->lock); +} diff --git a/libs/qmlglsink/qt6/qt6glwindow.h b/libs/qmlglsink/qt6/qt6glwindow.h new file mode 100644 index 000000000000..c964da23b0fd --- /dev/null +++ b/libs/qmlglsink/qt6/qt6glwindow.h @@ -0,0 +1,69 @@ +/* + * GStreamer + * Copyright (C) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright (C) 2022 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __QT6_GL_WINDOW_H__ +#define __QT6_GL_WINDOW_H__ + +#include +#include + +#include "gstqt6gl.h" +#include +#include +#include + +typedef struct _Qt6GLWindowPrivate Qt6GLWindowPrivate; + +class Qt6GLWindow : public QQuickWindow, protected QOpenGLFunctions +{ + Q_OBJECT +public: + Qt6GLWindow (QWindow * parent = NULL, QQuickWindow *source = NULL); + ~Qt6GLWindow (); + bool getGeometry (int * width, int * height); + + /* private for C interface ... */ + Qt6GLWindowPrivate *priv; + +private Q_SLOTS: + void beforeRendering (); + void afterRendering (); + void onSceneGraphInitialized (); + void onSceneGraphInvalidated (); + +private: + QQuickWindow * source; +}; + +extern "C" +{ +GstBuffer * qt6_gl_window_take_buffer (Qt6GLWindow * qt6_window, GstCaps ** updated_caps); +GstGLContext * qt6_gl_window_get_qt_context (Qt6GLWindow * qt6_window); +GstGLContext * qt6_gl_window_get_context (Qt6GLWindow * qt6_window); +gboolean qt6_gl_window_set_context (Qt6GLWindow * qt6_window, GstGLContext * context); +GstGLDisplay * qt6_gl_window_get_display (Qt6GLWindow * qt6_window); +gboolean qt6_gl_window_is_scenegraph_initialized (Qt6GLWindow * qt6_window); +void qt6_gl_window_use_default_fbo (Qt6GLWindow * qt6_window, gboolean useDefaultFbo); +void qt6_gl_window_unlock(Qt6GLWindow* qt6_window); +void qt6_gl_window_unlock_stop(Qt6GLWindow* qt6_window); +} + +#endif /* __QT6_GL_WINDOW_H__ */ diff --git a/libs/qmlglsink/qt6/resources.qrc b/libs/qmlglsink/qt6/resources.qrc new file mode 100644 index 000000000000..7a01e20f68f6 --- /dev/null +++ b/libs/qmlglsink/qt6/resources.qrc @@ -0,0 +1,7 @@ + + + vertex.vert.qsb + RGBA.frag.qsb + YUV_TRIPLANAR.frag.qsb + + diff --git a/libs/qmlglsink/qt6/vertex.vert b/libs/qmlglsink/qt6/vertex.vert new file mode 100644 index 000000000000..f67117731db2 --- /dev/null +++ b/libs/qmlglsink/qt6/vertex.vert @@ -0,0 +1,23 @@ +#version 440 + +layout(location = 0) in vec4 aVertex; +layout(location = 1) in vec2 aTexCoord; + +layout(location = 0) out vec2 vTexCoord; + +layout(std140, binding = 0) uniform buf { + mat4 qt_Matrix; + ivec4 swizzle; + mat4 color_matrix; + float qt_Opacity; +} ubuf; + +out gl_PerVertex { + vec4 gl_Position; +}; + +void main() +{ + gl_Position = ubuf.qt_Matrix * aVertex; + vTexCoord = aTexCoord; +} diff --git a/qmlglsink.pri b/qmlglsink.pri index 0d9af09a5b90..c522ee5efb32 100644 --- a/qmlglsink.pri +++ b/qmlglsink.pri @@ -1,33 +1,63 @@ LinuxBuild { - UseWayland: { - DEFINES += HAVE_QT_WAYLAND - } else { - DEFINES += HAVE_QT_X11 - } - DEFINES += HAVE_QT_EGLFS HAVE_QT_QPA_HEADER + DEFINES += HAVE_QT_WAYLAND HAVE_QT_X11 HAVE_QT_EGLFS HAVE_QT_QPA_HEADER } else:MacBuild { DEFINES += HAVE_QT_MAC } else:iOSBuild { DEFINES += HAVE_QT_IOS } else:WindowsBuild { - DEFINES += HAVE_QT_WIN32 HAVE_QT_QPA_HEADER + DEFINES += HAVE_QT_WIN32 LIBS += opengl32.lib user32.lib } else:AndroidBuild { DEFINES += HAVE_QT_ANDROID } -SOURCES += \ - libs/qmlglsink/qt6/gstplugin.cc \ - libs/qmlglsink/qt6/gstqml6glsink.cc \ - libs/qmlglsink/qt6/gstqsg6glnode.cc \ - libs/qmlglsink/qt6/gstqt6element.cc \ - libs/qmlglsink/qt6/gstqt6glutility.cc \ - libs/qmlglsink/qt6/qt6glitem.cc +# DEFINES += HAVE_QT_QPA_HEADER +# QT_QPA_HEADER= -HEADERS += \ - libs/qmlglsink/qt6/gstqml6glsink.h \ - libs/qmlglsink/qt6/gstqsg6glnode.h \ - libs/qmlglsink/qt6/gstqt6elements.h \ - libs/qmlglsink/qt6/gstqt6gl.h \ - libs/qmlglsink/qt6/gstqt6glutility.h \ - libs/qmlglsink/qt6/qt6glitem.h +LinuxBuild { + SOURCES += \ + libs/qmlglsink/qt6-linux/gstplugin.cc \ + libs/qmlglsink/qt6-linux/gstqml6glsink.cc \ + libs/qmlglsink/qt6-linux/gstqsg6glnode.cc \ + libs/qmlglsink/qt6-linux/gstqt6element.cc \ + libs/qmlglsink/qt6-linux/gstqt6glutility.cc \ + libs/qmlglsink/qt6-linux/qt6glitem.cc + + HEADERS += \ + libs/qmlglsink/qt6-linux/gstqml6glsink.h \ + libs/qmlglsink/qt6-linux/gstqsg6glnode.h \ + libs/qmlglsink/qt6-linux/gstqt6elements.h \ + libs/qmlglsink/qt6-linux/gstqt6gl.h \ + libs/qmlglsink/qt6-linux/gstqt6glutility.h \ + libs/qmlglsink/qt6-linux/qt6glitem.h + + INCLUDEPATH += libs/qmlglsink/qt6-linux +} else { + SOURCES += \ + libs/qmlglsink/qt6/gstplugin.cc \ + libs/qmlglsink/qt6/gstqml6glmixer.cc \ + libs/qmlglsink/qt6/gstqml6gloverlay.cc \ + libs/qmlglsink/qt6/gstqml6glsink.cc \ + libs/qmlglsink/qt6/gstqml6glsrc.cc \ + libs/qmlglsink/qt6/gstqsg6material.cc \ + libs/qmlglsink/qt6/gstqt6element.cc \ + libs/qmlglsink/qt6/gstqt6glutility.cc \ + libs/qmlglsink/qt6/qt6glitem.cc \ + libs/qmlglsink/qt6/qt6glrenderer.cc \ + libs/qmlglsink/qt6/qt6glwindow.cc + + HEADERS += \ + libs/qmlglsink/qt6/gstqml6glmixer.h \ + libs/qmlglsink/qt6/gstqml6gloverlay.h \ + libs/qmlglsink/qt6/gstqml6glsink.h \ + libs/qmlglsink/qt6/gstqml6glsrc.h \ + libs/qmlglsink/qt6/gstqsg6material.h \ + libs/qmlglsink/qt6/gstqt6elements.h \ + libs/qmlglsink/qt6/gstqt6gl.h \ + libs/qmlglsink/qt6/gstqt6glutility.h \ + libs/qmlglsink/qt6/qt6glitem.h \ + libs/qmlglsink/qt6/qt6glrenderer.h \ + libs/qmlglsink/qt6/qt6glwindow.h + + INCLUDEPATH += libs/qmlglsink/qt6 +} diff --git a/src/VideoReceiver/GStreamer.cc b/src/VideoReceiver/GStreamer.cc index 743994437abd..f536893f6f40 100644 --- a/src/VideoReceiver/GStreamer.cc +++ b/src/VideoReceiver/GStreamer.cc @@ -77,12 +77,14 @@ G_BEGIN_DECLS #if defined(Q_OS_ANDROID) || defined(Q_OS_IOS) GST_PLUGIN_STATIC_DECLARE(coreelements); GST_PLUGIN_STATIC_DECLARE(playback); - GST_PLUGIN_STATIC_DECLARE(libav); + #ifndef QGC_CMAKE_GST + GST_PLUGIN_STATIC_DECLARE(libav); + GST_PLUGIN_STATIC_DECLARE(x264); + #endif GST_PLUGIN_STATIC_DECLARE(rtp); GST_PLUGIN_STATIC_DECLARE(rtsp); GST_PLUGIN_STATIC_DECLARE(udp); GST_PLUGIN_STATIC_DECLARE(videoparsersbad); - GST_PLUGIN_STATIC_DECLARE(x264); GST_PLUGIN_STATIC_DECLARE(rtpmanager); GST_PLUGIN_STATIC_DECLARE(isomp4); GST_PLUGIN_STATIC_DECLARE(matroska); @@ -208,12 +210,14 @@ GStreamer::initialize(int argc, char* argv[], int debuglevel) #if defined(Q_OS_ANDROID) || defined(Q_OS_IOS) GST_PLUGIN_STATIC_REGISTER(coreelements); GST_PLUGIN_STATIC_REGISTER(playback); - GST_PLUGIN_STATIC_REGISTER(libav); + #ifndef QGC_CMAKE_GST + GST_PLUGIN_STATIC_REGISTER(libav); + GST_PLUGIN_STATIC_REGISTER(x264); + #endif GST_PLUGIN_STATIC_REGISTER(rtp); GST_PLUGIN_STATIC_REGISTER(rtsp); GST_PLUGIN_STATIC_REGISTER(udp); GST_PLUGIN_STATIC_REGISTER(videoparsersbad); - GST_PLUGIN_STATIC_REGISTER(x264); GST_PLUGIN_STATIC_REGISTER(rtpmanager); GST_PLUGIN_STATIC_REGISTER(isomp4); GST_PLUGIN_STATIC_REGISTER(matroska); diff --git a/src/VideoReceiver/VideoReceiver.pri b/src/VideoReceiver/VideoReceiver.pri index 5a15a5ba3bdf..d08f7a5a91cf 100644 --- a/src/VideoReceiver/VideoReceiver.pri +++ b/src/VideoReceiver/VideoReceiver.pri @@ -12,9 +12,7 @@ # LinuxBuild { - UseWayland { - QT += waylandclient - } + # QT += waylandclient CONFIG += link_pkgconfig packagesExist(gstreamer-1.0) { PKGCONFIG += gstreamer-1.0 gstreamer-video-1.0 gstreamer-gl-1.0 egl @@ -70,16 +68,17 @@ LinuxBuild { QMAKE_POST_LINK += $$escape_expand(\\n) xcopy \"$$GST_ROOT_WIN\\lib\\gstreamer-1.0\\*.dll\" \"$$DESTDIR_WIN\\gstreamer-plugins\\\" /Y $$escape_expand(\\n) } } else:AndroidBuild { - #- gstreamer assumed to be installed in $$PWD/../../gstreamer-1.0-android-universal-1.18.5/*** + #- gstreamer assumed to be installed in $$PWD/../../gstreamer-1.0-android-universal*/*** + GST_ROOT = $$PWD/../../gstreamer-1.0-android-universal*/ contains(ANDROID_TARGET_ARCH, armeabi-v7a) { - GST_ROOT = $$PWD/../../gstreamer-1.0-android-universal-1.18.5/armv7 + GST_ROOT += armv7 } else:contains(ANDROID_TARGET_ARCH, arm64-v8a) { - GST_ROOT = $$PWD/../../gstreamer-1.0-android-universal-1.18.5/arm64 + GST_ROOT += arm64 } else:contains(ANDROID_TARGET_ARCH, x86_64) { - GST_ROOT = $$PWD/../../gstreamer-1.0-android-universal-1.18.5/x86_64 + GST_ROOT += x86_64 } else { message(Unknown ANDROID_TARGET_ARCH $$ANDROID_TARGET_ARCH) - GST_ROOT = $$PWD/../../gstreamer-1.0-android-universal-1.18.5/x86 + GST_ROOT += x86 } exists($$GST_ROOT) { QMAKE_CXXFLAGS += -pthread diff --git a/tools/setup/ubuntu.sh b/tools/setup/ubuntu.sh index 742ab7495b82..a60759324ba9 100644 --- a/tools/setup/ubuntu.sh +++ b/tools/setup/ubuntu.sh @@ -3,6 +3,7 @@ set -e sudo apt-get update -y --quiet + sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \ build-essential \ ccache \ @@ -17,13 +18,31 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends i make \ ninja-build \ rsync \ - libsdl2-dev \ - libgstreamer-plugins-base1.0-dev \ - libgstreamer1.0-0:amd64 \ - libgstreamer1.0-dev \ binutils \ - patchelf \ - libxcb-xinerama0 \ + patchelf + +sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet install \ + libxcb-xinerama0 \ libxkbcommon-x11-0 \ - libxcb-cursor0 \ - libdrm-dev + libxcb-cursor0 + +sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet install \ + libsdl2-dev \ + libdrm-dev + +sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet install \ + libgstreamer1.0-dev \ + libgstreamer-plugins-base1.0-dev \ + libgstreamer-plugins-bad1.0-dev \ + gstreamer1.0-plugins-base \ + gstreamer1.0-plugins-good \ + gstreamer1.0-plugins-bad \ + gstreamer1.0-plugins-ugly \ + gstreamer1.0-libav \ + gstreamer1.0-tools \ + gstreamer1.0-x \ + gstreamer1.0-alsa \ + gstreamer1.0-gl \ + gstreamer1.0-gtk3 \ + gstreamer1.0-qt5 \ + gstreamer1.0-pulseaudio