From e7d28b96da929754d3f65f78ab88db76e6ca98b7 Mon Sep 17 00:00:00 2001 From: checkraisefold Date: Fri, 10 May 2024 12:22:58 -0700 Subject: Windows screenshare/video call support, general call improvements (#1725) * Initial support for d3d11 qml video item * Windows screenshare support, D3D11 * misc fixes * further window visibility checks * preview updates * fix qml preview * fix compositor * add libnice plugin dep * re-run formatter * final formatter fix [skip ci] * fix tumbleweed build --------- Co-authored-by: Joseph Donofry --- src/voip/CallManager.cpp | 103 +++++++++++++++++++----- src/voip/CallManager.h | 4 + src/voip/WebRTCSession.cpp | 191 ++++++++++++++++++++++++++++++++++----------- src/voip/WebRTCSession.h | 3 +- 4 files changed, 237 insertions(+), 64 deletions(-) (limited to 'src') diff --git a/src/voip/CallManager.cpp b/src/voip/CallManager.cpp index ac3ec8ee..58aaa1a4 100644 --- a/src/voip/CallManager.cpp +++ b/src/voip/CallManager.cpp @@ -34,6 +34,10 @@ #include #endif +#ifdef Q_OS_WINDOWS +#include +#endif + #ifdef GSTREAMER_AVAILABLE extern "C" { @@ -82,12 +86,12 @@ CallManager::CallManager(QObject *parent) { #ifdef GSTREAMER_AVAILABLE std::string errorMessage; - if (session_.havePlugins(true, true, ScreenShareType::XDP, &errorMessage)) { - screenShareTypes_.push_back(ScreenShareType::XDP); - screenShareType_ = ScreenShareType::XDP; - } - if (std::getenv("DISPLAY")) { + if (QGuiApplication::platformName() == QStringLiteral("windows") && + session_.havePlugins(true, true, ScreenShareType::D3D11, &errorMessage)) { + screenShareType_ = ScreenShareType::D3D11; + screenShareTypes_.push_back(ScreenShareType::D3D11); + } else if (std::getenv("DISPLAY")) { screenShareTypes_.push_back(ScreenShareType::X11); if (QGuiApplication::platformName() != QStringLiteral("wayland")) { // Selected by default @@ -96,6 +100,12 @@ CallManager::CallManager(QObject *parent) std::swap(screenShareTypes_[0], screenShareTypes_[1]); } } + + if (QGuiApplication::platformName() != QStringLiteral("windows") && + session_.havePlugins(true, true, ScreenShareType::XDP, &errorMessage)) { + screenShareTypes_.push_back(ScreenShareType::XDP); + screenShareType_ = ScreenShareType::XDP; + } #endif connect( @@ -254,7 +264,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w #ifdef GSTREAMER_AVAILABLE if (callType == CallType::SCREEN) { - if (screenShareType_ == ScreenShareType::X11) { + if (screenShareType_ == ScreenShareType::X11 || + screenShareType_ == ScreenShareType::D3D11) { if (windows_.empty() || windowIndex >= windows_.size()) { nhlog::ui()->error("WebRTC: window index out of range"); return; @@ -270,8 +281,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w #endif if (haveCallInvite_) { - nhlog::ui()->debug( - "WebRTC: Discarding outbound call for inbound call. localUser is polite party"); + nhlog::ui()->debug("WebRTC: Discarding outbound call for inbound call. " + "localUser is polite party"); if (callParty_ == callee->user_id) { if (callType == callType_) acceptInvite(); @@ -305,7 +316,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w playRingtone(QUrl(QStringLiteral("qrc:/media/media/ringback.ogg")), true); uint32_t shareWindowId = - callType == CallType::SCREEN && screenShareType_ == ScreenShareType::X11 + callType == CallType::SCREEN && + (screenShareType_ == ScreenShareType::X11 || screenShareType_ == ScreenShareType::D3D11) ? windows_[windowIndex].second : 0; if (!session_.createOffer(callType, screenShareType_, shareWindowId)) { @@ -337,7 +349,7 @@ callHangUpReasonString(CallHangUp::Reason reason) return "User"; } } -} +} // namespace void CallManager::hangUp(CallHangUp::Reason reason) @@ -511,8 +523,8 @@ CallManager::handleEvent(const RoomEvent &callInviteEvent) void CallManager::acceptInvite() { - // if call was accepted/rejected elsewhere and m.call.select_answer is received - // before acceptInvite + // if call was accepted/rejected elsewhere and m.call.select_answer is + // received before acceptInvite if (!haveCallInvite_) return; @@ -699,7 +711,8 @@ CallManager::handleEvent(const RoomEvent &callRejectEvent) if (callRejectEvent.content.call_id == callid_) { if (session_.state() == webrtc::State::OFFERSENT) { - // only accept reject if webrtc is in OFFERSENT state, else call has been accepted + // only accept reject if webrtc is in OFFERSENT state, else call has been + // accepted emit newMessage( roomid_, CallSelectAnswer{ @@ -861,7 +874,7 @@ bool CallManager::screenShareReady() const { #ifdef GSTREAMER_AVAILABLE - if (screenShareType_ == ScreenShareType::X11) { + if (screenShareType_ == ScreenShareType::X11 || screenShareType_ == ScreenShareType::D3D11) { return true; } else { return ScreenCastPortal::instance().ready(); @@ -875,12 +888,15 @@ QStringList CallManager::screenShareTypeList() { QStringList ret; - ret.reserve(2); + ret.reserve(3); for (ScreenShareType type : screenShareTypes_) { switch (type) { case ScreenShareType::X11: ret.append(tr("X11")); break; + case ScreenShareType::D3D11: + ret.append("DirectX 11"); + break; case ScreenShareType::XDP: ret.append(tr("PipeWire")); break; @@ -893,8 +909,10 @@ CallManager::screenShareTypeList() QStringList CallManager::windowList() { - if (!(std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::X11) != - screenShareTypes_.end())) { + if (std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::X11) == + screenShareTypes_.end() && + std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::D3D11) == + screenShareTypes_.end()) { return {}; } @@ -949,6 +967,32 @@ CallManager::windowList() } xcb_ewmh_get_windows_reply_wipe(&clients); } +#endif +#ifdef Q_OS_WINDOWS + for (HWND windowHandle = GetTopWindow(nullptr); windowHandle != nullptr; + windowHandle = GetNextWindow(windowHandle, GW_HWNDNEXT)) { + if (!IsWindowVisible(windowHandle)) + continue; + + int titleLength = GetWindowTextLengthW(windowHandle); + if (titleLength == 0) + continue; + + if (GetWindowLong(windowHandle, GWL_EXSTYLE) & WS_EX_TOOLWINDOW) + continue; + + TITLEBARINFO titleInfo; + titleInfo.cbSize = sizeof(titleInfo); + GetTitleBarInfo(windowHandle, &titleInfo); + if (titleInfo.rgstate[0] & STATE_SYSTEM_INVISIBLE) + continue; + + wchar_t *windowTitle = new wchar_t[titleLength + 1]; + GetWindowTextW(windowHandle, windowTitle, titleLength + 1); + + windows_.push_back( + {QString::fromWCharArray(windowTitle), reinterpret_cast(windowHandle)}); + } #endif QStringList ret; assert(windows_.size() < std::numeric_limits::max()); @@ -1007,11 +1051,13 @@ make_preview_sink() { if (QGuiApplication::platformName() == QStringLiteral("wayland")) { return gst_element_factory_make("waylandsink", nullptr); + } else if (QGuiApplication::platformName() == QStringLiteral("windows")) { + return gst_element_factory_make("d3d11videosink", nullptr); } else { return gst_element_factory_make("ximagesink", nullptr); } } -} +} // namespace #endif void @@ -1032,6 +1078,12 @@ CallManager::previewWindow(unsigned int index) const return; } + if (screenShareType_ == ScreenShareType::D3D11 && + (windows_.empty() || index >= windows_.size())) { + nhlog::ui()->error("D3D11 screencast not available"); + return; + } + auto settings = ChatPage::instance()->userSettings(); pipe_ = gst_pipeline_new(nullptr); @@ -1066,6 +1118,19 @@ CallManager::previewWindow(unsigned int index) const gst_bin_add(GST_BIN(pipe_), ximagesrc); screencastsrc = ximagesrc; + } else if (screenShareType_ == ScreenShareType::D3D11) { + GstElement *d3d11screensrc = gst_element_factory_make("d3d11screencapturesrc", nullptr); + if (!d3d11screensrc) { + nhlog::ui()->error("Failed to create d3d11screencapturesrc"); + gst_object_unref(pipe_); + pipe_ = nullptr; + return; + } + g_object_set(d3d11screensrc, "window-handle", windows_[index].second, nullptr); + g_object_set(d3d11screensrc, "show-cursor", !settings->screenShareHideCursor(), nullptr); + + gst_bin_add(GST_BIN(pipe_), d3d11screensrc); + screencastsrc = d3d11screensrc; } else { ScreenCastPortal &sc_portal = ScreenCastPortal::instance(); const ScreenCastPortal::Stream *stream = sc_portal.getStream(); @@ -1171,6 +1236,6 @@ getTurnURIs(const mtx::responses::TurnServer &turnServer) } return ret; } -} +} // namespace #include "moc_CallManager.cpp" diff --git a/src/voip/CallManager.h b/src/voip/CallManager.h index c0fd0831..2490fbf8 100644 --- a/src/voip/CallManager.h +++ b/src/voip/CallManager.h @@ -133,7 +133,11 @@ private: QTimer turnServerTimer_; QMediaPlayer player_; std::vector screenShareTypes_; +#ifndef Q_OS_WINDOWS std::vector> windows_; +#else + std::vector> windows_; +#endif std::vector rejectCallPartyIDs_; template diff --git a/src/voip/WebRTCSession.cpp b/src/voip/WebRTCSession.cpp index a10e24ef..ba1d5424 100644 --- a/src/voip/WebRTCSession.cpp +++ b/src/voip/WebRTCSession.cpp @@ -23,6 +23,7 @@ #include "voip/ScreenCastPortal.h" #ifdef GSTREAMER_AVAILABLE +#include "MainWindow.h" extern "C" { #include "gst/gl/gstgldisplay.h" @@ -330,33 +331,66 @@ GstElement * newVideoSinkChain(GstElement *pipe) { // use compositor for now; acceleration needs investigation - GstElement *queue = gst_element_factory_make("queue", nullptr); - GstElement *compositor = gst_element_factory_make("compositor", "compositor"); - GstElement *glupload = gst_element_factory_make("glupload", nullptr); - GstElement *qmlglsink = gst_element_factory_make("qml6glsink", nullptr); - GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr); + GstElement *queue = gst_element_factory_make("queue", nullptr); + + auto graphicsApi = MainWindow::instance()->graphicsApi(); + GstElement *compositor = gst_element_factory_make( + graphicsApi == QSGRendererInterface::OpenGL ? "compositor" : "d3d11compositor", "compositor"); g_object_set(compositor, "background", 1, nullptr); - g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr); - g_object_set(glsinkbin, "sink", qmlglsink, nullptr); - gst_bin_add_many(GST_BIN(pipe), queue, compositor, glupload, glsinkbin, nullptr); - gst_element_link_many(queue, compositor, glupload, glsinkbin, nullptr); - gst_element_sync_state_with_parent(queue); - gst_element_sync_state_with_parent(compositor); - gst_element_sync_state_with_parent(glupload); - gst_element_sync_state_with_parent(glsinkbin); - - // to propagate context (hopefully) - gst_element_set_state(qmlglsink, GST_STATE_READY); - - // Workaround: On wayland, when egl is used, gstreamer might terminate the display connection. - // Prevent that by "leaking" a reference to the display. See - // https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3743 - if (QGuiApplication::platformName() == QStringLiteral("wayland")) { - auto context = gst_element_get_context(qmlglsink, "gst.gl.GLDisplay"); - if (context) { - GstGLDisplay *display; - gst_context_get_gl_display(context, &display); + switch (graphicsApi) { + case QSGRendererInterface::OpenGL: { + GstElement *glupload = gst_element_factory_make("glupload", nullptr); + GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr); + GstElement *qmlglsink = gst_element_factory_make("qml6glsink", nullptr); + GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr); + + g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr); + g_object_set(glsinkbin, "sink", qmlglsink, nullptr); + gst_bin_add_many( + GST_BIN(pipe), queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr); + gst_element_link_many(queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr); + + gst_element_sync_state_with_parent(queue); + gst_element_sync_state_with_parent(compositor); + gst_element_sync_state_with_parent(glupload); + gst_element_sync_state_with_parent(glcolorconvert); + gst_element_sync_state_with_parent(glsinkbin); + + // to propagate context (hopefully) + gst_element_set_state(qmlglsink, GST_STATE_READY); + + // Workaround: On wayland, when egl is used, gstreamer might terminate the display + // connection. Prevent that by "leaking" a reference to the display. See + // https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3743 + if (QGuiApplication::platformName() == QStringLiteral("wayland")) { + auto context = gst_element_get_context(qmlglsink, "gst.gl.GLDisplay"); + if (context) { + GstGLDisplay *display; + gst_context_get_gl_display(context, &display); + } } + } break; + case QSGRendererInterface::Direct3D11: { + GstElement *d3d11upload = gst_element_factory_make("d3d11upload", nullptr); + GstElement *d3d11colorconvert = gst_element_factory_make("d3d11colorconvert", nullptr); + GstElement *qmld3d11sink = gst_element_factory_make("qml6d3d11sink", nullptr); + + g_object_set(qmld3d11sink, "widget", WebRTCSession::instance().getVideoItem(), nullptr); + gst_bin_add_many( + GST_BIN(pipe), queue, d3d11upload, compositor, d3d11colorconvert, qmld3d11sink, nullptr); + gst_element_link_many( + queue, d3d11upload, compositor, d3d11colorconvert, qmld3d11sink, nullptr); + + gst_element_sync_state_with_parent(queue); + gst_element_sync_state_with_parent(compositor); + gst_element_sync_state_with_parent(d3d11upload); + gst_element_sync_state_with_parent(d3d11colorconvert); + + // to propagate context (hopefully) + gst_element_set_state(qmld3d11sink, GST_STATE_READY); + } break; + default: + break; } return queue; @@ -606,20 +640,20 @@ WebRTCSession::havePlugins(bool isVideo, if (!initialised_ && !init(errorMessage)) return false; - static constexpr std::initializer_list audio_elements = { - "audioconvert", - "audioresample", - "autoaudiosink", - "capsfilter", - "decodebin", - "opusenc", - "queue", - "rtpopuspay", - "volume", - "webrtcbin", - }; - - static constexpr std::initializer_list video_elements = { + static constexpr std::initializer_list audio_elements = {"audioconvert", + "audioresample", + "autoaudiosink", + "capsfilter", + "decodebin", + "opusenc", + "queue", + "rtpopuspay", + "volume", + "webrtcbin", + "nicesrc", + "nicesink"}; + + static constexpr std::initializer_list gl_video_elements = { "compositor", "glsinkbin", "glupload", @@ -631,6 +665,19 @@ WebRTCSession::havePlugins(bool isVideo, "vp8enc", }; + static constexpr std::initializer_list d3d11_video_elements = { + "compositor", + "d3d11colorconvert", + "d3d11videosink", + "d3d11upload", + "qml6d3d11sink", + "rtpvp8pay", + "tee", + "videoconvert", + "videoscale", + "vp8enc", + }; + std::string strError("Missing GStreamer elements: "); GstRegistry *registry = gst_registry_get(); @@ -654,8 +701,19 @@ WebRTCSession::havePlugins(bool isVideo, haveVoicePlugins_ = check_plugins(audio_elements); // check both elements at once - if (isVideo) - haveVideoPlugins_ = check_plugins(video_elements); + if (isVideo) { + switch (MainWindow::instance()->graphicsApi()) { + case QSGRendererInterface::OpenGL: + haveVideoPlugins_ = check_plugins(gl_video_elements); + break; + case QSGRendererInterface::Direct3D11: + haveVideoPlugins_ = check_plugins(d3d11_video_elements); + break; + default: + haveVideoPlugins_ = false; + break; + } + } bool haveScreensharePlugins = false; if (isScreenshare) { @@ -663,6 +721,8 @@ WebRTCSession::havePlugins(bool isVideo, if (haveScreensharePlugins) { if (QGuiApplication::platformName() == QStringLiteral("wayland")) { haveScreensharePlugins = check_plugins({"waylandsink"}); + } else if (QGuiApplication::platformName() == QStringLiteral("windows")) { + haveScreensharePlugins = check_plugins({"d3d11videosink"}); } else { haveScreensharePlugins = check_plugins({"ximagesink"}); } @@ -670,6 +730,9 @@ WebRTCSession::havePlugins(bool isVideo, if (haveScreensharePlugins) { if (screenShareType == ScreenShareType::X11) { haveScreensharePlugins = check_plugins({"ximagesrc"}); + } else if (screenShareType == ScreenShareType::D3D11) { + haveScreensharePlugins = + check_plugins({"d3d11screencapturesrc", "d3d11download", "d3d11convert"}); } else { haveScreensharePlugins = check_plugins({"pipewiresrc"}); } @@ -685,9 +748,19 @@ WebRTCSession::havePlugins(bool isVideo, } if (isVideo || isScreenshare) { - // load qmlglsink to register GStreamer's GstGLVideoItem QML type - GstElement *qmlglsink = gst_element_factory_make("qml6glsink", nullptr); - gst_object_unref(qmlglsink); + switch (MainWindow::instance()->graphicsApi()) { + case QSGRendererInterface::OpenGL: { + // load qmlglsink to register GStreamer's GstGLVideoItem QML type + GstElement *qmlglsink = gst_element_factory_make("qml6glsink", nullptr); + gst_object_unref(qmlglsink); + } break; + case QSGRendererInterface::Direct3D11: { + GstElement *qmld3d11sink = gst_element_factory_make("qml6d3d11sink", nullptr); + gst_object_unref(qmld3d11sink); + } break; + default: + break; + } } return true; } @@ -1027,6 +1100,36 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType) gst_bin_add(GST_BIN(pipe_), ximagesrc); screencastsrc = ximagesrc; + } else if (screenShareType_ == ScreenShareType::D3D11) { + GstElement *d3d11screensrc = + gst_element_factory_make("d3d11screencapturesrc", "screenshare"); + if (!d3d11screensrc) { + nhlog::ui()->error("WebRTC: failed to create d3d11screencapturesrc"); + gst_object_unref(pipe_); + pipe_ = nullptr; + return false; + } + g_object_set( + d3d11screensrc, "window-handle", static_cast(shareWindowId_), nullptr); + g_object_set( + d3d11screensrc, "show-cursor", !settings->screenShareHideCursor(), nullptr); + g_object_set(d3d11screensrc, "do-timestamp", (gboolean)1, nullptr); + gst_bin_add(GST_BIN(pipe_), d3d11screensrc); + + GstElement *d3d11convert = gst_element_factory_make("d3d11convert", nullptr); + gst_bin_add(GST_BIN(pipe_), d3d11convert); + if (!gst_element_link(d3d11screensrc, d3d11convert)) { + nhlog::ui()->error("WebRTC: failed to link d3d11screencapturesrc -> d3d11convert"); + return false; + } + + GstElement *d3d11download = gst_element_factory_make("d3d11download", nullptr); + gst_bin_add(GST_BIN(pipe_), d3d11download); + if (!gst_element_link(d3d11convert, d3d11download)) { + nhlog::ui()->error("WebRTC: failed to link d3d11convert -> d3d11download"); + return false; + } + screencastsrc = d3d11download; } else { ScreenCastPortal &sc_portal = ScreenCastPortal::instance(); GstElement *pipewiresrc = gst_element_factory_make("pipewiresrc", "screenshare"); diff --git a/src/voip/WebRTCSession.h b/src/voip/WebRTCSession.h index 3357bff7..20c32110 100644 --- a/src/voip/WebRTCSession.h +++ b/src/voip/WebRTCSession.h @@ -31,7 +31,8 @@ Q_ENUM_NS(CallType) enum class ScreenShareType { X11, - XDP + XDP, + D3D11 }; Q_ENUM_NS(ScreenShareType) -- cgit v1.2.3