Feature : Video support for one-one call.

- Set Mediastreamer plugin folder.
- CameraGui component to manage video.
- Sticker component to switch between initials/avatar and video.
- Remote video detection on Call.
- Fix binary shader files to support at least Qt 6.4.
- Use MSQOgl  Mediatsreamer2 filter and activate video capabilities.
- Add a preview on Call view.
This commit is contained in:
Julien Wadel 2023-12-06 16:55:26 +01:00
parent a43430fa34
commit a93e646ce4
20 changed files with 493 additions and 16 deletions

View file

@ -41,15 +41,16 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG -DQT_QML_DEBUG -DQT_
set(CMAKE_INCLUDE_CURRENT_DIR ON)#useful for config.h
include(application_info.cmake)
if(APPLE)
if(MS2_PLUGINS_LOCATION)
set(MSPLUGINS_DIR ${MS2_PLUGINS_LOCATION})
else()
set(MSPLUGINS_DIR "Frameworks/mediastreamer2.framework/Versions/A/Libraries")
endif()
if(MEDIASTREAMER2_PLUGINS_LOCATION)
set(MSPLUGINS_DIR ${MEDIASTREAMER2_PLUGINS_LOCATION})
elseif(APPLE)
set(MSPLUGINS_DIR "Frameworks/mediastreamer2.framework/Versions/A/Libraries")
else()
set(MSPLUGINS_DIR "plugins/mediastreamer")
set(MSPLUGINS_DIR "${CMAKE_INSTALL_LIBDIR}/mediastreamer/plugins")
endif()
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake" "${CMAKE_CURRENT_BINARY_DIR}/config.h")
if(${Qt6_VERSION} VERSION_LESS "6.3.0")

View file

@ -36,6 +36,7 @@
#include "core/account/AccountProxy.hpp"
#include "core/call/CallCore.hpp"
#include "core/call/CallGui.hpp"
#include "core/camera/CameraGui.hpp"
#include "core/friend/FriendCore.hpp"
#include "core/friend/FriendGui.hpp"
#include "core/logger/QtLogger.hpp"
@ -123,6 +124,7 @@ void App::init() {
},
Qt::QueuedConnection);
mEngine->load(url);
// mEngine->load(u"qrc:/Linphone/view/Prototype/CameraPrototype.qml"_qs);
}
void App::initCppInterfaces() {
@ -150,7 +152,7 @@ void App::initCppInterfaces() {
qmlRegisterType<FriendGui>(Constants::MainQmlUri, 1, 0, "FriendGui");
qmlRegisterUncreatableType<FriendCore>(Constants::MainQmlUri, 1, 0, "FriendCore", QLatin1String("Uncreatable"));
qmlRegisterType<MagicSearchProxy>(Constants::MainQmlUri, 1, 0, "MagicSearchProxy");
qmlRegisterType<CameraGui>(Constants::MainQmlUri, 1, 0, "CameraGui");
LinphoneEnums::registerMetaTypes();
}

View file

@ -6,6 +6,8 @@ list(APPEND _LINPHONEAPP_SOURCES
core/App.cpp
core/call/CallCore.cpp
core/call/CallGui.cpp
core/camera/CameraGui.cpp
core/camera/CameraDummy.cpp
core/friend/FriendCore.cpp
core/friend/FriendGui.cpp
core/logger/QtLogger.cpp

View file

@ -75,6 +75,9 @@ void CallCore::setSelf(QSharedPointer<CallCore> me) {
mAccountModelConnection->makeConnectToModel(&CallModel::microphoneMutedChanged, [this](bool isMuted) {
mAccountModelConnection->invokeToCore([this, isMuted]() { setMicrophoneMuted(isMuted); });
});
mAccountModelConnection->makeConnectToModel(&CallModel::remoteVideoEnabledChanged, [this](bool enabled) {
mAccountModelConnection->invokeToCore([this, enabled]() { setRemoteVideoEnabled(enabled); });
});
// mAccountModelConnection->makeConnect(this, &CallCore::lSetSpeakerMuted, [this](bool isMuted) {
// mAccountModelConnection->invokeToModel([this, isMuted]() { mCallModel->setSpeakerMuted(isMuted); });
// });
@ -243,6 +246,17 @@ void CallCore::setPeerSecured(bool secured) {
}
}
bool CallCore::getRemoteVideoEnabled() const {
return mRemoteVideoEnabled;
}
void CallCore::setRemoteVideoEnabled(bool enabled) {
if (mRemoteVideoEnabled != enabled) {
mRemoteVideoEnabled = enabled;
emit remoteVideoEnabledChanged(mRemoteVideoEnabled);
}
}
LinphoneEnums::CallState CallCore::getTransferState() const {
return mTransferState;
}
@ -254,3 +268,7 @@ void CallCore::setTransferState(LinphoneEnums::CallState state, const QString &m
emit transferStateChanged();
}
}
std::shared_ptr<CallModel> CallCore::getModel() const {
return mCallModel;
}

View file

@ -42,6 +42,8 @@ class CallCore : public QObject, public AbstractObject {
Q_PROPERTY(bool paused READ getPaused WRITE lSetPaused NOTIFY pausedChanged)
Q_PROPERTY(QString peerAddress MEMBER mPeerAddress CONSTANT)
Q_PROPERTY(bool peerSecured READ getPeerSecured WRITE setPeerSecured NOTIFY peerSecuredChanged)
Q_PROPERTY(
bool remoteVideoEnabled READ getRemoteVideoEnabled WRITE setRemoteVideoEnabled NOTIFY remoteVideoEnabledChanged)
Q_PROPERTY(LinphoneEnums::CallState transferState READ getTransferState NOTIFY transferStateChanged)
public:
@ -78,9 +80,14 @@ public:
bool getPeerSecured() const;
void setPeerSecured(bool secured);
bool getRemoteVideoEnabled() const;
void setRemoteVideoEnabled(bool enabled);
LinphoneEnums::CallState getTransferState() const;
void setTransferState(LinphoneEnums::CallState state, const QString &message);
std::shared_ptr<CallModel> getModel() const;
signals:
void statusChanged(LinphoneEnums::CallStatus status);
void stateChanged(LinphoneEnums::CallState state);
@ -93,6 +100,7 @@ signals:
void pausedChanged();
void transferStateChanged();
void peerSecuredChanged();
void remoteVideoEnabledChanged(bool remoteVideoEnabled);
// Linphone commands
void lAccept(bool withVideo); // Accept an incoming call
@ -137,6 +145,7 @@ private:
bool mMicrophoneMuted;
bool mCameraEnabled;
bool mPaused = false;
bool mRemoteVideoEnabled = false;
QSharedPointer<SafeConnection<CallCore, CallModel>> mAccountModelConnection;
DECLARE_ABSTRACT_OBJECT

View file

@ -0,0 +1,41 @@
/*
* Copyright (c) 2010-2024 Belledonne Communications SARL.
*
* This file is part of linphone-desktop
* (see https://www.linphone.org).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <QOpenGLFramebufferObject>
#include <QQuickWindow>
#include <QThread>
#include <QTimer>
#include "CameraDummy.hpp"
// =============================================================================
CameraDummy::CameraDummy() {
}
QOpenGLFramebufferObject *CameraDummy::createFramebufferObject(const QSize &size) {
return new QOpenGLFramebufferObject(size);
}
void CameraDummy::render() {
}
void CameraDummy::synchronize(QQuickFramebufferObject *item) {
}

View file

@ -0,0 +1,37 @@
/*
* Copyright (c) 2024 Belledonne Communications SARL.
*
* This file is part of linphone-desktop
* (see https://www.linphone.org).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef CAMERA_DUMMY_H_
#define CAMERA_DUMMY_H_
#include <QMutex>
#include <QQuickFramebufferObject>
// =============================================================================
class CameraDummy : public QQuickFramebufferObject::Renderer {
public:
CameraDummy();
QOpenGLFramebufferObject *createFramebufferObject(const QSize &size) override;
void render() override;
void synchronize(QQuickFramebufferObject *item) override;
};
#endif

View file

@ -0,0 +1,114 @@
/*
* Copyright (c) 2010-2024 Belledonne Communications SARL.
*
* This file is part of linphone-desktop
* (see https://www.linphone.org).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <QOpenGLFramebufferObject>
#include <QQuickWindow>
#include <QThread>
#include <QTimer>
#include "CameraDummy.hpp"
#include "CameraGui.hpp"
#include "core/App.hpp"
#include "core/call/CallCore.hpp"
#include "core/call/CallGui.hpp"
DEFINE_ABSTRACT_OBJECT(CameraGui)
// =============================================================================
CameraGui::CameraGui(QQuickItem *parent) : QQuickFramebufferObject(parent) {
mustBeInMainThread(getClassName());
// The fbo content must be y-mirrored because the ms rendering is y-inverted.
setMirrorVertically(true);
mRefreshTimer.setInterval(1000 / mMaxFps);
connect(&mRefreshTimer, &QTimer::timeout, this, &QQuickFramebufferObject::update, Qt::QueuedConnection);
mRefreshTimer.start();
}
CameraGui::~CameraGui() {
mustBeInMainThread("~" + getClassName());
}
QQuickFramebufferObject::Renderer *CameraGui::createRenderer() const {
QQuickFramebufferObject::Renderer *renderer = NULL;
// A renderer is mandatory, we cannot wait async.
switch (getSourceLocation()) {
case CorePreview:
App::postModelSync([this, &renderer]() {
auto coreModel = CoreModel::getInstance();
if (coreModel) {
auto core = coreModel->getCore();
if (!core) return;
core->enableVideoPreview(true);
renderer = (QQuickFramebufferObject::Renderer *)core->createNativePreviewWindowId();
if (renderer) core->setNativePreviewWindowId(renderer);
}
});
break;
case Call:
App::postModelSync([this, &renderer]() {
auto call = mCallGui->getCore()->getModel()->getMonitor();
if (call) {
// qInfo() << "[Camera] (" << mQmlName << ") Setting Camera to CallModel";
renderer = (QQuickFramebufferObject::Renderer *)call->createNativeVideoWindowId();
if (renderer) call->setNativeVideoWindowId(renderer);
}
});
default: {
}
}
if (!renderer) {
QTimer::singleShot(1, this, &CameraGui::isNotReady);
renderer = new CameraDummy(); // Used to fill a renderer to avoid pushing a NULL.
QTimer::singleShot(1000, this, &CameraGui::requestNewRenderer);
} else QTimer::singleShot(1, this, &CameraGui::isReady); // Hack because of constness of createRenderer()
return renderer;
}
bool CameraGui::getIsReady() const {
return mIsReady;
}
void CameraGui::setIsReady(bool isReady) {
if (mIsReady != isReady) {
mIsReady = isReady;
emit isReadyChanged(mIsReady);
}
}
void CameraGui::isReady() {
setIsReady(true);
}
void CameraGui::isNotReady() {
setIsReady(false);
}
CallGui *CameraGui::getCallGui() const {
return mCallGui;
}
void CameraGui::setCallGui(CallGui *callGui) {
if (mCallGui != callGui) {
mCallGui = callGui;
emit callGuiChanged(mCallGui);
}
}
CameraGui::WindowIdLocation CameraGui::getSourceLocation() const {
if (mCallGui != nullptr) return Call;
else return CorePreview;
}

View file

@ -0,0 +1,70 @@
/*
* Copyright (c) 2010-2024 Belledonne Communications SARL.
*
* This file is part of linphone-desktop
* (see https://www.linphone.org).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef CAMERA_GUI_H_
#define CAMERA_GUI_H_
#include <memory>
#include "tool/AbstractObject.hpp"
#include <QMutex>
#include <QQuickFramebufferObject>
#include <QTimer>
// =============================================================================
class CallGui;
class CameraGui : public QQuickFramebufferObject, public AbstractObject {
Q_OBJECT
Q_PROPERTY(bool isReady READ getIsReady NOTIFY isReadyChanged)
Q_PROPERTY(CallGui *call READ getCallGui WRITE setCallGui NOTIFY callGuiChanged);
public:
CameraGui(QQuickItem *parent = Q_NULLPTR);
virtual ~CameraGui();
QQuickFramebufferObject::Renderer *createRenderer() const override;
bool getIsReady() const;
void setIsReady(bool isReady);
void isReady();
void isNotReady();
CallGui *getCallGui() const;
void setCallGui(CallGui *callGui);
typedef enum { None = -1, CorePreview = 0, Call, Device, Player, Core } WindowIdLocation;
WindowIdLocation getSourceLocation() const;
signals:
void requestNewRenderer();
void isReadyChanged(bool isReady);
void callGuiChanged(CallGui *callGui);
private:
bool mIsReady = false;
QTimer mRefreshTimer;
int mMaxFps = 30;
CallGui *mCallGui = nullptr;
DECLARE_ABSTRACT_OBJECT
};
#endif

View file

@ -146,6 +146,12 @@ void CallModel::onInfoMessageReceived(const std::shared_ptr<linphone::Call> &cal
void CallModel::onStateChanged(const std::shared_ptr<linphone::Call> &call,
linphone::Call::State state,
const std::string &message) {
if (state == linphone::Call::State::StreamsRunning) {
// After UpdatedByRemote, video direction could be changed.
auto params = call->getRemoteParams();
emit remoteVideoEnabledChanged(params && params->videoEnabled());
emit cameraEnabledChanged(call->cameraEnabled());
}
emit stateChanged(state, message);
}

View file

@ -55,6 +55,7 @@ signals:
void cameraEnabledChanged(bool enabled);
void durationChanged(int);
void pausedChanged(bool paused);
void remoteVideoEnabledChanged(bool remoteVideoEnabled);
private:
QTimer mDurationTimer;

View file

@ -78,6 +78,18 @@ void CoreModel::start() {
setPathsAfterCreation();
mCore->enableFriendListSubscription(true);
mCore->enableRecordAware(true);
mCore->setVideoDisplayFilter("MSQOGL");
mCore->usePreviewWindow(true);
// Force capture/display.
// Useful if the app was built without video support.
// (The capture/display attributes are reset by the core in this case.)
auto config = mCore->getConfig();
if (mCore->videoSupported()) {
config->setInt("video", "capture", 1);
config->setInt("video", "display", 1);
}
mCore->enableVideoPreview(false); // SDK doesn't write the state in configuration if not ready.
config->setInt("video", "show_local", 0); // So : write ourself to turn off camera before starting the core.
mCore->start();
setPathAfterStart();
mIterateTimer->start();

View file

@ -60,6 +60,10 @@ public:
if (mMonitor && mSelf) mMonitor->addListener(self);
}
std::shared_ptr<LinphoneClass> getMonitor() const {
return mMonitor;
}
protected:
std::shared_ptr<LinphoneClass> mMonitor;
std::shared_ptr<ListenerClass> mSelf = nullptr;

View file

@ -312,6 +312,11 @@ Window {
ColumnLayout {
anchors.centerIn: parent
spacing: 2
Sticker{
Layout.fillHeight: true
Layout.fillWidth: true
call: mainWindow.call
}
// Avatar {
// Layout.alignment: Qt.AlignCenter
// visible: mainWindow.isInContactList
@ -505,7 +510,6 @@ Window {
|| mainWindow.callState == LinphoneEnums.CallState.IncomingReceived
? bottomButtonsLayout.columns - 1 : 0
BottomButton {
enabled: false
enabledIcon: AppIcons.videoCamera
disabledIcon: AppIcons.videoCameraSlash
checked: !mainWindow.call.core.cameraEnabled
@ -527,5 +531,15 @@ Window {
}
}
}
}
Sticker{
height: 100
width: 100
anchors.right: parent.right
anchors.bottom: parent.bottom
visible: mainWindow.call.core.cameraEnabled
AccountProxy{
id: accounts
}
account: accounts.defaultAccount
}
}

View file

@ -17,6 +17,7 @@ list(APPEND _LINPHONEAPP_QML_FILES
view/Item/BusyIndicator.qml
view/Item/Button.qml
view/Item/Carousel.qml
view/Item/CheckBox.qml
view/Item/ComboBox.qml
@ -24,6 +25,7 @@ list(APPEND _LINPHONEAPP_QML_FILES
view/Item/Contact/Avatar.qml
view/Item/Contact/Contact.qml
view/Item/Contact/ContactDescription.qml
view/Item/Contact/Sticker.qml
view/Item/DesktopPopup.qml
view/Item/DigitInput.qml
@ -59,6 +61,7 @@ list(APPEND _LINPHONEAPP_QML_FILES
view/Prototype/PhoneNumberPrototype.qml
view/Prototype/AccountsPrototype.qml
view/Prototype/CallPrototype.qml
view/Prototype/CameraPrototype.qml
view/Prototype/FriendPrototype.qml
view/Prototype/ItemPrototype.qml
)

View file

@ -6,17 +6,24 @@ import QtQuick.Effects
import Linphone
import UtilsCpp
// Avatar using initial of the username in case
// they don't have any profile picture
// Fill contact, account or call
// Initials will be displayed if there isn't any avatar.
// TODO : get FriendGui from Call.
StackView{
id: mainItem
property FriendGui contact
property AccountGui account
property string address: account ? account.core.identityAddress : ''
property AccountGui account: null
property FriendGui contact: null
property CallGui call: null
property string address: account
? account.core.identityAddress
: call
? call.core.peerAddress
: ''
property var displayNameObj: UtilsCpp.getDisplayName(address)
property bool haveAvatar: (account && account.core.pictureUri )
|| (contact && contact.core.pictureUri)
onHaveAvatarChanged: replace(haveAvatar ? avatar : initials, StackView.Immediate)
initialItem: haveAvatar ? avatar : initials

View file

@ -0,0 +1,57 @@
import QtQuick
import QtQuick.Layouts
import QtQuick.Controls as Control
import Linphone
import UtilsCpp 1.0
// Display a sticker from a call or from an account.
// The Avatar is shown while the camera become available.
// The loader restart in case of resetting the renderer. This allow to display the avatar while loading.
// TODO: sizes, colors, decorations
Rectangle{
id: mainItem
height: 300
width: 200
property CallGui call: null
property AccountGui account: null
color: 'gray'
Avatar{
anchors.centerIn: parent
height: 100
width: height
account: mainItem.account
call: mainItem.call
visible: !cameraLoader.active || cameraLoader.status != Loader.Ready || !cameraLoader.item.isReady
}
Loader{
id: cameraLoader
anchors.fill: parent
Timer{
id: resetTimer
interval: 1
onTriggered: {cameraLoader.active=false; cameraLoader.active=true;}
}
active: mainItem.visible && (!call || call.core.remoteVideoEnabled)
sourceComponent: cameraComponent
}
Component{
id: cameraComponent
Item{
height: cameraLoader.height
width: cameraLoader.width
property bool isReady: cameraItem.visible
CameraGui{
id: cameraItem
anchors.fill: parent
visible: isReady
call: mainItem.call
onRequestNewRenderer: {
console.log("Request new renderer")
resetTimer.restart()
}
}
}
}
}

View file

@ -0,0 +1,79 @@
import QtQuick
import QtQuick.Layouts
import QtQuick.Controls as Control
import Linphone
import UtilsCpp 1.0
Window{
id: mainItem
height: 400
width: 800
visible: true
Rectangle{
anchors.centerIn: parent
height: 300
width: 200
color: 'gray'
Avatar{
anchors.centerIn: parent
height: 100
width: height
address: 'sip:jul@toto.com'
}
Loader{
id: cameraLoader
anchors.fill: parent
Timer{
id: resetTimer
interval: 1
onTriggered: {cameraLoader.active=false; cameraLoader.active=true;}
}
active: true
sourceComponent: cameraComponent
}
Component{
id: cameraComponent
Rectangle{
height: cameraLoader.height
width: cameraLoader.width
color: 'red'
CameraGui{
id: cameraItem
anchors.fill: parent
visible: isReady
onVisibleChanged: console.log('Ready?'+visible)
onRequestNewRenderer: {
console.log("Request new renderer")
resetTimer.restart()
}
}
}
}
}
/*
Control.StackView{
id: stackView
anchors.fill: parent
initialItem: cameraComponent
Component{
id: avatarComponent
Avatar{
}
}
Component{
id: cameraComponent
CameraGui{
id: cameraItem
onRequestNewRenderer: {
console.log("Request new renderer")
stackView.replace(cameraComponent, Control.StackView.Immediate)
}
}
}
}
*/
}