mirrors/linphone-desktop
1
0
Fork 0
mirror of https://gitlab.linphone.org/BC/public/linphone-desktop.git synced 2026-01-17 11:28:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Crash fix when switching timelines containing video files.

Browse source
This commit is contained in:
Julien Wadel 2023-05-11 16:08:09 +02:00
parent 25a5f33356
commit b2219c66ed
8 changed files with 304 additions and 184 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
linphone-app/CMakeLists.txt
View file

@ -261,6 +261,7 @@ set(SOURCES
src/components/other/images/ImageModel.cpp
src/components/other/images/ImageListModel.cpp
src/components/other/images/ImageProxyModel.cpp
src/components/other/images/VideoFrameGrabber.cpp
src/components/other/text-to-speech/TextToSpeech.cpp
src/components/other/timeZone/TimeZoneModel.cpp
src/components/other/timeZone/TimeZoneListModel.cpp
@ -402,6 +403,7 @@ set(HEADERS
src/components/other/images/ImageModel.hpp
src/components/other/images/ImageListModel.hpp
src/components/other/images/ImageProxyModel.hpp
src/components/other/images/VideoFrameGrabber.hpp
src/components/other/desktop-tools/DesktopTools.hpp
src/components/other/text-to-speech/TextToSpeech.hpp
src/components/other/timeZone/TimeZoneModel.hpp

23
linphone-app/src/app/providers/ThumbnailProvider.cpp
View file

@ -27,14 +27,21 @@
const QString ThumbnailProvider::ProviderId = "thumbnail";
ThumbnailProvider::ThumbnailProvider () : QQuickImageProvider(
QQmlImageProviderBase::Image,
QQmlImageProviderBase::ForceAsynchronousImageLoading
) {
ThumbnailAsyncImageResponse::ThumbnailAsyncImageResponse(const QString &id, const QSize &requestedSize) {
mPath = id;
connect(&mListener, &VideoFrameGrabberListener::imageGrabbed, this, &ThumbnailAsyncImageResponse::imageGrabbed);
ImageModel::retrieveImageAsync(id, &mListener);
}
QImage ThumbnailProvider::requestImage (const QString &id, QSize *size, const QSize &) {
QImage image = ImageModel::createThumbnail(id);
*size = image.size();
return image;
void ThumbnailAsyncImageResponse::imageGrabbed(QImage image) {
mImage = ImageModel::createThumbnail(mPath, image);
emit finished();
}
QQuickTextureFactory *ThumbnailAsyncImageResponse::textureFactory() const {
return QQuickTextureFactory::textureFactoryForImage(mImage);
}
QQuickImageResponse *ThumbnailProvider::requestImageResponse(const QString &id, const QSize &requestedSize){
ThumbnailAsyncImageResponse *response = new ThumbnailAsyncImageResponse(id, requestedSize);
return response;
}

29
linphone-app/src/app/providers/ThumbnailProvider.hpp
View file

@ -21,17 +21,32 @@
#ifndef THUMBNAIL_PROVIDER_H_
#define THUMBNAIL_PROVIDER_H_
#include <QQuickImageProvider>
#include <QQuickAsyncImageProvider>
#include "components/other/images/VideoFrameGrabber.hpp"
// Thumbnails are created asynchronously with QQuickAsyncImageProvider and not QQuickImageProvider.
// This ensure to have async objects like QMediaPlayer and QAbstractVideoSurface while keeping them in the main thread (mandatory for VideoSurface).
// If not, there seems to have some deadlocks in Qt library when GUI objects are deleted while still playing media.
// =============================================================================
class ThumbnailProvider : public QQuickImageProvider {
class ThumbnailAsyncImageResponse : public QQuickImageResponse {
public:
ThumbnailProvider ();
ThumbnailAsyncImageResponse(const QString &id, const QSize &requestedSize);
QQuickTextureFactory *textureFactory() const override; // Convert QImage into texture. If Image is null, then sourceSize will be egal to 0. So there will be no errors.
void imageGrabbed(QImage image);
QImage mImage;
QString mPath;
VideoFrameGrabberListener mListener;
};
QImage requestImage (const QString &id, QSize *size, const QSize &requestedSize) override;
static const QString ProviderId;
class ThumbnailProvider : public QQuickAsyncImageProvider {
public:
virtual QQuickImageResponse *requestImageResponse(const QString &id, const QSize &requestedSize) override;
static const QString ProviderId;
};
#endif // THUMBNAIL_PROVIDER_H_

193
linphone-app/src/components/other/images/ImageModel.cpp
View file

@ -30,76 +30,10 @@
#include "components/Components.hpp"
#include "components/core/CoreManager.hpp"
#include "utils/QExifImageHeader.hpp"
#include "VideoFrameGrabber.hpp"
#include <QVideoSurfaceFormat>
VideoFrameGrabber::VideoFrameGrabber( QObject *parent)
: QAbstractVideoSurface(parent){
}
QList<QVideoFrame::PixelFormat> VideoFrameGrabber::supportedPixelFormats(QAbstractVideoBuffer::HandleType handleType) const {
Q_UNUSED(handleType);
return QList<QVideoFrame::PixelFormat>()
<< QVideoFrame::Format_ARGB32
<< QVideoFrame::Format_ARGB32_Premultiplied
<< QVideoFrame::Format_RGB32
<< QVideoFrame::Format_RGB24
<< QVideoFrame::Format_RGB565
<< QVideoFrame::Format_RGB555
<< QVideoFrame::Format_ARGB8565_Premultiplied
<< QVideoFrame::Format_BGRA32
<< QVideoFrame::Format_BGRA32_Premultiplied
<< QVideoFrame::Format_BGR32
<< QVideoFrame::Format_BGR24
<< QVideoFrame::Format_BGR565
<< QVideoFrame::Format_BGR555
<< QVideoFrame::Format_BGRA5658_Premultiplied
<< QVideoFrame::Format_AYUV444
<< QVideoFrame::Format_AYUV444_Premultiplied
<< QVideoFrame::Format_YUV444
<< QVideoFrame::Format_YUV420P
<< QVideoFrame::Format_YV12
<< QVideoFrame::Format_UYVY
<< QVideoFrame::Format_YUYV
<< QVideoFrame::Format_NV12
<< QVideoFrame::Format_NV21
<< QVideoFrame::Format_IMC1
<< QVideoFrame::Format_IMC2
<< QVideoFrame::Format_IMC3
<< QVideoFrame::Format_IMC4
<< QVideoFrame::Format_Y8
<< QVideoFrame::Format_Y16
<< QVideoFrame::Format_Jpeg
<< QVideoFrame::Format_CameraRaw
<< QVideoFrame::Format_AdobeDng;
}
bool VideoFrameGrabber::isFormatSupported(const QVideoSurfaceFormat &format) const {
const QImage::Format imageFormat = QVideoFrame::imageFormatFromPixelFormat(format.pixelFormat());
const QSize size = format.frameSize();
return imageFormat != QImage::Format_Invalid
&& !size.isEmpty()
&& format.handleType() == QAbstractVideoBuffer::NoHandle;
}
bool VideoFrameGrabber::start(const QVideoSurfaceFormat &format){
return QAbstractVideoSurface::start(format);
}
void VideoFrameGrabber::stop() {
QAbstractVideoSurface::stop();
}
bool VideoFrameGrabber::present(const QVideoFrame &frame){
if (frame.isValid()) {
emit frameAvailable(frame.image());
return true;
}else
return false;
}
// =============================================================================
ImageModel::ImageModel (const QString& id, const QString& path, const QString& description, QObject * parent) : QObject(parent) {
@ -147,11 +81,50 @@ void ImageModel::setUrl(const QUrl& url){
setPath(url.toString(QUrl::RemoveScheme));
}
QImage ImageModel::createThumbnail(const QString& path){
QImage ImageModel::createThumbnail(const QString& path, QImage originalImage){
QImage thumbnail;
if (!originalImage.isNull()){
int rotation = 0;
QExifImageHeader exifImageHeader;
if (exifImageHeader.loadFromJpeg(path))
rotation = int(exifImageHeader.value(QExifImageHeader::ImageTag::Orientation).toShort());
// Fill with color to replace transparency with white color instead of black (default).
QImage image(originalImage.size(), originalImage.format());
image.fill(QColor(Qt::white).rgb());
QPainter painter(&image);
painter.drawImage(0, 0, originalImage);
//--------------------
double factor = image.width() / (double)image.height();
Qt::AspectRatioMode aspectRatio = Qt::KeepAspectRatio;
if(factor < 0.2 || factor > 5)
aspectRatio = Qt::KeepAspectRatioByExpanding;
thumbnail = image.scaled(
Constants::ThumbnailImageFileWidth, Constants::ThumbnailImageFileHeight,
aspectRatio , Qt::SmoothTransformation
);
if(aspectRatio == Qt::KeepAspectRatioByExpanding)
thumbnail = thumbnail.copy(0,0,Constants::ThumbnailImageFileWidth, Constants::ThumbnailImageFileHeight);
if (rotation != 0) {
QTransform transform;
if (rotation == 3 || rotation == 4)
transform.rotate(180);
else if (rotation == 5 || rotation == 6)
transform.rotate(90);
else if (rotation == 7 || rotation == 8)
transform.rotate(-90);
thumbnail = thumbnail.transformed(transform);
if (rotation == 2 || rotation == 4 || rotation == 5 || rotation == 7)
thumbnail = thumbnail.mirrored(true, false);
}
}
return thumbnail;
}
void ImageModel::retrieveImageAsync(const QString& path, VideoFrameGrabberListener* requester){
QImage thumbnail;
if(QFileInfo(path).isFile()){
QImage originalImage(path);
if( originalImage.isNull()){// Try to determine format from headers
QImageReader reader(path);
reader.setDecideFormatFromContent(true);
@ -159,83 +132,13 @@ QImage ImageModel::createThumbnail(const QString& path){
if(!format.isEmpty())
originalImage = QImage(path, format);
else if(Utils::isVideo(path)){
QObject context;
int mediaStep = 0;
QMediaPlayer player(&context);
VideoFrameGrabber grabber(&context);
// Media connections
QObject::connect(&player, QOverload<QMediaPlayer::Error>::of(&QMediaPlayer::error), &context, [&context, &mediaStep, path](QMediaPlayer::Error error) mutable{
mediaStep = -1;
});
QObject::connect(&player, &QMediaPlayer::mediaStatusChanged, &context, [&context, &player, &mediaStep](QMediaPlayer::MediaStatus status) mutable{
switch(status){
case QMediaPlayer::LoadedMedia : if(mediaStep == 0){
if( player.isVideoAvailable() )
mediaStep = 1;
else
mediaStep = -1;
}
break;
case QMediaPlayer::UnknownMediaStatus:
case QMediaPlayer::InvalidMedia:
case QMediaPlayer::EndOfMedia:
mediaStep = -1;
break;
default:{}
}
});
QObject::connect(&grabber, &VideoFrameGrabber::frameAvailable, &context, [&context,&originalImage, &player](QImage frame) mutable{
originalImage = frame.copy();
player.stop();
}, Qt::DirectConnection);
// Processing
player.setVideoOutput(&grabber);
player.setMedia(QUrl::fromLocalFile(path));
do{
qApp->processEvents();
if(mediaStep == 1){
mediaStep = 2;
player.setPosition(player.duration() / 2);
player.play();
}
}while(mediaStep >= 0 );
VideoFrameGrabber *grabber = new VideoFrameGrabber();
connect(grabber, &VideoFrameGrabber::grabFinished, requester, &VideoFrameGrabberListener::imageGrabbed);
grabber->requestFrame(path);
}
}
if (!originalImage.isNull()){
int rotation = 0;
QExifImageHeader exifImageHeader;
if (exifImageHeader.loadFromJpeg(path))
rotation = int(exifImageHeader.value(QExifImageHeader::ImageTag::Orientation).toShort());
// Fill with color to replace transparency with white color instead of black (default).
QImage image(originalImage.size(), originalImage.format());
image.fill(QColor(Qt::white).rgb());
QPainter painter(&image);
painter.drawImage(0, 0, originalImage);
//--------------------
double factor = image.width() / (double)image.height();
Qt::AspectRatioMode aspectRatio = Qt::KeepAspectRatio;
if(factor < 0.2 || factor > 5)
aspectRatio = Qt::KeepAspectRatioByExpanding;
thumbnail = image.scaled(
Constants::ThumbnailImageFileWidth, Constants::ThumbnailImageFileHeight,
aspectRatio , Qt::SmoothTransformation
);
if(aspectRatio == Qt::KeepAspectRatioByExpanding)
thumbnail = thumbnail.copy(0,0,Constants::ThumbnailImageFileWidth, Constants::ThumbnailImageFileHeight);
if (rotation != 0) {
QTransform transform;
if (rotation == 3 || rotation == 4)
transform.rotate(180);
else if (rotation == 5 || rotation == 6)
transform.rotate(90);
else if (rotation == 7 || rotation == 8)
transform.rotate(-90);
thumbnail = thumbnail.transformed(transform);
if (rotation == 2 || rotation == 4 || rotation == 5 || rotation == 7)
thumbnail = thumbnail.mirrored(true, false);
}
if(!originalImage.isNull()){
emit requester->imageGrabbed(originalImage);
}
}
return thumbnail;
}
}

32
linphone-app/src/components/other/images/ImageModel.hpp
View file

@ -29,35 +29,20 @@
#include "utils/LinphoneEnums.hpp"
#include <QAbstractVideoSurface>
#include <QMediaPlayer>
class VideoFrameGrabber : public QAbstractVideoSurface {
Q_OBJECT
public:
VideoFrameGrabber(QObject *parent = 0);
QList<QVideoFrame::PixelFormat> supportedPixelFormats(
QAbstractVideoBuffer::HandleType handleType = QAbstractVideoBuffer::NoHandle) const override;
bool isFormatSupported(const QVideoSurfaceFormat &format) const override;
bool start(const QVideoSurfaceFormat &format) override;
void stop() override;
bool present(const QVideoFrame &frame) override;
signals:
void frameAvailable(QImage frame);
};
class VideoFrameGrabberListener;
class ImageModel : public QObject {
Q_OBJECT
Q_OBJECT
public:
ImageModel (const QString& id, const QString& path, const QString& description, QObject * parent = nullptr);
ImageModel (const QString& id, const QString& path, const QString& description, QObject * parent = nullptr);
Q_PROPERTY(QString path MEMBER mPath WRITE setPath NOTIFY pathChanged)
Q_PROPERTY(QString description MEMBER mDescription WRITE setDescription NOTIFY descriptionChanged)
Q_PROPERTY(QString id MEMBER mId NOTIFY idChanged)
QString getPath() const;
QString getDescription() const;
QString getId() const;
@ -66,13 +51,14 @@ public:
void setDescription(const QString& description);
Q_INVOKABLE void setUrl(const QUrl& url);
static QImage createThumbnail(const QString& path);
static QImage createThumbnail(const QString& path, QImage originalImage); // Build the thumbnail from an image.
static void retrieveImageAsync(const QString& path, VideoFrameGrabberListener* requester); // Get an image from the path. When it is ready, the signal imageGrabbed() is send to the listener. It can be direct if this is not a media file.
signals:
void pathChanged();
void descriptionChanged();
void idChanged();
private:
QString mId;
QString mPath;

141
linphone-app/src/components/other/images/VideoFrameGrabber.cpp Normal file
View file

@ -0,0 +1,141 @@
/*
* Copyright (c) 2023 Belledonne Communications SARL.
*
* This file is part of linphone-desktop
* (see https://www.linphone.org).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "VideoFrameGrabber.hpp"
#include <QVideoSurfaceFormat>
VideoFrameGrabberListener::VideoFrameGrabberListener(){
}
VideoFrameGrabber::VideoFrameGrabber( QObject *parent)
: QAbstractVideoSurface(parent){
QObject::connect(&player, QOverload<QMediaPlayer::Error>::of(&QMediaPlayer::error), this, [this](QMediaPlayer::Error error) mutable{
end();
}, Qt::DirectConnection);
QObject::connect(&player, &QMediaPlayer::mediaStatusChanged, this, [this](QMediaPlayer::MediaStatus status) mutable{
switch(status){
case QMediaPlayer::LoadedMedia : if(!mLoadedMedia){
mLoadedMedia = true;
if( player.isVideoAvailable() ){
player.setPosition(player.duration() / 2);
player.play();
}else{
end();
}
}
break;
case QMediaPlayer::UnknownMediaStatus:
case QMediaPlayer::InvalidMedia:
case QMediaPlayer::EndOfMedia:
case QMediaPlayer::NoMedia:
end();
break;
default:{}
}
}, Qt::DirectConnection);
QObject::connect(this, &VideoFrameGrabber::frameAvailable, this, [this](QImage frame) mutable{
mResult = frame.copy();
player.setMedia(QUrl());
}, Qt::DirectConnection);
player.setVideoOutput(this);
}
void VideoFrameGrabber::requestFrame(const QString& path){
mLoadedMedia = false;
mPath = path;
player.setMedia(QUrl::fromLocalFile(mPath));
}
void VideoFrameGrabber::end(){
if(player.mediaStatus() != QMediaPlayer::NoMedia){
player.setMedia(QUrl());
}else if(!mResultSent){// Avoid sending multiple times before destroying the object
mResultSent = true;
emit grabFinished(mResult);
deleteLater();
}
}
QList<QVideoFrame::PixelFormat> VideoFrameGrabber::supportedPixelFormats(QAbstractVideoBuffer::HandleType handleType) const {
Q_UNUSED(handleType);
return QList<QVideoFrame::PixelFormat>()
<< QVideoFrame::Format_ARGB32
<< QVideoFrame::Format_ARGB32_Premultiplied
<< QVideoFrame::Format_RGB32
<< QVideoFrame::Format_RGB24
<< QVideoFrame::Format_RGB565
<< QVideoFrame::Format_RGB555
<< QVideoFrame::Format_ARGB8565_Premultiplied
<< QVideoFrame::Format_BGRA32
<< QVideoFrame::Format_BGRA32_Premultiplied
<< QVideoFrame::Format_BGR32
<< QVideoFrame::Format_BGR24
<< QVideoFrame::Format_BGR565
<< QVideoFrame::Format_BGR555
<< QVideoFrame::Format_BGRA5658_Premultiplied
<< QVideoFrame::Format_AYUV444
<< QVideoFrame::Format_AYUV444_Premultiplied
<< QVideoFrame::Format_YUV444
<< QVideoFrame::Format_YUV420P
<< QVideoFrame::Format_YV12
<< QVideoFrame::Format_UYVY
<< QVideoFrame::Format_YUYV
<< QVideoFrame::Format_NV12
<< QVideoFrame::Format_NV21
<< QVideoFrame::Format_IMC1
<< QVideoFrame::Format_IMC2
<< QVideoFrame::Format_IMC3
<< QVideoFrame::Format_IMC4
<< QVideoFrame::Format_Y8
<< QVideoFrame::Format_Y16
<< QVideoFrame::Format_Jpeg
<< QVideoFrame::Format_CameraRaw
<< QVideoFrame::Format_AdobeDng;
}
bool VideoFrameGrabber::isFormatSupported(const QVideoSurfaceFormat &format) const {
const QImage::Format imageFormat = QVideoFrame::imageFormatFromPixelFormat(format.pixelFormat());
const QSize size = format.frameSize();
return imageFormat != QImage::Format_Invalid
&& !size.isEmpty()
&& format.handleType() == QAbstractVideoBuffer::NoHandle;
}
bool VideoFrameGrabber::start(const QVideoSurfaceFormat &format){
return QAbstractVideoSurface::start(format);
}
void VideoFrameGrabber::stop() {
QAbstractVideoSurface::stop();
}
bool VideoFrameGrabber::present(const QVideoFrame &frame){
if (frame.isValid()) {
emit frameAvailable(frame.image());
return true;
}else
return false;
}

66
linphone-app/src/components/other/images/VideoFrameGrabber.hpp Normal file
View file

@ -0,0 +1,66 @@
/*
* Copyright (c) 2023 Belledonne Communications SARL.
*
* This file is part of linphone-desktop
* (see https://www.linphone.org).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef VIDEO_FRAME_GRABBER_H
#define VIDEO_FRAME_GRABBER_H
#include <QAbstractVideoSurface>
#include <QMediaPlayer>
// Call VideoFrameGrabber::requestFrame() and wait for imageGrabbed() to get the image.
// You will need to link your listener with connect(grabber, &VideoFrameGrabber::grabFinished, listener, &VideoFrameGrabberListener::imageGrabbed);
class VideoFrameGrabberListener: public QObject{
Q_OBJECT
public:
VideoFrameGrabberListener();
signals:
void imageGrabbed(QImage image);
};
class VideoFrameGrabber : public QAbstractVideoSurface {
Q_OBJECT
public:
VideoFrameGrabber(QObject *parent = 0);
void requestFrame(const QString& path); // Function to call.
void end();
QList<QVideoFrame::PixelFormat> supportedPixelFormats(
QAbstractVideoBuffer::HandleType handleType = QAbstractVideoBuffer::NoHandle) const override;
bool isFormatSupported(const QVideoSurfaceFormat &format) const override;
bool start(const QVideoSurfaceFormat &format) override;
void stop() override;
bool present(const QVideoFrame &frame) override;
QMediaPlayer player;
bool mLoadedMedia = false;
bool mResultSent = false;
QString mPath;
QImage mResult;
signals:
void frameAvailable(QImage frame);
void grabFinished(QImage frame);
};
#endif

2
linphone-app/ui/modules/Linphone/File/FileView.qml
View file

@ -255,7 +255,7 @@ Item {
id: waitingProvider
anchors.fill: parent
sourceComponent: thumbnailProvider.sourceComponent == thumbnailImage && thumbnailProvider.item.status != Image.Ready
sourceComponent: thumbnailProvider.sourceComponent == thumbnailImage && (thumbnailProvider.item.status != Image.Ready || thumbnailProvider.item.sourceSize.height == 0)
? extension
: undefined
states: State {