From 0374f0de5ec881569e463505b232b3870c0fd9d2 Mon Sep 17 00:00:00 2001 From: Lev Zelenskiy Date: Thu, 9 Feb 2012 16:50:30 +1000 Subject: [PATCH] GStreamer backend changes for media probing API. QGstreamerPlayerSession: Using GStreamer buffer probes to access media data. Change-Id: Ibc056283fdedaebba90456cc4e86ab63eae5f5f7 Reviewed-by: Michael Goddard --- src/gsttools/qgstutils.cpp | 87 +++++++++- src/gsttools/qvideosurfacegstsink.cpp | 23 +-- src/multimedia/gsttools_headers/qgstutils_p.h | 2 + .../gsttools_headers/qvideosurfacegstsink_p.h | 1 + src/plugins/gstreamer/gstreamer.pro | 4 + .../mediaplayer/qgstreamerplayerservice.cpp | 36 +++++ .../mediaplayer/qgstreamerplayersession.cpp | 179 ++++++++++++++++++++- .../mediaplayer/qgstreamerplayersession.h | 28 ++++ .../gstreamer/qgstreameraudioprobecontrol.cpp | 86 ++++++++++ .../gstreamer/qgstreameraudioprobecontrol.h | 71 ++++++++ .../gstreamer/qgstreamervideoprobecontrol.cpp | 117 ++++++++++++++ .../gstreamer/qgstreamervideoprobecontrol.h | 75 +++++++++ .../tst_qmediaplayerbackend.cpp | 92 +++++++++-- 13 files changed, 777 insertions(+), 24 deletions(-) create mode 100644 src/plugins/gstreamer/qgstreameraudioprobecontrol.cpp create mode 100644 src/plugins/gstreamer/qgstreameraudioprobecontrol.h create mode 100644 src/plugins/gstreamer/qgstreamervideoprobecontrol.cpp create mode 100644 src/plugins/gstreamer/qgstreamervideoprobecontrol.h diff --git a/src/gsttools/qgstutils.cpp b/src/gsttools/qgstutils.cpp index 5176cd9..5c12763 100644 --- a/src/gsttools/qgstutils.cpp +++ b/src/gsttools/qgstutils.cpp @@ -45,6 +45,7 @@ #include #include #include +#include QT_BEGIN_NAMESPACE @@ -143,7 +144,7 @@ QSize QGstUtils::capsResolution(const GstCaps *caps) /*! Returns aspect ratio corrected resolution of \a caps. - If caps doesn't have a valid size, and ampty QSize is returned. + If caps doesn't have a valid size, an empty QSize is returned. */ QSize QGstUtils::capsCorrectedResolution(const GstCaps *caps) { @@ -166,4 +167,88 @@ QSize QGstUtils::capsCorrectedResolution(const GstCaps *caps) return size; } +/*! + *Returns audio format for caps. + If caps doesn't have a valid audio format, an empty QAudioFormat is returned. +*/ + +QAudioFormat QGstUtils::audioFormatForCaps(const GstCaps *caps) +{ + const GstStructure *structure = gst_caps_get_structure(caps, 0); + + QAudioFormat format; + + if (qstrcmp(gst_structure_get_name(structure), "audio/x-raw-int") == 0) { + + format.setCodec("audio/pcm"); + + int endianness = 0; + gst_structure_get_int(structure, "endianness", &endianness); + if (endianness == 1234) + format.setByteOrder(QAudioFormat::LittleEndian); + else if (endianness == 4321) + format.setByteOrder(QAudioFormat::BigEndian); + + gboolean isSigned = FALSE; + gst_structure_get_boolean(structure, "signed", &isSigned); + if (isSigned) + format.setSampleType(QAudioFormat::SignedInt); + else + format.setSampleType(QAudioFormat::UnSignedInt); + + // Number of bits allocated per sample. + int width = 0; + gst_structure_get_int(structure, "width", &width); + + // The number of bits used per sample. This must be less than or equal to the width. + int depth = 0; + gst_structure_get_int(structure, "depth", &depth); + + if (width != depth) { + // Unsupported sample layout. + return QAudioFormat(); + } + format.setSampleSize(width); + + int rate = 0; + gst_structure_get_int(structure, "rate", &rate); + format.setSampleRate(rate); + + int channels = 0; + gst_structure_get_int(structure, "channels", &channels); + format.setChannelCount(channels); + + } else if (qstrcmp(gst_structure_get_name(structure), "audio/x-raw-float") == 0) { + + format.setCodec("audio/pcm"); + + int endianness = 0; + gst_structure_get_int(structure, "endianness", &endianness); + if (endianness == 1234) + format.setByteOrder(QAudioFormat::LittleEndian); + else if (endianness == 4321) + format.setByteOrder(QAudioFormat::BigEndian); + + format.setSampleType(QAudioFormat::Float); + + int width = 0; + gst_structure_get_int(structure, "width", &width); + + format.setSampleSize(width); + + int rate = 0; + gst_structure_get_int(structure, "rate", &rate); + format.setSampleRate(rate); + + int channels = 0; + gst_structure_get_int(structure, "channels", &channels); + format.setChannelCount(channels); + + } else { + return QAudioFormat(); + } + + return format; +} + QT_END_NAMESPACE diff --git a/src/gsttools/qvideosurfacegstsink.cpp b/src/gsttools/qvideosurfacegstsink.cpp index da11ac7..8928462 100644 --- a/src/gsttools/qvideosurfacegstsink.cpp +++ b/src/gsttools/qvideosurfacegstsink.cpp @@ -199,16 +199,7 @@ GstFlowReturn QVideoSurfaceGstDelegate::render(GstBuffer *buffer) m_format.frameSize(), m_format.pixelFormat()); - qint64 startTime = GST_BUFFER_TIMESTAMP(buffer); - - if (startTime >= 0) { - m_frame.setStartTime(startTime/G_GINT64_CONSTANT (1000000)); - - qint64 duration = GST_BUFFER_DURATION(buffer); - - if (duration >= 0) - m_frame.setEndTime((startTime + duration)/G_GINT64_CONSTANT (1000000)); - } + QVideoSurfaceGstSink::setFrameTimeStamps(&m_frame, buffer); m_renderReturn = GST_FLOW_OK; @@ -685,6 +676,18 @@ QVideoSurfaceFormat QVideoSurfaceGstSink::formatForCaps(GstCaps *caps, int *byte return QVideoSurfaceFormat(); } +void QVideoSurfaceGstSink::setFrameTimeStamps(QVideoFrame *frame, GstBuffer *buffer) +{ + qint64 startTime = GST_BUFFER_TIMESTAMP(buffer); + if (startTime >= 0) { + frame->setStartTime(startTime/G_GINT64_CONSTANT (1000000)); + + qint64 duration = GST_BUFFER_DURATION(buffer); + if (duration >= 0) + frame->setEndTime((startTime + duration)/G_GINT64_CONSTANT (1000000)); + } +} + void QVideoSurfaceGstSink::handleShowPrerollChange(GObject *o, GParamSpec *p, gpointer d) { Q_UNUSED(o); diff --git a/src/multimedia/gsttools_headers/qgstutils_p.h b/src/multimedia/gsttools_headers/qgstutils_p.h index c9b5d14..e9220d7 100644 --- a/src/multimedia/gsttools_headers/qgstutils_p.h +++ b/src/multimedia/gsttools_headers/qgstutils_p.h @@ -55,6 +55,7 @@ #include #include +#include QT_BEGIN_NAMESPACE @@ -67,6 +68,7 @@ namespace QGstUtils { QSize capsResolution(const GstCaps *caps); QSize capsCorrectedResolution(const GstCaps *caps); + QAudioFormat audioFormatForCaps(const GstCaps *caps); } QT_END_NAMESPACE diff --git a/src/multimedia/gsttools_headers/qvideosurfacegstsink_p.h b/src/multimedia/gsttools_headers/qvideosurfacegstsink_p.h index 9dc57bb..143f65d 100644 --- a/src/multimedia/gsttools_headers/qvideosurfacegstsink_p.h +++ b/src/multimedia/gsttools_headers/qvideosurfacegstsink_p.h @@ -135,6 +135,7 @@ public: static QVideoSurfaceGstSink *createSink(QAbstractVideoSurface *surface); static QVideoSurfaceFormat formatForCaps(GstCaps *caps, int *bytesPerLine = 0); + static void setFrameTimeStamps(QVideoFrame *frame, GstBuffer *buffer); static void handleShowPrerollChange(GObject *o, GParamSpec *p, gpointer d); diff --git a/src/plugins/gstreamer/gstreamer.pro b/src/plugins/gstreamer/gstreamer.pro index 2488e3d..14d40fd 100644 --- a/src/plugins/gstreamer/gstreamer.pro +++ b/src/plugins/gstreamer/gstreamer.pro @@ -63,6 +63,8 @@ HEADERS += \ qgstreamervideoinputdevicecontrol.h \ gstvideoconnector.h \ qgstcodecsinfo.h \ + qgstreamervideoprobecontrol.h \ + qgstreameraudioprobecontrol.h \ SOURCES += \ qgstreamervideorendererinterface.cpp \ @@ -72,6 +74,8 @@ SOURCES += \ qgstreamervideoinputdevicecontrol.cpp \ qgstcodecsinfo.cpp \ gstvideoconnector.c \ + qgstreamervideoprobecontrol.cpp \ + qgstreameraudioprobecontrol.cpp \ contains(config_test_xvideo, yes):!isEmpty(QT.widgets.name): { diff --git a/src/plugins/gstreamer/mediaplayer/qgstreamerplayerservice.cpp b/src/plugins/gstreamer/mediaplayer/qgstreamerplayerservice.cpp index c08032f..f660cef 100644 --- a/src/plugins/gstreamer/mediaplayer/qgstreamerplayerservice.cpp +++ b/src/plugins/gstreamer/mediaplayer/qgstreamerplayerservice.cpp @@ -64,6 +64,8 @@ #endif #include "qgstreamerstreamscontrol.h" +#include "qgstreameraudioprobecontrol.h" +#include "qgstreamervideoprobecontrol.h" #include #include @@ -115,6 +117,24 @@ QMediaControl *QGstreamerPlayerService::requestControl(const char *name) if (qstrcmp(name,QMediaStreamsControl_iid) == 0) return m_streamsControl; + if (qstrcmp(name,QMediaVideoProbeControl_iid) == 0) { + if (m_session) { + QGstreamerVideoProbeControl *probe = new QGstreamerVideoProbeControl(this); + m_session->addProbe(probe); + return probe; + } + return 0; + } + + if (qstrcmp(name,QMediaAudioProbeControl_iid) == 0) { + if (m_session) { + QGstreamerAudioProbeControl *probe = new QGstreamerAudioProbeControl(this); + m_session->addProbe(probe); + return probe; + } + return 0; + } + if (!m_videoOutput) { if (qstrcmp(name, QVideoRendererControl_iid) == 0) m_videoOutput = m_videoRenderer; @@ -140,6 +160,22 @@ void QGstreamerPlayerService::releaseControl(QMediaControl *control) m_videoOutput = 0; m_control->setVideoOutput(0); } + + QGstreamerVideoProbeControl* videoProbe = qobject_cast(control); + if (videoProbe) { + if (m_session) + m_session->removeProbe(videoProbe); + delete videoProbe; + return; + } + + QGstreamerAudioProbeControl* audioProbe = qobject_cast(control); + if (audioProbe) { + if (m_session) + m_session->removeProbe(audioProbe); + delete audioProbe; + return; + } } QT_END_NAMESPACE diff --git a/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.cpp b/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.cpp index 93ce494..708465d 100644 --- a/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.cpp +++ b/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.cpp @@ -42,6 +42,8 @@ #include "qgstreamerplayersession.h" #include +#include "qgstreameraudioprobecontrol.h" +#include "qgstreamervideoprobecontrol.h" #include "qgstreamervideorendererinterface.h" #include "gstvideoconnector.h" #include @@ -89,6 +91,7 @@ QGstreamerPlayerSession::QGstreamerPlayerSession(QObject *parent) m_videoSink(0), m_pendingVideoSink(0), m_nullVideoSink(0), + m_audioSink(0), m_bus(0), m_videoOutput(0), m_renderer(0), @@ -96,6 +99,8 @@ QGstreamerPlayerSession::QGstreamerPlayerSession(QObject *parent) #if defined(HAVE_GST_APPSRC) m_appSrc(0), #endif + m_videoBufferProbeId(-1), + m_audioBufferProbeId(-1), m_volume(100), m_playbackRate(1.0), m_muted(false), @@ -128,6 +133,12 @@ QGstreamerPlayerSession::QGstreamerPlayerSession(QObject *parent) flags |= GST_PLAY_FLAG_NATIVE_VIDEO; #endif g_object_set(G_OBJECT(m_playbin), "flags", flags, NULL); + + m_audioSink = gst_element_factory_make("autoaudiosink", "audiosink"); + if (m_audioSink) { + g_object_set(G_OBJECT(m_playbin), "audio-sink", m_audioSink, NULL); + addAudioBufferProbe(); + } } else { m_usePlaybin2 = false; m_playbin = gst_element_factory_make("playbin", NULL); @@ -181,6 +192,9 @@ QGstreamerPlayerSession::~QGstreamerPlayerSession() if (m_playbin) { stop(); + removeVideoBufferProbe(); + removeAudioBufferProbe(); + delete m_busHelper; gst_object_unref(GST_OBJECT(m_bus)); gst_object_unref(GST_OBJECT(m_playbin)); @@ -511,7 +525,8 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput) qDebug() << "The pipeline has not started yet, pending state:" << m_pendingState; #endif //the pipeline has not started yet - m_pendingVideoSink = 0; + flushVideoProbes(); + m_pendingVideoSink = 0; gst_element_set_state(m_videoSink, GST_STATE_NULL); gst_element_set_state(m_playbin, GST_STATE_NULL); @@ -522,6 +537,8 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput) gst_element_unlink(m_videoIdentity, m_videoSink); } + removeVideoBufferProbe(); + gst_bin_remove(GST_BIN(m_videoOutputBin), m_videoSink); m_videoSink = videoSink; @@ -544,6 +561,8 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput) g_object_set(G_OBJECT(m_videoSink), "show-preroll-frame", value, NULL); } + addVideoBufferProbe(); + switch (m_pendingState) { case QMediaPlayer::PausedState: gst_element_set_state(m_playbin, GST_STATE_PAUSED); @@ -554,6 +573,9 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput) default: break; } + + resumeVideoProbes(); + } else { if (m_pendingVideoSink) { #ifdef DEBUG_PLAYBIN @@ -630,6 +652,8 @@ void QGstreamerPlayerSession::finishVideoOutputChange() gst_element_unlink(m_videoIdentity, m_videoSink); } + removeVideoBufferProbe(); + gst_bin_remove(GST_BIN(m_videoOutputBin), m_videoSink); m_videoSink = m_pendingVideoSink; @@ -637,6 +661,8 @@ void QGstreamerPlayerSession::finishVideoOutputChange() gst_bin_add(GST_BIN(m_videoOutputBin), m_videoSink); + addVideoBufferProbe(); + m_usingColorspaceElement = false; bool linked = gst_element_link(m_videoIdentity, m_videoSink); if (!linked) { @@ -680,12 +706,18 @@ void QGstreamerPlayerSession::finishVideoOutputChange() if (m_usingColorspaceElement) gst_element_set_state(m_colorSpace, state); - gst_element_set_state(m_videoSink, state); + gst_element_set_state(m_videoSink, state); + + if (state == GST_STATE_NULL) + flushVideoProbes(); // Set state change that was deferred due the video output // change being pending gst_element_set_state(m_playbin, state); + if (state != GST_STATE_NULL) + resumeVideoProbes(); + //don't have to wait here, it will unblock eventually if (gst_pad_is_blocked(srcPad)) gst_pad_set_blocked_async(srcPad, false, &block_pad_cb, 0); @@ -766,8 +798,10 @@ bool QGstreamerPlayerSession::play() m_pendingState = m_state = QMediaPlayer::StoppedState; emit stateChanged(m_state); - } else + } else { + resumeVideoProbes(); return true; + } } return false; @@ -789,6 +823,7 @@ bool QGstreamerPlayerSession::pause() emit stateChanged(m_state); } else { + resumeVideoProbes(); return true; } } @@ -803,9 +838,11 @@ void QGstreamerPlayerSession::stop() #endif m_everPlayed = false; if (m_playbin) { + if (m_renderer) m_renderer->stopRenderer(); + flushVideoProbes(); gst_element_set_state(m_playbin, GST_STATE_NULL); m_lastPosition = 0; @@ -1621,4 +1658,140 @@ void QGstreamerPlayerSession::showPrerollFrames(bool enabled) } } +void QGstreamerPlayerSession::addProbe(QGstreamerVideoProbeControl* probe) +{ + QMutexLocker locker(&m_videoProbeMutex); + + if (m_videoProbes.contains(probe)) + return; + + m_videoProbes.append(probe); +} + +void QGstreamerPlayerSession::removeProbe(QGstreamerVideoProbeControl* probe) +{ + QMutexLocker locker(&m_videoProbeMutex); + m_videoProbes.removeOne(probe); + // Do not emit flush signal in this case. + // Assume user releases any outstanding references to video frames. +} + +gboolean QGstreamerPlayerSession::padVideoBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data) +{ + Q_UNUSED(pad); + + QGstreamerPlayerSession *session = reinterpret_cast(user_data); + QMutexLocker locker(&session->m_videoProbeMutex); + + if (session->m_videoProbes.isEmpty()) + return TRUE; + + foreach (QGstreamerVideoProbeControl* probe, session->m_videoProbes) + probe->bufferProbed(buffer); + + return TRUE; +} + +void QGstreamerPlayerSession::addProbe(QGstreamerAudioProbeControl* probe) +{ + QMutexLocker locker(&m_audioProbeMutex); + + if (m_audioProbes.contains(probe)) + return; + + m_audioProbes.append(probe); +} + +void QGstreamerPlayerSession::removeProbe(QGstreamerAudioProbeControl* probe) +{ + QMutexLocker locker(&m_audioProbeMutex); + m_audioProbes.removeOne(probe); +} + +gboolean QGstreamerPlayerSession::padAudioBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data) +{ + Q_UNUSED(pad); + + QGstreamerPlayerSession *session = reinterpret_cast(user_data); + QMutexLocker locker(&session->m_audioProbeMutex); + + if (session->m_audioProbes.isEmpty()) + return TRUE; + + foreach (QGstreamerAudioProbeControl* probe, session->m_audioProbes) + probe->bufferProbed(buffer); + + return TRUE; +} + +void QGstreamerPlayerSession::removeVideoBufferProbe() +{ + if (m_videoBufferProbeId == -1) + return; + + if (!m_videoSink) { + m_videoBufferProbeId = -1; + return; + } + + GstPad *pad = gst_element_get_static_pad(m_videoSink, "sink"); + if (pad) + gst_pad_remove_buffer_probe(pad, m_videoBufferProbeId); + + m_videoBufferProbeId = -1; +} + +void QGstreamerPlayerSession::addVideoBufferProbe() +{ + Q_ASSERT(m_videoBufferProbeId == -1); + if (!m_videoSink) + return; + + GstPad *pad = gst_element_get_static_pad(m_videoSink, "sink"); + if (pad) + m_videoBufferProbeId = gst_pad_add_buffer_probe(pad, G_CALLBACK(padVideoBufferProbe), this); +} + +void QGstreamerPlayerSession::removeAudioBufferProbe() +{ + if (m_audioBufferProbeId == -1) + return; + + if (!m_audioSink) { + m_audioBufferProbeId = -1; + return; + } + + GstPad *pad = gst_element_get_static_pad(m_audioSink, "sink"); + if (pad) + gst_pad_remove_buffer_probe(pad, m_audioBufferProbeId); + + m_audioBufferProbeId = -1; +} + +void QGstreamerPlayerSession::addAudioBufferProbe() +{ + Q_ASSERT(m_audioBufferProbeId == -1); + if (!m_audioSink) + return; + + GstPad *pad = gst_element_get_static_pad(m_audioSink, "sink"); + if (pad) + m_audioBufferProbeId = gst_pad_add_buffer_probe(pad, G_CALLBACK(padAudioBufferProbe), this); +} + +void QGstreamerPlayerSession::flushVideoProbes() +{ + QMutexLocker locker(&m_videoProbeMutex); + foreach (QGstreamerVideoProbeControl* probe, m_videoProbes) + probe->startFlushing(); +} + +void QGstreamerPlayerSession::resumeVideoProbes() +{ + QMutexLocker locker(&m_videoProbeMutex); + foreach (QGstreamerVideoProbeControl* probe, m_videoProbes) + probe->stopFlushing(); +} + QT_END_NAMESPACE diff --git a/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.h b/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.h index 8a54e8c..27b6f34 100644 --- a/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.h +++ b/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.h @@ -43,11 +43,13 @@ #define QGSTREAMERPLAYERSESSION_H #include +#include #include #include "qgstreamerplayercontrol.h" #include #include #include +#include #if defined(HAVE_GST_APPSRC) #include "qgstappsrc.h" @@ -61,6 +63,8 @@ class QGstreamerBusHelper; class QGstreamerMessage; class QGstreamerVideoRendererInterface; +class QGstreamerVideoProbeControl; +class QGstreamerAudioProbeControl; class QGstreamerPlayerSession : public QObject, public QGstreamerBusMessageFilter @@ -119,6 +123,14 @@ public: bool isLiveSource() const; + void addProbe(QGstreamerVideoProbeControl* probe); + void removeProbe(QGstreamerVideoProbeControl* probe); + static gboolean padVideoBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data); + + void addProbe(QGstreamerAudioProbeControl* probe); + void removeProbe(QGstreamerAudioProbeControl* probe); + static gboolean padAudioBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data); + public slots: void loadFromUri(const QNetworkRequest &url); void loadFromStream(const QNetworkRequest &url, QIODevice *stream); @@ -169,6 +181,13 @@ private: static void handleElementAdded(GstBin *bin, GstElement *element, QGstreamerPlayerSession *session); void processInvalidMedia(QMediaPlayer::Error errorCode, const QString& errorString); + void removeVideoBufferProbe(); + void addVideoBufferProbe(); + void removeAudioBufferProbe(); + void addAudioBufferProbe(); + void flushVideoProbes(); + void resumeVideoProbes(); + QNetworkRequest m_request; QMediaPlayer::State m_state; QMediaPlayer::State m_pendingState; @@ -184,6 +203,8 @@ private: GstElement* m_pendingVideoSink; GstElement* m_nullVideoSink; + GstElement* m_audioSink; + GstBus* m_bus; QObject *m_videoOutput; QGstreamerVideoRendererInterface *m_renderer; @@ -199,6 +220,13 @@ private: QList m_streamTypes; QMap m_playbin2StreamOffset; + QList m_videoProbes; + QMutex m_videoProbeMutex; + int m_videoBufferProbeId; + + QList m_audioProbes; + QMutex m_audioProbeMutex; + int m_audioBufferProbeId; int m_volume; qreal m_playbackRate; diff --git a/src/plugins/gstreamer/qgstreameraudioprobecontrol.cpp b/src/plugins/gstreamer/qgstreameraudioprobecontrol.cpp new file mode 100644 index 0000000..d1f15d8 --- /dev/null +++ b/src/plugins/gstreamer/qgstreameraudioprobecontrol.cpp @@ -0,0 +1,86 @@ +/**************************************************************************** +** +** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qgstreameraudioprobecontrol.h" +#include + +QGstreamerAudioProbeControl::QGstreamerAudioProbeControl(QObject *parent) + : QMediaAudioProbeControl(parent) +{ + +} + +QGstreamerAudioProbeControl::~QGstreamerAudioProbeControl() +{ + +} + +void QGstreamerAudioProbeControl::bufferProbed(GstBuffer* buffer) +{ + GstCaps* caps = gst_buffer_get_caps(buffer); + if (!caps) + return; + + QAudioFormat format = QGstUtils::audioFormatForCaps(caps); + gst_caps_unref(caps); + if (!format.isValid()) + return; + + QAudioBuffer audioBuffer = QAudioBuffer(QByteArray((const char*)buffer->data, buffer->size), format); + + { + QMutexLocker locker(&m_bufferMutex); + m_pendingBuffer = audioBuffer; + QMetaObject::invokeMethod(this, "bufferProbed", Qt::QueuedConnection); + } +} + +void QGstreamerAudioProbeControl::bufferProbed() +{ + QAudioBuffer audioBuffer; + { + QMutexLocker locker(&m_bufferMutex); + if (!m_pendingBuffer.isValid()) + return; + audioBuffer = m_pendingBuffer; + } + emit audioBufferProbed(audioBuffer); +} diff --git a/src/plugins/gstreamer/qgstreameraudioprobecontrol.h b/src/plugins/gstreamer/qgstreameraudioprobecontrol.h new file mode 100644 index 0000000..90fc584 --- /dev/null +++ b/src/plugins/gstreamer/qgstreameraudioprobecontrol.h @@ -0,0 +1,71 @@ +/**************************************************************************** +** +** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QGSTREAMERAUDIOPROBECONTROL_H +#define QGSTREAMERAUDIOPROBECONTROL_H + +#include +#include +#include +#include "qaudiobuffer.h" + +QT_BEGIN_NAMESPACE + +class QGstreamerAudioProbeControl : public QMediaAudioProbeControl +{ + Q_OBJECT +public: + explicit QGstreamerAudioProbeControl(QObject *parent); + virtual ~QGstreamerAudioProbeControl(); + + void bufferProbed(GstBuffer* buffer); + +private slots: + void bufferProbed(); + +private: + QAudioBuffer m_pendingBuffer; + QMutex m_bufferMutex; +}; + +QT_END_NAMESPACE + +#endif // QGSTREAMERAUDIOPROBECONTROL_H diff --git a/src/plugins/gstreamer/qgstreamervideoprobecontrol.cpp b/src/plugins/gstreamer/qgstreamervideoprobecontrol.cpp new file mode 100644 index 0000000..6d586c4 --- /dev/null +++ b/src/plugins/gstreamer/qgstreamervideoprobecontrol.cpp @@ -0,0 +1,117 @@ +/**************************************************************************** +** +** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qgstreamervideoprobecontrol.h" +#include +#include + +QGstreamerVideoProbeControl::QGstreamerVideoProbeControl(QObject *parent) + : QMediaVideoProbeControl(parent) + , m_flushing(false) + , m_frameProbed(false) +{ + +} + +QGstreamerVideoProbeControl::~QGstreamerVideoProbeControl() +{ + +} + +void QGstreamerVideoProbeControl::startFlushing() +{ + m_flushing = true; + + { + QMutexLocker locker(&m_frameMutex); + m_pendingFrame = QVideoFrame(); + } + + // only emit flush if at least one frame was probed + if (m_frameProbed) + emit flush(); +} + +void QGstreamerVideoProbeControl::stopFlushing() +{ + m_flushing = false; +} + +void QGstreamerVideoProbeControl::bufferProbed(GstBuffer* buffer) +{ + if (m_flushing) + return; + + GstCaps* caps = gst_buffer_get_caps(buffer); + if (!caps) + return; + + int bytesPerLine = 0; + QVideoSurfaceFormat format = QVideoSurfaceGstSink::formatForCaps(caps, &bytesPerLine); + gst_caps_unref(caps); + if (!format.isValid() || !bytesPerLine) + return; + + QVideoFrame frame = QVideoFrame(new QGstVideoBuffer(buffer, bytesPerLine), + format.frameSize(), format.pixelFormat()); + + QVideoSurfaceGstSink::setFrameTimeStamps(&frame, buffer); + + m_frameProbed = true; + + { + QMutexLocker locker(&m_frameMutex); + m_pendingFrame = frame; + QMetaObject::invokeMethod(this, "frameProbed", Qt::QueuedConnection); + } +} + +void QGstreamerVideoProbeControl::frameProbed() +{ + QVideoFrame frame; + { + QMutexLocker locker(&m_frameMutex); + if (!m_pendingFrame.isValid()) + return; + frame = m_pendingFrame; + } + emit videoFrameProbed(frame); +} diff --git a/src/plugins/gstreamer/qgstreamervideoprobecontrol.h b/src/plugins/gstreamer/qgstreamervideoprobecontrol.h new file mode 100644 index 0000000..499983b --- /dev/null +++ b/src/plugins/gstreamer/qgstreamervideoprobecontrol.h @@ -0,0 +1,75 @@ +/**************************************************************************** +** +** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QGSTREAMERVIDEOPROBECONTROL_H +#define QGSTREAMERVIDEOPROBECONTROL_H + +#include +#include +#include +#include "qvideoframe.h" + +QT_BEGIN_NAMESPACE + +class QGstreamerVideoProbeControl : public QMediaVideoProbeControl +{ + Q_OBJECT +public: + explicit QGstreamerVideoProbeControl(QObject *parent); + virtual ~QGstreamerVideoProbeControl(); + + void bufferProbed(GstBuffer* buffer); + void startFlushing(); + void stopFlushing(); + +private slots: + void frameProbed(); + +private: + bool m_flushing; + bool m_frameProbed; // true if at least one frame was probed + QVideoFrame m_pendingFrame; + QMutex m_frameMutex; +}; + +QT_END_NAMESPACE + +#endif // QGSTREAMERVIDEOPROBECONTROL_H diff --git a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp index a09c4f3..9c22457 100644 --- a/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp +++ b/tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp @@ -44,6 +44,8 @@ #include #include "qmediaservice.h" #include "qmediaplayer.h" +#include "qaudioprobe.h" +#include "qvideoprobe.h" //TESTED_COMPONENT=src/multimedia @@ -78,6 +80,7 @@ private slots: void volumeAcrossFiles_data(); void volumeAcrossFiles(); void seekPauseSeek(); + void probes(); private: //one second local wav file @@ -92,7 +95,7 @@ class TestVideoSurface : public QAbstractVideoSurface { Q_OBJECT public: - explicit TestVideoSurface() { } + TestVideoSurface() { } //video surface QList supportedPixelFormats( @@ -102,12 +105,26 @@ public: void stop(); bool present(const QVideoFrame &frame); - QList& frameList() { return m_frameList; } - -private: QList m_frameList; }; +class ProbeDataHandler : public QObject +{ + Q_OBJECT + +public: + ProbeDataHandler() : isVideoFlushCalled(false) { } + + QList m_frameList; + QList m_bufferList; + bool isVideoFlushCalled; + +public slots: + void processFrame(const QVideoFrame&); + void processBuffer(const QAudioBuffer&); + void flushVideo(); + void flushAudio(); +}; void tst_QMediaPlayerBackend::init() { @@ -453,21 +470,21 @@ void tst_QMediaPlayerBackend::seekPauseSeek() player.setMedia(QUrl::fromLocalFile(videoFile.absoluteFilePath())); QCOMPARE(player.state(), QMediaPlayer::StoppedState); - QVERIFY(surface->frameList().isEmpty()); // frame must not appear until we call pause() or play() + QVERIFY(surface->m_frameList.isEmpty()); // frame must not appear until we call pause() or play() positionSpy.clear(); player.setPosition((qint64)7000); QTRY_VERIFY(!positionSpy.isEmpty() && qAbs(player.position() - (qint64)7000) < (qint64)500); QCOMPARE(player.state(), QMediaPlayer::StoppedState); QTest::qWait(250); // wait a bit to ensure the frame is not rendered - QVERIFY(surface->frameList().isEmpty()); // still no frame, we must call pause() or play() to see a frame + QVERIFY(surface->m_frameList.isEmpty()); // still no frame, we must call pause() or play() to see a frame player.pause(); QTRY_COMPARE(player.state(), QMediaPlayer::PausedState); // it might take some time for the operation to be completed - QTRY_COMPARE(surface->frameList().size(), 1); // we must see a frame at position 7000 here + QTRY_COMPARE(surface->m_frameList.size(), 1); // we must see a frame at position 7000 here { - QVideoFrame frame = surface->frameList().back(); + QVideoFrame frame = surface->m_frameList.back(); QVERIFY(qAbs(frame.startTime() - (qint64)7000) < (qint64)500); QCOMPARE(frame.width(), 160); QCOMPARE(frame.height(), 120); @@ -486,10 +503,10 @@ void tst_QMediaPlayerBackend::seekPauseSeek() player.setPosition((qint64)12000); QTRY_VERIFY(!positionSpy.isEmpty() && qAbs(player.position() - (qint64)12000) < (qint64)500); QCOMPARE(player.state(), QMediaPlayer::PausedState); - QCOMPARE(surface->frameList().size(), 2); + QCOMPARE(surface->m_frameList.size(), 2); { - QVideoFrame frame = surface->frameList().back(); + QVideoFrame frame = surface->m_frameList.back(); QVERIFY(qAbs(frame.startTime() - (qint64)12000) < (qint64)500); QCOMPARE(frame.width(), 160); QCOMPARE(frame.height(), 120); @@ -504,6 +521,41 @@ void tst_QMediaPlayerBackend::seekPauseSeek() } } +void tst_QMediaPlayerBackend::probes() +{ + QMediaPlayer *player = new QMediaPlayer; + + TestVideoSurface *surface = new TestVideoSurface; + player->setVideoOutput(surface); + + QVideoProbe *videoProbe = new QVideoProbe; + QAudioProbe *audioProbe = new QAudioProbe; + + ProbeDataHandler probeHandler; + connect(videoProbe, SIGNAL(videoFrameProbed(const QVideoFrame&)), &probeHandler, SLOT(processFrame(QVideoFrame))); + connect(videoProbe, SIGNAL(flush()), &probeHandler, SLOT(flushVideo())); + connect(audioProbe, SIGNAL(audioBufferProbed(const QAudioBuffer&)), &probeHandler, SLOT(processBuffer(QAudioBuffer))); + connect(audioProbe, SIGNAL(flush()), &probeHandler, SLOT(flushAudio())); + + QVERIFY(videoProbe->setSource(player)); + QVERIFY(audioProbe->setSource(player)); + + QFileInfo videoFile(QLatin1String(TESTDATA_DIR "testdata/colors.mp4")); + QVERIFY(videoFile.exists()); + player->setMedia(QUrl::fromLocalFile(videoFile.absoluteFilePath())); + QTRY_COMPARE(player->mediaStatus(), QMediaPlayer::LoadedMedia); + + player->pause(); + QTRY_COMPARE(surface->m_frameList.size(), 1); + QVERIFY(!probeHandler.m_frameList.isEmpty()); + QTRY_VERIFY(!probeHandler.m_bufferList.isEmpty()); + + delete player; + QTRY_VERIFY(probeHandler.isVideoFlushCalled); + delete videoProbe; + delete audioProbe; +} + QList TestVideoSurface::supportedPixelFormats( QAbstractVideoBuffer::HandleType handleType) const { @@ -538,6 +590,26 @@ bool TestVideoSurface::present(const QVideoFrame &frame) } +void ProbeDataHandler::processFrame(const QVideoFrame &frame) +{ + m_frameList.append(frame); +} + +void ProbeDataHandler::processBuffer(const QAudioBuffer &buffer) +{ + m_bufferList.append(buffer); +} + +void ProbeDataHandler::flushVideo() +{ + isVideoFlushCalled = true; +} + +void ProbeDataHandler::flushAudio() +{ + +} + QTEST_MAIN(tst_QMediaPlayerBackend) #include "tst_qmediaplayerbackend.moc" -- 2.7.4