From: Andy Nichols Date: Thu, 15 Aug 2013 13:08:35 +0000 (+0200) Subject: AVFoundation: Use CoreAnimation to render video to QVideoWidget X-Git-Tag: upstream/5.2.95+rc1~59 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=51d0f852b9be9c5753e28e2bb4a7f2f6c35dc533;p=platform%2Fupstream%2Fqtmultimedia.git AVFoundation: Use CoreAnimation to render video to QVideoWidget Previously a QGLWidget was used as a target for the AVFVideoFrameRenderer. This was uncessary as it is possible to render directly on top of the QWidget using the CoreAnimation Framework. Change-Id: I08923c85fd56c8874c1d8c187ae5145e220fab92 Reviewed-by: Richard Moe Gustavsen --- diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h index 5b52e8e..b339747 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h @@ -56,15 +56,11 @@ class QOpenGLFramebufferObject; class QWindow; class QOpenGLContext; class QAbstractVideoSurface; -class QGLWidget; class AVFVideoFrameRenderer : public QObject { public: AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QObject *parent = 0); -#ifndef QT_NO_WIDGETS - AVFVideoFrameRenderer(QGLWidget *glWidget, const QSize &size, QObject *parent = 0); -#endif virtual ~AVFVideoFrameRenderer(); @@ -76,9 +72,6 @@ private: void renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo); CARenderer *m_videoLayerRenderer; -#ifndef QT_NO_WIDGETS - QGLWidget *m_glWidget; -#endif QAbstractVideoSurface *m_surface; QOpenGLFramebufferObject *m_fbo[2]; QWindow *m_offscreenSurface; diff --git a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm index 210dd56..fb63392 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm @@ -45,10 +45,6 @@ #include #include -#ifndef QT_NO_WIDGETS -#include -#endif - #ifdef QT_DEBUG_AVF #include #endif @@ -76,31 +72,6 @@ AVFVideoFrameRenderer::AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QOb m_offscreenSurface->setGeometry(0, 0, 1, 1); m_offscreenSurface->create(); } -#ifndef QT_NO_WIDGETS -AVFVideoFrameRenderer::AVFVideoFrameRenderer(QGLWidget *glWidget, const QSize &size, QObject *parent) - : QObject(parent) - , m_videoLayerRenderer(0) - , m_glWidget(glWidget) - , m_surface(0) - , m_offscreenSurface(0) - , m_glContext(0) - , m_targetSize(size) - , m_currentBuffer(1) - , m_isContextShared(true) -{ - m_fbo[0] = 0; - m_fbo[1] = 0; - - //Create Hidden QWindow surface to create context in this thread - m_offscreenSurface = new QWindow(); - m_offscreenSurface->setSurfaceType(QWindow::OpenGLSurface); - //Needs geometry to be a valid surface, but size is not important - m_offscreenSurface->setGeometry(0, 0, 1, 1); - m_offscreenSurface->create(); - - -} -#endif AVFVideoFrameRenderer::~AVFVideoFrameRenderer() { @@ -168,10 +139,6 @@ QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *lay if (m_surface) { //QOpenGLContext *renderThreadContext = 0; shareContext = qobject_cast(m_surface->property("GLContext").value()); -#ifndef QT_NO_WIDGETS - } else { - shareContext = m_glWidget->context()->contextHandle(); -#endif } m_glContext = new QOpenGLContext(); m_glContext->setFormat(m_offscreenSurface->requestedFormat()); diff --git a/src/plugins/avfoundation/mediaplayer/avfvideowidget.h b/src/plugins/avfoundation/mediaplayer/avfvideowidget.h index 460d530..12b8c26 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideowidget.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideowidget.h @@ -42,43 +42,44 @@ #ifndef AVFVIDEOWIDGET_H #define AVFVIDEOWIDGET_H -#include -#include +#include -QT_BEGIN_NAMESPACE +@class AVPlayerLayer; +#if defined(Q_OS_OSX) +@class NSView; +#else +@class UIView; +#endif -class QOpenGLShaderProgram; +QT_BEGIN_NAMESPACE -class AVFVideoWidget : public QGLWidget +class AVFVideoWidget : public QWidget { public: - AVFVideoWidget(QWidget *parent, const QGLFormat &format); + AVFVideoWidget(QWidget *parent); virtual ~AVFVideoWidget(); - void initializeGL(); - void resizeGL(int w, int h); - void paintGL(); - - void setTexture(GLuint texture); - QSize sizeHint() const; - void setNativeSize(const QSize &size); - + Qt::AspectRatioMode aspectRatioMode() const; void setAspectRatioMode(Qt::AspectRatioMode mode); + void setPlayerLayer(AVPlayerLayer *layer); + +protected: + void resizeEvent(QResizeEvent *); + void paintEvent(QPaintEvent *); private: - QRect displayRect() const; + void updateAspectRatio(); + void updatePlayerLayerBounds(const QSize &size); - GLuint m_textureId; QSize m_nativeSize; Qt::AspectRatioMode m_aspectRatioMode; - - QOpenGLShaderProgram *m_shaderProgram; - QMatrix4x4 m_transformMatrix; - - int m_matrixLocation; - int m_vertexCoordEntry; - int m_textureCoordEntry; + AVPlayerLayer *m_playerLayer; +#if defined(Q_OS_OSX) + NSView *m_nativeView; +#else + UIView *m_nativeView; +#endif }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideowidget.mm b/src/plugins/avfoundation/mediaplayer/avfvideowidget.mm index 518a6bc..2e4de37 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideowidget.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideowidget.mm @@ -41,15 +41,19 @@ #include "avfvideowidget.h" #include -#include + +#include +#include +#include +#include QT_USE_NAMESPACE -AVFVideoWidget::AVFVideoWidget(QWidget *parent, const QGLFormat &format) - : QGLWidget(format, parent) - , m_textureId(0) +AVFVideoWidget::AVFVideoWidget(QWidget *parent) + : QWidget(parent) , m_aspectRatioMode(Qt::KeepAspectRatio) - , m_shaderProgram(0) + , m_playerLayer(0) + , m_nativeView(0) { setAutoFillBackground(false); } @@ -59,143 +63,114 @@ AVFVideoWidget::~AVFVideoWidget() #ifdef QT_DEBUG_AVF qDebug() << Q_FUNC_INFO; #endif - delete m_shaderProgram; + + if (m_playerLayer) + [m_playerLayer release]; } -void AVFVideoWidget::initializeGL() +QSize AVFVideoWidget::sizeHint() const { - glClearColor(0.0f, 0.0f, 0.0f, 1.0f); - - m_shaderProgram = new QOpenGLShaderProgram; - - static const char *textureVertexProgram = - "uniform highp mat4 matrix;\n" - "attribute highp vec3 vertexCoordEntry;\n" - "attribute highp vec2 textureCoordEntry;\n" - "varying highp vec2 textureCoord;\n" - "void main() {\n" - " textureCoord = textureCoordEntry;\n" - " gl_Position = matrix * vec4(vertexCoordEntry, 1);\n" - "}\n"; - - static const char *textureFragmentProgram = - "uniform sampler2D texture;\n" - "varying highp vec2 textureCoord;\n" - "void main() {\n" - " gl_FragColor = texture2D(texture, textureCoord);\n" - "}\n"; - - m_shaderProgram->addShaderFromSourceCode(QOpenGLShader::Vertex, textureVertexProgram); - m_shaderProgram->addShaderFromSourceCode(QOpenGLShader::Fragment, textureFragmentProgram); - m_shaderProgram->link(); + return m_nativeSize; } -void AVFVideoWidget::resizeGL(int w, int h) +Qt::AspectRatioMode AVFVideoWidget::aspectRatioMode() const { - glViewport(0, 0, GLsizei(w), GLsizei(h)); - updateGL(); + return m_aspectRatioMode; } -void AVFVideoWidget::paintGL() +void AVFVideoWidget::setAspectRatioMode(Qt::AspectRatioMode mode) { - glClear(GL_COLOR_BUFFER_BIT); - if (!m_textureId) - return; - - QRect targetRect = displayRect(); - GLfloat x1 = targetRect.left(); - GLfloat x2 = targetRect.right(); - GLfloat y1 = targetRect.bottom(); - GLfloat y2 = targetRect.top(); - GLfloat zValue = 0; + if (m_aspectRatioMode != mode) { + m_aspectRatioMode = mode; - const GLfloat textureCoordinates[] = { - 0, 0, - 1, 0, - 1, 1, - 0, 1 - }; + updateAspectRatio(); + } +} - const GLfloat vertexCoordinates[] = { - x1, y1, zValue, - x2, y1, zValue, - x2, y2, zValue, - x1, y2, zValue - }; +void AVFVideoWidget::setPlayerLayer(AVPlayerLayer *layer) +{ + if (m_playerLayer == layer) + return; - //Set matrix to transfrom geometry values into gl coordinate space. - m_transformMatrix.setToIdentity(); - m_transformMatrix.scale( 2.0f / size().width(), 2.0f / size().height() ); - m_transformMatrix.translate(-size().width() / 2.0f, -size().height() / 2.0f); + if (!m_nativeView) { + //make video widget a native window +#if defined(Q_OS_OSX) + m_nativeView = (NSView*)this->winId(); + [m_nativeView setWantsLayer:YES]; +#else + m_nativeView = (UIView*)this->winId(); +#endif + } - m_shaderProgram->bind(); + if (m_playerLayer) { + [m_playerLayer removeFromSuperlayer]; + [m_playerLayer release]; + } - m_vertexCoordEntry = m_shaderProgram->attributeLocation("vertexCoordEntry"); - m_textureCoordEntry = m_shaderProgram->attributeLocation("textureCoordEntry"); - m_matrixLocation = m_shaderProgram->uniformLocation("matrix"); + m_playerLayer = layer; - //attach the data! - glEnableVertexAttribArray(m_vertexCoordEntry); - glEnableVertexAttribArray(m_textureCoordEntry); + CALayer *nativeLayer = [m_nativeView layer]; - glVertexAttribPointer(m_vertexCoordEntry, 3, GL_FLOAT, GL_FALSE, 0, vertexCoordinates); - glVertexAttribPointer(m_textureCoordEntry, 2, GL_FLOAT, GL_FALSE, 0, textureCoordinates); - m_shaderProgram->setUniformValue(m_matrixLocation, m_transformMatrix); + if (layer) { + [layer retain]; - glBindTexture(GL_TEXTURE_2D, m_textureId); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + m_nativeSize = QSize(m_playerLayer.bounds.size.width, + m_playerLayer.bounds.size.height); - glDrawArrays(GL_TRIANGLE_FAN, 0, 4); + updateAspectRatio(); + [nativeLayer addSublayer:m_playerLayer]; + updatePlayerLayerBounds(this->size()); + } - glBindTexture(GL_TEXTURE_2D, 0); + NSArray *sublayers = [nativeLayer sublayers]; + qDebug() << "playerlayer: " << "at z:" << [m_playerLayer zPosition] + << " frame: " << m_playerLayer.frame.size.width << "x" << m_playerLayer.frame.size.height; + qDebug() << "superlayer: " << "at z:" << [nativeLayer zPosition] + << " frame: " << nativeLayer.frame.size.width << "x" << nativeLayer.frame.size.height; + int i = 0; + for (CALayer *layer in sublayers) { + qDebug() << "layer " << i << ": at z:" << [layer zPosition] + << " frame: " << layer.frame.size.width << "x" << layer.frame.size.height; + i++; + } - glDisableVertexAttribArray(m_vertexCoordEntry); - glDisableVertexAttribArray(m_textureCoordEntry); - m_shaderProgram->release(); } -void AVFVideoWidget::setTexture(GLuint texture) +void AVFVideoWidget::resizeEvent(QResizeEvent *event) { - m_textureId = texture; - - if (isVisible()) { - makeCurrent(); - updateGL(); - } + updatePlayerLayerBounds(event->size()); + QWidget::resizeEvent(event); } -QSize AVFVideoWidget::sizeHint() const +void AVFVideoWidget::paintEvent(QPaintEvent *event) { - return m_nativeSize; -} + QPainter painter(this); + painter.fillRect(rect(), Qt::black); -void AVFVideoWidget::setNativeSize(const QSize &size) -{ - m_nativeSize = size; + QWidget::paintEvent(event); } -void AVFVideoWidget::setAspectRatioMode(Qt::AspectRatioMode mode) +void AVFVideoWidget::updateAspectRatio() { - if (m_aspectRatioMode != mode) { - m_aspectRatioMode = mode; - update(); + if (m_playerLayer) { + switch (m_aspectRatioMode) { + case Qt::IgnoreAspectRatio: + [m_playerLayer setVideoGravity:AVLayerVideoGravityResize]; + break; + case Qt::KeepAspectRatio: + [m_playerLayer setVideoGravity:AVLayerVideoGravityResizeAspect]; + break; + case Qt::KeepAspectRatioByExpanding: + [m_playerLayer setVideoGravity:AVLayerVideoGravityResizeAspectFill]; + break; + default: + break; + } } } -QRect AVFVideoWidget::displayRect() const +void AVFVideoWidget::updatePlayerLayerBounds(const QSize &size) { - QRect displayRect = rect(); - - if (m_aspectRatioMode == Qt::KeepAspectRatio) { - QSize size = m_nativeSize; - size.scale(displayRect.size(), Qt::KeepAspectRatio); - - displayRect = QRect(QPoint(0, 0), size); - displayRect.moveCenter(rect().center()); - } - return displayRect; + m_playerLayer.bounds = CGRectMake(0.0f, 0.0f, (float)size.width(), (float)size.height()); } diff --git a/src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.h b/src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.h index 5230d75..89d3e7f 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.h +++ b/src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.h @@ -45,13 +45,11 @@ #include #include "avfvideooutput.h" -#import +@class AVPlayerLayer; QT_BEGIN_NAMESPACE -class AVFDisplayLink; class AVFVideoWidget; -class AVFVideoFrameRenderer; class AVFVideoWidgetControl : public QVideoWidgetControl, public AVFVideoOutput { @@ -83,24 +81,15 @@ public: int saturation() const; void setSaturation(int saturation); -private Q_SLOTS: - void updateVideoFrame(const CVTimeStamp &ts); - private: - void setupVideoOutput(); - - AVFDisplayLink *m_displayLink; AVFVideoWidget *m_videoWidget; - AVFVideoFrameRenderer *m_frameRenderer; - QSize m_nativeSize; - Qt::AspectRatioMode m_aspectRatioMode; + bool m_fullscreen; int m_brightness; int m_contrast; int m_hue; int m_saturation; - void *m_playerLayer; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.mm b/src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.mm index f50117d..0d62c39 100644 --- a/src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.mm +++ b/src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.mm @@ -40,10 +40,7 @@ ****************************************************************************/ #include "avfvideowidgetcontrol.h" - #include "avfvideowidget.h" -#include "avfvideoframerenderer.h" -#include "avfdisplaylink.h" #ifdef QT_DEBUG_AVF #include @@ -55,22 +52,13 @@ QT_USE_NAMESPACE AVFVideoWidgetControl::AVFVideoWidgetControl(QObject *parent) : QVideoWidgetControl(parent) - , m_frameRenderer(0) - , m_aspectRatioMode(Qt::KeepAspectRatio) , m_fullscreen(false) , m_brightness(0) , m_contrast(0) , m_hue(0) , m_saturation(0) - , m_playerLayer(0) { - QGLFormat format = QGLFormat::defaultFormat(); - format.setSwapInterval(1); // Vertical sync (avoid tearing) - format.setDoubleBuffer(true); - m_videoWidget = new AVFVideoWidget(0, format); - - m_displayLink = new AVFDisplayLink(this); - connect(m_displayLink, SIGNAL(tick(CVTimeStamp)), this, SLOT(updateVideoFrame(CVTimeStamp))); + m_videoWidget = new AVFVideoWidget(0); } AVFVideoWidgetControl::~AVFVideoWidgetControl() @@ -78,10 +66,6 @@ AVFVideoWidgetControl::~AVFVideoWidgetControl() #ifdef QT_DEBUG_AVF qDebug() << Q_FUNC_INFO; #endif - m_displayLink->stop(); - if (m_playerLayer) - [(AVPlayerLayer*)m_playerLayer release]; - delete m_videoWidget; } @@ -91,26 +75,8 @@ void AVFVideoWidgetControl::setLayer(void *playerLayer) qDebug() << Q_FUNC_INFO << playerLayer; #endif - if (m_playerLayer == playerLayer) - return; - - [(AVPlayerLayer*)playerLayer retain]; - [(AVPlayerLayer*)m_playerLayer release]; - - m_playerLayer = playerLayer; - - //If there is no layer to render, stop scheduling updates - if (m_playerLayer == 0) { - m_displayLink->stop(); - return; - } + m_videoWidget->setPlayerLayer((AVPlayerLayer*)playerLayer); - setupVideoOutput(); - - //make sure we schedule updates - if (!m_displayLink->isActive()) { - m_displayLink->start(); - } } QWidget *AVFVideoWidgetControl::videoWidget() @@ -130,12 +96,11 @@ void AVFVideoWidgetControl::setFullScreen(bool fullScreen) Qt::AspectRatioMode AVFVideoWidgetControl::aspectRatioMode() const { - return m_aspectRatioMode; + return m_videoWidget->aspectRatioMode(); } void AVFVideoWidgetControl::setAspectRatioMode(Qt::AspectRatioMode mode) { - m_aspectRatioMode = mode; m_videoWidget->setAspectRatioMode(mode); } @@ -179,41 +144,4 @@ void AVFVideoWidgetControl::setSaturation(int saturation) m_saturation = saturation; } -void AVFVideoWidgetControl::updateVideoFrame(const CVTimeStamp &ts) -{ - Q_UNUSED(ts) - - AVPlayerLayer *playerLayer = (AVPlayerLayer*)m_playerLayer; - - if (!playerLayer) { - qWarning("updateVideoFrame called without AVPlayerLayer (which shouldn't happen)"); - return; - } - - //Don't try to render a layer that is not ready - if (!playerLayer.readyForDisplay) - return; - - GLuint textureId = m_frameRenderer->renderLayerToTexture(playerLayer); - - //Make sure we have a valid texture - if (textureId == 0) { - qWarning("renderLayerToTexture failed"); - return; - } - - m_videoWidget->setTexture(textureId); -} - -void AVFVideoWidgetControl::setupVideoOutput() -{ - CGRect layerBounds = [(AVPlayerLayer*)m_playerLayer bounds]; - m_nativeSize = QSize(layerBounds.size.width, layerBounds.size.height); - m_videoWidget->setNativeSize(m_nativeSize); - - if (m_frameRenderer) - delete m_frameRenderer; - - m_frameRenderer = new AVFVideoFrameRenderer(m_videoWidget, m_nativeSize, this); -} - +#include "moc_avfvideowidgetcontrol.cpp" diff --git a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro index 21edf89..f0bd18c 100644 --- a/src/plugins/avfoundation/mediaplayer/mediaplayer.pro +++ b/src/plugins/avfoundation/mediaplayer/mediaplayer.pro @@ -44,7 +44,7 @@ OBJECTIVE_SOURCES += \ LIBS += -framework QuartzCore -framework AppKit qtHaveModule(widgets) { - QT += multimediawidgets-private opengl + QT += multimediawidgets-private HEADERS += \ avfvideowidgetcontrol.h \ avfvideowidget.h