From b7f4b2decbd04addbe7d647e65a4f353a56bdb61 Mon Sep 17 00:00:00 2001 From: Lev Zelenskiy Date: Thu, 19 Jul 2012 14:38:45 +1000 Subject: [PATCH] Videoprobe implementation for wmf backend. Change-Id: Ia597af428764229a76c0059ae7a57eb302aee63d Reviewed-by: Dmytro Poplavskiy Reviewed-by: Michael Goddard --- src/plugins/wmf/mftvideo.cpp | 647 +++++++++++++++++++++++++ src/plugins/wmf/mftvideo.h | 114 +++++ src/plugins/wmf/player/mfplayerservice.cpp | 16 + src/plugins/wmf/player/mfplayersession.cpp | 234 ++++++++- src/plugins/wmf/player/mfplayersession.h | 8 + src/plugins/wmf/player/mfvideoprobecontrol.cpp | 56 +++ src/plugins/wmf/player/mfvideoprobecontrol.h | 61 +++ src/plugins/wmf/player/player.pri | 6 +- src/plugins/wmf/wmf.pro | 6 +- 9 files changed, 1139 insertions(+), 9 deletions(-) create mode 100644 src/plugins/wmf/mftvideo.cpp create mode 100644 src/plugins/wmf/mftvideo.h create mode 100644 src/plugins/wmf/player/mfvideoprobecontrol.cpp create mode 100644 src/plugins/wmf/player/mfvideoprobecontrol.h diff --git a/src/plugins/wmf/mftvideo.cpp b/src/plugins/wmf/mftvideo.cpp new file mode 100644 index 0000000..6fc4660 --- /dev/null +++ b/src/plugins/wmf/mftvideo.cpp @@ -0,0 +1,647 @@ +/**************************************************************************** +** +** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "mftvideo.h" +#include "mfvideoprobecontrol.h" +#include +#include +#include +#include +#include +#include + +// This MFT sends all samples it processes to connected video probes. +// Sample is sent to probes in ProcessInput. +// In ProcessOutput this MFT simply returns the original sample. + +// The implementation is based on a boilerplate from the MF SDK example. + +MFTransform::MFTransform(): + m_cRef(1), + m_inputType(0), + m_outputType(0), + m_sample(0), + m_bytesPerLine(0) +{ +} + +MFTransform::~MFTransform() +{ + if (m_inputType) + m_inputType->Release(); + + if (m_outputType) + m_outputType->Release(); +} + +void MFTransform::addProbe(MFVideoProbeControl *probe) +{ + QMutexLocker locker(&m_videoProbeMutex); + + if (m_videoProbes.contains(probe)) + return; + + m_videoProbes.append(probe); +} + +void MFTransform::removeProbe(MFVideoProbeControl *probe) +{ + QMutexLocker locker(&m_videoProbeMutex); + m_videoProbes.removeOne(probe); +} + +STDMETHODIMP MFTransform::QueryInterface(REFIID riid, void** ppv) +{ + if (!ppv) + return E_POINTER; + if (riid == IID_IMFTransform) { + *ppv = static_cast(this); + } else if (riid == IID_IUnknown) { + *ppv = static_cast(this); + } else { + *ppv = NULL; + return E_NOINTERFACE; + } + AddRef(); + return S_OK; +} + +STDMETHODIMP_(ULONG) MFTransform::AddRef() +{ + return InterlockedIncrement(&m_cRef); +} + +STDMETHODIMP_(ULONG) MFTransform::Release() +{ + ULONG cRef = InterlockedDecrement(&m_cRef); + if (cRef == 0) { + delete this; + } + return cRef; +} + +STDMETHODIMP MFTransform::GetStreamLimits(DWORD *pdwInputMinimum, DWORD *pdwInputMaximum, DWORD *pdwOutputMinimum, DWORD *pdwOutputMaximum) +{ + if (!pdwInputMinimum || !pdwInputMaximum || !pdwOutputMinimum || !pdwOutputMaximum) + return E_POINTER; + *pdwInputMinimum = 1; + *pdwInputMaximum = 1; + *pdwOutputMinimum = 1; + *pdwOutputMaximum = 1; + return S_OK; +} + +STDMETHODIMP MFTransform::GetStreamCount(DWORD *pcInputStreams, DWORD *pcOutputStreams) +{ + if (!pcInputStreams || !pcOutputStreams) + return E_POINTER; + + *pcInputStreams = 1; + *pcOutputStreams = 1; + return S_OK; +} + +STDMETHODIMP MFTransform::GetStreamIDs(DWORD dwInputIDArraySize, DWORD *pdwInputIDs, DWORD dwOutputIDArraySize, DWORD *pdwOutputIDs) +{ + // streams are numbered consecutively + Q_UNUSED(dwInputIDArraySize); + Q_UNUSED(pdwInputIDs); + Q_UNUSED(dwOutputIDArraySize); + Q_UNUSED(pdwOutputIDs); + return E_NOTIMPL; +} + +STDMETHODIMP MFTransform::GetInputStreamInfo(DWORD dwInputStreamID, MFT_INPUT_STREAM_INFO *pStreamInfo) +{ + if (dwInputStreamID > 0) + return MF_E_INVALIDSTREAMNUMBER; + + if (!pStreamInfo) + return E_POINTER; + + pStreamInfo->cbSize = 0; + pStreamInfo->hnsMaxLatency = 0; + pStreamInfo->dwFlags = MFT_INPUT_STREAM_WHOLE_SAMPLES | MFT_INPUT_STREAM_SINGLE_SAMPLE_PER_BUFFER; + pStreamInfo->cbMaxLookahead = 0; + pStreamInfo->cbAlignment = 0; + return S_OK; +} + +STDMETHODIMP MFTransform::GetOutputStreamInfo(DWORD dwOutputStreamID, MFT_OUTPUT_STREAM_INFO *pStreamInfo) +{ + if (dwOutputStreamID > 0) + return MF_E_INVALIDSTREAMNUMBER; + + if (!pStreamInfo) + return E_POINTER; + + pStreamInfo->cbSize = 0; + pStreamInfo->dwFlags = MFT_OUTPUT_STREAM_WHOLE_SAMPLES | MFT_OUTPUT_STREAM_SINGLE_SAMPLE_PER_BUFFER; + pStreamInfo->cbAlignment = 0; + + return S_OK; +} + +STDMETHODIMP MFTransform::GetAttributes(IMFAttributes **pAttributes) +{ + // This MFT does not support attributes. + Q_UNUSED(pAttributes); + return E_NOTIMPL; +} + +STDMETHODIMP MFTransform::GetInputStreamAttributes(DWORD dwInputStreamID, IMFAttributes **pAttributes) +{ + // This MFT does not support input stream attributes. + Q_UNUSED(dwInputStreamID); + Q_UNUSED(pAttributes); + return E_NOTIMPL; +} + +STDMETHODIMP MFTransform::GetOutputStreamAttributes(DWORD dwOutputStreamID, IMFAttributes **pAttributes) +{ + // This MFT does not support output stream attributes. + Q_UNUSED(dwOutputStreamID); + Q_UNUSED(pAttributes); + return E_NOTIMPL; +} + +STDMETHODIMP MFTransform::DeleteInputStream(DWORD dwStreamID) +{ + // This MFT has a fixed number of input streams. + Q_UNUSED(dwStreamID); + return E_NOTIMPL; +} + +STDMETHODIMP MFTransform::AddInputStreams(DWORD cStreams, DWORD *adwStreamIDs) +{ + // This MFT has a fixed number of input streams. + Q_UNUSED(cStreams); + Q_UNUSED(adwStreamIDs); + return E_NOTIMPL; +} + +STDMETHODIMP MFTransform::GetInputAvailableType(DWORD dwInputStreamID, DWORD dwTypeIndex, IMFMediaType **ppType) +{ + // This MFT does not have a list of preferred input types + Q_UNUSED(dwInputStreamID); + Q_UNUSED(dwTypeIndex); + Q_UNUSED(ppType); + return E_NOTIMPL; +} + +STDMETHODIMP MFTransform::GetOutputAvailableType(DWORD dwOutputStreamID,DWORD dwTypeIndex, IMFMediaType **ppType) +{ + // This MFT does not have a list of preferred output types + Q_UNUSED(dwOutputStreamID); + Q_UNUSED(dwTypeIndex); + Q_UNUSED(ppType); + return E_NOTIMPL; +} + +STDMETHODIMP MFTransform::SetInputType(DWORD dwInputStreamID, IMFMediaType *pType, DWORD dwFlags) +{ + if (dwInputStreamID > 0) + return MF_E_INVALIDSTREAMNUMBER; + + QMutexLocker locker(&m_mutex); + + DWORD flags = 0; + if (m_outputType && m_outputType->IsEqual(pType, &flags) != S_OK) { + return MF_E_INVALIDMEDIATYPE; + } + + if (dwFlags == MFT_SET_TYPE_TEST_ONLY) + return pType ? S_OK : E_POINTER; + + if (m_inputType) + m_inputType->Release(); + + m_inputType = pType; + + if (m_inputType) + m_inputType->AddRef(); + + return S_OK; +} + +STDMETHODIMP MFTransform::SetOutputType(DWORD dwOutputStreamID, IMFMediaType *pType, DWORD dwFlags) +{ + if (dwOutputStreamID > 0) + return MF_E_INVALIDSTREAMNUMBER; + + QMutexLocker locker(&m_mutex); + + DWORD flags = 0; + if (m_inputType && m_inputType->IsEqual(pType, &flags) != S_OK) { + return MF_E_INVALIDMEDIATYPE; + } + + if (dwFlags == MFT_SET_TYPE_TEST_ONLY) + return pType ? S_OK : E_POINTER; + + if (m_outputType) + m_outputType->Release(); + + m_outputType = pType; + + if (m_outputType) { + m_outputType->AddRef(); + m_format = videoFormatForMFMediaType(m_outputType, &m_bytesPerLine); + } + + return S_OK; +} + +STDMETHODIMP MFTransform::GetInputCurrentType(DWORD dwInputStreamID, IMFMediaType **ppType) +{ + if (dwInputStreamID > 0) + return MF_E_INVALIDSTREAMNUMBER; + + if (ppType == NULL) + return E_POINTER; + + QMutexLocker locker(&m_mutex); + + if (!m_inputType) + return MF_E_TRANSFORM_TYPE_NOT_SET; + + *ppType = m_inputType; + (*ppType)->AddRef(); + + return S_OK; +} + +STDMETHODIMP MFTransform::GetOutputCurrentType(DWORD dwOutputStreamID, IMFMediaType **ppType) +{ + if (dwOutputStreamID > 0) + return MF_E_INVALIDSTREAMNUMBER; + + if (ppType == NULL) + return E_POINTER; + + QMutexLocker locker(&m_mutex); + + if (!m_outputType) { + if (m_inputType) { + *ppType = m_inputType; + (*ppType)->AddRef(); + return S_OK; + } + return MF_E_TRANSFORM_TYPE_NOT_SET; + } + + *ppType = m_outputType; + (*ppType)->AddRef(); + + return S_OK; +} + +STDMETHODIMP MFTransform::GetInputStatus(DWORD dwInputStreamID, DWORD *pdwFlags) +{ + if (dwInputStreamID > 0) + return MF_E_INVALIDSTREAMNUMBER; + + if (!pdwFlags) + return E_POINTER; + + QMutexLocker locker(&m_mutex); + + if (!m_inputType) + return MF_E_TRANSFORM_TYPE_NOT_SET; + + if (m_sample) + *pdwFlags = 0; + else + *pdwFlags = MFT_INPUT_STATUS_ACCEPT_DATA; + + return S_OK; +} + +STDMETHODIMP MFTransform::GetOutputStatus(DWORD *pdwFlags) +{ + if (!pdwFlags) + return E_POINTER; + + QMutexLocker locker(&m_mutex); + + if (!m_outputType) + return MF_E_TRANSFORM_TYPE_NOT_SET; + + if (m_sample) + *pdwFlags = MFT_OUTPUT_STATUS_SAMPLE_READY; + else + *pdwFlags = 0; + + return S_OK; +} + +STDMETHODIMP MFTransform::SetOutputBounds(LONGLONG hnsLowerBound, LONGLONG hnsUpperBound) +{ + Q_UNUSED(hnsLowerBound); + Q_UNUSED(hnsUpperBound); + return E_NOTIMPL; +} + +STDMETHODIMP MFTransform::ProcessEvent(DWORD dwInputStreamID, IMFMediaEvent *pEvent) +{ + // This MFT ignores all events, and the pipeline should send all events downstream. + Q_UNUSED(dwInputStreamID); + Q_UNUSED(pEvent); + return E_NOTIMPL; +} + +STDMETHODIMP MFTransform::ProcessMessage(MFT_MESSAGE_TYPE eMessage, ULONG_PTR ulParam) +{ + Q_UNUSED(ulParam); + + HRESULT hr = S_OK; + + switch (eMessage) + { + case MFT_MESSAGE_COMMAND_FLUSH: + hr = OnFlush(); + break; + + case MFT_MESSAGE_COMMAND_DRAIN: + // Drain: Tells the MFT not to accept any more input until + // all of the pending output has been processed. That is our + // default behevior already, so there is nothing to do. + break; + + case MFT_MESSAGE_SET_D3D_MANAGER: + // The pipeline should never send this message unless the MFT + // has the MF_SA_D3D_AWARE attribute set to TRUE. However, if we + // do get this message, it's invalid and we don't implement it. + hr = E_NOTIMPL; + break; + + // The remaining messages do not require any action from this MFT. + case MFT_MESSAGE_NOTIFY_BEGIN_STREAMING: + case MFT_MESSAGE_NOTIFY_END_STREAMING: + case MFT_MESSAGE_NOTIFY_END_OF_STREAM: + case MFT_MESSAGE_NOTIFY_START_OF_STREAM: + break; + } + + return hr; +} + +STDMETHODIMP MFTransform::ProcessInput(DWORD dwInputStreamID, IMFSample *pSample, DWORD dwFlags) +{ + if (dwInputStreamID > 0) + return MF_E_INVALIDSTREAMNUMBER; + + if (dwFlags != 0) + return E_INVALIDARG; // dwFlags is reserved and must be zero. + + QMutexLocker locker(&m_mutex); + + if (!m_inputType || !m_outputType) + return MF_E_TRANSFORM_TYPE_NOT_SET; + + if (m_sample) + return MF_E_NOTACCEPTING; + + // Validate the number of buffers. There should only be a single buffer to hold the video frame. + DWORD dwBufferCount = 0; + HRESULT hr = pSample->GetBufferCount(&dwBufferCount); + if (FAILED(hr)) + return hr; + + if (dwBufferCount == 0) + return E_FAIL; + + if (dwBufferCount > 1) + return MF_E_SAMPLE_HAS_TOO_MANY_BUFFERS; + + m_sample = pSample; + m_sample->AddRef(); + + QMutexLocker lockerProbe(&m_videoProbeMutex); + + if (!m_videoProbes.isEmpty()) { + QVideoFrame frame = makeVideoFrame(); + + foreach (MFVideoProbeControl* probe, m_videoProbes) + probe->bufferProbed(frame); + } + + return S_OK; +} + +STDMETHODIMP MFTransform::ProcessOutput(DWORD dwFlags, DWORD cOutputBufferCount, MFT_OUTPUT_DATA_BUFFER *pOutputSamples, DWORD *pdwStatus) +{ + if (dwFlags != 0) + return E_INVALIDARG; + + if (pOutputSamples == NULL || pdwStatus == NULL) + return E_POINTER; + + if (cOutputBufferCount != 1) + return E_INVALIDARG; + + QMutexLocker locker(&m_mutex); + + if (!m_sample) + return MF_E_TRANSFORM_NEED_MORE_INPUT; + + if (pOutputSamples[0].pSample) + pOutputSamples[0].pSample->Release(); + + pOutputSamples[0].pSample = m_sample; + pOutputSamples[0].pSample->AddRef(); + + pOutputSamples[0].dwStatus = 0; + *pdwStatus = 0; + + m_sample->Release(); + m_sample = 0; + + return S_OK; +} + +HRESULT MFTransform::OnFlush() +{ + QMutexLocker locker(&m_mutex); + + if (m_sample) { + m_sample->Release(); + m_sample = 0; + } + return S_OK; +} + +QVideoFrame::PixelFormat MFTransform::formatFromSubtype(const GUID& subtype) +{ + if (subtype == MFVideoFormat_ARGB32) + return QVideoFrame::Format_ARGB32; + else if (subtype == MFVideoFormat_RGB32) + return QVideoFrame::Format_RGB32; + else if (subtype == MFVideoFormat_RGB24) + return QVideoFrame::Format_RGB24; + else if (subtype == MFVideoFormat_RGB565) + return QVideoFrame::Format_RGB565; + else if (subtype == MFVideoFormat_RGB555) + return QVideoFrame::Format_RGB555; + else if (subtype == MFVideoFormat_AYUV) + return QVideoFrame::Format_AYUV444; + else if (subtype == MFVideoFormat_I420) + return QVideoFrame::Format_YUV420P; + else if (subtype == MFVideoFormat_UYVY) + return QVideoFrame::Format_UYVY; + else if (subtype == MFVideoFormat_YV12) + return QVideoFrame::Format_YV12; + else if (subtype == MFVideoFormat_NV12) + return QVideoFrame::Format_NV12; + + return QVideoFrame::Format_Invalid; +} + +QVideoSurfaceFormat MFTransform::videoFormatForMFMediaType(IMFMediaType *mediaType, int *bytesPerLine) +{ + UINT32 stride; + if (FAILED(mediaType->GetUINT32(MF_MT_DEFAULT_STRIDE, &stride))) { + *bytesPerLine = 0; + return QVideoSurfaceFormat(); + } + + *bytesPerLine = (int)stride; + + QSize size; + UINT32 width, height; + if (FAILED(MFGetAttributeSize(mediaType, MF_MT_FRAME_SIZE, &width, &height))) + return QVideoSurfaceFormat(); + + size.setWidth(width); + size.setHeight(height); + + GUID subtype = GUID_NULL; + if (FAILED(mediaType->GetGUID(MF_MT_SUBTYPE, &subtype))) + return QVideoSurfaceFormat(); + + QVideoFrame::PixelFormat pixelFormat = formatFromSubtype(subtype); + QVideoSurfaceFormat format(size, pixelFormat); + + UINT32 num, den; + if (SUCCEEDED(MFGetAttributeRatio(mediaType, MF_MT_PIXEL_ASPECT_RATIO, &num, &den))) { + format.setPixelAspectRatio(num, den); + } + if (SUCCEEDED(MFGetAttributeRatio(mediaType, MF_MT_FRAME_RATE, &num, &den))) { + format.setFrameRate(qreal(num)/den); + } + + return format; +} + +QVideoFrame MFTransform::makeVideoFrame() +{ + QVideoFrame frame; + + if (!m_format.isValid()) + return frame; + + IMFMediaBuffer *buffer = 0; + + do { + if (FAILED(m_sample->ConvertToContiguousBuffer(&buffer))) + break; + + QByteArray array = dataFromBuffer(buffer, m_format.frameHeight(), &m_bytesPerLine); + if (array.isEmpty()) + break; + + // Wrapping IMFSample or IMFMediaBuffer in a QVideoFrame is not possible because we cannot hold + // IMFSample for a "long" time without affecting the rest of the topology. + // If IMFSample is held for more than 5 frames decoder starts to reuse it even though it hasn't been released it yet. + // That is why we copy data from IMFMediaBuffer here. + frame = QVideoFrame(new QMemoryVideoBuffer(array, m_bytesPerLine), m_format.frameSize(), m_format.pixelFormat()); + + LONGLONG startTime = -1; + if (SUCCEEDED(m_sample->GetSampleTime(&startTime))) { + frame.setStartTime(startTime); + + LONGLONG duration = -1; + if (SUCCEEDED(m_sample->GetSampleDuration(&duration))) + frame.setEndTime(startTime + duration); + } + } while (false); + + if (buffer) + buffer->Release(); + + return frame; +} + +QByteArray MFTransform::dataFromBuffer(IMFMediaBuffer *buffer, int height, int *bytesPerLine) +{ + QByteArray array; + BYTE *bytes; + DWORD length; + HRESULT hr = buffer->Lock(&bytes, NULL, &length); + if (SUCCEEDED(hr)) { + array = QByteArray((const char *)bytes, (int)length); + buffer->Unlock(); + } else { + // try to lock as Direct3DSurface + IDirect3DSurface9 *surface = 0; + do { + if (FAILED(MFGetService(buffer, MR_BUFFER_SERVICE, IID_IDirect3DSurface9, (void**)&surface))) + break; + + D3DLOCKED_RECT rect; + if (FAILED(surface->LockRect(&rect, NULL, D3DLOCK_READONLY))) + break; + + if (bytesPerLine) + *bytesPerLine = (int)rect.Pitch; + + array = QByteArray((const char *)rect.pBits, rect.Pitch * height); + surface->UnlockRect(); + } while (false); + + if (surface) { + surface->Release(); + surface = 0; + } + } + + return array; +} diff --git a/src/plugins/wmf/mftvideo.h b/src/plugins/wmf/mftvideo.h new file mode 100644 index 0000000..7f4b15d --- /dev/null +++ b/src/plugins/wmf/mftvideo.h @@ -0,0 +1,114 @@ +/**************************************************************************** +** +** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef MFTRANSFORM_H +#define MFTRANSFORM_H + +#include +#include +#include +#include +#include + +QT_USE_NAMESPACE + +class MFVideoProbeControl; + +class MFTransform: public IMFTransform +{ +public: + MFTransform(); + ~MFTransform(); + + void addProbe(MFVideoProbeControl* probe); + void removeProbe(MFVideoProbeControl* probe); + + // IUnknown methods + STDMETHODIMP QueryInterface(REFIID iid, void** ppv); + STDMETHODIMP_(ULONG) AddRef(); + STDMETHODIMP_(ULONG) Release(); + + // IMFTransform methods + STDMETHODIMP GetStreamLimits(DWORD *pdwInputMinimum, DWORD *pdwInputMaximum, DWORD *pdwOutputMinimum, DWORD *pdwOutputMaximum); + STDMETHODIMP GetStreamCount(DWORD *pcInputStreams, DWORD *pcOutputStreams); + STDMETHODIMP GetStreamIDs(DWORD dwInputIDArraySize, DWORD *pdwInputIDs, DWORD dwOutputIDArraySize, DWORD *pdwOutputIDs); + STDMETHODIMP GetInputStreamInfo(DWORD dwInputStreamID, MFT_INPUT_STREAM_INFO *pStreamInfo); + STDMETHODIMP GetOutputStreamInfo(DWORD dwOutputStreamID, MFT_OUTPUT_STREAM_INFO *pStreamInfo); + STDMETHODIMP GetAttributes(IMFAttributes **pAttributes); + STDMETHODIMP GetInputStreamAttributes(DWORD dwInputStreamID, IMFAttributes **pAttributes); + STDMETHODIMP GetOutputStreamAttributes(DWORD dwOutputStreamID, IMFAttributes **pAttributes); + STDMETHODIMP DeleteInputStream(DWORD dwStreamID); + STDMETHODIMP AddInputStreams(DWORD cStreams, DWORD *adwStreamIDs); + STDMETHODIMP GetInputAvailableType(DWORD dwInputStreamID, DWORD dwTypeIndex, IMFMediaType **ppType); + STDMETHODIMP GetOutputAvailableType(DWORD dwOutputStreamID,DWORD dwTypeIndex, IMFMediaType **ppType); + STDMETHODIMP SetInputType(DWORD dwInputStreamID, IMFMediaType *pType, DWORD dwFlags); + STDMETHODIMP SetOutputType(DWORD dwOutputStreamID, IMFMediaType *pType, DWORD dwFlags); + STDMETHODIMP GetInputCurrentType(DWORD dwInputStreamID, IMFMediaType **ppType); + STDMETHODIMP GetOutputCurrentType(DWORD dwOutputStreamID, IMFMediaType **ppType); + STDMETHODIMP GetInputStatus(DWORD dwInputStreamID, DWORD *pdwFlags); + STDMETHODIMP GetOutputStatus(DWORD *pdwFlags); + STDMETHODIMP SetOutputBounds(LONGLONG hnsLowerBound, LONGLONG hnsUpperBound); + STDMETHODIMP ProcessEvent(DWORD dwInputStreamID, IMFMediaEvent *pEvent); + STDMETHODIMP ProcessMessage(MFT_MESSAGE_TYPE eMessage, ULONG_PTR ulParam); + STDMETHODIMP ProcessInput(DWORD dwInputStreamID, IMFSample *pSample, DWORD dwFlags); + STDMETHODIMP ProcessOutput(DWORD dwFlags, DWORD cOutputBufferCount, MFT_OUTPUT_DATA_BUFFER *pOutputSamples, DWORD *pdwStatus); + +private: + HRESULT OnFlush(); + static QVideoFrame::PixelFormat formatFromSubtype(const GUID& subtype); + static QVideoSurfaceFormat videoFormatForMFMediaType(IMFMediaType *mediaType, int *bytesPerLine); + QVideoFrame makeVideoFrame(); + QByteArray dataFromBuffer(IMFMediaBuffer *buffer, int height, int *bytesPerLine); + + long m_cRef; + IMFMediaType *m_inputType; + IMFMediaType *m_outputType; + IMFSample *m_sample; + QMutex m_mutex; + + QList m_videoProbes; + QMutex m_videoProbeMutex; + + QVideoSurfaceFormat m_format; + int m_bytesPerLine; +}; + +#endif diff --git a/src/plugins/wmf/player/mfplayerservice.cpp b/src/plugins/wmf/player/mfplayerservice.cpp index aad1abe..6ab9a99 100644 --- a/src/plugins/wmf/player/mfplayerservice.cpp +++ b/src/plugins/wmf/player/mfplayerservice.cpp @@ -50,6 +50,7 @@ #include "mfvideorenderercontrol.h" #include "mfaudioendpointcontrol.h" #include "mfaudioprobecontrol.h" +#include "mfvideoprobecontrol.h" #include "mfplayerservice.h" #include "mfplayersession.h" #include "mfmetadatacontrol.h" @@ -113,6 +114,13 @@ QMediaControl* MFPlayerService::requestControl(const char *name) return probe; } return 0; + } else if (qstrcmp(name,QMediaVideoProbeControl_iid) == 0) { + if (m_session) { + MFVideoProbeControl *probe = new MFVideoProbeControl(this); + m_session->addProbe(probe); + return probe; + } + return 0; } return 0; @@ -141,6 +149,14 @@ void MFPlayerService::releaseControl(QMediaControl *control) delete audioProbe; return; } + + MFVideoProbeControl* videoProbe = qobject_cast(control); + if (videoProbe) { + if (m_session) + m_session->removeProbe(videoProbe); + delete videoProbe; + return; + } } MFAudioEndpointControl* MFPlayerService::audioEndpointControl() const diff --git a/src/plugins/wmf/player/mfplayersession.cpp b/src/plugins/wmf/player/mfplayersession.cpp index bd51bba..b65fc94 100644 --- a/src/plugins/wmf/player/mfplayersession.cpp +++ b/src/plugins/wmf/player/mfplayersession.cpp @@ -64,6 +64,7 @@ #include #include "sourceresolver.h" #include "samplegrabber.h" +#include "mftvideo.h" //#define DEBUG_MEDIAFOUNDATION //#define TEST_STREAMING @@ -419,6 +420,7 @@ MFPlayerSession::MFPlayerSession(MFPlayerService *playerService) , m_mediaTypes(0) , m_audioSampleGrabber(0) , m_audioSampleGrabberNode(0) + , m_videoProbeMFT(0) { m_hCloseEvent = CreateEvent(NULL, FALSE, FALSE, NULL); m_sourceResolver = new SourceResolver(); @@ -442,6 +444,7 @@ MFPlayerSession::MFPlayerSession(MFPlayerService *playerService) m_varStart.uhVal.QuadPart = 0; m_audioSampleGrabber = new AudioSampleGrabberCallback; + m_videoProbeMFT = new MFTransform; } void MFPlayerSession::close() @@ -485,9 +488,20 @@ void MFPlayerSession::removeProbe(MFAudioProbeControl *probe) m_audioSampleGrabber->removeProbe(probe); } +void MFPlayerSession::addProbe(MFVideoProbeControl* probe) +{ + m_videoProbeMFT->addProbe(probe); +} + +void MFPlayerSession::removeProbe(MFVideoProbeControl* probe) +{ + m_videoProbeMFT->removeProbe(probe); +} + MFPlayerSession::~MFPlayerSession() { m_audioSampleGrabber->Release(); + m_videoProbeMFT->Release(); } @@ -581,6 +595,9 @@ void MFPlayerSession::setupPlaybackTopology(IMFMediaSource *source, IMFPresentat return; } + // Remember output node id for a first video stream + TOPOID outputNodeId = -1; + // For each stream, create the topology nodes and add them to the topology. DWORD succeededCount = 0; for (DWORD i = 0; i < cSourceStreams; i++) @@ -595,12 +612,17 @@ void MFPlayerSession::setupPlaybackTopology(IMFMediaSource *source, IMFPresentat if (sourceNode) { IMFTopologyNode *outputNode = addOutputNode(streamDesc, mediaType, topology, 0); if (outputNode) { - // Only install audio sample grabber for the first audio stream. - if (mediaType != Audio || m_audioSampleGrabberNode || !setupAudioSampleGrabber(topology, sourceNode, outputNode)) { - hr = sourceNode->ConnectOutput(0, outputNode, 0); - } else { - hr = S_OK; + bool connected = false; + if (mediaType == Audio) { + if (!m_audioSampleGrabberNode) + connected = setupAudioSampleGrabber(topology, sourceNode, outputNode); + } else if (mediaType == Video && outputNodeId == -1) { + // Remember video output node ID. + outputNode->GetTopoNodeID(&outputNodeId); } + + if (!connected) + hr = sourceNode->ConnectOutput(0, outputNode, 0); if (FAILED(hr)) { emit error(QMediaPlayer::FormatError, tr("Unable to play some stream"), false); } @@ -627,6 +649,10 @@ void MFPlayerSession::setupPlaybackTopology(IMFMediaSource *source, IMFPresentat changeStatus(QMediaPlayer::InvalidMedia); emit error(QMediaPlayer::ResourceError, tr("Unable to play"), true); } else { + if (outputNodeId != -1) { + topology = insertMFT(topology, outputNodeId); + } + hr = m_session->SetTopology(MFSESSION_SETTOPOLOGY_IMMEDIATE, topology); if (FAILED(hr)) { changeStatus(QMediaPlayer::UnknownMediaStatus); @@ -837,6 +863,204 @@ QAudioFormat MFPlayerSession::audioFormatForMFMediaType(IMFMediaType *mediaType) return format; } +// BindOutputNode +// Sets the IMFStreamSink pointer on an output node. +// IMFActivate pointer in the output node must be converted to an +// IMFStreamSink pointer before the topology loader resolves the topology. +HRESULT BindOutputNode(IMFTopologyNode *pNode) +{ + IUnknown *nodeObject = NULL; + IMFActivate *activate = NULL; + IMFStreamSink *stream = NULL; + IMFMediaSink *sink = NULL; + + HRESULT hr = pNode->GetObject(&nodeObject); + if (FAILED(hr)) + return hr; + + hr = nodeObject->QueryInterface(IID_PPV_ARGS(&activate)); + if (SUCCEEDED(hr)) { + DWORD dwStreamID = 0; + + // Try to create the media sink. + hr = activate->ActivateObject(IID_PPV_ARGS(&sink)); + if (SUCCEEDED(hr)) + dwStreamID = MFGetAttributeUINT32(pNode, MF_TOPONODE_STREAMID, 0); + + if (SUCCEEDED(hr)) { + // First check if the media sink already has a stream sink with the requested ID. + hr = sink->GetStreamSinkById(dwStreamID, &stream); + if (FAILED(hr)) { + // Create the stream sink. + hr = sink->AddStreamSink(dwStreamID, NULL, &stream); + } + } + + // Replace the node's object pointer with the stream sink. + if (SUCCEEDED(hr)) { + hr = pNode->SetObject(stream); + } + } else { + hr = nodeObject->QueryInterface(IID_PPV_ARGS(&stream)); + } + + if (nodeObject) + nodeObject->Release(); + if (activate) + activate->Release(); + if (stream) + stream->Release(); + if (sink) + sink->Release(); + return hr; +} + +// BindOutputNodes +// Sets the IMFStreamSink pointers on all of the output nodes in a topology. +HRESULT BindOutputNodes(IMFTopology *pTopology) +{ + DWORD cNodes = 0; + + IMFCollection *collection = NULL; + IUnknown *element = NULL; + IMFTopologyNode *node = NULL; + + // Get the collection of output nodes. + HRESULT hr = pTopology->GetOutputNodeCollection(&collection); + + // Enumerate all of the nodes in the collection. + if (SUCCEEDED(hr)) + hr = collection->GetElementCount(&cNodes); + + if (SUCCEEDED(hr)) { + for (DWORD i = 0; i < cNodes; i++) { + hr = collection->GetElement(i, &element); + if (FAILED(hr)) + break; + + hr = element->QueryInterface(IID_IMFTopologyNode, (void**)&node); + if (FAILED(hr)) + break; + + // Bind this node. + hr = BindOutputNode(node); + if (FAILED(hr)) + break; + } + } + + if (collection) + collection->Release(); + if (element) + element->Release(); + if (node) + node->Release(); + return hr; +} + +// This method binds output nodes to complete the topology, +// then loads the topology and inserts MFT between the output node +// and a filter connected to the output node. +IMFTopology *MFPlayerSession::insertMFT(IMFTopology *topology, TOPOID outputNodeId) +{ + bool isNewTopology = false; + + IMFTopoLoader *topoLoader = 0; + IMFTopology *resolvedTopology = 0; + IMFCollection *outputNodes = 0; + + do { + if (FAILED(BindOutputNodes(topology))) + break; + + if (FAILED(MFCreateTopoLoader(&topoLoader))) + break; + + if (FAILED(topoLoader->Load(topology, &resolvedTopology, NULL))) + break; + + // Get all output nodes and search for video output node. + if (FAILED(resolvedTopology->GetOutputNodeCollection(&outputNodes))) + break; + + DWORD elementCount = 0; + if (FAILED(outputNodes->GetElementCount(&elementCount))) + break; + + for (DWORD n = 0; n < elementCount; n++) { + IUnknown *element = 0; + IMFTopologyNode *node = 0; + IMFTopologyNode *inputNode = 0; + IMFTopologyNode *mftNode = 0; + + do { + if (FAILED(outputNodes->GetElement(n, &element))) + break; + + if (FAILED(element->QueryInterface(IID_IMFTopologyNode, (void**)&node))) + break; + + TOPOID id; + if (FAILED(node->GetTopoNodeID(&id))) + break; + + if (id != outputNodeId) + break; + + // Insert MFT between the output node and the node connected to it. + DWORD outputIndex = 0; + if (FAILED(node->GetInput(0, &inputNode, &outputIndex))) + break; + + if (FAILED(MFCreateTopologyNode(MF_TOPOLOGY_TRANSFORM_NODE, &mftNode))) + break; + + if (FAILED(mftNode->SetObject(m_videoProbeMFT))) + break; + + if (FAILED(resolvedTopology->AddNode(mftNode))) + break; + + if (FAILED(inputNode->ConnectOutput(0, mftNode, 0))) + break; + + if (FAILED(mftNode->ConnectOutput(0, node, 0))) + break; + + isNewTopology = true; + } while (false); + + if (mftNode) + mftNode->Release(); + if (inputNode) + inputNode->Release(); + if (node) + node->Release(); + if (element) + element->Release(); + + if (isNewTopology) + break; + } + } while (false); + + if (outputNodes) + outputNodes->Release(); + + if (topoLoader) + topoLoader->Release(); + + if (isNewTopology) { + topology->Release(); + return resolvedTopology; + } + + if (resolvedTopology) + resolvedTopology->Release(); + + return topology; +} + void MFPlayerSession::stop(bool immediate) { #ifdef DEBUG_MEDIAFOUNDATION diff --git a/src/plugins/wmf/player/mfplayersession.h b/src/plugins/wmf/player/mfplayersession.h index 969fd24..fb3f7dd 100644 --- a/src/plugins/wmf/player/mfplayersession.h +++ b/src/plugins/wmf/player/mfplayersession.h @@ -55,6 +55,7 @@ #include #include #include +#include QT_BEGIN_NAMESPACE class QMediaContent; @@ -72,7 +73,9 @@ class MFPlayerControl; class MFMetaDataControl; class MFPlayerService; class AudioSampleGrabberCallback; +class MFTransform; class MFAudioProbeControl; +class MFVideoProbeControl; class MFPlayerSession : public QObject, public IMFAsyncCallback { @@ -120,6 +123,8 @@ public: void addProbe(MFAudioProbeControl* probe); void removeProbe(MFAudioProbeControl* probe); + void addProbe(MFVideoProbeControl* probe); + void removeProbe(MFVideoProbeControl* probe); Q_SIGNALS: void error(QMediaPlayer::Error error, QString errorString, bool isFatal); @@ -223,6 +228,9 @@ private: QAudioFormat audioFormatForMFMediaType(IMFMediaType *mediaType) const; AudioSampleGrabberCallback *m_audioSampleGrabber; IMFTopologyNode *m_audioSampleGrabberNode; + + IMFTopology *insertMFT(IMFTopology *topology, TOPOID outputNodeId); + MFTransform *m_videoProbeMFT; }; diff --git a/src/plugins/wmf/player/mfvideoprobecontrol.cpp b/src/plugins/wmf/player/mfvideoprobecontrol.cpp new file mode 100644 index 0000000..c16dcd9 --- /dev/null +++ b/src/plugins/wmf/player/mfvideoprobecontrol.cpp @@ -0,0 +1,56 @@ +/**************************************************************************** +** +** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "mfvideoprobecontrol.h" + +MFVideoProbeControl::MFVideoProbeControl(QObject *parent) + : QMediaVideoProbeControl(parent) +{ +} + +MFVideoProbeControl::~MFVideoProbeControl() +{ +} + +void MFVideoProbeControl::bufferProbed(const QVideoFrame& frame) +{ + QMetaObject::invokeMethod(this, "videoFrameProbed", Qt::QueuedConnection, Q_ARG(QVideoFrame, frame)); +} diff --git a/src/plugins/wmf/player/mfvideoprobecontrol.h b/src/plugins/wmf/player/mfvideoprobecontrol.h new file mode 100644 index 0000000..3387258 --- /dev/null +++ b/src/plugins/wmf/player/mfvideoprobecontrol.h @@ -0,0 +1,61 @@ +/**************************************************************************** +** +** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** GNU Lesser General Public License Usage +** This file may be used under the terms of the GNU Lesser General Public +** License version 2.1 as published by the Free Software Foundation and +** appearing in the file LICENSE.LGPL included in the packaging of this +** file. Please review the following information to ensure the GNU Lesser +** General Public License version 2.1 requirements will be met: +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU General +** Public License version 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of this +** file. Please review the following information to ensure the GNU General +** Public License version 3.0 requirements will be met: +** http://www.gnu.org/copyleft/gpl.html. +** +** Other Usage +** Alternatively, this file may be used in accordance with the terms and +** conditions contained in a signed written agreement between you and Nokia. +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef MFVIDEOPROBECONTROL_H +#define MFVIDEOPROBECONTROL_H + +#include +#include +#include + +QT_USE_NAMESPACE + +class MFVideoProbeControl : public QMediaVideoProbeControl +{ + Q_OBJECT +public: + explicit MFVideoProbeControl(QObject *parent); + virtual ~MFVideoProbeControl(); + + void bufferProbed(const QVideoFrame& frame); +}; + +#endif diff --git a/src/plugins/wmf/player/player.pri b/src/plugins/wmf/player/player.pri index 8f28b52..ac8dc9f 100644 --- a/src/plugins/wmf/player/player.pri +++ b/src/plugins/wmf/player/player.pri @@ -11,7 +11,8 @@ HEADERS += \ $$PWD/mfvideorenderercontrol.h \ $$PWD/mfaudioendpointcontrol.h \ $$PWD/mfmetadatacontrol.h \ - $$PWD/mfaudioprobecontrol.h + $$PWD/mfaudioprobecontrol.h \ + $$PWD/mfvideoprobecontrol.h SOURCES += \ $$PWD/mfplayerservice.cpp \ @@ -20,7 +21,8 @@ SOURCES += \ $$PWD/mfvideorenderercontrol.cpp \ $$PWD/mfaudioendpointcontrol.cpp \ $$PWD/mfmetadatacontrol.cpp \ - $$PWD/mfaudioprobecontrol.cpp + $$PWD/mfaudioprobecontrol.cpp \ + $$PWD/mfvideoprobecontrol.cpp !isEmpty(QT.widgets.name):!simulator { HEADERS += $$PWD/evr9videowindowcontrol.h diff --git a/src/plugins/wmf/wmf.pro b/src/plugins/wmf/wmf.pro index 87555dd..2292173 100644 --- a/src/plugins/wmf/wmf.pro +++ b/src/plugins/wmf/wmf.pro @@ -19,13 +19,15 @@ HEADERS += \ wmfserviceplugin.h \ mfstream.h \ sourceresolver.h \ - samplegrabber.h + samplegrabber.h \ + mftvideo.h SOURCES += \ wmfserviceplugin.cpp \ mfstream.cpp \ sourceresolver.cpp \ - samplegrabber.cpp + samplegrabber.cpp \ + mftvideo.cpp include (player/player.pri) include (decoder/decoder.pri) -- 2.7.4