From 996f02a531318f5aa3004d876fb1b3f2af429e3b Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Sat, 18 May 2013 13:04:31 -0700 Subject: [PATCH] Multiple Media Foundation video i/o fixes. Video i/o tests enabled for media foundation; Negative stride support added to VideoCapture; Error handling improved, dead lock in case of playback error fixed; Some code refacotring done. --- modules/highgui/src/cap_msmf.cpp | 178 ++++++++++++++------------------- modules/highgui/test/test_precomp.hpp | 6 +- modules/highgui/test/test_video_io.cpp | 38 ++++++- 3 files changed, 112 insertions(+), 110 deletions(-) diff --git a/modules/highgui/src/cap_msmf.cpp b/modules/highgui/src/cap_msmf.cpp index 15f2bdc..814fb75 100644 --- a/modules/highgui/src/cap_msmf.cpp +++ b/modules/highgui/src/cap_msmf.cpp @@ -54,7 +54,6 @@ #include #include #include -#include #include #include #include @@ -72,6 +71,8 @@ #pragma comment(lib, "Mfreadwrite") #pragma comment(lib, "MinCore_Downlevel") +// for ComPtr usage +#include using namespace Microsoft::WRL; struct IMFMediaType; @@ -112,7 +113,7 @@ struct MediaType unsigned int width; unsigned int MF_MT_YUV_MATRIX; unsigned int MF_MT_VIDEO_LIGHTING; - unsigned int MF_MT_DEFAULT_STRIDE; + int MF_MT_DEFAULT_STRIDE; // stride is negative if image is bottom-up unsigned int MF_MT_VIDEO_CHROMA_SITING; GUID MF_MT_AM_FORMAT_TYPE; wchar_t *pMF_MT_AM_FORMAT_TYPEName; @@ -226,6 +227,7 @@ private: DWORD dwSampleSize); STDMETHODIMP OnShutdown(); }; + /// Class for controlling of thread of the grabbing raw data from video device class ImageGrabberThread { @@ -249,6 +251,7 @@ private: bool igt_stop; unsigned int igt_DeviceID; }; + // Structure for collecting info about one parametr of current video device struct Parametr { @@ -260,6 +263,7 @@ struct Parametr long Flag; Parametr(); }; + // Structure for collecting info about 17 parametrs of current video device struct CamParametrs { @@ -281,11 +285,13 @@ struct CamParametrs Parametr Iris; Parametr Focus; }; + typedef std::wstring String; typedef std::vector vectorNum; typedef std::map SUBTYPEMap; typedef std::map FrameRateMap; typedef void(*emergensyStopEventCallback)(int, void *); + /// Class for controlling of video device class videoDevice { @@ -329,7 +335,7 @@ private: IMFMediaSource *vd_pSource; emergensyStopEventCallback vd_func; void *vd_userData; - long enumerateCaptureFormats(IMFMediaSource *pSource); + HRESULT enumerateCaptureFormats(IMFMediaSource *pSource); long setDeviceFormat(IMFMediaSource *pSource, unsigned long dwFormatIndex); void buildLibraryofTypes(); int findType(unsigned int size, unsigned int frameRate = 0); @@ -337,6 +343,7 @@ private: long initDevice(); long checkDevice(IMFAttributes *pAttributes, IMFActivate **pDevice); }; + /// Class for managing of list of video devices class videoDevices { @@ -352,6 +359,7 @@ private: std::vector vds_Devices; videoDevices(void); }; + // Class for creating of Media Foundation context class Media_Foundation { @@ -362,6 +370,7 @@ public: private: Media_Foundation(void); }; + /// The only visiable class for controlling of video devices in format singelton class videoInput { @@ -411,23 +420,27 @@ public: bool isFrameNew(int deviceID); // Writing of Raw Data pixels from video device with deviceID with correction of RedAndBlue flipping flipRedAndBlue and vertical flipping flipImage bool getPixels(int deviceID, unsigned char * pixels, bool flipRedAndBlue = false, bool flipImage = false); + static void processPixels(unsigned char * src, unsigned char * dst, unsigned int width, unsigned int height, unsigned int bpp, bool bRGB, bool bFlip); private: bool accessToDevices; videoInput(void); - void processPixels(unsigned char * src, unsigned char * dst, unsigned int width, unsigned int height, unsigned int bpp, bool bRGB, bool bFlip); void updateListOfDevices(); }; + DebugPrintOut::DebugPrintOut(void):verbose(true) { } + DebugPrintOut::~DebugPrintOut(void) { } + DebugPrintOut& DebugPrintOut::getInstance() { static DebugPrintOut instance; return instance; } + void DebugPrintOut::printOut(const wchar_t *format, ...) { if(verbose) @@ -448,14 +461,17 @@ void DebugPrintOut::printOut(const wchar_t *format, ...) va_end (args); } } + void DebugPrintOut::setVerbose(bool state) { verbose = state; } + LPCWSTR GetGUIDNameConstNew(const GUID& guid); HRESULT GetGUIDNameNew(const GUID& guid, WCHAR **ppwsz); HRESULT LogAttributeValueByIndexNew(IMFAttributes *pAttr, DWORD index); HRESULT SpecialCaseAttributeValueNew(GUID guid, const PROPVARIANT& var, MediaType &out); + unsigned int *GetParametr(GUID guid, MediaType &out) { if(guid == MF_MT_YUV_MATRIX) @@ -463,7 +479,7 @@ unsigned int *GetParametr(GUID guid, MediaType &out) if(guid == MF_MT_VIDEO_LIGHTING) return &(out.MF_MT_VIDEO_LIGHTING); if(guid == MF_MT_DEFAULT_STRIDE) - return &(out.MF_MT_DEFAULT_STRIDE); + return (unsigned int*)&(out.MF_MT_DEFAULT_STRIDE); if(guid == MF_MT_VIDEO_CHROMA_SITING) return &(out.MF_MT_VIDEO_CHROMA_SITING); if(guid == MF_MT_VIDEO_NOMINAL_RANGE) @@ -480,6 +496,7 @@ unsigned int *GetParametr(GUID guid, MediaType &out) return &(out.MF_MT_INTERLACE_MODE); return NULL; } + HRESULT LogAttributeValueByIndexNew(IMFAttributes *pAttr, DWORD index, MediaType &out) { WCHAR *pGuidName = NULL; @@ -566,6 +583,7 @@ done: PropVariantClear(&var); return hr; } + HRESULT GetGUIDNameNew(const GUID& guid, WCHAR **ppwsz) { HRESULT hr = S_OK; @@ -625,6 +643,10 @@ HRESULT LogVideoAreaNew(const PROPVARIANT& var) } HRESULT SpecialCaseAttributeValueNew(GUID guid, const PROPVARIANT& var, MediaType &out) { + if (guid == MF_MT_DEFAULT_STRIDE) + { + out.MF_MT_DEFAULT_STRIDE = var.intVal; + } else if (guid == MF_MT_FRAME_SIZE) { UINT32 uHigh = 0, uLow = 0; @@ -1039,6 +1061,7 @@ HRESULT ImageGrabber::startGrabbing(void) hr = S_OK; goto done; } + printf("media foundation event: %d\n", met); if (met == MESessionEnded) { DPO->printOut(L"IMAGEGRABBER VIDEODEVICE %i: MESessionEnded \n", ig_DeviceID); @@ -1055,16 +1078,21 @@ HRESULT ImageGrabber::startGrabbing(void) DPO->printOut(L"IMAGEGRABBER VIDEODEVICE %i: MEVideoCaptureDeviceRemoved \n", ig_DeviceID); break; } + if ((met == MEError) || (met == MENonFatalError)) + { + pEvent->GetStatus(&hrStatus); + DPO->printOut(L"IMAGEGRABBER VIDEODEVICE %i: MEError | MENonFatalError: %u\n", ig_DeviceID, hrStatus); + break; + } } + DPO->printOut(L"IMAGEGRABBER VIDEODEVICE %i: Finish startGrabbing \n", ig_DeviceID); +done: if (ig_Synchronous) { SetEvent(ig_hFinish); } - DPO->printOut(L"IMAGEGRABBER VIDEODEVICE %i: Finish startGrabbing \n", ig_DeviceID); - -done: SafeRelease(&ig_pSession); SafeRelease(&ig_pTopology); @@ -2010,7 +2038,7 @@ videoDevice::~videoDevice(void) CoTaskMemFree(vd_pFriendlyName); } -long videoDevice::enumerateCaptureFormats(IMFMediaSource *pSource) +HRESULT videoDevice::enumerateCaptureFormats(IMFMediaSource *pSource) { ComPtr pPD = NULL; ComPtr pSD = NULL; @@ -3002,9 +3030,7 @@ protected: IplImage* frame; bool isOpened; - long enumerateCaptureFormats(IMFMediaSource *pSource); - void processPixels(unsigned char * src, unsigned char * dst, unsigned int width, - unsigned int height, unsigned int bpp, bool bRGB, bool bFlip); + HRESULT enumerateCaptureFormats(IMFMediaSource *pSource); }; CvCaptureFile_MSMF::CvCaptureFile_MSMF(): @@ -3034,10 +3060,10 @@ bool CvCaptureFile_MSMF::open(const char* filename) MF_OBJECT_TYPE ObjectType = MF_OBJECT_INVALID; - IMFSourceResolver* pSourceResolver = NULL; + ComPtr pSourceResolver = NULL; IUnknown* pUnkSource = NULL; - hr = MFCreateSourceResolver(&pSourceResolver); + hr = MFCreateSourceResolver(pSourceResolver.GetAddressOf()); if (SUCCEEDED(hr)) { @@ -3056,10 +3082,12 @@ bool CvCaptureFile_MSMF::open(const char* filename) hr = pUnkSource->QueryInterface(IID_PPV_ARGS(&videoFileSource)); } - SafeRelease(&pSourceResolver); SafeRelease(&pUnkSource); - enumerateCaptureFormats(videoFileSource); + if (SUCCEEDED(hr)) + { + hr = enumerateCaptureFormats(videoFileSource); + } if (SUCCEEDED(hr)) { @@ -3071,9 +3099,9 @@ bool CvCaptureFile_MSMF::open(const char* filename) grabberThread->start(); } - isOpened = true; + isOpened = SUCCEEDED(hr); - return true; + return isOpened; } void CvCaptureFile_MSMF::close() @@ -3207,6 +3235,8 @@ double CvCaptureFile_MSMF::getProperty(int property_id) return captureFormats[captureFormatIndex].width; case CV_CAP_PROP_FRAME_HEIGHT: return captureFormats[captureFormatIndex].height; + case CV_CAP_PROP_FRAME_COUNT: + return 30; case CV_CAP_PROP_FOURCC: // FIXME: implement method in VideoInput back end //return VI.getFourcc(index); @@ -3282,77 +3312,18 @@ IplImage* CvCaptureFile_MSMF::retrieveFrame(int) RawImage *RIOut = grabberThread->getImageGrabber()->getRawImage(); unsigned int size = bytes * width * height; + bool verticalFlip = captureFormats[captureFormatIndex].MF_MT_DEFAULT_STRIDE < 0; + if(RIOut && size == RIOut->getSize()) { - processPixels(RIOut->getpPixels(), (unsigned char*)frame->imageData, width, height, bytes, false, false); + videoInput::processPixels(RIOut->getpPixels(), (unsigned char*)frame->imageData, width, + height, bytes, false, verticalFlip); } return frame; } -void CvCaptureFile_MSMF::processPixels(unsigned char * src, unsigned char * dst, unsigned int width, - unsigned int height, unsigned int bpp, bool bRGB, bool bFlip) -{ - unsigned int widthInBytes = width * bpp; - unsigned int numBytes = widthInBytes * height; - int *dstInt, *srcInt; - if(!bRGB) - { - if(bFlip) - { - for(unsigned int y = 0; y < height; y++) - { - dstInt = (int *)(dst + (y * widthInBytes)); - srcInt = (int *)(src + ( (height -y -1) * widthInBytes)); - memcpy(dstInt, srcInt, widthInBytes); - } - } - else - { - memcpy(dst, src, numBytes); - } - } - else - { - if(bFlip) - { - unsigned int x = 0; - unsigned int y = (height - 1) * widthInBytes; - src += y; - for(unsigned int i = 0; i < numBytes; i+=3) - { - if(x >= width) - { - x = 0; - src -= widthInBytes*2; - } - *dst = *(src+2); - dst++; - *dst = *(src+1); - dst++; - *dst = *src; - dst++; - src+=3; - x++; - } - } - else - { - for(unsigned int i = 0; i < numBytes; i+=3) - { - *dst = *(src+2); - dst++; - *dst = *(src+1); - dst++; - *dst = *src; - dst++; - src+=3; - } - } - } -} - -long CvCaptureFile_MSMF::enumerateCaptureFormats(IMFMediaSource *pSource) +HRESULT CvCaptureFile_MSMF::enumerateCaptureFormats(IMFMediaSource *pSource) { ComPtr pPD = NULL; ComPtr pSD = NULL; @@ -3364,12 +3335,6 @@ long CvCaptureFile_MSMF::enumerateCaptureFormats(IMFMediaSource *pSource) goto done; } - DWORD cnt; - - pPD->GetStreamDescriptorCount(&cnt); - - printf("Stream count: %d\n", cnt); - BOOL fSelected; hr = pPD->GetStreamDescriptorByIndex(0, &fSelected, pSD.GetAddressOf()); if (FAILED(hr)) @@ -3423,10 +3388,21 @@ CvCapture* cvCreateCameraCapture_MSMF( int index ) CvCapture* cvCreateFileCapture_MSMF (const char* filename) { CvCaptureFile_MSMF* capture = new CvCaptureFile_MSMF; - if( capture->open(filename) ) - return capture; - delete capture; - return 0; + try + { + if( capture->open(filename) ) + return capture; + else + { + delete capture; + return NULL; + } + } + catch(...) + { + delete capture; + throw; + } } // @@ -3440,10 +3416,10 @@ class CvVideoWriter_MSMF : public CvVideoWriter public: CvVideoWriter_MSMF(); virtual ~CvVideoWriter_MSMF(); - virtual bool open( const char* filename, int fourcc, - double fps, CvSize frameSize, bool isColor ); + virtual bool open(const char* filename, int fourcc, + double fps, CvSize frameSize, bool isColor); virtual void close(); - virtual bool writeFrame( const IplImage* img); + virtual bool writeFrame(const IplImage* img); private: UINT32 videoWidth; @@ -3533,7 +3509,7 @@ bool CvVideoWriter_MSMF::open( const char* filename, int fourcc, videoWidth = frameSize.width; videoHeight = frameSize.height; fps = _fps; - bitRate = videoWidth*videoHeight; // 1-bit per pixel + bitRate = fps*videoWidth*videoHeight; // 1-bit per pixel encodingFormat = FourCC2GUID(fourcc); inputFormat = MFVideoFormat_RGB32; @@ -3596,14 +3572,7 @@ bool CvVideoWriter_MSMF::writeFrame(const IplImage* img) BYTE g = rowStart[colIdx * img->nChannels + 1]; BYTE r = rowStart[colIdx * img->nChannels + 2]; - // On ARM devices data is stored starting from the last line - // (and not the first line) so you have to revert them on the Y axis -#if _M_ARM - int targetRow = videoHeight - rowIdx - 1; - target[(targetRow * videoWidth) + colIdx] = (r << 16) + (g << 8) + b; -#else target[rowIdx*img->width+colIdx] = (r << 16) + (g << 8) + b; -#endif } } @@ -3677,6 +3646,7 @@ HRESULT CvVideoWriter_MSMF::InitializeSinkWriter(const char* filename) { hr = MFSetAttributeRatio(mediaTypeOut.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); } + if (SUCCEEDED(hr)) { hr = sinkWriter->AddStream(mediaTypeOut.Get(), &streamIndex); @@ -3711,7 +3681,7 @@ HRESULT CvVideoWriter_MSMF::InitializeSinkWriter(const char* filename) { hr = MFSetAttributeRatio(mediaTypeIn.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); } - if (SUCCEEDED(hr)) + if (SUCCEEDED(hr)) { hr = sinkWriter->SetInputMediaType(streamIndex, mediaTypeIn.Get(), NULL); } diff --git a/modules/highgui/test/test_precomp.hpp b/modules/highgui/test/test_precomp.hpp index 0d0bd80..be06c06 100644 --- a/modules/highgui/test/test_precomp.hpp +++ b/modules/highgui/test/test_precomp.hpp @@ -47,7 +47,8 @@ defined(HAVE_QUICKTIME) || \ defined(HAVE_AVFOUNDATION) || \ /*defined(HAVE_OPENNI) || too specialized */ \ - defined(HAVE_FFMPEG) + defined(HAVE_FFMPEG) || \ + defined(HAVE_MSMF) # define BUILD_WITH_VIDEO_INPUT_SUPPORT 1 #else # define BUILD_WITH_VIDEO_INPUT_SUPPORT 0 @@ -57,7 +58,8 @@ defined(HAVE_GSTREAMER) || \ defined(HAVE_QUICKTIME) || \ defined(HAVE_AVFOUNDATION) || \ - defined(HAVE_FFMPEG) + defined(HAVE_FFMPEG) || \ + defined(HAVE_MSMF) # define BUILD_WITH_VIDEO_OUTPUT_SUPPORT 1 #else # define BUILD_WITH_VIDEO_OUTPUT_SUPPORT 0 diff --git a/modules/highgui/test/test_video_io.cpp b/modules/highgui/test/test_video_io.cpp index b0c2e53..34ec0bd 100644 --- a/modules/highgui/test/test_video_io.cpp +++ b/modules/highgui/test/test_video_io.cpp @@ -54,6 +54,33 @@ string fourccToString(int fourcc) return format("%c%c%c%c", fourcc & 255, (fourcc >> 8) & 255, (fourcc >> 16) & 255, (fourcc >> 24) & 255); } +#ifdef HAVE_MSMF +const VideoFormat g_specific_fmt_list[] = +{ +/* VideoFormat("avi", 'dv25'), + VideoFormat("avi", 'dv50'), + VideoFormat("avi", 'dvc '), + VideoFormat("avi", 'dvh1'), + VideoFormat("avi", 'dvhd'), + VideoFormat("avi", 'dvsd'), + VideoFormat("avi", 'dvsl'), + VideoFormat("avi", 'M4S2'), */ + VideoFormat("wmv", 'WMV3'), + // VideoFormat("avi", 'H264'), + // VideoFormat("avi", 'MJPG'), + // VideoFormat("avi", 'MP43'), + // VideoFormat("avi", 'MP4S'), + // VideoFormat("avi", 'MP4V'), +/* VideoFormat("avi", 'MPG1'), + VideoFormat("avi", 'MSS1'), + VideoFormat("avi", 'MSS2'), + VideoFormat("avi", 'WMV1'), + VideoFormat("avi", 'WMV2'), + VideoFormat("avi", 'WMV3'), + VideoFormat("avi", 'WVC1'), */ + VideoFormat() +}; +#else const VideoFormat g_specific_fmt_list[] = { VideoFormat("avi", CV_FOURCC('X', 'V', 'I', 'D')), @@ -63,17 +90,17 @@ const VideoFormat g_specific_fmt_list[] = VideoFormat("mkv", CV_FOURCC('X', 'V', 'I', 'D')), VideoFormat("mkv", CV_FOURCC('M', 'P', 'E', 'G')), VideoFormat("mkv", CV_FOURCC('M', 'J', 'P', 'G')), - VideoFormat("mov", CV_FOURCC('m', 'p', '4', 'v')), VideoFormat() }; +#endif } class CV_HighGuiTest : public cvtest::BaseTest { protected: - void ImageTest(const string& dir); + void ImageTest (const string& dir); void VideoTest (const string& dir, const cvtest::VideoFormat& fmt); void SpecificImageTest (const string& dir); void SpecificVideoTest (const string& dir, const cvtest::VideoFormat& fmt); @@ -291,8 +318,11 @@ void CV_HighGuiTest::VideoTest(const string& dir, const cvtest::VideoFormat& fmt if (psnr < thresDbell) { printf("Too low psnr = %gdb\n", psnr); - // imwrite("img.png", img); - // imwrite("img1.png", img1); + //imwrite("original.png", img); + //imwrite("after_test.png", img1); + //Mat diff; + //absdiff(img, img1, diff); + //imwrite("diff.png", diff); ts->set_failed_test_info(ts->FAIL_MISMATCH); break; } -- 2.7.4