1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
5 // Copyright (C) 2015, Itseez, Inc., all rights reserved.
6 // Third party copyrights are property of their respective owners.
13 # define NO_VA_SUPPORT_ERROR CV_Error(cv::Error::StsBadFunc, "OpenCV was build without VA support (libva)")
18 ////////////////////////////////////////////////////////////////////////
19 // CL-VA Interoperability
22 # include "opencv2/core/opencl/runtime/opencl_core.hpp"
23 # include "opencv2/core.hpp"
24 # include "opencv2/core/ocl.hpp"
25 # include "opencl_kernels_core.hpp"
28 #if defined(HAVE_VA_INTEL) && defined(HAVE_OPENCL)
29 # include <CL/va_ext.h>
30 #endif // HAVE_VA_INTEL && HAVE_OPENCL
32 namespace cv { namespace va_intel {
34 #if defined(HAVE_VA_INTEL) && defined(HAVE_OPENCL)
36 static clGetDeviceIDsFromVA_APIMediaAdapterINTEL_fn clGetDeviceIDsFromVA_APIMediaAdapterINTEL = NULL;
37 static clCreateFromVA_APIMediaSurfaceINTEL_fn clCreateFromVA_APIMediaSurfaceINTEL = NULL;
38 static clEnqueueAcquireVA_APIMediaSurfacesINTEL_fn clEnqueueAcquireVA_APIMediaSurfacesINTEL = NULL;
39 static clEnqueueReleaseVA_APIMediaSurfacesINTEL_fn clEnqueueReleaseVA_APIMediaSurfacesINTEL = NULL;
41 static bool contextInitialized = false;
43 #endif // HAVE_VA_INTEL && HAVE_OPENCL
47 Context& initializeContextFromVA(VADisplay display, bool tryInterop)
49 (void)display; (void)tryInterop;
53 # if (defined(HAVE_VA_INTEL) && defined(HAVE_OPENCL))
54 contextInitialized = false;
58 cl_int status = clGetPlatformIDs(0, NULL, &numPlatforms);
59 if (status != CL_SUCCESS)
60 CV_Error(cv::Error::OpenCLInitError, "OpenCL: Can't get number of platforms");
61 if (numPlatforms == 0)
62 CV_Error(cv::Error::OpenCLInitError, "OpenCL: No available platforms");
64 std::vector<cl_platform_id> platforms(numPlatforms);
65 status = clGetPlatformIDs(numPlatforms, &platforms[0], NULL);
66 if (status != CL_SUCCESS)
67 CV_Error(cv::Error::OpenCLInitError, "OpenCL: Can't get platform Id list");
69 // For CL-VA interop, we must find platform/device with "cl_intel_va_api_media_sharing" extension.
70 // With standard initialization procedure, we should examine platform extension string for that.
71 // But in practice, the platform ext string doesn't contain it, while device ext string does.
72 // Follow Intel procedure (see tutorial), we should obtain device IDs by extension call.
73 // Note that we must obtain function pointers using specific platform ID, and can't provide pointers in advance.
74 // So, we iterate and select the first platform, for which we got non-NULL pointers, device, and CL context.
77 cl_context context = 0;
78 cl_device_id device = 0;
80 for (int i = 0; i < (int)numPlatforms; ++i)
82 // Get extension function pointers
84 clGetDeviceIDsFromVA_APIMediaAdapterINTEL = (clGetDeviceIDsFromVA_APIMediaAdapterINTEL_fn)
85 clGetExtensionFunctionAddressForPlatform(platforms[i], "clGetDeviceIDsFromVA_APIMediaAdapterINTEL");
86 clCreateFromVA_APIMediaSurfaceINTEL = (clCreateFromVA_APIMediaSurfaceINTEL_fn)
87 clGetExtensionFunctionAddressForPlatform(platforms[i], "clCreateFromVA_APIMediaSurfaceINTEL");
88 clEnqueueAcquireVA_APIMediaSurfacesINTEL = (clEnqueueAcquireVA_APIMediaSurfacesINTEL_fn)
89 clGetExtensionFunctionAddressForPlatform(platforms[i], "clEnqueueAcquireVA_APIMediaSurfacesINTEL");
90 clEnqueueReleaseVA_APIMediaSurfacesINTEL = (clEnqueueReleaseVA_APIMediaSurfacesINTEL_fn)
91 clGetExtensionFunctionAddressForPlatform(platforms[i], "clEnqueueReleaseVA_APIMediaSurfacesINTEL");
93 if (((void*)clGetDeviceIDsFromVA_APIMediaAdapterINTEL == NULL) ||
94 ((void*)clCreateFromVA_APIMediaSurfaceINTEL == NULL) ||
95 ((void*)clEnqueueAcquireVA_APIMediaSurfacesINTEL == NULL) ||
96 ((void*)clEnqueueReleaseVA_APIMediaSurfacesINTEL == NULL))
103 cl_uint numDevices = 0;
105 status = clGetDeviceIDsFromVA_APIMediaAdapterINTEL(platforms[i], CL_VA_API_DISPLAY_INTEL, display,
106 CL_PREFERRED_DEVICES_FOR_VA_API_INTEL, 0, NULL, &numDevices);
107 if ((status != CL_SUCCESS) || !(numDevices > 0))
109 numDevices = 1; // initializeContextFromHandle() expects only 1 device
110 status = clGetDeviceIDsFromVA_APIMediaAdapterINTEL(platforms[i], CL_VA_API_DISPLAY_INTEL, display,
111 CL_PREFERRED_DEVICES_FOR_VA_API_INTEL, numDevices, &device, NULL);
112 if (status != CL_SUCCESS)
115 // Creating CL-VA media sharing OpenCL context
117 cl_context_properties props[] = {
118 CL_CONTEXT_VA_API_DISPLAY_INTEL, (cl_context_properties) display,
119 CL_CONTEXT_INTEROP_USER_SYNC, CL_FALSE, // no explicit sync required
123 context = clCreateContext(props, numDevices, &device, NULL, NULL, &status);
124 if (status != CL_SUCCESS)
126 clReleaseDevice(device);
137 contextInitialized = true;
138 Context& ctx = Context::getDefault(false);
139 initializeContextFromHandle(ctx, platforms[found], context, device);
143 # endif // HAVE_VA_INTEL && HAVE_OPENCL
145 Context& ctx = Context::getDefault(true);
151 #if defined(HAVE_VA_INTEL) && defined(HAVE_OPENCL)
152 static bool ocl_convert_nv12_to_bgr(cl_mem clImageY, cl_mem clImageUV, cl_mem clBuffer, int step, int cols, int rows)
155 k.create("YUV2BGR_NV12_8u", cv::ocl::core::cvtclr_dx_oclsrc, "");
159 k.args(clImageY, clImageUV, clBuffer, step, cols, rows);
161 size_t globalsize[] = { (size_t)cols, (size_t)rows };
162 return k.run(2, globalsize, 0, false);
165 static bool ocl_convert_bgr_to_nv12(cl_mem clBuffer, int step, int cols, int rows, cl_mem clImageY, cl_mem clImageUV)
168 k.create("BGR2YUV_NV12_8u", cv::ocl::core::cvtclr_dx_oclsrc, "");
172 k.args(clBuffer, step, cols, rows, clImageY, clImageUV);
174 size_t globalsize[] = { (size_t)cols, (size_t)rows };
175 return k.run(2, globalsize, 0, false);
177 #endif // HAVE_VA_INTEL && HAVE_OPENCL
179 } // namespace cv::va_intel::ocl
182 const int NCHANNELS = 3;
184 static void copy_convert_nv12_to_bgr(const VAImage& image, const unsigned char* buffer, Mat& bgr)
186 const float d1 = 16.0f;
187 const float d2 = 128.0f;
189 static const float coeffs[5] =
198 const size_t srcOffsetY = image.offsets[0];
199 const size_t srcOffsetUV = image.offsets[1];
201 const size_t srcStepY = image.pitches[0];
202 const size_t srcStepUV = image.pitches[1];
204 const size_t dstStep = bgr.step;
206 const unsigned char* srcY0 = buffer + srcOffsetY;
207 const unsigned char* srcUV = buffer + srcOffsetUV;
209 unsigned char* dst0 = bgr.data;
211 for (int y = 0; y < bgr.rows; y += 2)
213 const unsigned char* srcY1 = srcY0 + srcStepY;
214 unsigned char *dst1 = dst0 + dstStep;
216 for (int x = 0; x < bgr.cols; x += 2)
218 float Y0 = float(srcY0[x+0]);
219 float Y1 = float(srcY0[x+1]);
220 float Y2 = float(srcY1[x+0]);
221 float Y3 = float(srcY1[x+1]);
223 float U = float(srcUV[2*(x/2)+0]) - d2;
224 float V = float(srcUV[2*(x/2)+1]) - d2;
226 Y0 = std::max(0.0f, Y0 - d1) * coeffs[0];
227 Y1 = std::max(0.0f, Y1 - d1) * coeffs[0];
228 Y2 = std::max(0.0f, Y2 - d1) * coeffs[0];
229 Y3 = std::max(0.0f, Y3 - d1) * coeffs[0];
231 float ruv = coeffs[4]*V;
232 float guv = coeffs[3]*V + coeffs[2]*U;
233 float buv = coeffs[1]*U;
235 dst0[(x+0)*NCHANNELS+0] = saturate_cast<unsigned char>(Y0 + buv);
236 dst0[(x+0)*NCHANNELS+1] = saturate_cast<unsigned char>(Y0 + guv);
237 dst0[(x+0)*NCHANNELS+2] = saturate_cast<unsigned char>(Y0 + ruv);
239 dst0[(x+1)*NCHANNELS+0] = saturate_cast<unsigned char>(Y1 + buv);
240 dst0[(x+1)*NCHANNELS+1] = saturate_cast<unsigned char>(Y1 + guv);
241 dst0[(x+1)*NCHANNELS+2] = saturate_cast<unsigned char>(Y1 + ruv);
243 dst1[(x+0)*NCHANNELS+0] = saturate_cast<unsigned char>(Y2 + buv);
244 dst1[(x+0)*NCHANNELS+1] = saturate_cast<unsigned char>(Y2 + guv);
245 dst1[(x+0)*NCHANNELS+2] = saturate_cast<unsigned char>(Y2 + ruv);
247 dst1[(x+1)*NCHANNELS+0] = saturate_cast<unsigned char>(Y3 + buv);
248 dst1[(x+1)*NCHANNELS+1] = saturate_cast<unsigned char>(Y3 + guv);
249 dst1[(x+1)*NCHANNELS+2] = saturate_cast<unsigned char>(Y3 + ruv);
252 srcY0 = srcY1 + srcStepY;
254 dst0 = dst1 + dstStep;
258 static void copy_convert_bgr_to_nv12(const VAImage& image, const Mat& bgr, unsigned char* buffer)
260 const float d1 = 16.0f;
261 const float d2 = 128.0f;
263 static const float coeffs[8] =
265 0.256999969f, 0.50399971f, 0.09799957f, -0.1479988098f,
266 -0.2909994125f, 0.438999176f, -0.3679990768f, -0.0709991455f
269 const size_t dstOffsetY = image.offsets[0];
270 const size_t dstOffsetUV = image.offsets[1];
272 const size_t dstStepY = image.pitches[0];
273 const size_t dstStepUV = image.pitches[1];
275 const size_t srcStep = bgr.step;
277 const unsigned char* src0 = bgr.data;
279 unsigned char* dstY0 = buffer + dstOffsetY;
280 unsigned char* dstUV = buffer + dstOffsetUV;
282 for (int y = 0; y < bgr.rows; y += 2)
284 const unsigned char *src1 = src0 + srcStep;
285 unsigned char* dstY1 = dstY0 + dstStepY;
287 for (int x = 0; x < bgr.cols; x += 2)
289 float B0 = float(src0[(x+0)*NCHANNELS+0]);
290 float G0 = float(src0[(x+0)*NCHANNELS+1]);
291 float R0 = float(src0[(x+0)*NCHANNELS+2]);
293 float B1 = float(src0[(x+1)*NCHANNELS+0]);
294 float G1 = float(src0[(x+1)*NCHANNELS+1]);
295 float R1 = float(src0[(x+1)*NCHANNELS+2]);
297 float B2 = float(src1[(x+0)*NCHANNELS+0]);
298 float G2 = float(src1[(x+0)*NCHANNELS+1]);
299 float R2 = float(src1[(x+0)*NCHANNELS+2]);
301 float B3 = float(src1[(x+1)*NCHANNELS+0]);
302 float G3 = float(src1[(x+1)*NCHANNELS+1]);
303 float R3 = float(src1[(x+1)*NCHANNELS+2]);
305 float Y0 = coeffs[0]*R0 + coeffs[1]*G0 + coeffs[2]*B0 + d1;
306 float Y1 = coeffs[0]*R1 + coeffs[1]*G1 + coeffs[2]*B1 + d1;
307 float Y2 = coeffs[0]*R2 + coeffs[1]*G2 + coeffs[2]*B2 + d1;
308 float Y3 = coeffs[0]*R3 + coeffs[1]*G3 + coeffs[2]*B3 + d1;
310 float U = coeffs[3]*R0 + coeffs[4]*G0 + coeffs[5]*B0 + d2;
311 float V = coeffs[5]*R0 + coeffs[6]*G0 + coeffs[7]*B0 + d2;
313 dstY0[x+0] = saturate_cast<unsigned char>(Y0);
314 dstY0[x+1] = saturate_cast<unsigned char>(Y1);
315 dstY1[x+0] = saturate_cast<unsigned char>(Y2);
316 dstY1[x+1] = saturate_cast<unsigned char>(Y3);
318 dstUV[2*(x/2)+0] = saturate_cast<unsigned char>(U);
319 dstUV[2*(x/2)+1] = saturate_cast<unsigned char>(V);
322 src0 = src1 + srcStep;
323 dstY0 = dstY1 + dstStepY;
329 void convertToVASurface(VADisplay display, InputArray src, VASurfaceID surface, Size size)
331 (void)display; (void)src; (void)surface; (void)size;
332 #if !defined(HAVE_VA)
335 const int stype = CV_8UC3;
337 int srcType = src.type();
338 CV_Assert(srcType == stype);
340 Size srcSize = src.size();
341 CV_Assert(srcSize.width == size.width && srcSize.height == size.height);
343 # if (defined(HAVE_VA_INTEL) && defined(HAVE_OPENCL))
344 if (contextInitialized)
346 UMat u = src.getUMat();
348 // TODO Add support for roi
349 CV_Assert(u.offset == 0);
350 CV_Assert(u.isContinuous());
352 cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ);
354 using namespace cv::ocl;
355 Context& ctx = Context::getDefault();
356 cl_context context = (cl_context)ctx.ptr();
360 cl_mem clImageY = clCreateFromVA_APIMediaSurfaceINTEL(context, CL_MEM_WRITE_ONLY, &surface, 0, &status);
361 if (status != CL_SUCCESS)
362 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromVA_APIMediaSurfaceINTEL failed (Y plane)");
363 cl_mem clImageUV = clCreateFromVA_APIMediaSurfaceINTEL(context, CL_MEM_WRITE_ONLY, &surface, 1, &status);
364 if (status != CL_SUCCESS)
365 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromVA_APIMediaSurfaceINTEL failed (UV plane)");
367 cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr();
369 cl_mem images[2] = { clImageY, clImageUV };
370 status = clEnqueueAcquireVA_APIMediaSurfacesINTEL(q, 2, images, 0, NULL, NULL);
371 if (status != CL_SUCCESS)
372 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireVA_APIMediaSurfacesINTEL failed");
373 if (!ocl::ocl_convert_bgr_to_nv12(clBuffer, (int)u.step[0], u.cols, u.rows, clImageY, clImageUV))
374 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: ocl_convert_bgr_to_nv12 failed");
375 clEnqueueReleaseVA_APIMediaSurfacesINTEL(q, 2, images, 0, NULL, NULL);
376 if (status != CL_SUCCESS)
377 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseVA_APIMediaSurfacesINTEL failed");
379 status = clFinish(q); // TODO Use events
380 if (status != CL_SUCCESS)
381 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clFinish failed");
383 status = clReleaseMemObject(clImageY); // TODO RAII
384 if (status != CL_SUCCESS)
385 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed (Y plane)");
386 status = clReleaseMemObject(clImageUV);
387 if (status != CL_SUCCESS)
388 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed (UV plane)");
391 # endif // HAVE_VA_INTEL && HAVE_OPENCL
393 Mat m = src.getMat();
395 // TODO Add support for roi
396 CV_Assert(m.data == m.datastart);
397 CV_Assert(m.isContinuous());
401 status = vaSyncSurface(display, surface);
402 if (status != VA_STATUS_SUCCESS)
403 CV_Error(cv::Error::StsError, "VA-API: vaSyncSurface failed");
406 status = vaDeriveImage(display, surface, &image);
407 if (status != VA_STATUS_SUCCESS)
408 CV_Error(cv::Error::StsError, "VA-API: vaDeriveImage failed");
410 unsigned char* buffer = 0;
411 status = vaMapBuffer(display, image.buf, (void **)&buffer);
412 if (status != VA_STATUS_SUCCESS)
413 CV_Error(cv::Error::StsError, "VA-API: vaMapBuffer failed");
415 CV_Assert(image.format.fourcc == VA_FOURCC_NV12);
417 copy_convert_bgr_to_nv12(image, m, buffer);
419 status = vaUnmapBuffer(display, image.buf);
420 if (status != VA_STATUS_SUCCESS)
421 CV_Error(cv::Error::StsError, "VA-API: vaUnmapBuffer failed");
423 status = vaDestroyImage(display, image.image_id);
424 if (status != VA_STATUS_SUCCESS)
425 CV_Error(cv::Error::StsError, "VA-API: vaDestroyImage failed");
430 void convertFromVASurface(VADisplay display, VASurfaceID surface, Size size, OutputArray dst)
432 (void)display; (void)surface; (void)dst; (void)size;
433 #if !defined(HAVE_VA)
436 const int dtype = CV_8UC3;
438 // TODO Need to specify ACCESS_WRITE here somehow to prevent useless data copying!
439 dst.create(size, dtype);
441 # if (defined(HAVE_VA_INTEL) && defined(HAVE_OPENCL))
442 if (contextInitialized)
444 UMat u = dst.getUMat();
446 // TODO Add support for roi
447 CV_Assert(u.offset == 0);
448 CV_Assert(u.isContinuous());
450 cl_mem clBuffer = (cl_mem)u.handle(ACCESS_WRITE);
452 using namespace cv::ocl;
453 Context& ctx = Context::getDefault();
454 cl_context context = (cl_context)ctx.ptr();
458 cl_mem clImageY = clCreateFromVA_APIMediaSurfaceINTEL(context, CL_MEM_READ_ONLY, &surface, 0, &status);
459 if (status != CL_SUCCESS)
460 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromVA_APIMediaSurfaceINTEL failed (Y plane)");
461 cl_mem clImageUV = clCreateFromVA_APIMediaSurfaceINTEL(context, CL_MEM_READ_ONLY, &surface, 1, &status);
462 if (status != CL_SUCCESS)
463 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromVA_APIMediaSurfaceINTEL failed (UV plane)");
465 cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr();
467 cl_mem images[2] = { clImageY, clImageUV };
468 status = clEnqueueAcquireVA_APIMediaSurfacesINTEL(q, 2, images, 0, NULL, NULL);
469 if (status != CL_SUCCESS)
470 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireVA_APIMediaSurfacesINTEL failed");
471 if (!ocl::ocl_convert_nv12_to_bgr(clImageY, clImageUV, clBuffer, (int)u.step[0], u.cols, u.rows))
472 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: ocl_convert_nv12_to_bgr failed");
473 status = clEnqueueReleaseVA_APIMediaSurfacesINTEL(q, 2, images, 0, NULL, NULL);
474 if (status != CL_SUCCESS)
475 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseVA_APIMediaSurfacesINTEL failed");
477 status = clFinish(q); // TODO Use events
478 if (status != CL_SUCCESS)
479 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clFinish failed");
481 status = clReleaseMemObject(clImageY); // TODO RAII
482 if (status != CL_SUCCESS)
483 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed (Y plane)");
484 status = clReleaseMemObject(clImageUV);
485 if (status != CL_SUCCESS)
486 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clReleaseMem failed (UV plane)");
489 # endif // HAVE_VA_INTEL && HAVE_OPENCL
491 Mat m = dst.getMat();
493 // TODO Add support for roi
494 CV_Assert(m.data == m.datastart);
495 CV_Assert(m.isContinuous());
499 status = vaSyncSurface(display, surface);
500 if (status != VA_STATUS_SUCCESS)
501 CV_Error(cv::Error::StsError, "VA-API: vaSyncSurface failed");
504 status = vaDeriveImage(display, surface, &image);
505 if (status != VA_STATUS_SUCCESS)
506 CV_Error(cv::Error::StsError, "VA-API: vaDeriveImage failed");
508 unsigned char* buffer = 0;
509 status = vaMapBuffer(display, image.buf, (void **)&buffer);
510 if (status != VA_STATUS_SUCCESS)
511 CV_Error(cv::Error::StsError, "VA-API: vaMapBuffer failed");
513 CV_Assert(image.format.fourcc == VA_FOURCC_NV12);
515 copy_convert_nv12_to_bgr(image, buffer, m);
517 status = vaUnmapBuffer(display, image.buf);
518 if (status != VA_STATUS_SUCCESS)
519 CV_Error(cv::Error::StsError, "VA-API: vaUnmapBuffer failed");
521 status = vaDestroyImage(display, image.image_id);
522 if (status != VA_STATUS_SUCCESS)
523 CV_Error(cv::Error::StsError, "VA-API: vaDestroyImage failed");
528 }} // namespace cv::va_intel