1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html
7 #include "opencl_kernels_core.hpp"
9 namespace cv { namespace hal {
12 template<typename T> struct VSplit2;
13 template<typename T> struct VSplit3;
14 template<typename T> struct VSplit4;
16 #define SPLIT2_KERNEL_TEMPLATE(name, data_type, reg_type, load_func, store_func) \
18 struct name<data_type> \
20 void operator()(const data_type* src, data_type* dst0, \
21 data_type* dst1) const \
23 reg_type r = load_func(src); \
24 store_func(dst0, r.val[0]); \
25 store_func(dst1, r.val[1]); \
29 #define SPLIT3_KERNEL_TEMPLATE(name, data_type, reg_type, load_func, store_func) \
31 struct name<data_type> \
33 void operator()(const data_type* src, data_type* dst0, data_type* dst1, \
34 data_type* dst2) const \
36 reg_type r = load_func(src); \
37 store_func(dst0, r.val[0]); \
38 store_func(dst1, r.val[1]); \
39 store_func(dst2, r.val[2]); \
43 #define SPLIT4_KERNEL_TEMPLATE(name, data_type, reg_type, load_func, store_func) \
45 struct name<data_type> \
47 void operator()(const data_type* src, data_type* dst0, data_type* dst1, \
48 data_type* dst2, data_type* dst3) const \
50 reg_type r = load_func(src); \
51 store_func(dst0, r.val[0]); \
52 store_func(dst1, r.val[1]); \
53 store_func(dst2, r.val[2]); \
54 store_func(dst3, r.val[3]); \
58 SPLIT2_KERNEL_TEMPLATE(VSplit2, uchar , uint8x16x2_t, vld2q_u8 , vst1q_u8 );
59 SPLIT2_KERNEL_TEMPLATE(VSplit2, ushort, uint16x8x2_t, vld2q_u16, vst1q_u16);
60 SPLIT2_KERNEL_TEMPLATE(VSplit2, int , int32x4x2_t, vld2q_s32, vst1q_s32);
61 SPLIT2_KERNEL_TEMPLATE(VSplit2, int64 , int64x1x2_t, vld2_s64 , vst1_s64 );
63 SPLIT3_KERNEL_TEMPLATE(VSplit3, uchar , uint8x16x3_t, vld3q_u8 , vst1q_u8 );
64 SPLIT3_KERNEL_TEMPLATE(VSplit3, ushort, uint16x8x3_t, vld3q_u16, vst1q_u16);
65 SPLIT3_KERNEL_TEMPLATE(VSplit3, int , int32x4x3_t, vld3q_s32, vst1q_s32);
66 SPLIT3_KERNEL_TEMPLATE(VSplit3, int64 , int64x1x3_t, vld3_s64 , vst1_s64 );
68 SPLIT4_KERNEL_TEMPLATE(VSplit4, uchar , uint8x16x4_t, vld4q_u8 , vst1q_u8 );
69 SPLIT4_KERNEL_TEMPLATE(VSplit4, ushort, uint16x8x4_t, vld4q_u16, vst1q_u16);
70 SPLIT4_KERNEL_TEMPLATE(VSplit4, int , int32x4x4_t, vld4q_s32, vst1q_s32);
71 SPLIT4_KERNEL_TEMPLATE(VSplit4, int64 , int64x1x4_t, vld4_s64 , vst1_s64 );
78 VSplit2() : support(false) { }
79 void operator()(const T *, T *, T *) const { }
87 VSplit3() : support(false) { }
88 void operator()(const T *, T *, T *, T *) const { }
96 VSplit4() : support(false) { }
97 void operator()(const T *, T *, T *, T *, T *) const { }
102 #define SPLIT2_KERNEL_TEMPLATE(data_type, reg_type, cast_type, _mm_deinterleave, flavor) \
104 struct VSplit2<data_type> \
108 ELEMS_IN_VEC = 16 / sizeof(data_type) \
113 support = checkHardwareSupport(CV_CPU_SSE2); \
116 void operator()(const data_type * src, \
117 data_type * dst0, data_type * dst1) const \
119 reg_type v_src0 = _mm_loadu_##flavor((cast_type const *)(src)); \
120 reg_type v_src1 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC)); \
121 reg_type v_src2 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 2)); \
122 reg_type v_src3 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 3)); \
124 _mm_deinterleave(v_src0, v_src1, v_src2, v_src3); \
126 _mm_storeu_##flavor((cast_type *)(dst0), v_src0); \
127 _mm_storeu_##flavor((cast_type *)(dst0 + ELEMS_IN_VEC), v_src1); \
128 _mm_storeu_##flavor((cast_type *)(dst1), v_src2); \
129 _mm_storeu_##flavor((cast_type *)(dst1 + ELEMS_IN_VEC), v_src3); \
135 #define SPLIT3_KERNEL_TEMPLATE(data_type, reg_type, cast_type, _mm_deinterleave, flavor) \
137 struct VSplit3<data_type> \
141 ELEMS_IN_VEC = 16 / sizeof(data_type) \
146 support = checkHardwareSupport(CV_CPU_SSE2); \
149 void operator()(const data_type * src, \
150 data_type * dst0, data_type * dst1, data_type * dst2) const \
152 reg_type v_src0 = _mm_loadu_##flavor((cast_type const *)(src)); \
153 reg_type v_src1 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC)); \
154 reg_type v_src2 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 2)); \
155 reg_type v_src3 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 3)); \
156 reg_type v_src4 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 4)); \
157 reg_type v_src5 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 5)); \
159 _mm_deinterleave(v_src0, v_src1, v_src2, \
160 v_src3, v_src4, v_src5); \
162 _mm_storeu_##flavor((cast_type *)(dst0), v_src0); \
163 _mm_storeu_##flavor((cast_type *)(dst0 + ELEMS_IN_VEC), v_src1); \
164 _mm_storeu_##flavor((cast_type *)(dst1), v_src2); \
165 _mm_storeu_##flavor((cast_type *)(dst1 + ELEMS_IN_VEC), v_src3); \
166 _mm_storeu_##flavor((cast_type *)(dst2), v_src4); \
167 _mm_storeu_##flavor((cast_type *)(dst2 + ELEMS_IN_VEC), v_src5); \
173 #define SPLIT4_KERNEL_TEMPLATE(data_type, reg_type, cast_type, _mm_deinterleave, flavor) \
175 struct VSplit4<data_type> \
179 ELEMS_IN_VEC = 16 / sizeof(data_type) \
184 support = checkHardwareSupport(CV_CPU_SSE2); \
187 void operator()(const data_type * src, data_type * dst0, data_type * dst1, \
188 data_type * dst2, data_type * dst3) const \
190 reg_type v_src0 = _mm_loadu_##flavor((cast_type const *)(src)); \
191 reg_type v_src1 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC)); \
192 reg_type v_src2 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 2)); \
193 reg_type v_src3 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 3)); \
194 reg_type v_src4 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 4)); \
195 reg_type v_src5 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 5)); \
196 reg_type v_src6 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 6)); \
197 reg_type v_src7 = _mm_loadu_##flavor((cast_type const *)(src + ELEMS_IN_VEC * 7)); \
199 _mm_deinterleave(v_src0, v_src1, v_src2, v_src3, \
200 v_src4, v_src5, v_src6, v_src7); \
202 _mm_storeu_##flavor((cast_type *)(dst0), v_src0); \
203 _mm_storeu_##flavor((cast_type *)(dst0 + ELEMS_IN_VEC), v_src1); \
204 _mm_storeu_##flavor((cast_type *)(dst1), v_src2); \
205 _mm_storeu_##flavor((cast_type *)(dst1 + ELEMS_IN_VEC), v_src3); \
206 _mm_storeu_##flavor((cast_type *)(dst2), v_src4); \
207 _mm_storeu_##flavor((cast_type *)(dst2 + ELEMS_IN_VEC), v_src5); \
208 _mm_storeu_##flavor((cast_type *)(dst3), v_src6); \
209 _mm_storeu_##flavor((cast_type *)(dst3 + ELEMS_IN_VEC), v_src7); \
215 SPLIT2_KERNEL_TEMPLATE( uchar, __m128i, __m128i, _mm_deinterleave_epi8, si128);
216 SPLIT2_KERNEL_TEMPLATE(ushort, __m128i, __m128i, _mm_deinterleave_epi16, si128);
217 SPLIT2_KERNEL_TEMPLATE( int, __m128, float, _mm_deinterleave_ps, ps);
219 SPLIT3_KERNEL_TEMPLATE( uchar, __m128i, __m128i, _mm_deinterleave_epi8, si128);
220 SPLIT3_KERNEL_TEMPLATE(ushort, __m128i, __m128i, _mm_deinterleave_epi16, si128);
221 SPLIT3_KERNEL_TEMPLATE( int, __m128, float, _mm_deinterleave_ps, ps);
223 SPLIT4_KERNEL_TEMPLATE( uchar, __m128i, __m128i, _mm_deinterleave_epi8, si128);
224 SPLIT4_KERNEL_TEMPLATE(ushort, __m128i, __m128i, _mm_deinterleave_epi16, si128);
225 SPLIT4_KERNEL_TEMPLATE( int, __m128, float, _mm_deinterleave_ps, ps);
229 template<typename T> static void
230 split_( const T* src, T** dst, int len, int cn )
232 int k = cn % 4 ? cn % 4 : 4;
240 memcpy(dst0, src, len * sizeof(T));
244 for( i = 0, j = 0 ; i < len; i++, j += cn )
250 T *dst0 = dst[0], *dst1 = dst[1];
256 int inc_i = (sizeof(T) == 8)? 1: 16/sizeof(T);
257 int inc_j = 2 * inc_i;
260 for( ; i < len - inc_i; i += inc_i, j += inc_j)
261 vsplit(src + j, dst0 + i, dst1 + i);
266 int inc_i = 32/sizeof(T);
267 int inc_j = 2 * inc_i;
272 for( ; i <= len - inc_i; i += inc_i, j += inc_j)
273 vsplit(src + j, dst0 + i, dst1 + i);
277 for( ; i < len; i++, j += cn )
285 T *dst0 = dst[0], *dst1 = dst[1], *dst2 = dst[2];
291 int inc_i = (sizeof(T) == 8)? 1: 16/sizeof(T);
292 int inc_j = 3 * inc_i;
295 for( ; i <= len - inc_i; i += inc_i, j += inc_j)
296 vsplit(src + j, dst0 + i, dst1 + i, dst2 + i);
301 int inc_i = 32/sizeof(T);
302 int inc_j = 3 * inc_i;
308 for( ; i <= len - inc_i; i += inc_i, j += inc_j)
309 vsplit(src + j, dst0 + i, dst1 + i, dst2 + i);
313 for( ; i < len; i++, j += cn )
322 T *dst0 = dst[0], *dst1 = dst[1], *dst2 = dst[2], *dst3 = dst[3];
328 int inc_i = (sizeof(T) == 8)? 1: 16/sizeof(T);
329 int inc_j = 4 * inc_i;
332 for( ; i <= len - inc_i; i += inc_i, j += inc_j)
333 vsplit(src + j, dst0 + i, dst1 + i, dst2 + i, dst3 + i);
338 int inc_i = 32/sizeof(T);
339 int inc_j = 4 * inc_i;
344 for( ; i <= len - inc_i; i += inc_i, j += inc_j)
345 vsplit(src + j, dst0 + i, dst1 + i, dst2 + i, dst3 + i);
349 for( ; i < len; i++, j += cn )
351 dst0[i] = src[j]; dst1[i] = src[j+1];
352 dst2[i] = src[j+2]; dst3[i] = src[j+3];
356 for( ; k < cn; k += 4 )
358 T *dst0 = dst[k], *dst1 = dst[k+1], *dst2 = dst[k+2], *dst3 = dst[k+3];
359 for( i = 0, j = k; i < len; i++, j += cn )
361 dst0[i] = src[j]; dst1[i] = src[j+1];
362 dst2[i] = src[j+2]; dst3[i] = src[j+3];
367 void split8u(const uchar* src, uchar** dst, int len, int cn )
369 CALL_HAL(split8u, cv_hal_split8u, src,dst, len, cn)
370 split_(src, dst, len, cn);
373 void split16u(const ushort* src, ushort** dst, int len, int cn )
375 CALL_HAL(split16u, cv_hal_split16u, src,dst, len, cn)
376 split_(src, dst, len, cn);
379 void split32s(const int* src, int** dst, int len, int cn )
381 CALL_HAL(split32s, cv_hal_split32s, src,dst, len, cn)
382 split_(src, dst, len, cn);
385 void split64s(const int64* src, int64** dst, int len, int cn )
387 CALL_HAL(split64s, cv_hal_split64s, src,dst, len, cn)
388 split_(src, dst, len, cn);
393 /****************************************************************************************\
395 \****************************************************************************************/
397 typedef void (*SplitFunc)(const uchar* src, uchar** dst, int len, int cn);
399 static SplitFunc getSplitFunc(int depth)
401 static SplitFunc splitTab[] =
403 (SplitFunc)GET_OPTIMIZED(cv::hal::split8u), (SplitFunc)GET_OPTIMIZED(cv::hal::split8u), (SplitFunc)GET_OPTIMIZED(cv::hal::split16u), (SplitFunc)GET_OPTIMIZED(cv::hal::split16u),
404 (SplitFunc)GET_OPTIMIZED(cv::hal::split32s), (SplitFunc)GET_OPTIMIZED(cv::hal::split32s), (SplitFunc)GET_OPTIMIZED(cv::hal::split64s), 0
407 return splitTab[depth];
413 static bool ipp_split(const Mat& src, Mat* mv, int channels)
416 CV_INSTRUMENT_REGION_IPP()
418 if(channels != 3 && channels != 4)
423 IppiSize size = ippiSize(src.size());
424 void *dstPtrs[4] = {NULL};
425 size_t dstStep = mv[0].step;
426 for(int i = 0; i < channels; i++)
428 dstPtrs[i] = mv[i].ptr();
429 if(dstStep != mv[i].step)
433 return CV_INSTRUMENT_FUN_IPP(llwiCopySplit, src.ptr(), (int)src.step, dstPtrs, (int)dstStep, size, (int)src.elemSize1(), channels, 0) >= 0;
437 const Mat *arrays[5] = {NULL};
438 uchar *ptrs[5] = {NULL};
441 for(int i = 1; i < channels; i++)
443 arrays[i] = &mv[i-1];
446 NAryMatIterator it(arrays, ptrs);
447 IppiSize size = { (int)it.size, 1 };
449 for( size_t i = 0; i < it.nplanes; i++, ++it )
451 if(CV_INSTRUMENT_FUN_IPP(llwiCopySplit, ptrs[0], 0, (void**)&ptrs[1], 0, size, (int)src.elemSize1(), channels, 0) < 0)
457 CV_UNUSED(src); CV_UNUSED(mv); CV_UNUSED(channels);
464 void cv::split(const Mat& src, Mat* mv)
466 CV_INSTRUMENT_REGION()
468 int k, depth = src.depth(), cn = src.channels();
475 for( k = 0; k < cn; k++ )
477 mv[k].create(src.dims, src.size, depth);
480 CV_IPP_RUN_FAST(ipp_split(src, mv, cn));
482 SplitFunc func = getSplitFunc(depth);
483 CV_Assert( func != 0 );
485 size_t esz = src.elemSize(), esz1 = src.elemSize1();
486 size_t blocksize0 = (BLOCK_SIZE + esz-1)/esz;
487 AutoBuffer<uchar> _buf((cn+1)*(sizeof(Mat*) + sizeof(uchar*)) + 16);
488 const Mat** arrays = (const Mat**)_buf.data();
489 uchar** ptrs = (uchar**)alignPtr(arrays + cn + 1, 16);
492 for( k = 0; k < cn; k++ )
494 arrays[k+1] = &mv[k];
497 NAryMatIterator it(arrays, ptrs, cn+1);
498 size_t total = it.size;
499 size_t blocksize = std::min((size_t)CV_SPLIT_MERGE_MAX_BLOCK_SIZE(cn), cn <= 4 ? total : std::min(total, blocksize0));
501 for( size_t i = 0; i < it.nplanes; i++, ++it )
503 for( size_t j = 0; j < total; j += blocksize )
505 size_t bsz = std::min(total - j, blocksize);
506 func( ptrs[0], &ptrs[1], (int)bsz, cn );
508 if( j + blocksize < total )
511 for( k = 0; k < cn; k++ )
512 ptrs[k+1] += bsz*esz1;
522 static bool ocl_split( InputArray _m, OutputArrayOfArrays _mv )
524 int type = _m.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type),
525 rowsPerWI = ocl::Device::getDefault().isIntel() ? 4 : 1;
527 String dstargs, processelem, indexdecl;
528 for (int i = 0; i < cn; ++i)
530 dstargs += format("DECLARE_DST_PARAM(%d)", i);
531 indexdecl += format("DECLARE_INDEX(%d)", i);
532 processelem += format("PROCESS_ELEM(%d)", i);
535 ocl::Kernel k("split", ocl::core::split_merge_oclsrc,
536 format("-D T=%s -D OP_SPLIT -D cn=%d -D DECLARE_DST_PARAMS=%s"
537 " -D PROCESS_ELEMS_N=%s -D DECLARE_INDEX_N=%s",
538 ocl::memopTypeToStr(depth), cn, dstargs.c_str(),
539 processelem.c_str(), indexdecl.c_str()));
543 Size size = _m.size();
544 _mv.create(cn, 1, depth);
545 for (int i = 0; i < cn; ++i)
546 _mv.create(size, depth, i);
548 std::vector<UMat> dst;
549 _mv.getUMatVector(dst);
551 int argidx = k.set(0, ocl::KernelArg::ReadOnly(_m.getUMat()));
552 for (int i = 0; i < cn; ++i)
553 argidx = k.set(argidx, ocl::KernelArg::WriteOnlyNoSize(dst[i]));
554 k.set(argidx, rowsPerWI);
556 size_t globalsize[2] = { (size_t)size.width, ((size_t)size.height + rowsPerWI - 1) / rowsPerWI };
557 return k.run(2, globalsize, NULL, false);
564 void cv::split(InputArray _m, OutputArrayOfArrays _mv)
566 CV_INSTRUMENT_REGION()
568 CV_OCL_RUN(_m.dims() <= 2 && _mv.isUMatVector(),
578 CV_Assert( !_mv.fixedType() || _mv.empty() || _mv.type() == m.depth() );
580 int depth = m.depth(), cn = m.channels();
581 _mv.create(cn, 1, depth);
582 for (int i = 0; i < cn; ++i)
583 _mv.create(m.dims, m.size.p, depth, i);
585 std::vector<Mat> dst;
586 _mv.getMatVector(dst);