1 /*M///////////////////////////////////////////////////////////////////////////////////////
3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
5 // By downloading, copying, installing or using the software you agree to this license.
6 // If you do not agree to this license, do not download, install,
7 // copy or use the software.
11 // For Open Source Computer Vision Library
13 // Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
14 // Copyright (C) 2009, Willow Garage Inc., all rights reserved.
15 // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
16 // Copyright (C) 2015, Itseez Inc., all rights reserved.
17 // Third party copyrights are property of their respective owners.
19 // Redistribution and use in source and binary forms, with or without modification,
20 // are permitted provided that the following conditions are met:
22 // * Redistribution's of source code must retain the above copyright notice,
23 // this list of conditions and the following disclaimer.
25 // * Redistribution's in binary form must reproduce the above copyright notice,
26 // this list of conditions and the following disclaimer in the documentation
27 // and/or other materials provided with the distribution.
29 // * The name of the copyright holders may not be used to endorse or promote products
30 // derived from this software without specific prior written permission.
32 // This software is provided by the copyright holders and contributors "as is" and
33 // any express or implied warranties, including, but not limited to, the implied
34 // warranties of merchantability and fitness for a particular purpose are disclaimed.
35 // In no event shall the Intel Corporation or contributors be liable for any direct,
36 // indirect, incidental, special, exemplary, or consequential damages
37 // (including, but not limited to, procurement of substitute goods or services;
38 // loss of use, data, or profits; or business interruption) however caused
39 // and on any theory of liability, whether in contract, strict liability,
40 // or tort (including negligence or otherwise) arising in any way out of
41 // the use of this software, even if advised of the possibility of such damage.
45 #ifndef OPENCV_HAL_INTRIN_CPP_HPP
46 #define OPENCV_HAL_INTRIN_CPP_HPP
51 #include "opencv2/core/saturate.hpp"
57 CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN
60 /** @addtogroup core_hal_intrin
62 "Universal intrinsics" is a types and functions set intended to simplify vectorization of code on
63 different platforms. Currently there are two supported SIMD extensions: __SSE/SSE2__ on x86
64 architectures and __NEON__ on ARM architectures, both allow working with 128 bit registers
65 containing packed values of different types. In case when there is no SIMD extension available
66 during compilation, fallback C++ implementation of intrinsics will be chosen and code will work as
67 expected although it could be slower.
71 There are several types representing 128-bit register as a vector of packed values, each type is
72 implemented as a structure based on a one SIMD register.
74 - cv::v_uint8x16 and cv::v_int8x16: sixteen 8-bit integer values (unsigned/signed) - char
75 - cv::v_uint16x8 and cv::v_int16x8: eight 16-bit integer values (unsigned/signed) - short
76 - cv::v_uint32x4 and cv::v_int32x4: four 32-bit integer values (unsgined/signed) - int
77 - cv::v_uint64x2 and cv::v_int64x2: two 64-bit integer values (unsigned/signed) - int64
78 - cv::v_float32x4: four 32-bit floating point values (signed) - float
79 - cv::v_float64x2: two 64-bit floating point valies (signed) - double
82 cv::v_float64x2 is not implemented in NEON variant, if you want to use this type, don't forget to
83 check the CV_SIMD128_64F preprocessor definition:
90 ### Load and store operations
92 These operations allow to set contents of the register explicitly or by loading it from some memory
93 block and to save contents of the register to memory block.
96 @ref v_reg::v_reg(const _Tp *ptr) "from memory",
97 @ref v_reg::v_reg(_Tp s0, _Tp s1) "from two values", ...
98 - Other create methods:
99 @ref v_setall_s8, @ref v_setall_u8, ...,
100 @ref v_setzero_u8, @ref v_setzero_s8, ...
102 @ref v_load, @ref v_load_aligned, @ref v_load_halves,
103 @ref v_store, @ref v_store_aligned,
104 @ref v_store_high, @ref v_store_low
108 These operations allow to reorder or recombine elements in one or multiple vectors.
110 - Interleave, deinterleave (2, 3 and 4 channels): @ref v_load_deinterleave, @ref v_store_interleave
111 - Expand: @ref v_load_expand, @ref v_load_expand_q, @ref v_expand
112 - Pack: @ref v_pack, @ref v_pack_u, @ref v_rshr_pack, @ref v_rshr_pack_u,
113 @ref v_pack_store, @ref v_pack_u_store, @ref v_rshr_pack_store, @ref v_rshr_pack_u_store
114 - Recombine: @ref v_zip, @ref v_recombine, @ref v_combine_low, @ref v_combine_high
115 - Extract: @ref v_extract
118 ### Arithmetic, bitwise and comparison operations
120 Element-wise binary and unary operations.
123 @ref operator +(const v_reg &a, const v_reg &b) "+",
124 @ref operator -(const v_reg &a, const v_reg &b) "-",
125 @ref operator *(const v_reg &a, const v_reg &b) "*",
126 @ref operator /(const v_reg &a, const v_reg &b) "/",
129 - Non-saturating arithmetics: @ref v_add_wrap, @ref v_sub_wrap
132 @ref operator <<(const v_reg &a, int s) "<<",
133 @ref operator >>(const v_reg &a, int s) ">>",
134 @ref v_shl, @ref v_shr
137 @ref operator&(const v_reg &a, const v_reg &b) "&",
138 @ref operator |(const v_reg &a, const v_reg &b) "|",
139 @ref operator ^(const v_reg &a, const v_reg &b) "^",
140 @ref operator ~(const v_reg &a) "~"
143 @ref operator >(const v_reg &a, const v_reg &b) ">",
144 @ref operator >=(const v_reg &a, const v_reg &b) ">=",
145 @ref operator <(const v_reg &a, const v_reg &b) "<",
146 @ref operator <=(const v_reg &a, const v_reg &b) "<=",
147 @ref operator==(const v_reg &a, const v_reg &b) "==",
148 @ref operator !=(const v_reg &a, const v_reg &b) "!="
150 - min/max: @ref v_min, @ref v_max
154 Most of these operations return only one value.
156 - Reduce: @ref v_reduce_min, @ref v_reduce_max, @ref v_reduce_sum, @ref v_popcount
157 - Mask: @ref v_signmask, @ref v_check_all, @ref v_check_any, @ref v_select
161 - Some frequent operations: @ref v_sqrt, @ref v_invsqrt, @ref v_magnitude, @ref v_sqr_magnitude
162 - Absolute values: @ref v_abs, @ref v_absdiff
166 Different type conversions and casts:
168 - Rounding: @ref v_round, @ref v_floor, @ref v_ceil, @ref v_trunc,
169 - To float: @ref v_cvt_f32, @ref v_cvt_f64
170 - Reinterpret: @ref v_reinterpret_as_u8, @ref v_reinterpret_as_s8, ...
172 ### Matrix operations
174 In these operations vectors represent matrix rows/columns: @ref v_dotprod, @ref v_matmul, @ref v_transpose4x4
178 Most operations are implemented only for some subset of the available types, following matrices
179 shows the applicability of different operations to the types.
183 | Operations\\Types | uint 8x16 | int 8x16 | uint 16x8 | int 16x8 | uint 32x4 | int 32x4 |
184 |-------------------|:-:|:-:|:-:|:-:|:-:|:-:|
185 |load, store | x | x | x | x | x | x |
186 |interleave | x | x | x | x | x | x |
187 |expand | x | x | x | x | x | x |
188 |expand_q | x | x | | | | |
189 |add, sub | x | x | x | x | x | x |
190 |add_wrap, sub_wrap | x | x | x | x | | |
191 |mul | | | x | x | x | x |
192 |mul_expand | | | x | x | x | |
193 |compare | x | x | x | x | x | x |
194 |shift | | | x | x | x | x |
195 |dotprod | | | | x | | |
196 |logical | x | x | x | x | x | x |
197 |min, max | x | x | x | x | x | x |
198 |absdiff | x | x | x | x | x | x |
199 |reduce | | | | | x | x |
200 |mask | x | x | x | x | x | x |
201 |pack | x | x | x | x | x | x |
202 |pack_u | x | | x | | | |
203 |unpack | x | x | x | x | x | x |
204 |extract | x | x | x | x | x | x |
205 |cvt_flt32 | | | | | | x |
206 |cvt_flt64 | | | | | | x |
207 |transpose4x4 | | | | | x | x |
211 | Operations\\Types | uint 64x2 | int 64x2 |
212 |-------------------|:-:|:-:|
213 |load, store | x | x |
221 | Operations\\Types | float 32x4 | float 64x2 |
222 |-------------------|:-:|:-:|
223 |load, store | x | x |
237 |float math | x | x |
238 |transpose4x4 | x | |
243 template<typename _Tp, int n> struct v_reg
246 typedef _Tp lane_type;
247 typedef v_reg<typename V_TypeTraits<_Tp>::int_type, n> int_vec;
248 typedef v_reg<typename V_TypeTraits<_Tp>::abs_type, n> abs_vec;
252 /** @brief Constructor
254 Initializes register with data from memory
255 @param ptr pointer to memory block with data for register */
256 explicit v_reg(const _Tp* ptr) { for( int i = 0; i < n; i++ ) s[i] = ptr[i]; }
258 /** @brief Constructor
260 Initializes register with two 64-bit values */
261 v_reg(_Tp s0, _Tp s1) { s[0] = s0; s[1] = s1; }
263 /** @brief Constructor
265 Initializes register with four 32-bit values */
266 v_reg(_Tp s0, _Tp s1, _Tp s2, _Tp s3) { s[0] = s0; s[1] = s1; s[2] = s2; s[3] = s3; }
268 /** @brief Constructor
270 Initializes register with eight 16-bit values */
271 v_reg(_Tp s0, _Tp s1, _Tp s2, _Tp s3,
272 _Tp s4, _Tp s5, _Tp s6, _Tp s7)
274 s[0] = s0; s[1] = s1; s[2] = s2; s[3] = s3;
275 s[4] = s4; s[5] = s5; s[6] = s6; s[7] = s7;
278 /** @brief Constructor
280 Initializes register with sixteen 8-bit values */
281 v_reg(_Tp s0, _Tp s1, _Tp s2, _Tp s3,
282 _Tp s4, _Tp s5, _Tp s6, _Tp s7,
283 _Tp s8, _Tp s9, _Tp s10, _Tp s11,
284 _Tp s12, _Tp s13, _Tp s14, _Tp s15)
286 s[0] = s0; s[1] = s1; s[2] = s2; s[3] = s3;
287 s[4] = s4; s[5] = s5; s[6] = s6; s[7] = s7;
288 s[8] = s8; s[9] = s9; s[10] = s10; s[11] = s11;
289 s[12] = s12; s[13] = s13; s[14] = s14; s[15] = s15;
292 /** @brief Default constructor
294 Does not initialize anything*/
297 /** @brief Copy constructor */
298 v_reg(const v_reg<_Tp, n> & r)
300 for( int i = 0; i < n; i++ )
303 /** @brief Access first value
305 Returns value of the first lane according to register type, for example:
307 v_int32x4 r(1, 2, 3, 4);
308 int v = r.get0(); // returns 1
310 uint64_t v = r.get0(); // returns 1
313 _Tp get0() const { return s[0]; }
316 _Tp get(const int i) const { return s[i]; }
317 v_reg<_Tp, n> high() const
321 for( i = 0; i < n/2; i++ )
329 static v_reg<_Tp, n> zero()
332 for( int i = 0; i < n; i++ )
337 static v_reg<_Tp, n> all(_Tp s)
340 for( int i = 0; i < n; i++ )
345 template<typename _Tp2, int n2> v_reg<_Tp2, n2> reinterpret_as() const
347 size_t bytes = std::min(sizeof(_Tp2)*n2, sizeof(_Tp)*n);
349 std::memcpy(&c.s[0], &s[0], bytes);
357 /** @brief Sixteen 8-bit unsigned integer values */
358 typedef v_reg<uchar, 16> v_uint8x16;
359 /** @brief Sixteen 8-bit signed integer values */
360 typedef v_reg<schar, 16> v_int8x16;
361 /** @brief Eight 16-bit unsigned integer values */
362 typedef v_reg<ushort, 8> v_uint16x8;
363 /** @brief Eight 16-bit signed integer values */
364 typedef v_reg<short, 8> v_int16x8;
365 /** @brief Four 32-bit unsigned integer values */
366 typedef v_reg<unsigned, 4> v_uint32x4;
367 /** @brief Four 32-bit signed integer values */
368 typedef v_reg<int, 4> v_int32x4;
369 /** @brief Four 32-bit floating point values (single precision) */
370 typedef v_reg<float, 4> v_float32x4;
371 /** @brief Two 64-bit floating point values (double precision) */
372 typedef v_reg<double, 2> v_float64x2;
373 /** @brief Two 64-bit unsigned integer values */
374 typedef v_reg<uint64, 2> v_uint64x2;
375 /** @brief Two 64-bit signed integer values */
376 typedef v_reg<int64, 2> v_int64x2;
378 //! @brief Helper macro
379 //! @ingroup core_hal_intrin_impl
380 #define OPENCV_HAL_IMPL_BIN_OP(bin_op) \
381 template<typename _Tp, int n> inline v_reg<_Tp, n> \
382 operator bin_op (const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b) \
385 for( int i = 0; i < n; i++ ) \
386 c.s[i] = saturate_cast<_Tp>(a.s[i] bin_op b.s[i]); \
389 template<typename _Tp, int n> inline v_reg<_Tp, n>& \
390 operator bin_op##= (v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b) \
392 for( int i = 0; i < n; i++ ) \
393 a.s[i] = saturate_cast<_Tp>(a.s[i] bin_op b.s[i]); \
397 /** @brief Add values
400 OPENCV_HAL_IMPL_BIN_OP(+)
402 /** @brief Subtract values
405 OPENCV_HAL_IMPL_BIN_OP(-)
407 /** @brief Multiply values
409 For 16- and 32-bit integer types and floating types. */
410 OPENCV_HAL_IMPL_BIN_OP(*)
412 /** @brief Divide values
414 For floating types only. */
415 OPENCV_HAL_IMPL_BIN_OP(/)
417 //! @brief Helper macro
418 //! @ingroup core_hal_intrin_impl
419 #define OPENCV_HAL_IMPL_BIT_OP(bit_op) \
420 template<typename _Tp, int n> inline v_reg<_Tp, n> operator bit_op \
421 (const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b) \
424 typedef typename V_TypeTraits<_Tp>::int_type itype; \
425 for( int i = 0; i < n; i++ ) \
426 c.s[i] = V_TypeTraits<_Tp>::reinterpret_from_int((itype)(V_TypeTraits<_Tp>::reinterpret_int(a.s[i]) bit_op \
427 V_TypeTraits<_Tp>::reinterpret_int(b.s[i]))); \
430 template<typename _Tp, int n> inline v_reg<_Tp, n>& operator \
431 bit_op##= (v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b) \
433 typedef typename V_TypeTraits<_Tp>::int_type itype; \
434 for( int i = 0; i < n; i++ ) \
435 a.s[i] = V_TypeTraits<_Tp>::reinterpret_from_int((itype)(V_TypeTraits<_Tp>::reinterpret_int(a.s[i]) bit_op \
436 V_TypeTraits<_Tp>::reinterpret_int(b.s[i]))); \
440 /** @brief Bitwise AND
442 Only for integer types. */
443 OPENCV_HAL_IMPL_BIT_OP(&)
445 /** @brief Bitwise OR
447 Only for integer types. */
448 OPENCV_HAL_IMPL_BIT_OP(|)
450 /** @brief Bitwise XOR
452 Only for integer types.*/
453 OPENCV_HAL_IMPL_BIT_OP(^)
455 /** @brief Bitwise NOT
457 Only for integer types.*/
458 template<typename _Tp, int n> inline v_reg<_Tp, n> operator ~ (const v_reg<_Tp, n>& a)
461 for( int i = 0; i < n; i++ )
463 c.s[i] = V_TypeTraits<_Tp>::reinterpret_from_int(~V_TypeTraits<_Tp>::reinterpret_int(a.s[i]));
468 //! @brief Helper macro
469 //! @ingroup core_hal_intrin_impl
470 #define OPENCV_HAL_IMPL_MATH_FUNC(func, cfunc, _Tp2) \
471 template<typename _Tp, int n> inline v_reg<_Tp2, n> func(const v_reg<_Tp, n>& a) \
474 for( int i = 0; i < n; i++ ) \
475 c.s[i] = cfunc(a.s[i]); \
479 /** @brief Square root of elements
481 Only for floating point types.*/
482 OPENCV_HAL_IMPL_MATH_FUNC(v_sqrt, std::sqrt, _Tp)
485 OPENCV_HAL_IMPL_MATH_FUNC(v_sin, std::sin, _Tp)
486 OPENCV_HAL_IMPL_MATH_FUNC(v_cos, std::cos, _Tp)
487 OPENCV_HAL_IMPL_MATH_FUNC(v_exp, std::exp, _Tp)
488 OPENCV_HAL_IMPL_MATH_FUNC(v_log, std::log, _Tp)
491 /** @brief Absolute value of elements
493 Only for floating point types.*/
494 OPENCV_HAL_IMPL_MATH_FUNC(v_abs, (typename V_TypeTraits<_Tp>::abs_type)std::abs,
495 typename V_TypeTraits<_Tp>::abs_type)
497 /** @brief Round elements
499 Only for floating point types.*/
500 OPENCV_HAL_IMPL_MATH_FUNC(v_round, cvRound, int)
502 /** @brief Floor elements
504 Only for floating point types.*/
505 OPENCV_HAL_IMPL_MATH_FUNC(v_floor, cvFloor, int)
507 /** @brief Ceil elements
509 Only for floating point types.*/
510 OPENCV_HAL_IMPL_MATH_FUNC(v_ceil, cvCeil, int)
512 /** @brief Truncate elements
514 Only for floating point types.*/
515 OPENCV_HAL_IMPL_MATH_FUNC(v_trunc, int, int)
517 //! @brief Helper macro
518 //! @ingroup core_hal_intrin_impl
519 #define OPENCV_HAL_IMPL_MINMAX_FUNC(func, cfunc) \
520 template<typename _Tp, int n> inline v_reg<_Tp, n> func(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b) \
523 for( int i = 0; i < n; i++ ) \
524 c.s[i] = cfunc(a.s[i], b.s[i]); \
528 //! @brief Helper macro
529 //! @ingroup core_hal_intrin_impl
530 #define OPENCV_HAL_IMPL_REDUCE_MINMAX_FUNC(func, cfunc) \
531 template<typename _Tp, int n> inline _Tp func(const v_reg<_Tp, n>& a) \
534 for( int i = 1; i < n; i++ ) \
535 c = cfunc(c, a.s[i]); \
539 /** @brief Choose min values for each pair
546 {min(A1,B1) min(A2,B2) ...}
548 For all types except 64-bit integer. */
549 OPENCV_HAL_IMPL_MINMAX_FUNC(v_min, std::min)
551 /** @brief Choose max values for each pair
558 {max(A1,B1) max(A2,B2) ...}
560 For all types except 64-bit integer. */
561 OPENCV_HAL_IMPL_MINMAX_FUNC(v_max, std::max)
563 /** @brief Find one min value
567 {A1 A2 A3 ...} => min(A1,A2,A3,...)
569 For 32-bit integer and 32-bit floating point types. */
570 OPENCV_HAL_IMPL_REDUCE_MINMAX_FUNC(v_reduce_min, std::min)
572 /** @brief Find one max value
576 {A1 A2 A3 ...} => max(A1,A2,A3,...)
578 For 32-bit integer and 32-bit floating point types. */
579 OPENCV_HAL_IMPL_REDUCE_MINMAX_FUNC(v_reduce_max, std::max)
581 static const unsigned char popCountTable[] =
583 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
584 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
585 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
586 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
587 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
588 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
589 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
590 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
591 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
592 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
593 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
594 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
595 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
596 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
597 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
598 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8,
600 /** @brief Count the 1 bits in the vector and return 4 values
604 {A1 A2 A3 ...} => popcount(A1)
606 Any types but result will be in v_uint32x4*/
607 template<typename _Tp, int n> inline v_uint32x4 v_popcount(const v_reg<_Tp, n>& a)
610 b = v_reinterpret_as_u8(a);
611 for( int i = 0; i < v_uint8x16::nlanes; i++ )
613 b.s[i] = popCountTable[b.s[i]];
616 for( int i = 0; i < v_uint32x4::nlanes; i++ )
618 c.s[i] = b.s[i*4] + b.s[i*4+1] + b.s[i*4+2] + b.s[i*4+3];
625 template<typename _Tp, int n>
626 inline void v_minmax( const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b,
627 v_reg<_Tp, n>& minval, v_reg<_Tp, n>& maxval )
629 for( int i = 0; i < n; i++ )
631 minval.s[i] = std::min(a.s[i], b.s[i]);
632 maxval.s[i] = std::max(a.s[i], b.s[i]);
637 //! @brief Helper macro
638 //! @ingroup core_hal_intrin_impl
639 #define OPENCV_HAL_IMPL_CMP_OP(cmp_op) \
640 template<typename _Tp, int n> \
641 inline v_reg<_Tp, n> operator cmp_op(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b) \
643 typedef typename V_TypeTraits<_Tp>::int_type itype; \
645 for( int i = 0; i < n; i++ ) \
646 c.s[i] = V_TypeTraits<_Tp>::reinterpret_from_int((itype)-(int)(a.s[i] cmp_op b.s[i])); \
650 /** @brief Less-than comparison
652 For all types except 64-bit integer values. */
653 OPENCV_HAL_IMPL_CMP_OP(<)
655 /** @brief Greater-than comparison
657 For all types except 64-bit integer values. */
658 OPENCV_HAL_IMPL_CMP_OP(>)
660 /** @brief Less-than or equal comparison
662 For all types except 64-bit integer values. */
663 OPENCV_HAL_IMPL_CMP_OP(<=)
665 /** @brief Greater-than or equal comparison
667 For all types except 64-bit integer values. */
668 OPENCV_HAL_IMPL_CMP_OP(>=)
670 /** @brief Equal comparison
672 For all types except 64-bit integer values. */
673 OPENCV_HAL_IMPL_CMP_OP(==)
675 /** @brief Not equal comparison
677 For all types except 64-bit integer values. */
678 OPENCV_HAL_IMPL_CMP_OP(!=)
680 //! @brief Helper macro
681 //! @ingroup core_hal_intrin_impl
682 #define OPENCV_HAL_IMPL_ADD_SUB_OP(func, bin_op, cast_op, _Tp2) \
683 template<typename _Tp, int n> \
684 inline v_reg<_Tp2, n> func(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b) \
686 typedef _Tp2 rtype; \
688 for( int i = 0; i < n; i++ ) \
689 c.s[i] = cast_op(a.s[i] bin_op b.s[i]); \
693 /** @brief Add values without saturation
695 For 8- and 16-bit integer values. */
696 OPENCV_HAL_IMPL_ADD_SUB_OP(v_add_wrap, +, (_Tp), _Tp)
698 /** @brief Subtract values without saturation
700 For 8- and 16-bit integer values. */
701 OPENCV_HAL_IMPL_ADD_SUB_OP(v_sub_wrap, -, (_Tp), _Tp)
704 template<typename T> inline T _absdiff(T a, T b)
706 return a > b ? a - b : b - a;
710 /** @brief Absolute difference
712 Returns \f$ |a - b| \f$ converted to corresponding unsigned type.
715 v_int32x4 a, b; // {1, 2, 3, 4} and {4, 3, 2, 1}
716 v_uint32x4 c = v_absdiff(a, b); // result is {3, 1, 1, 3}
718 For 8-, 16-, 32-bit integer source types. */
719 template<typename _Tp, int n>
720 inline v_reg<typename V_TypeTraits<_Tp>::abs_type, n> v_absdiff(const v_reg<_Tp, n>& a, const v_reg<_Tp, n> & b)
722 typedef typename V_TypeTraits<_Tp>::abs_type rtype;
724 const rtype mask = (rtype)(std::numeric_limits<_Tp>::is_signed ? (1 << (sizeof(rtype)*8 - 1)) : 0);
725 for( int i = 0; i < n; i++ )
727 rtype ua = a.s[i] ^ mask;
728 rtype ub = b.s[i] ^ mask;
729 c.s[i] = _absdiff(ua, ub);
736 For 32-bit floating point values */
737 inline v_float32x4 v_absdiff(const v_float32x4& a, const v_float32x4& b)
740 for( int i = 0; i < c.nlanes; i++ )
741 c.s[i] = _absdiff(a.s[i], b.s[i]);
747 For 64-bit floating point values */
748 inline v_float64x2 v_absdiff(const v_float64x2& a, const v_float64x2& b)
751 for( int i = 0; i < c.nlanes; i++ )
752 c.s[i] = _absdiff(a.s[i], b.s[i]);
756 /** @brief Inversed square root
758 Returns \f$ 1/sqrt(a) \f$
759 For floating point types only. */
760 template<typename _Tp, int n>
761 inline v_reg<_Tp, n> v_invsqrt(const v_reg<_Tp, n>& a)
764 for( int i = 0; i < n; i++ )
765 c.s[i] = 1.f/std::sqrt(a.s[i]);
771 Returns \f$ sqrt(a^2 + b^2) \f$
772 For floating point types only. */
773 template<typename _Tp, int n>
774 inline v_reg<_Tp, n> v_magnitude(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b)
777 for( int i = 0; i < n; i++ )
778 c.s[i] = std::sqrt(a.s[i]*a.s[i] + b.s[i]*b.s[i]);
782 /** @brief Square of the magnitude
784 Returns \f$ a^2 + b^2 \f$
785 For floating point types only. */
786 template<typename _Tp, int n>
787 inline v_reg<_Tp, n> v_sqr_magnitude(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b)
790 for( int i = 0; i < n; i++ )
791 c.s[i] = a.s[i]*a.s[i] + b.s[i]*b.s[i];
795 /** @brief Multiply and add
797 Returns \f$ a*b + c \f$
798 For floating point types only. */
799 template<typename _Tp, int n>
800 inline v_reg<_Tp, n> v_muladd(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b,
801 const v_reg<_Tp, n>& c)
804 for( int i = 0; i < n; i++ )
805 d.s[i] = a.s[i]*b.s[i] + c.s[i];
809 /** @brief Dot product of elements
811 Multiply values in two registers and sum adjacent result pairs.
814 {A1 A2 ...} // 16-bit
815 x {B1 B2 ...} // 16-bit
817 {A1B1+A2B2 ...} // 32-bit
819 Implemented only for 16-bit signed source type (v_int16x8).
821 template<typename _Tp, int n> inline v_reg<typename V_TypeTraits<_Tp>::w_type, n/2>
822 v_dotprod(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b)
824 typedef typename V_TypeTraits<_Tp>::w_type w_type;
825 v_reg<w_type, n/2> c;
826 for( int i = 0; i < (n/2); i++ )
827 c.s[i] = (w_type)a.s[i*2]*b.s[i*2] + (w_type)a.s[i*2+1]*b.s[i*2+1];
831 /** @brief Multiply and expand
833 Multiply values two registers and store results in two registers with wider pack type.
837 x {E F G H} // 32-bit
844 v_uint32x4 a, b; // {1,2,3,4} and {2,2,2,2}
845 v_uint64x2 c, d; // results
846 v_mul_expand(a, b, c, d); // c, d = {2,4}, {6, 8}
848 Implemented only for 16- and unsigned 32-bit source types (v_int16x8, v_uint16x8, v_uint32x4).
850 template<typename _Tp, int n> inline void v_mul_expand(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b,
851 v_reg<typename V_TypeTraits<_Tp>::w_type, n/2>& c,
852 v_reg<typename V_TypeTraits<_Tp>::w_type, n/2>& d)
854 typedef typename V_TypeTraits<_Tp>::w_type w_type;
855 for( int i = 0; i < (n/2); i++ )
857 c.s[i] = (w_type)a.s[i]*b.s[i];
858 d.s[i] = (w_type)a.s[i+(n/2)]*b.s[i+(n/2)];
863 template<typename _Tp, int n> inline void v_hsum(const v_reg<_Tp, n>& a,
864 v_reg<typename V_TypeTraits<_Tp>::w_type, n/2>& c)
866 typedef typename V_TypeTraits<_Tp>::w_type w_type;
867 for( int i = 0; i < (n/2); i++ )
869 c.s[i] = (w_type)a.s[i*2] + a.s[i*2+1];
874 //! @brief Helper macro
875 //! @ingroup core_hal_intrin_impl
876 #define OPENCV_HAL_IMPL_SHIFT_OP(shift_op) \
877 template<typename _Tp, int n> inline v_reg<_Tp, n> operator shift_op(const v_reg<_Tp, n>& a, int imm) \
880 for( int i = 0; i < n; i++ ) \
881 c.s[i] = (_Tp)(a.s[i] shift_op imm); \
885 /** @brief Bitwise shift left
887 For 16-, 32- and 64-bit integer values. */
888 OPENCV_HAL_IMPL_SHIFT_OP(<<)
890 /** @brief Bitwise shift right
892 For 16-, 32- and 64-bit integer values. */
893 OPENCV_HAL_IMPL_SHIFT_OP(>>)
895 /** @brief Sum packed values
899 {A1 A2 A3 ...} => sum{A1,A2,A3,...}
901 For 32-bit integer and 32-bit floating point types.*/
902 template<typename _Tp, int n> inline typename V_TypeTraits<_Tp>::sum_type v_reduce_sum(const v_reg<_Tp, n>& a)
904 typename V_TypeTraits<_Tp>::sum_type c = a.s[0];
905 for( int i = 1; i < n; i++ )
910 /** @brief Sums all elements of each input vector, returns the vector of sums
914 result[0] = a[0] + a[1] + a[2] + a[3]
915 result[1] = b[0] + b[1] + b[2] + b[3]
916 result[2] = c[0] + c[1] + c[2] + c[3]
917 result[3] = d[0] + d[1] + d[2] + d[3]
920 inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b,
921 const v_float32x4& c, const v_float32x4& d)
924 r.s[0] = a.s[0] + a.s[1] + a.s[2] + a.s[3];
925 r.s[1] = b.s[0] + b.s[1] + b.s[2] + b.s[3];
926 r.s[2] = c.s[0] + c.s[1] + c.s[2] + c.s[3];
927 r.s[3] = d.s[0] + d.s[1] + d.s[2] + d.s[3];
931 /** @brief Get negative values mask
933 Returned value is a bit mask with bits set to 1 on places corresponding to negative packed values indexes.
936 v_int32x4 r; // set to {-1, -1, 1, 1}
937 int mask = v_signmask(r); // mask = 3 <== 00000000 00000000 00000000 00000011
939 For all types except 64-bit. */
940 template<typename _Tp, int n> inline int v_signmask(const v_reg<_Tp, n>& a)
943 for( int i = 0; i < n; i++ )
944 mask |= (V_TypeTraits<_Tp>::reinterpret_int(a.s[i]) < 0) << i;
948 /** @brief Check if all packed values are less than zero
950 Unsigned values will be casted to signed: `uchar 254 => char -2`.
951 For all types except 64-bit. */
952 template<typename _Tp, int n> inline bool v_check_all(const v_reg<_Tp, n>& a)
954 for( int i = 0; i < n; i++ )
955 if( V_TypeTraits<_Tp>::reinterpret_int(a.s[i]) >= 0 )
960 /** @brief Check if any of packed values is less than zero
962 Unsigned values will be casted to signed: `uchar 254 => char -2`.
963 For all types except 64-bit. */
964 template<typename _Tp, int n> inline bool v_check_any(const v_reg<_Tp, n>& a)
966 for( int i = 0; i < n; i++ )
967 if( V_TypeTraits<_Tp>::reinterpret_int(a.s[i]) < 0 )
972 /** @brief Bitwise select
974 Return value will be built by combining values a and b using the following scheme:
975 If the i-th bit in _mask_ is 1
976 select i-th bit from _a_
978 select i-th bit from _b_ */
979 template<typename _Tp, int n> inline v_reg<_Tp, n> v_select(const v_reg<_Tp, n>& mask,
980 const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b)
982 typedef V_TypeTraits<_Tp> Traits;
983 typedef typename Traits::int_type int_type;
985 for( int i = 0; i < n; i++ )
987 int_type m = Traits::reinterpret_int(mask.s[i]);
988 c.s[i] = Traits::reinterpret_from_int((Traits::reinterpret_int(a.s[i]) & m)
989 | (Traits::reinterpret_int(b.s[i]) & ~m));
994 /** @brief Expand values to the wider pack type
996 Copy contents of register to two registers with 2x wider pack type.
999 int32x4 int64x2 int64x2
1000 {A B C D} ==> {A B} , {C D}
1002 template<typename _Tp, int n> inline void v_expand(const v_reg<_Tp, n>& a,
1003 v_reg<typename V_TypeTraits<_Tp>::w_type, n/2>& b0,
1004 v_reg<typename V_TypeTraits<_Tp>::w_type, n/2>& b1)
1006 for( int i = 0; i < (n/2); i++ )
1009 b1.s[i] = a.s[i+(n/2)];
1014 template<typename _Tp, int n> inline v_reg<typename V_TypeTraits<_Tp>::int_type, n>
1015 v_reinterpret_as_int(const v_reg<_Tp, n>& a)
1017 v_reg<typename V_TypeTraits<_Tp>::int_type, n> c;
1018 for( int i = 0; i < n; i++ )
1019 c.s[i] = V_TypeTraits<_Tp>::reinterpret_int(a.s[i]);
1023 template<typename _Tp, int n> inline v_reg<typename V_TypeTraits<_Tp>::uint_type, n>
1024 v_reinterpret_as_uint(const v_reg<_Tp, n>& a)
1026 v_reg<typename V_TypeTraits<_Tp>::uint_type, n> c;
1027 for( int i = 0; i < n; i++ )
1028 c.s[i] = V_TypeTraits<_Tp>::reinterpret_uint(a.s[i]);
1033 /** @brief Interleave two vectors
1040 {A1 B1 A2 B2} and {A3 B3 A4 B4}
1042 For all types except 64-bit.
1044 template<typename _Tp, int n> inline void v_zip( const v_reg<_Tp, n>& a0, const v_reg<_Tp, n>& a1,
1045 v_reg<_Tp, n>& b0, v_reg<_Tp, n>& b1 )
1048 for( i = 0; i < n/2; i++ )
1050 b0.s[i*2] = a0.s[i];
1051 b0.s[i*2+1] = a1.s[i];
1055 b1.s[i*2-n] = a0.s[i];
1056 b1.s[i*2-n+1] = a1.s[i];
1060 /** @brief Load register contents from memory
1062 @param ptr pointer to memory block with data
1063 @return register object
1065 @note Returned type will be detected from passed pointer type, for example uchar ==> cv::v_uint8x16, int ==> cv::v_int32x4, etc.
1067 template<typename _Tp>
1068 inline v_reg<_Tp, V_SIMD128Traits<_Tp>::nlanes> v_load(const _Tp* ptr)
1070 return v_reg<_Tp, V_SIMD128Traits<_Tp>::nlanes>(ptr);
1073 /** @brief Load register contents from memory (aligned)
1075 similar to cv::v_load, but source memory block should be aligned (to 16-byte boundary)
1077 template<typename _Tp>
1078 inline v_reg<_Tp, V_SIMD128Traits<_Tp>::nlanes> v_load_aligned(const _Tp* ptr)
1080 return v_reg<_Tp, V_SIMD128Traits<_Tp>::nlanes>(ptr);
1083 /** @brief Load register contents from two memory blocks
1085 @param loptr memory block containing data for first half (0..n/2)
1086 @param hiptr memory block containing data for second half (n/2..n)
1089 int lo[2] = { 1, 2 }, hi[2] = { 3, 4 };
1090 v_int32x4 r = v_load_halves(lo, hi);
1093 template<typename _Tp>
1094 inline v_reg<_Tp, V_SIMD128Traits<_Tp>::nlanes> v_load_halves(const _Tp* loptr, const _Tp* hiptr)
1096 v_reg<_Tp, V_SIMD128Traits<_Tp>::nlanes> c;
1097 for( int i = 0; i < c.nlanes/2; i++ )
1100 c.s[i+c.nlanes/2] = hiptr[i];
1105 /** @brief Load register contents from memory with double expand
1107 Same as cv::v_load, but result pack type will be 2x wider than memory type.
1110 short buf[4] = {1, 2, 3, 4}; // type is int16
1111 v_int32x4 r = v_load_expand(buf); // r = {1, 2, 3, 4} - type is int32
1113 For 8-, 16-, 32-bit integer source types. */
1114 template<typename _Tp>
1115 inline v_reg<typename V_TypeTraits<_Tp>::w_type, V_SIMD128Traits<_Tp>::nlanes / 2>
1116 v_load_expand(const _Tp* ptr)
1118 typedef typename V_TypeTraits<_Tp>::w_type w_type;
1119 v_reg<w_type, V_SIMD128Traits<w_type>::nlanes> c;
1120 for( int i = 0; i < c.nlanes; i++ )
1127 /** @brief Load register contents from memory with quad expand
1129 Same as cv::v_load_expand, but result type is 4 times wider than source.
1131 char buf[4] = {1, 2, 3, 4}; // type is int8
1132 v_int32x4 r = v_load_q(buf); // r = {1, 2, 3, 4} - type is int32
1134 For 8-bit integer source types. */
1135 template<typename _Tp>
1136 inline v_reg<typename V_TypeTraits<_Tp>::q_type, V_SIMD128Traits<_Tp>::nlanes / 4>
1137 v_load_expand_q(const _Tp* ptr)
1139 typedef typename V_TypeTraits<_Tp>::q_type q_type;
1140 v_reg<q_type, V_SIMD128Traits<q_type>::nlanes> c;
1141 for( int i = 0; i < c.nlanes; i++ )
1148 /** @brief Load and deinterleave (2 channels)
1150 Load data from memory deinterleave and store to 2 registers.
1153 {A1 B1 A2 B2 ...} ==> {A1 A2 ...}, {B1 B2 ...}
1155 For all types except 64-bit. */
1156 template<typename _Tp, int n> inline void v_load_deinterleave(const _Tp* ptr, v_reg<_Tp, n>& a,
1160 for( i = i2 = 0; i < n; i++, i2 += 2 )
1167 /** @brief Load and deinterleave (3 channels)
1169 Load data from memory deinterleave and store to 3 registers.
1172 {A1 B1 C1 A2 B2 C2 ...} ==> {A1 A2 ...}, {B1 B2 ...}, {C1 C2 ...}
1174 For all types except 64-bit. */
1175 template<typename _Tp, int n> inline void v_load_deinterleave(const _Tp* ptr, v_reg<_Tp, n>& a,
1176 v_reg<_Tp, n>& b, v_reg<_Tp, n>& c)
1179 for( i = i3 = 0; i < n; i++, i3 += 3 )
1187 /** @brief Load and deinterleave (4 channels)
1189 Load data from memory deinterleave and store to 4 registers.
1192 {A1 B1 C1 D1 A2 B2 C2 D2 ...} ==> {A1 A2 ...}, {B1 B2 ...}, {C1 C2 ...}, {D1 D2 ...}
1194 For all types except 64-bit. */
1195 template<typename _Tp, int n>
1196 inline void v_load_deinterleave(const _Tp* ptr, v_reg<_Tp, n>& a,
1197 v_reg<_Tp, n>& b, v_reg<_Tp, n>& c,
1201 for( i = i4 = 0; i < n; i++, i4 += 4 )
1210 /** @brief Interleave and store (2 channels)
1212 Interleave and store data from 2 registers to memory.
1215 {A1 A2 ...}, {B1 B2 ...} ==> {A1 B1 A2 B2 ...}
1217 For all types except 64-bit. */
1218 template<typename _Tp, int n>
1219 inline void v_store_interleave( _Tp* ptr, const v_reg<_Tp, n>& a,
1220 const v_reg<_Tp, n>& b)
1223 for( i = i2 = 0; i < n; i++, i2 += 2 )
1230 /** @brief Interleave and store (3 channels)
1232 Interleave and store data from 3 registers to memory.
1235 {A1 A2 ...}, {B1 B2 ...}, {C1 C2 ...} ==> {A1 B1 C1 A2 B2 C2 ...}
1237 For all types except 64-bit. */
1238 template<typename _Tp, int n>
1239 inline void v_store_interleave( _Tp* ptr, const v_reg<_Tp, n>& a,
1240 const v_reg<_Tp, n>& b, const v_reg<_Tp, n>& c)
1243 for( i = i3 = 0; i < n; i++, i3 += 3 )
1251 /** @brief Interleave and store (4 channels)
1253 Interleave and store data from 4 registers to memory.
1256 {A1 A2 ...}, {B1 B2 ...}, {C1 C2 ...}, {D1 D2 ...} ==> {A1 B1 C1 D1 A2 B2 C2 D2 ...}
1258 For all types except 64-bit. */
1259 template<typename _Tp, int n> inline void v_store_interleave( _Tp* ptr, const v_reg<_Tp, n>& a,
1260 const v_reg<_Tp, n>& b, const v_reg<_Tp, n>& c,
1261 const v_reg<_Tp, n>& d)
1264 for( i = i4 = 0; i < n; i++, i4 += 4 )
1273 /** @brief Store data to memory
1275 Store register contents to memory.
1278 REG {A B C D} ==> MEM {A B C D}
1280 Pointer can be unaligned. */
1281 template<typename _Tp, int n>
1282 inline void v_store(_Tp* ptr, const v_reg<_Tp, n>& a)
1284 for( int i = 0; i < n; i++ )
1288 /** @brief Store data to memory (lower half)
1290 Store lower half of register contents to memory.
1293 REG {A B C D} ==> MEM {A B}
1295 template<typename _Tp, int n>
1296 inline void v_store_low(_Tp* ptr, const v_reg<_Tp, n>& a)
1298 for( int i = 0; i < (n/2); i++ )
1302 /** @brief Store data to memory (higher half)
1304 Store higher half of register contents to memory.
1307 REG {A B C D} ==> MEM {C D}
1309 template<typename _Tp, int n>
1310 inline void v_store_high(_Tp* ptr, const v_reg<_Tp, n>& a)
1312 for( int i = 0; i < (n/2); i++ )
1313 ptr[i] = a.s[i+(n/2)];
1316 /** @brief Store data to memory (aligned)
1318 Store register contents to memory.
1321 REG {A B C D} ==> MEM {A B C D}
1323 Pointer __should__ be aligned by 16-byte boundary. */
1324 template<typename _Tp, int n>
1325 inline void v_store_aligned(_Tp* ptr, const v_reg<_Tp, n>& a)
1327 for( int i = 0; i < n; i++ )
1331 /** @brief Combine vector from first elements of two vectors
1340 For all types except 64-bit. */
1341 template<typename _Tp, int n>
1342 inline v_reg<_Tp, n> v_combine_low(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b)
1345 for( int i = 0; i < (n/2); i++ )
1348 c.s[i+(n/2)] = b.s[i];
1353 /** @brief Combine vector from last elements of two vectors
1362 For all types except 64-bit. */
1363 template<typename _Tp, int n>
1364 inline v_reg<_Tp, n> v_combine_high(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b)
1367 for( int i = 0; i < (n/2); i++ )
1369 c.s[i] = a.s[i+(n/2)];
1370 c.s[i+(n/2)] = b.s[i+(n/2)];
1375 /** @brief Combine two vectors from lower and higher parts of two other vectors
1378 low = cv::v_combine_low(a, b);
1379 high = cv::v_combine_high(a, b);
1381 template<typename _Tp, int n>
1382 inline void v_recombine(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b,
1383 v_reg<_Tp, n>& low, v_reg<_Tp, n>& high)
1385 for( int i = 0; i < (n/2); i++ )
1388 low.s[i+(n/2)] = b.s[i];
1389 high.s[i] = a.s[i+(n/2)];
1390 high.s[i+(n/2)] = b.s[i+(n/2)];
1394 /** @brief Vector extract
1400 ========================
1401 shift = 1 {A2 A3 A4 B1}
1402 shift = 2 {A3 A4 B1 B2}
1403 shift = 3 {A4 B1 B2 B3}
1405 Restriction: 0 <= shift < nlanes
1410 c = v_extract<2>(a, b);
1412 For integer types only. */
1413 template<int s, typename _Tp, int n>
1414 inline v_reg<_Tp, n> v_extract(const v_reg<_Tp, n>& a, const v_reg<_Tp, n>& b)
1417 const int shift = n - s;
1419 for (; i < shift; ++i)
1422 r.s[i] = b.s[i-shift];
1428 Rounds each value. Input type is float vector ==> output type is int vector.*/
1429 template<int n> inline v_reg<int, n> v_round(const v_reg<float, n>& a)
1432 for( int i = 0; i < n; i++ )
1433 c.s[i] = cvRound(a.s[i]);
1439 Floor each value. Input type is float vector ==> output type is int vector.*/
1440 template<int n> inline v_reg<int, n> v_floor(const v_reg<float, n>& a)
1443 for( int i = 0; i < n; i++ )
1444 c.s[i] = cvFloor(a.s[i]);
1450 Ceil each value. Input type is float vector ==> output type is int vector.*/
1451 template<int n> inline v_reg<int, n> v_ceil(const v_reg<float, n>& a)
1454 for( int i = 0; i < n; i++ )
1455 c.s[i] = cvCeil(a.s[i]);
1461 Truncate each value. Input type is float vector ==> output type is int vector.*/
1462 template<int n> inline v_reg<int, n> v_trunc(const v_reg<float, n>& a)
1465 for( int i = 0; i < n; i++ )
1466 c.s[i] = (int)(a.s[i]);
1471 template<int n> inline v_reg<int, n*2> v_round(const v_reg<double, n>& a)
1474 for( int i = 0; i < n; i++ )
1476 c.s[i] = cvRound(a.s[i]);
1483 template<int n> inline v_reg<int, n*2> v_floor(const v_reg<double, n>& a)
1486 for( int i = 0; i < n; i++ )
1488 c.s[i] = cvFloor(a.s[i]);
1495 template<int n> inline v_reg<int, n*2> v_ceil(const v_reg<double, n>& a)
1498 for( int i = 0; i < n; i++ )
1500 c.s[i] = cvCeil(a.s[i]);
1507 template<int n> inline v_reg<int, n*2> v_trunc(const v_reg<double, n>& a)
1510 for( int i = 0; i < n; i++ )
1512 c.s[i] = cvCeil(a.s[i]);
1518 /** @brief Convert to float
1520 Supported input type is cv::v_int32x4. */
1521 template<int n> inline v_reg<float, n> v_cvt_f32(const v_reg<int, n>& a)
1524 for( int i = 0; i < n; i++ )
1525 c.s[i] = (float)a.s[i];
1529 /** @brief Convert to double
1531 Supported input type is cv::v_int32x4. */
1532 template<int n> inline v_reg<double, n> v_cvt_f64(const v_reg<int, n*2>& a)
1535 for( int i = 0; i < n; i++ )
1536 c.s[i] = (double)a.s[i];
1540 /** @brief Convert to double
1542 Supported input type is cv::v_float32x4. */
1543 template<int n> inline v_reg<double, n> v_cvt_f64(const v_reg<float, n*2>& a)
1546 for( int i = 0; i < n; i++ )
1547 c.s[i] = (double)a.s[i];
1551 /** @brief Transpose 4x4 matrix
1566 template<typename _Tp>
1567 inline void v_transpose4x4( v_reg<_Tp, 4>& a0, const v_reg<_Tp, 4>& a1,
1568 const v_reg<_Tp, 4>& a2, const v_reg<_Tp, 4>& a3,
1569 v_reg<_Tp, 4>& b0, v_reg<_Tp, 4>& b1,
1570 v_reg<_Tp, 4>& b2, v_reg<_Tp, 4>& b3 )
1572 b0 = v_reg<_Tp, 4>(a0.s[0], a1.s[0], a2.s[0], a3.s[0]);
1573 b1 = v_reg<_Tp, 4>(a0.s[1], a1.s[1], a2.s[1], a3.s[1]);
1574 b2 = v_reg<_Tp, 4>(a0.s[2], a1.s[2], a2.s[2], a3.s[2]);
1575 b3 = v_reg<_Tp, 4>(a0.s[3], a1.s[3], a2.s[3], a3.s[3]);
1578 //! @brief Helper macro
1579 //! @ingroup core_hal_intrin_impl
1580 #define OPENCV_HAL_IMPL_C_INIT_ZERO(_Tpvec, _Tp, suffix) \
1581 inline _Tpvec v_setzero_##suffix() { return _Tpvec::zero(); }
1583 //! @name Init with zero
1585 //! @brief Create new vector with zero elements
1586 OPENCV_HAL_IMPL_C_INIT_ZERO(v_uint8x16, uchar, u8)
1587 OPENCV_HAL_IMPL_C_INIT_ZERO(v_int8x16, schar, s8)
1588 OPENCV_HAL_IMPL_C_INIT_ZERO(v_uint16x8, ushort, u16)
1589 OPENCV_HAL_IMPL_C_INIT_ZERO(v_int16x8, short, s16)
1590 OPENCV_HAL_IMPL_C_INIT_ZERO(v_uint32x4, unsigned, u32)
1591 OPENCV_HAL_IMPL_C_INIT_ZERO(v_int32x4, int, s32)
1592 OPENCV_HAL_IMPL_C_INIT_ZERO(v_float32x4, float, f32)
1593 OPENCV_HAL_IMPL_C_INIT_ZERO(v_float64x2, double, f64)
1594 OPENCV_HAL_IMPL_C_INIT_ZERO(v_uint64x2, uint64, u64)
1595 OPENCV_HAL_IMPL_C_INIT_ZERO(v_int64x2, int64, s64)
1598 //! @brief Helper macro
1599 //! @ingroup core_hal_intrin_impl
1600 #define OPENCV_HAL_IMPL_C_INIT_VAL(_Tpvec, _Tp, suffix) \
1601 inline _Tpvec v_setall_##suffix(_Tp val) { return _Tpvec::all(val); }
1603 //! @name Init with value
1605 //! @brief Create new vector with elements set to a specific value
1606 OPENCV_HAL_IMPL_C_INIT_VAL(v_uint8x16, uchar, u8)
1607 OPENCV_HAL_IMPL_C_INIT_VAL(v_int8x16, schar, s8)
1608 OPENCV_HAL_IMPL_C_INIT_VAL(v_uint16x8, ushort, u16)
1609 OPENCV_HAL_IMPL_C_INIT_VAL(v_int16x8, short, s16)
1610 OPENCV_HAL_IMPL_C_INIT_VAL(v_uint32x4, unsigned, u32)
1611 OPENCV_HAL_IMPL_C_INIT_VAL(v_int32x4, int, s32)
1612 OPENCV_HAL_IMPL_C_INIT_VAL(v_float32x4, float, f32)
1613 OPENCV_HAL_IMPL_C_INIT_VAL(v_float64x2, double, f64)
1614 OPENCV_HAL_IMPL_C_INIT_VAL(v_uint64x2, uint64, u64)
1615 OPENCV_HAL_IMPL_C_INIT_VAL(v_int64x2, int64, s64)
1618 //! @brief Helper macro
1619 //! @ingroup core_hal_intrin_impl
1620 #define OPENCV_HAL_IMPL_C_REINTERPRET(_Tpvec, _Tp, suffix) \
1621 template<typename _Tp0, int n0> inline _Tpvec \
1622 v_reinterpret_as_##suffix(const v_reg<_Tp0, n0>& a) \
1623 { return a.template reinterpret_as<_Tp, _Tpvec::nlanes>(); }
1625 //! @name Reinterpret
1627 //! @brief Convert vector to different type without modifying underlying data.
1628 OPENCV_HAL_IMPL_C_REINTERPRET(v_uint8x16, uchar, u8)
1629 OPENCV_HAL_IMPL_C_REINTERPRET(v_int8x16, schar, s8)
1630 OPENCV_HAL_IMPL_C_REINTERPRET(v_uint16x8, ushort, u16)
1631 OPENCV_HAL_IMPL_C_REINTERPRET(v_int16x8, short, s16)
1632 OPENCV_HAL_IMPL_C_REINTERPRET(v_uint32x4, unsigned, u32)
1633 OPENCV_HAL_IMPL_C_REINTERPRET(v_int32x4, int, s32)
1634 OPENCV_HAL_IMPL_C_REINTERPRET(v_float32x4, float, f32)
1635 OPENCV_HAL_IMPL_C_REINTERPRET(v_float64x2, double, f64)
1636 OPENCV_HAL_IMPL_C_REINTERPRET(v_uint64x2, uint64, u64)
1637 OPENCV_HAL_IMPL_C_REINTERPRET(v_int64x2, int64, s64)
1640 //! @brief Helper macro
1641 //! @ingroup core_hal_intrin_impl
1642 #define OPENCV_HAL_IMPL_C_SHIFTL(_Tpvec, _Tp) \
1643 template<int n> inline _Tpvec v_shl(const _Tpvec& a) \
1646 //! @name Left shift
1648 //! @brief Shift left
1649 OPENCV_HAL_IMPL_C_SHIFTL(v_uint16x8, ushort)
1650 OPENCV_HAL_IMPL_C_SHIFTL(v_int16x8, short)
1651 OPENCV_HAL_IMPL_C_SHIFTL(v_uint32x4, unsigned)
1652 OPENCV_HAL_IMPL_C_SHIFTL(v_int32x4, int)
1653 OPENCV_HAL_IMPL_C_SHIFTL(v_uint64x2, uint64)
1654 OPENCV_HAL_IMPL_C_SHIFTL(v_int64x2, int64)
1657 //! @brief Helper macro
1658 //! @ingroup core_hal_intrin_impl
1659 #define OPENCV_HAL_IMPL_C_SHIFTR(_Tpvec, _Tp) \
1660 template<int n> inline _Tpvec v_shr(const _Tpvec& a) \
1663 //! @name Right shift
1665 //! @brief Shift right
1666 OPENCV_HAL_IMPL_C_SHIFTR(v_uint16x8, ushort)
1667 OPENCV_HAL_IMPL_C_SHIFTR(v_int16x8, short)
1668 OPENCV_HAL_IMPL_C_SHIFTR(v_uint32x4, unsigned)
1669 OPENCV_HAL_IMPL_C_SHIFTR(v_int32x4, int)
1670 OPENCV_HAL_IMPL_C_SHIFTR(v_uint64x2, uint64)
1671 OPENCV_HAL_IMPL_C_SHIFTR(v_int64x2, int64)
1674 //! @brief Helper macro
1675 //! @ingroup core_hal_intrin_impl
1676 #define OPENCV_HAL_IMPL_C_RSHIFTR(_Tpvec, _Tp) \
1677 template<int n> inline _Tpvec v_rshr(const _Tpvec& a) \
1680 for( int i = 0; i < _Tpvec::nlanes; i++ ) \
1681 c.s[i] = (_Tp)((a.s[i] + ((_Tp)1 << (n - 1))) >> n); \
1685 //! @name Rounding shift
1687 //! @brief Rounding shift right
1688 OPENCV_HAL_IMPL_C_RSHIFTR(v_uint16x8, ushort)
1689 OPENCV_HAL_IMPL_C_RSHIFTR(v_int16x8, short)
1690 OPENCV_HAL_IMPL_C_RSHIFTR(v_uint32x4, unsigned)
1691 OPENCV_HAL_IMPL_C_RSHIFTR(v_int32x4, int)
1692 OPENCV_HAL_IMPL_C_RSHIFTR(v_uint64x2, uint64)
1693 OPENCV_HAL_IMPL_C_RSHIFTR(v_int64x2, int64)
1696 //! @brief Helper macro
1697 //! @ingroup core_hal_intrin_impl
1698 #define OPENCV_HAL_IMPL_C_PACK(_Tpvec, _Tpnvec, _Tpn, pack_suffix) \
1699 inline _Tpnvec v_##pack_suffix(const _Tpvec& a, const _Tpvec& b) \
1702 for( int i = 0; i < _Tpvec::nlanes; i++ ) \
1704 c.s[i] = saturate_cast<_Tpn>(a.s[i]); \
1705 c.s[i+_Tpvec::nlanes] = saturate_cast<_Tpn>(b.s[i]); \
1712 //! @brief Pack values from two vectors to one
1714 //! Return vector type have twice more elements than input vector types. Variant with _u_ suffix also
1715 //! converts to corresponding unsigned type.
1717 //! - pack: for 16-, 32- and 64-bit integer input types
1718 //! - pack_u: for 16- and 32-bit signed integer input types
1719 OPENCV_HAL_IMPL_C_PACK(v_uint16x8, v_uint8x16, uchar, pack)
1720 OPENCV_HAL_IMPL_C_PACK(v_int16x8, v_int8x16, schar, pack)
1721 OPENCV_HAL_IMPL_C_PACK(v_uint32x4, v_uint16x8, ushort, pack)
1722 OPENCV_HAL_IMPL_C_PACK(v_int32x4, v_int16x8, short, pack)
1723 OPENCV_HAL_IMPL_C_PACK(v_uint64x2, v_uint32x4, unsigned, pack)
1724 OPENCV_HAL_IMPL_C_PACK(v_int64x2, v_int32x4, int, pack)
1725 OPENCV_HAL_IMPL_C_PACK(v_int16x8, v_uint8x16, uchar, pack_u)
1726 OPENCV_HAL_IMPL_C_PACK(v_int32x4, v_uint16x8, ushort, pack_u)
1729 //! @brief Helper macro
1730 //! @ingroup core_hal_intrin_impl
1731 #define OPENCV_HAL_IMPL_C_RSHR_PACK(_Tpvec, _Tp, _Tpnvec, _Tpn, pack_suffix) \
1732 template<int n> inline _Tpnvec v_rshr_##pack_suffix(const _Tpvec& a, const _Tpvec& b) \
1735 for( int i = 0; i < _Tpvec::nlanes; i++ ) \
1737 c.s[i] = saturate_cast<_Tpn>((a.s[i] + ((_Tp)1 << (n - 1))) >> n); \
1738 c.s[i+_Tpvec::nlanes] = saturate_cast<_Tpn>((b.s[i] + ((_Tp)1 << (n - 1))) >> n); \
1743 //! @name Pack with rounding shift
1745 //! @brief Pack values from two vectors to one with rounding shift
1747 //! Values from the input vectors will be shifted right by _n_ bits with rounding, converted to narrower
1748 //! type and returned in the result vector. Variant with _u_ suffix converts to unsigned type.
1750 //! - pack: for 16-, 32- and 64-bit integer input types
1751 //! - pack_u: for 16- and 32-bit signed integer input types
1752 OPENCV_HAL_IMPL_C_RSHR_PACK(v_uint16x8, ushort, v_uint8x16, uchar, pack)
1753 OPENCV_HAL_IMPL_C_RSHR_PACK(v_int16x8, short, v_int8x16, schar, pack)
1754 OPENCV_HAL_IMPL_C_RSHR_PACK(v_uint32x4, unsigned, v_uint16x8, ushort, pack)
1755 OPENCV_HAL_IMPL_C_RSHR_PACK(v_int32x4, int, v_int16x8, short, pack)
1756 OPENCV_HAL_IMPL_C_RSHR_PACK(v_uint64x2, uint64, v_uint32x4, unsigned, pack)
1757 OPENCV_HAL_IMPL_C_RSHR_PACK(v_int64x2, int64, v_int32x4, int, pack)
1758 OPENCV_HAL_IMPL_C_RSHR_PACK(v_int16x8, short, v_uint8x16, uchar, pack_u)
1759 OPENCV_HAL_IMPL_C_RSHR_PACK(v_int32x4, int, v_uint16x8, ushort, pack_u)
1762 //! @brief Helper macro
1763 //! @ingroup core_hal_intrin_impl
1764 #define OPENCV_HAL_IMPL_C_PACK_STORE(_Tpvec, _Tp, _Tpnvec, _Tpn, pack_suffix) \
1765 inline void v_##pack_suffix##_store(_Tpn* ptr, const _Tpvec& a) \
1767 for( int i = 0; i < _Tpvec::nlanes; i++ ) \
1768 ptr[i] = saturate_cast<_Tpn>(a.s[i]); \
1771 //! @name Pack and store
1773 //! @brief Store values from the input vector into memory with pack
1775 //! Values will be stored into memory with saturating conversion to narrower type.
1776 //! Variant with _u_ suffix converts to corresponding unsigned type.
1778 //! - pack: for 16-, 32- and 64-bit integer input types
1779 //! - pack_u: for 16- and 32-bit signed integer input types
1780 OPENCV_HAL_IMPL_C_PACK_STORE(v_uint16x8, ushort, v_uint8x16, uchar, pack)
1781 OPENCV_HAL_IMPL_C_PACK_STORE(v_int16x8, short, v_int8x16, schar, pack)
1782 OPENCV_HAL_IMPL_C_PACK_STORE(v_uint32x4, unsigned, v_uint16x8, ushort, pack)
1783 OPENCV_HAL_IMPL_C_PACK_STORE(v_int32x4, int, v_int16x8, short, pack)
1784 OPENCV_HAL_IMPL_C_PACK_STORE(v_uint64x2, uint64, v_uint32x4, unsigned, pack)
1785 OPENCV_HAL_IMPL_C_PACK_STORE(v_int64x2, int64, v_int32x4, int, pack)
1786 OPENCV_HAL_IMPL_C_PACK_STORE(v_int16x8, short, v_uint8x16, uchar, pack_u)
1787 OPENCV_HAL_IMPL_C_PACK_STORE(v_int32x4, int, v_uint16x8, ushort, pack_u)
1790 //! @brief Helper macro
1791 //! @ingroup core_hal_intrin_impl
1792 #define OPENCV_HAL_IMPL_C_RSHR_PACK_STORE(_Tpvec, _Tp, _Tpnvec, _Tpn, pack_suffix) \
1793 template<int n> inline void v_rshr_##pack_suffix##_store(_Tpn* ptr, const _Tpvec& a) \
1795 for( int i = 0; i < _Tpvec::nlanes; i++ ) \
1796 ptr[i] = saturate_cast<_Tpn>((a.s[i] + ((_Tp)1 << (n - 1))) >> n); \
1799 //! @name Pack and store with rounding shift
1801 //! @brief Store values from the input vector into memory with pack
1803 //! Values will be shifted _n_ bits right with rounding, converted to narrower type and stored into
1804 //! memory. Variant with _u_ suffix converts to unsigned type.
1806 //! - pack: for 16-, 32- and 64-bit integer input types
1807 //! - pack_u: for 16- and 32-bit signed integer input types
1808 OPENCV_HAL_IMPL_C_RSHR_PACK_STORE(v_uint16x8, ushort, v_uint8x16, uchar, pack)
1809 OPENCV_HAL_IMPL_C_RSHR_PACK_STORE(v_int16x8, short, v_int8x16, schar, pack)
1810 OPENCV_HAL_IMPL_C_RSHR_PACK_STORE(v_uint32x4, unsigned, v_uint16x8, ushort, pack)
1811 OPENCV_HAL_IMPL_C_RSHR_PACK_STORE(v_int32x4, int, v_int16x8, short, pack)
1812 OPENCV_HAL_IMPL_C_RSHR_PACK_STORE(v_uint64x2, uint64, v_uint32x4, unsigned, pack)
1813 OPENCV_HAL_IMPL_C_RSHR_PACK_STORE(v_int64x2, int64, v_int32x4, int, pack)
1814 OPENCV_HAL_IMPL_C_RSHR_PACK_STORE(v_int16x8, short, v_uint8x16, uchar, pack_u)
1815 OPENCV_HAL_IMPL_C_RSHR_PACK_STORE(v_int32x4, int, v_uint16x8, ushort, pack_u)
1818 /** @brief Matrix multiplication
1825 {D0 D1 D2 D3} x |V3|
1826 ====================
1827 {R0 R1 R2 R3}, where:
1828 R0 = A0V0 + A1V1 + A2V2 + A3V3,
1829 R1 = B0V0 + B1V1 + B2V2 + B3V3
1833 inline v_float32x4 v_matmul(const v_float32x4& v, const v_float32x4& m0,
1834 const v_float32x4& m1, const v_float32x4& m2,
1835 const v_float32x4& m3)
1837 return v_float32x4(v.s[0]*m0.s[0] + v.s[1]*m1.s[0] + v.s[2]*m2.s[0] + v.s[3]*m3.s[0],
1838 v.s[0]*m0.s[1] + v.s[1]*m1.s[1] + v.s[2]*m2.s[1] + v.s[3]*m3.s[1],
1839 v.s[0]*m0.s[2] + v.s[1]*m1.s[2] + v.s[2]*m2.s[2] + v.s[3]*m3.s[2],
1840 v.s[0]*m0.s[3] + v.s[1]*m1.s[3] + v.s[2]*m2.s[3] + v.s[3]*m3.s[3]);
1845 //! @name Check SIMD support
1847 //! @brief Check CPU capability of SIMD operation
1848 static inline bool hasSIMD128()
1856 CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END