{
int i = 0;
double result = 0;
- #if CV_ENABLE_UNROLLED
+
+ #if CV_ENABLE_UNROLLED
for( ; i <= len - 4; i += 4 )
result += (double)src1[i]*src2[i] + (double)src1[i+1]*src2[i+1] +
(double)src1[i+2]*src2[i+2] + (double)src1[i+3]*src2[i+3];
{
int j, len0 = len & -4, blockSize0 = (1 << 13), blockSize;
__m128i z = _mm_setzero_si128();
+ CV_DECL_ALIGNED(16) int buf[4];
+
while( i < len0 )
{
blockSize = std::min(len0 - i, blockSize0);
- __m128i s = _mm_setzero_si128();
+ __m128i s = z;
j = 0;
for( ; j <= blockSize - 16; j += 16 )
{
s0 = _mm_madd_epi16(s0, s1);
s = _mm_add_epi32(s, s0);
}
- CV_DECL_ALIGNED(16) int buf[4];
+
_mm_store_si128((__m128i*)buf, s);
r += buf[0] + buf[1] + buf[2] + buf[3];
i += blockSize;
}
}
+#elif CV_NEON
+ int len0 = len & -8, blockSize0 = (1 << 15), blockSize;
+ uint32x4_t v_zero = vdupq_n_u32(0u);
+ CV_DECL_ALIGNED(16) uint buf[4];
+
+ while( i < len0 )
+ {
+ blockSize = std::min(len0 - i, blockSize0);
+ uint32x4_t v_sum = v_zero;
+
+ int j = 0;
+ for( ; j <= blockSize - 16; j += 16 )
+ {
+ uint8x16_t v_src1 = vld1q_u8(src1 + j), v_src2 = vld1q_u8(src2 + j);
+
+ uint16x8_t v_src10 = vmovl_u8(vget_low_u8(v_src1)), v_src20 = vmovl_u8(vget_low_u8(v_src2));
+ v_sum = vmlal_u16(v_sum, vget_low_u16(v_src10), vget_low_u16(v_src20));
+ v_sum = vmlal_u16(v_sum, vget_high_u16(v_src10), vget_high_u16(v_src20));
+
+ v_src10 = vmovl_u8(vget_high_u8(v_src1));
+ v_src20 = vmovl_u8(vget_high_u8(v_src2));
+ v_sum = vmlal_u16(v_sum, vget_low_u16(v_src10), vget_low_u16(v_src20));
+ v_sum = vmlal_u16(v_sum, vget_high_u16(v_src10), vget_high_u16(v_src20));
+ }
+
+ for( ; j <= blockSize - 8; j += 8 )
+ {
+ uint16x8_t v_src1 = vmovl_u8(vld1_u8(src1 + j)), v_src2 = vmovl_u8(vld1_u8(src2 + j));
+ v_sum = vmlal_u16(v_sum, vget_low_u16(v_src1), vget_low_u16(v_src2));
+ v_sum = vmlal_u16(v_sum, vget_high_u16(v_src1), vget_high_u16(v_src2));
+ }
+
+ vst1q_u32(buf, v_sum);
+ r += buf[0] + buf[1] + buf[2] + buf[3];
+
+ src1 += blockSize;
+ src2 += blockSize;
+ i += blockSize;
+ }
#endif
return r + dotProd_(src1, src2, len - i);
}
static double dotProd_8s(const schar* src1, const schar* src2, int len)
{
- return dotProd_(src1, src2, len);
+ int i = 0;
+ double r = 0.0;
+
+#if CV_NEON
+ int len0 = len & -8, blockSize0 = (1 << 14), blockSize;
+ int32x4_t v_zero = vdupq_n_s32(0);
+ CV_DECL_ALIGNED(16) int buf[4];
+
+ while( i < len0 )
+ {
+ blockSize = std::min(len0 - i, blockSize0);
+ int32x4_t v_sum = v_zero;
+
+ int j = 0;
+ for( ; j <= blockSize - 16; j += 16 )
+ {
+ int8x16_t v_src1 = vld1q_s8(src1 + j), v_src2 = vld1q_s8(src2 + j);
+
+ int16x8_t v_src10 = vmovl_s8(vget_low_s8(v_src1)), v_src20 = vmovl_s8(vget_low_s8(v_src2));
+ v_sum = vmlal_s16(v_sum, vget_low_s16(v_src10), vget_low_s16(v_src20));
+ v_sum = vmlal_s16(v_sum, vget_high_s16(v_src10), vget_high_s16(v_src20));
+
+ v_src10 = vmovl_s8(vget_high_s8(v_src1));
+ v_src20 = vmovl_s8(vget_high_s8(v_src2));
+ v_sum = vmlal_s16(v_sum, vget_low_s16(v_src10), vget_low_s16(v_src20));
+ v_sum = vmlal_s16(v_sum, vget_high_s16(v_src10), vget_high_s16(v_src20));
+ }
+
+ for( ; j <= blockSize - 8; j += 8 )
+ {
+ int16x8_t v_src1 = vmovl_s8(vld1_s8(src1 + j)), v_src2 = vmovl_s8(vld1_s8(src2 + j));
+ v_sum = vmlal_s16(v_sum, vget_low_s16(v_src1), vget_low_s16(v_src2));
+ v_sum = vmlal_s16(v_sum, vget_high_s16(v_src1), vget_high_s16(v_src2));
+ }
+
+ vst1q_s32(buf, v_sum);
+ r += buf[0] + buf[1] + buf[2] + buf[3];
+
+ src1 += blockSize;
+ src2 += blockSize;
+ i += blockSize;
+ }
+#endif
+
+ return r + dotProd_(src1, src2, len - i);
}
static double dotProd_16u(const ushort* src1, const ushort* src2, int len)
static double dotProd_32f(const float* src1, const float* src2, int len)
{
+ double r = 0.0;
+ int i = 0;
+
#if (ARITHM_USE_IPP == 1)
- double r = 0;
if (0 <= ippsDotProd_32f64f(src1, src2, len, &r))
return r;
setIppErrorStatus();
+#elif CV_NEON
+ int len0 = len & -4, blockSize0 = (1 << 15), blockSize;
+ float32x4_t v_zero = vdupq_n_f32(0.0f);
+ CV_DECL_ALIGNED(16) float buf[4];
+
+ while( i < len0 )
+ {
+ blockSize = std::min(len0 - i, blockSize0);
+ float32x4_t v_sum = v_zero;
+
+ int j = 0;
+ for( ; j <= blockSize - 4; j += 4 )
+ v_sum = vmlaq_f32(v_sum, vld1q_f32(src1 + j), vld1q_f32(src2 + j));
+
+ vst1q_f32(buf, v_sum);
+ r += buf[0] + buf[1] + buf[2] + buf[3];
+
+ src1 += blockSize;
+ src2 += blockSize;
+ i += blockSize;
+ }
#endif
- return dotProd_(src1, src2, len);
+ return r + dotProd_(src1, src2, len - i);
}
static double dotProd_64f(const double* src1, const double* src2, int len)