use _mm_sad_epu8 instead of _mm_movemask_epi8 in countNonZero. Therefore, we do not...

diff --git a/modules/core/src/stat.cpp b/modules/core/src/stat.cpp
index 8a6bd54..81f9a24 100644 (file)
--- a/modules/core/src/stat.cpp
+++ b/modules/core/src/stat.cpp
@@ -504,54 +504,21 @@ static int countNonZero_(const T* src, int len )
     return nz;
 }
 
-#if CV_SSE2
-
-static const uchar * initPopcountTable()
-{
-    static uchar tab[256];
-    static volatile bool initialized = false;
-    if( !initialized )
-    {
-        // we compute inverse popcount table,
-        // since we pass (img[x] == 0) mask as index in the table.
-        unsigned int j = 0u;
-#if CV_POPCNT
-        if (checkHardwareSupport(CV_CPU_POPCNT))
-        {
-            for( ; j < 256u; j++ )
-                tab[j] = (uchar)(8 - _mm_popcnt_u32(j));
-        }
-#endif
-        for( ; j < 256u; j++ )
-        {
-            int val = 0;
-            for( int mask = 1; mask < 256; mask += mask )
-                val += (j & mask) == 0;
-            tab[j] = (uchar)val;
-        }
-        initialized = true;
-    }
-
-    return tab;
-}
-
-#endif
-
 static int countNonZero8u( const uchar* src, int len )
 {
     int i=0, nz = 0;
 #if CV_SSE2
     if(USE_SSE2)//5x-6x
     {
-        __m128i pattern = _mm_setzero_si128 ();
-        static const uchar * tab = initPopcountTable();
+        __m128i v_zero = _mm_setzero_si128();
+        __m128i sum = _mm_setzero_si128();
 
         for (; i<=len-16; i+=16)
         {
             __m128i r0 = _mm_loadu_si128((const __m128i*)(src+i));
-            int val = _mm_movemask_epi8(_mm_cmpeq_epi8(r0, pattern));
-            nz += tab[val & 255] + tab[val >> 8];
+            sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_cmpeq_epi8(r0, v_zero)), v_zero));
         }
+        nz = i - _mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum)));
     }
 #elif CV_NEON
     int len0 = len & -16, blockSize1 = (1 << 8) - 16, blockSize0 = blockSize1 << 6;
@@ -598,15 +565,15 @@ static int countNonZero16u( const ushort* src, int len )
     if (USE_SSE2)
     {
         __m128i v_zero = _mm_setzero_si128 ();
-        static const uchar * tab = initPopcountTable();
+        __m128i sum = _mm_setzero_si128();
 
         for ( ; i <= len - 8; i += 8)
         {
-            __m128i v_src = _mm_loadu_si128((const __m128i*)(src + i));
-            int val = _mm_movemask_epi8(_mm_packs_epi16(_mm_cmpeq_epi16(v_src, v_zero), v_zero));
-            nz += tab[val];
+            __m128i r0 = _mm_loadu_si128((const __m128i*)(src + i));
+            sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_cmpeq_epi16(r0, v_zero)), v_zero));
         }
 
+        nz = i - (_mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum))) >> 1);
         src += i;
     }
 #elif CV_NEON
@@ -649,20 +616,15 @@ static int countNonZero32s( const int* src, int len )
     if (USE_SSE2)
     {
         __m128i v_zero = _mm_setzero_si128 ();
-        static const uchar * tab = initPopcountTable();
+        __m128i sum = _mm_setzero_si128();
 
-        for ( ; i <= len - 8; i += 8)
+        for ( ; i <= len - 4; i += 4)
         {
-            __m128i v_src = _mm_loadu_si128((const __m128i*)(src + i));
-            __m128i v_dst0 = _mm_cmpeq_epi32(v_src, v_zero);
-
-            v_src = _mm_loadu_si128((const __m128i*)(src + i + 4));
-            __m128i v_dst1 = _mm_cmpeq_epi32(v_src, v_zero);
-
-            int val = _mm_movemask_epi8(_mm_packs_epi16(_mm_packs_epi32(v_dst0, v_dst1), v_zero));
-            nz += tab[val];
+            __m128i r0 = _mm_loadu_si128((const __m128i*)(src + i));
+            sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_cmpeq_epi32(r0, v_zero)), v_zero));
         }
 
+        nz = i - (_mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum))) >> 2);
         src += i;
     }
 #elif CV_NEON
@@ -706,19 +668,17 @@ static int countNonZero32f( const float* src, int len )
 #if CV_SSE2
     if (USE_SSE2)
     {
-        __m128i v_zero_i = _mm_setzero_si128();
         __m128 v_zero_f = _mm_setzero_ps();
-        static const uchar * tab = initPopcountTable();
+        __m128i v_zero = _mm_setzero_si128 ();
+        __m128i sum = _mm_setzero_si128();
 
-        for ( ; i <= len - 8; i += 8)
+        for ( ; i <= len - 4; i += 4)
         {
-            __m128i v_dst0 = _mm_castps_si128(_mm_cmpeq_ps(_mm_loadu_ps(src + i), v_zero_f));
-            __m128i v_dst1 = _mm_castps_si128(_mm_cmpeq_ps(_mm_loadu_ps(src + i + 4), v_zero_f));
-
-            int val = _mm_movemask_epi8(_mm_packs_epi16(_mm_packs_epi32(v_dst0, v_dst1), v_zero_i));
-            nz += tab[val];
+            __m128 r0 = _mm_loadu_ps(src + i);
+            sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_castps_si128(_mm_cmpeq_ps(r0, v_zero_f))), v_zero));
         }
 
+        nz = i - (_mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum))) >> 2);
         src += i;
     }
 #elif CV_NEON
@@ -758,32 +718,7 @@ static int countNonZero32f( const float* src, int len )
 
 static int countNonZero64f( const double* src, int len )
 {
-    int i = 0, nz = 0;
-#if CV_SSE2
-    if (USE_SSE2)
-    {
-        __m128i v_zero_i = _mm_setzero_si128();
-        __m128d v_zero_d = _mm_setzero_pd();
-        static const uchar * tab = initPopcountTable();
-
-        for ( ; i <= len - 8; i += 8)
-        {
-            __m128i v_dst0 = _mm_castpd_si128(_mm_cmpeq_pd(_mm_loadu_pd(src + i), v_zero_d));
-            __m128i v_dst1 = _mm_castpd_si128(_mm_cmpeq_pd(_mm_loadu_pd(src + i + 2), v_zero_d));
-            __m128i v_dst2 = _mm_castpd_si128(_mm_cmpeq_pd(_mm_loadu_pd(src + i + 4), v_zero_d));
-            __m128i v_dst3 = _mm_castpd_si128(_mm_cmpeq_pd(_mm_loadu_pd(src + i + 6), v_zero_d));
-
-            v_dst0 = _mm_packs_epi32(v_dst0, v_dst1);
-            v_dst1 = _mm_packs_epi32(v_dst2, v_dst3);
-
-            int val = _mm_movemask_epi8(_mm_packs_epi16(_mm_packs_epi32(v_dst0, v_dst1), v_zero_i));
-            nz += tab[val];
-        }
-
-        src += i;
-    }
-#endif
-    return nz + countNonZero_(src, len - i);
+    return countNonZero_(src, len);
 }
 
 typedef int (*CountNonZeroFunc)(const uchar*, int);