From 20473344b223023e1adea692cb93ae4a4c96bbc8 Mon Sep 17 00:00:00 2001 From: mtklein Date: Mon, 25 Jan 2016 09:26:54 -0800 Subject: [PATCH] spin off some safe parts from AVX2 CL (reviewed here https://codereview.chromium.org/1532613002/) BUG=skia: GOLD_TRYBOT_URL= https://gold.skia.org/search2?unt=true&query=source_type%3Dgm&master=false&issue=1628333003 CQ_EXTRA_TRYBOTS=client.skia:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD-Trybot Review URL: https://codereview.chromium.org/1628333003 --- gyp/opts.gyp | 4 +- src/opts/SkOpts_sse41.cpp | 212 +++++++++++++++++++++++++++------------------- 2 files changed, 127 insertions(+), 89 deletions(-) diff --git a/gyp/opts.gyp b/gyp/opts.gyp index 396a37e..ae4b294 100644 --- a/gyp/opts.gyp +++ b/gyp/opts.gyp @@ -149,7 +149,7 @@ ], 'sources': [ '<@(avx_sources)' ], 'msvs_settings': { 'VCCLCompilerTool': { 'EnableEnhancedInstructionSet': '3' } }, - 'xcode_settings': { 'OTHER_CFLAGS': [ '-mavx' ] }, + 'xcode_settings': { 'OTHER_CPLUSPLUSFLAGS': [ '-mavx' ] }, 'conditions': [ [ 'not skia_android_framework', { 'cflags': [ '-mavx' ] }], ], @@ -167,7 +167,7 @@ ], 'sources': [ '<@(avx2_sources)' ], 'msvs_settings': { 'VCCLCompilerTool': { 'EnableEnhancedInstructionSet': '5' } }, - 'xcode_settings': { 'OTHER_CFLAGS': [ '-mavx2' ] }, + 'xcode_settings': { 'OTHER_CPLUSPLUSFLAGS': [ '-mavx2' ] }, 'conditions': [ [ 'not skia_android_framework', { 'cflags': [ '-mavx2' ] }], ], diff --git a/src/opts/SkOpts_sse41.cpp b/src/opts/SkOpts_sse41.cpp index 16ba87a..f097e56 100644 --- a/src/opts/SkOpts_sse41.cpp +++ b/src/opts/SkOpts_sse41.cpp @@ -12,88 +12,67 @@ #ifndef SK_SUPPORT_LEGACY_X86_BLITS -// This file deals mostly with unpacked 8-bit values, -// i.e. values between 0 and 255, but in 16-bit lanes with 0 at the top. - -// So __m128i typically represents 1 or 2 pixels, and m128ix2 represents 4. -struct m128ix2 { __m128i lo, hi; }; - -// unpack{lo,hi}() get our raw pixels unpacked, from half of 4 packed pixels to 2 unpacked pixels. -static inline __m128i unpacklo(__m128i x) { return _mm_cvtepu8_epi16(x); } -static inline __m128i unpackhi(__m128i x) { return _mm_unpackhi_epi8(x, _mm_setzero_si128()); } - -// pack() converts back, from 4 unpacked pixels to 4 packed pixels. -static inline __m128i pack(__m128i lo, __m128i hi) { return _mm_packus_epi16(lo, hi); } - -// These nextN() functions abstract over the difference between iterating over -// an array of values and returning a constant value, for uint8_t and uint32_t. -// The nextN() taking pointers increment that pointer past where they read. -// -// nextN() returns N unpacked pixels or 4N unpacked coverage values. - -static inline __m128i next1(uint8_t val) { return _mm_set1_epi16(val); } -static inline __m128i next2(uint8_t val) { return _mm_set1_epi16(val); } -static inline m128ix2 next4(uint8_t val) { return { next2(val), next2(val) }; } - -static inline __m128i next1(uint32_t val) { return unpacklo(_mm_cvtsi32_si128(val)); } -static inline __m128i next2(uint32_t val) { return unpacklo(_mm_set1_epi32(val)); } -static inline m128ix2 next4(uint32_t val) { return { next2(val), next2(val) }; } +namespace sk_sse41 { -static inline __m128i next1(const uint8_t*& ptr) { return _mm_set1_epi16(*ptr++); } -static inline __m128i next2(const uint8_t*& ptr) { - auto r = _mm_cvtsi32_si128(*(const uint16_t*)ptr); - ptr += 2; - const int _ = ~0; - return _mm_shuffle_epi8(r, _mm_setr_epi8(0,_,0,_,0,_,0,_, 1,_,1,_,1,_,1,_)); -} -static inline m128ix2 next4(const uint8_t*& ptr) { - auto r = _mm_cvtsi32_si128(*(const uint32_t*)ptr); +// An SSE register holding at most 64 bits of useful data in the low lanes. +struct m64i { + __m128i v; + /*implicit*/ m64i(__m128i v) : v(v) {} + operator __m128i() const { return v; } +}; + +// Load 4, 2, or 1 constant pixels or coverages (4x replicated). +static __m128i next4(uint32_t val) { return _mm_set1_epi32(val); } +static m64i next2(uint32_t val) { return _mm_set1_epi32(val); } +static m64i next1(uint32_t val) { return _mm_set1_epi32(val); } + +static __m128i next4(uint8_t val) { return _mm_set1_epi8(val); } +static m64i next2(uint8_t val) { return _mm_set1_epi8(val); } +static m64i next1(uint8_t val) { return _mm_set1_epi8(val); } + +// Load 4, 2, or 1 variable pixels or coverages (4x replicated), +// incrementing the pointer past what we read. +static __m128i next4(const uint32_t*& ptr) { + auto r = _mm_loadu_si128((const __m128i*)ptr); ptr += 4; - const int _ = ~0; - auto lo = _mm_shuffle_epi8(r, _mm_setr_epi8(0,_,0,_,0,_,0,_, 1,_,1,_,1,_,1,_)), - hi = _mm_shuffle_epi8(r, _mm_setr_epi8(2,_,2,_,2,_,2,_, 3,_,3,_,3,_,3,_)); - return { lo, hi }; + return r; } - -static inline __m128i next1(const uint32_t*& ptr) { return unpacklo(_mm_cvtsi32_si128(*ptr++)); } -static inline __m128i next2(const uint32_t*& ptr) { - auto r = unpacklo(_mm_loadl_epi64((const __m128i*)ptr)); +static m64i next2(const uint32_t*& ptr) { + auto r = _mm_loadl_epi64((const __m128i*)ptr); ptr += 2; return r; } -static inline m128ix2 next4(const uint32_t*& ptr) { - auto packed = _mm_loadu_si128((const __m128i*)ptr); - ptr += 4; - return { unpacklo(packed), unpackhi(packed) }; +static m64i next1(const uint32_t*& ptr) { + auto r = _mm_cvtsi32_si128(*ptr); + ptr += 1; + return r; } -// Divide by 255 with rounding. -// (x+127)/255 == ((x+128)*257)>>16. -// Sometimes we can be more efficient by breaking this into two parts. -static inline __m128i div255_part1(__m128i x) { return _mm_add_epi16(x, _mm_set1_epi16(128)); } -static inline __m128i div255_part2(__m128i x) { return _mm_mulhi_epu16(x, _mm_set1_epi16(257)); } -static inline __m128i div255(__m128i x) { return div255_part2(div255_part1(x)); } - -// (x*y+127)/255, a byte multiply. -static inline __m128i scale(__m128i x, __m128i y) { - return div255(_mm_mullo_epi16(x, y)); +// xyzw -> xxxx yyyy zzzz wwww +static __m128i replicate_coverage(__m128i xyzw) { + const uint8_t mask[] = { 0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3 }; + return _mm_shuffle_epi8(xyzw, _mm_load_si128((const __m128i*)mask)); } -// (255 - x). -static inline __m128i inv(__m128i x) { - return _mm_xor_si128(_mm_set1_epi16(0x00ff), x); // This seems a bit faster than _mm_sub_epi16. +static __m128i next4(const uint8_t*& ptr) { + auto r = replicate_coverage(_mm_cvtsi32_si128(*(const uint32_t*)ptr)); + ptr += 4; + return r; } - -// ARGB argb -> AAAA aaaa -static inline __m128i alphas(__m128i px) { - const int a = 2 * (SK_A32_SHIFT/8); // SK_A32_SHIFT is typically 24, so this is typically 6. - const int _ = ~0; - return _mm_shuffle_epi8(px, _mm_setr_epi8(a+0,_,a+0,_,a+0,_,a+0,_, a+8,_,a+8,_,a+8,_,a+8,_)); +static m64i next2(const uint8_t*& ptr) { + auto r = replicate_coverage(_mm_cvtsi32_si128(*(const uint16_t*)ptr)); + ptr += 2; + return r; +} +static m64i next1(const uint8_t*& ptr) { + auto r = replicate_coverage(_mm_cvtsi32_si128(*ptr)); + ptr += 1; + return r; } // For i = 0...n, tgt = fn(dst,src,cov), where Dst,Src,and Cov can be constants or arrays. template -static inline void loop(int n, uint32_t* t, const Dst dst, const Src src, const Cov cov, Fn&& fn) { +static void loop(int n, uint32_t* t, const Dst dst, const Src src, const Cov cov, Fn&& fn) { // We don't want to muck with the callers' pointers, so we make them const and copy here. Dst d = dst; Src s = src; @@ -102,30 +81,85 @@ static inline void loop(int n, uint32_t* t, const Dst dst, const Src src, const // Writing this as a single while-loop helps hoist loop invariants from fn. while (n) { if (n >= 4) { - auto d4 = next4(d), - s4 = next4(s), - c4 = next4(c); - auto lo = fn(d4.lo, s4.lo, c4.lo), - hi = fn(d4.hi, s4.hi, c4.hi); - _mm_storeu_si128((__m128i*)t, pack(lo,hi)); + _mm_storeu_si128((__m128i*)t, fn(next4(d), next4(s), next4(c))); t += 4; n -= 4; continue; } if (n & 2) { - auto r = fn(next2(d), next2(s), next2(c)); - _mm_storel_epi64((__m128i*)t, pack(r,r)); + _mm_storel_epi64((__m128i*)t, fn(next2(d), next2(s), next2(c))); t += 2; } if (n & 1) { - auto r = fn(next1(d), next1(s), next1(c)); - *t = _mm_cvtsi128_si32(pack(r,r)); + *t = _mm_cvtsi128_si32(fn(next1(d), next1(s), next1(c))); } return; } } -namespace sk_sse41 { +// packed +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // +// unpacked + +// Everything on the packed side of the squiggly line deals with densely packed 8-bit data, +// e.g. [BGRA bgra ... ] for pixels or [ CCCC cccc ... ] for coverage. +// +// Everything on the unpacked side of the squiggly line deals with unpacked 8-bit data, +// e.g [B_G_ R_A_ b_g_ r_a_ ] for pixels or [ C_C_ C_C_ c_c_ c_c_ c_c_ ] for coverage, +// where _ is a zero byte. +// +// Adapt / adapt(fn) allow the two sides to interoperate, +// by unpacking arguments, calling fn, then packing the results. +// +// This lets us write most of our code in terms of unpacked inputs (considerably simpler) +// and all the packing and unpacking is handled automatically. + +template +struct Adapt { + Fn fn; + + __m128i operator()(__m128i d, __m128i s, __m128i c) { + auto lo = [](__m128i x) { return _mm_unpacklo_epi8(x, _mm_setzero_si128()); }; + auto hi = [](__m128i x) { return _mm_unpackhi_epi8(x, _mm_setzero_si128()); }; + return _mm_packus_epi16(fn(lo(d), lo(s), lo(c)), + fn(hi(d), hi(s), hi(c))); + } + + m64i operator()(const m64i& d, const m64i& s, const m64i& c) { + auto lo = [](__m128i x) { return _mm_unpacklo_epi8(x, _mm_setzero_si128()); }; + auto r = fn(lo(d), lo(s), lo(c)); + return _mm_packus_epi16(r, r); + } +}; + +template +static Adapt adapt(Fn&& fn) { return { fn }; } + +// These helpers all work exclusively with unpacked 8-bit values, +// except div255() with is 16-bit -> unpacked 8-bit, and mul255() which is the reverse. + +// Divide by 255 with rounding. +// (x+127)/255 == ((x+128)*257)>>16. +// Sometimes we can be more efficient by breaking this into two parts. +static __m128i div255_part1(__m128i x) { return _mm_add_epi16(x, _mm_set1_epi16(128)); } +static __m128i div255_part2(__m128i x) { return _mm_mulhi_epu16(x, _mm_set1_epi16(257)); } +static __m128i div255(__m128i x) { return div255_part2(div255_part1(x)); } + +// (x*y+127)/255, a byte multiply. +static __m128i scale(__m128i x, __m128i y) { return div255(_mm_mullo_epi16(x, y)); } + +// (255 * x). +static __m128i mul255(__m128i x) { return _mm_sub_epi16(_mm_slli_epi16(x, 8), x); } + +// (255 - x). +static __m128i inv(__m128i x) { return _mm_xor_si128(_mm_set1_epi16(0x00ff), x); } + +// ARGB argb -> AAAA aaaa +static __m128i alphas(__m128i px) { + const int a = 2 * (SK_A32_SHIFT/8); // SK_A32_SHIFT is typically 24, so this is typically 6. + const int _ = ~0; + return _mm_shuffle_epi8(px, _mm_setr_epi8(a+0,_,a+0,_,a+0,_,a+0,_, a+8,_,a+8,_,a+8,_,a+8,_)); +} // SrcOver, with a constant source and full coverage. static void blit_row_color32(SkPMColor* tgt, const SkPMColor* dst, int n, SkPMColor src) { @@ -134,14 +168,14 @@ static void blit_row_color32(SkPMColor* tgt, const SkPMColor* dst, int n, SkPMCo // But we can go one step further to ((s*255 + 128 + d*inv(alphas(s)))*257)>>16. // This lets us hoist (s*255+128) and inv(alphas(s)) out of the loop. - __m128i s = next2(src), - s_255_128 = div255_part1(_mm_mullo_epi16(s, _mm_set1_epi16(255))), + __m128i s = _mm_unpacklo_epi8(_mm_set1_epi32(src), _mm_setzero_si128()), + s_255_128 = div255_part1(mul255(s)), A = inv(alphas(s)); const uint8_t cov = 0xff; - loop(n, tgt, dst, src, cov, [=](__m128i d, __m128i, __m128i) { + loop(n, tgt, dst, src, cov, adapt([=](__m128i d, __m128i, __m128i) { return div255_part2(_mm_add_epi16(s_255_128, _mm_mullo_epi16(d, A))); - }); + })); } // SrcOver, with a constant source and variable coverage. @@ -152,23 +186,26 @@ static void blit_mask_d32_a8(SkPMColor* dst, size_t dstRB, if (SkColorGetA(color) == 0xFF) { const SkPMColor src = SkSwizzle_BGRA_to_PMColor(color); while (h --> 0) { - loop(w, dst, (const SkPMColor*)dst, src, cov, [](__m128i d, __m128i s, __m128i c) { + loop(w, dst, (const SkPMColor*)dst, src, cov, + adapt([](__m128i d, __m128i s, __m128i c) { // Src blend mode: a simple lerp from d to s by c. // TODO: try a pmaddubsw version? - return div255(_mm_add_epi16(_mm_mullo_epi16(inv(c),d), _mm_mullo_epi16(c,s))); - }); + return div255(_mm_add_epi16(_mm_mullo_epi16(inv(c),d), + _mm_mullo_epi16( c ,s))); + })); dst += dstRB / sizeof(*dst); cov += covRB / sizeof(*cov); } } else { const SkPMColor src = SkPreMultiplyColor(color); while (h --> 0) { - loop(w, dst, (const SkPMColor*)dst, src, cov, [](__m128i d, __m128i s, __m128i c) { + loop(w, dst, (const SkPMColor*)dst, src, cov, + adapt([](__m128i d, __m128i s, __m128i c) { // SrcOver blend mode, with coverage folded into source alpha. __m128i sc = scale(s,c), AC = inv(alphas(sc)); return _mm_add_epi16(sc, scale(d,AC)); - }); + })); dst += dstRB / sizeof(*dst); cov += covRB / sizeof(*cov); } @@ -176,6 +213,7 @@ static void blit_mask_d32_a8(SkPMColor* dst, size_t dstRB, } } // namespace sk_sse41 + #endif namespace SkOpts { -- 2.7.4