2 * Copyright 2016 Google Inc.
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
8 #ifndef SkRasterPipeline_opts_DEFINED
9 #define SkRasterPipeline_opts_DEFINED
11 #include "SkColorPriv.h"
12 #include "SkColorLookUpTable.h"
13 #include "SkColorSpaceXform_A2B.h"
14 #include "SkColorSpaceXformPriv.h"
18 #include "SkPM4fPriv.h"
19 #include "SkRasterPipeline.h"
22 #include "../jumper/SkJumper.h"
26 static constexpr int N = 4;
28 using SkNf = SkNx<N, float>;
29 using SkNi = SkNx<N, int32_t>;
30 using SkNu = SkNx<N, uint32_t>;
31 using SkNh = SkNx<N, uint16_t>;
32 using SkNb = SkNx<N, uint8_t>;
34 using Fn = void(SK_VECTORCALL *)(size_t x_tail, void** p, SkNf,SkNf,SkNf,SkNf,
36 // x_tail encodes two values x and tail as x*N+tail, where 0 <= tail < N.
37 // x is the induction variable we're walking along, incrementing by N each step.
38 // tail == 0 means work with a full N pixels; otherwise use only the low tail pixels.
40 // p is our program, a sequence of Fn to call interlaced with any void* context pointers. E.g.
55 #define SI static inline
57 // Basically, return *(*ptr)++, maybe faster than the compiler can do it.
58 SI void* load_and_increment(void*** ptr) {
59 // We do this often enough that it's worth hyper-optimizing.
60 // x86 can do this in one instruction if ptr is in rsi.
61 // (This is why p is the second argument to Fn: it's passed in rsi.)
62 #if defined(__GNUC__) && defined(__x86_64__)
64 __asm__("lodsq" : "=a"(rax), "+S"(*ptr));
71 // Stages are logically a pipeline, and physically are contiguous in an array.
72 // To get to the next stage, we just increment our pointer to the next array element.
73 SI void SK_VECTORCALL next(size_t x_tail, void** p, SkNf r, SkNf g, SkNf b, SkNf a,
74 SkNf dr, SkNf dg, SkNf db, SkNf da) {
75 auto next = (Fn)load_and_increment(&p);
76 next(x_tail,p, r,g,b,a, dr,dg,db,da);
79 // Stages defined below always call next.
80 // This is always the last stage, a backstop that actually returns to the caller when done.
81 SI void SK_VECTORCALL just_return(size_t, void**, SkNf, SkNf, SkNf, SkNf,
82 SkNf, SkNf, SkNf, SkNf) {}
85 static SK_ALWAYS_INLINE void name##_kernel(size_t x, size_t tail, \
86 SkNf& r, SkNf& g, SkNf& b, SkNf& a, \
87 SkNf& dr, SkNf& dg, SkNf& db, SkNf& da); \
88 SI void SK_VECTORCALL name(size_t x_tail, void** p, \
89 SkNf r, SkNf g, SkNf b, SkNf a, \
90 SkNf dr, SkNf dg, SkNf db, SkNf da) { \
91 name##_kernel(x_tail/N, x_tail%N, r,g,b,a, dr,dg,db,da); \
92 next(x_tail,p, r,g,b,a, dr,dg,db,da); \
94 static SK_ALWAYS_INLINE void name##_kernel(size_t x, size_t tail, \
95 SkNf& r, SkNf& g, SkNf& b, SkNf& a, \
96 SkNf& dr, SkNf& dg, SkNf& db, SkNf& da)
98 #define STAGE_CTX(name, Ctx) \
99 static SK_ALWAYS_INLINE void name##_kernel(Ctx ctx, size_t x, size_t tail, \
100 SkNf& r, SkNf& g, SkNf& b, SkNf& a, \
101 SkNf& dr, SkNf& dg, SkNf& db, SkNf& da); \
102 SI void SK_VECTORCALL name(size_t x_tail, void** p, \
103 SkNf r, SkNf g, SkNf b, SkNf a, \
104 SkNf dr, SkNf dg, SkNf db, SkNf da) { \
105 auto ctx = (Ctx)load_and_increment(&p); \
106 name##_kernel(ctx, x_tail/N, x_tail%N, r,g,b,a, dr,dg,db,da); \
107 next(x_tail,p, r,g,b,a, dr,dg,db,da); \
109 static SK_ALWAYS_INLINE void name##_kernel(Ctx ctx, size_t x, size_t tail, \
110 SkNf& r, SkNf& g, SkNf& b, SkNf& a, \
111 SkNf& dr, SkNf& dg, SkNf& db, SkNf& da)
113 // Many xfermodes apply the same logic to each channel.
114 #define RGBA_XFERMODE(name) \
115 static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \
116 const SkNf& d, const SkNf& da); \
117 SI void SK_VECTORCALL name(size_t x_tail, void** p, \
118 SkNf r, SkNf g, SkNf b, SkNf a, \
119 SkNf dr, SkNf dg, SkNf db, SkNf da) { \
120 r = name##_kernel(r,a,dr,da); \
121 g = name##_kernel(g,a,dg,da); \
122 b = name##_kernel(b,a,db,da); \
123 a = name##_kernel(a,a,da,da); \
124 next(x_tail,p, r,g,b,a, dr,dg,db,da); \
126 static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \
127 const SkNf& d, const SkNf& da)
129 // Most of the rest apply the same logic to color channels and use srcover's alpha logic.
130 #define RGB_XFERMODE(name) \
131 static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \
132 const SkNf& d, const SkNf& da); \
133 SI void SK_VECTORCALL name(size_t x_tail, void** p, \
134 SkNf r, SkNf g, SkNf b, SkNf a, \
135 SkNf dr, SkNf dg, SkNf db, SkNf da) { \
136 r = name##_kernel(r,a,dr,da); \
137 g = name##_kernel(g,a,dg,da); \
138 b = name##_kernel(b,a,db,da); \
139 a = a + (da * (1.0f-a)); \
140 next(x_tail,p, r,g,b,a, dr,dg,db,da); \
142 static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \
143 const SkNf& d, const SkNf& da)
145 template <typename T>
146 SI SkNx<N,T> load(size_t tail, const T* src) {
149 memset(buf, 0, 8*sizeof(T));
150 switch (tail & (N-1)) {
151 case 7: buf[6] = src[6];
152 case 6: buf[5] = src[5];
153 case 5: buf[4] = src[4];
154 case 4: buf[3] = src[3];
155 case 3: buf[2] = src[2];
156 case 2: buf[1] = src[1];
159 return SkNx<N,T>::Load(buf);
161 return SkNx<N,T>::Load(src);
163 template <typename T>
164 SI SkNx<N,T> gather(size_t tail, const T* src, const SkNi& offset) {
167 switch (tail & (N-1)) {
168 case 7: buf[6] = src[offset[6]];
169 case 6: buf[5] = src[offset[5]];
170 case 5: buf[4] = src[offset[4]];
171 case 4: buf[3] = src[offset[3]];
172 case 3: buf[2] = src[offset[2]];
173 case 2: buf[1] = src[offset[1]];
175 buf[0] = src[offset[0]];
176 return SkNx<N,T>::Load(buf);
179 for (size_t i = 0; i < N; i++) {
180 buf[i] = src[offset[i]];
182 return SkNx<N,T>::Load(buf);
184 template <typename T>
185 SI void store(size_t tail, const SkNx<N,T>& v, T* dst) {
187 switch (tail & (N-1)) {
188 case 7: dst[6] = v[6];
189 case 6: dst[5] = v[5];
190 case 5: dst[4] = v[4];
191 case 4: dst[3] = v[3];
192 case 3: dst[2] = v[2];
193 case 2: dst[1] = v[1];
201 SI SkNf SkNf_fma(const SkNf& f, const SkNf& m, const SkNf& a) { return SkNx_fma(f,m,a); }
203 SI SkNi SkNf_round(const SkNf& x, const SkNf& scale) {
204 // Every time I try, _mm_cvtps_epi32 benches as slower than using FMA and _mm_cvttps_epi32. :/
205 return SkNx_cast<int>(SkNf_fma(x,scale, 0.5f));
208 SI SkNf SkNf_from_byte(const SkNi& x) {
209 // Same trick as in store_8888: 0x470000BB == 32768.0f + BB/256.0f for all bytes BB.
210 auto v = 0x47000000 | x;
211 // Read this as (pun_float(v) - 32768.0f) * (256/255.0f), redistributed to be an FMA.
212 return SkNf_fma(SkNf::Load(&v), 256/255.0f, -32768*256/255.0f);
214 SI SkNf SkNf_from_byte(const SkNu& x) { return SkNf_from_byte(SkNi::Load(&x)); }
215 SI SkNf SkNf_from_byte(const SkNb& x) { return SkNf_from_byte(SkNx_cast<int>(x)); }
217 SI void from_8888(const SkNu& _8888, SkNf* r, SkNf* g, SkNf* b, SkNf* a) {
218 *r = SkNf_from_byte((_8888 ) & 0xff);
219 *g = SkNf_from_byte((_8888 >> 8) & 0xff);
220 *b = SkNf_from_byte((_8888 >> 16) & 0xff);
221 *a = SkNf_from_byte((_8888 >> 24) );
223 SI void from_4444(const SkNh& _4444, SkNf* r, SkNf* g, SkNf* b, SkNf* a) {
224 auto _32_bit = SkNx_cast<int>(_4444);
226 *r = SkNx_cast<float>(_32_bit & (0xF << SK_R4444_SHIFT)) * (1.0f / (0xF << SK_R4444_SHIFT));
227 *g = SkNx_cast<float>(_32_bit & (0xF << SK_G4444_SHIFT)) * (1.0f / (0xF << SK_G4444_SHIFT));
228 *b = SkNx_cast<float>(_32_bit & (0xF << SK_B4444_SHIFT)) * (1.0f / (0xF << SK_B4444_SHIFT));
229 *a = SkNx_cast<float>(_32_bit & (0xF << SK_A4444_SHIFT)) * (1.0f / (0xF << SK_A4444_SHIFT));
231 SI void from_565(const SkNh& _565, SkNf* r, SkNf* g, SkNf* b) {
232 auto _32_bit = SkNx_cast<int>(_565);
234 *r = SkNx_cast<float>(_32_bit & SK_R16_MASK_IN_PLACE) * (1.0f / SK_R16_MASK_IN_PLACE);
235 *g = SkNx_cast<float>(_32_bit & SK_G16_MASK_IN_PLACE) * (1.0f / SK_G16_MASK_IN_PLACE);
236 *b = SkNx_cast<float>(_32_bit & SK_B16_MASK_IN_PLACE) * (1.0f / SK_B16_MASK_IN_PLACE);
238 SI void from_f16(const void* px, SkNf* r, SkNf* g, SkNf* b, SkNf* a) {
240 SkNh::Load4(px, &rh, &gh, &bh, &ah);
242 *r = SkHalfToFloat_finite_ftz(rh);
243 *g = SkHalfToFloat_finite_ftz(gh);
244 *b = SkHalfToFloat_finite_ftz(bh);
245 *a = SkHalfToFloat_finite_ftz(ah);
249 a = SkNf::Max(a, 0.0f);
250 r = SkNf::Max(r, 0.0f);
251 g = SkNf::Max(g, 0.0f);
252 b = SkNf::Max(b, 0.0f);
255 a = SkNf::Min(a, 1.0f);
256 r = SkNf::Min(r, 1.0f);
257 g = SkNf::Min(g, 1.0f);
258 b = SkNf::Min(b, 1.0f);
261 a = SkNf::Min(a, 1.0f);
268 auto scale = (a == 0.0f).thenElse(0.0f, 1.0f/a);
279 STAGE_CTX(set_rgb, const float*) {
284 STAGE(swap_rb) { SkTSwap(r,b); }
286 STAGE(move_src_dst) {
292 STAGE(move_dst_src) {
306 r = sk_linear_from_srgb_math(r);
307 g = sk_linear_from_srgb_math(g);
308 b = sk_linear_from_srgb_math(b);
311 r = sk_linear_to_srgb_needs_round(r);
312 g = sk_linear_to_srgb_needs_round(g);
313 b = sk_linear_to_srgb_needs_round(b);
317 auto from_2dot2 = [](const SkNf& x) {
318 // x^(141/64) = x^(2.20312) is a great approximation of the true value, x^(2.2).
319 // (note: x^(35/16) = x^(2.1875) is an okay one as well and would be quicker)
320 auto x16 = x.rsqrt().rsqrt().rsqrt().rsqrt(); // x^(1/16) = x^(4/64);
321 auto x64 = x16.rsqrt().rsqrt(); // x^(1/64)
323 // x^(141/64) = x^(128/64) * x^(12/64) * x^(1/64)
324 return SkNf::Max((x*x) * (x16*x16*x16) * (x64), 0.0f);
332 auto to_2dot2 = [](const SkNf& x) {
333 // x^(29/64) is a very good approximation of the true value, x^(1/2.2).
334 auto x2 = x.rsqrt(), // x^(-1/2)
335 x32 = x2.rsqrt().rsqrt().rsqrt().rsqrt(), // x^(-1/32)
336 x64 = x32.rsqrt(); // x^(+1/64)
339 return SkNf::Max(x2.invert() * x32 * x64.invert(), 0.0f); // Watch out for NaN.
347 // The default shader produces a constant color (from the SkPaint).
348 STAGE_CTX(constant_color, const SkPM4f*) {
355 // Set up registers with values relevant to shaders.
356 STAGE_CTX(seed_shader, const int*) {
359 static const float dx[] = { 0,1,2,3,4,5,6,7 };
360 r = x + 0.5f + SkNf::Load(dx); // dst pixel center x coordinates
361 g = y + 0.5f; // dst pixel center y coordinate(s)
364 dr = dg = db = da = 0.0f;
367 // s' = sc for a scalar c.
368 STAGE_CTX(scale_1_float, const float*) {
376 // s' = sc for 8-bit c.
377 STAGE_CTX(scale_u8, const uint8_t**) {
379 SkNf c = SkNf_from_byte(load(tail, ptr));
387 SI SkNf lerp(const SkNf& from, const SkNf& to, const SkNf& cov) {
388 return SkNf_fma(to-from, cov, from);
391 // s' = d(1-c) + sc, for a scalar c.
392 STAGE_CTX(lerp_1_float, const float*) {
401 // s' = d(1-c) + sc for 8-bit c.
402 STAGE_CTX(lerp_u8, const uint8_t**) {
404 SkNf c = SkNf_from_byte(load(tail, ptr));
412 // s' = d(1-c) + sc for 565 c.
413 STAGE_CTX(lerp_565, const uint16_t**) {
416 from_565(load(tail, ptr), &cr, &cg, &cb);
424 STAGE_CTX(load_a8, const uint8_t**) {
427 a = SkNf_from_byte(load(tail, ptr));
429 STAGE_CTX(store_a8, uint8_t**) {
431 store(tail, SkNx_cast<uint8_t>(SkNf_round(255.0f, a)), ptr);
434 STAGE_CTX(load_g8, const uint8_t**) {
436 r = g = b = SkNf_from_byte(load(tail, ptr));
440 STAGE_CTX(load_565, const uint16_t**) {
442 from_565(load(tail, ptr), &r,&g,&b);
445 STAGE_CTX(store_565, uint16_t**) {
447 store(tail, SkNx_cast<uint16_t>( SkNf_round(r, SK_R16_MASK) << SK_R16_SHIFT
448 | SkNf_round(g, SK_G16_MASK) << SK_G16_SHIFT
449 | SkNf_round(b, SK_B16_MASK) << SK_B16_SHIFT), ptr);
452 STAGE_CTX(load_4444, const uint16_t**) {
454 from_4444(load(tail, ptr), &r,&g,&b,&a);
456 STAGE_CTX(store_4444, uint16_t**) {
458 store(tail, SkNx_cast<uint16_t>( SkNf_round(r, 0xF) << SK_R4444_SHIFT
459 | SkNf_round(g, 0xF) << SK_G4444_SHIFT
460 | SkNf_round(b, 0xF) << SK_B4444_SHIFT
461 | SkNf_round(a, 0xF) << SK_A4444_SHIFT), ptr);
464 STAGE_CTX(load_f16, const uint64_t**) {
467 const void* src = ptr;
468 SkNx<N, uint64_t> px;
470 px = load(tail, ptr);
473 from_f16(src, &r, &g, &b, &a);
475 STAGE_CTX(store_f16, uint64_t**) {
478 SkNx<N, uint64_t> px;
479 SkNh::Store4(tail ? (void*)&px : (void*)ptr, SkFloatToHalf_finite_ftz(r),
480 SkFloatToHalf_finite_ftz(g),
481 SkFloatToHalf_finite_ftz(b),
482 SkFloatToHalf_finite_ftz(a));
484 store(tail, px, ptr);
488 STAGE_CTX(load_f32, const SkPM4f**) {
491 const void* src = ptr;
494 px = load(tail, ptr);
497 SkNf::Load4(src, &r, &g, &b, &a);
499 STAGE_CTX(store_f32, SkPM4f**) {
503 SkNf::Store4(tail ? (void*)&px : (void*)ptr, r,g,b,a);
505 store(tail, px, ptr);
510 STAGE_CTX(load_8888, const uint32_t**) {
512 from_8888(load(tail, ptr), &r, &g, &b, &a);
514 STAGE_CTX(store_8888, uint32_t**) {
515 auto byte = [](const SkNf& x, int ix) {
516 // Here's a neat trick: 0x47000000 == 32768.0f, and 0x470000ff == 32768.0f + (255/256.0f).
517 auto v = SkNf_fma(255/256.0f, x, 32768.0f);
519 case 0: return SkNi::Load(&v) & 0xff; // R
520 case 3: return SkNi::Load(&v) << 24; // A
522 return (SkNi::Load(&v) & 0xff) << (8*ix); // B or G
526 store(tail, byte(r,0)|byte(g,1)|byte(b,2)|byte(a,3), (int*)ptr);
529 STAGE_CTX(load_u16_be, const uint64_t**) {
531 const void* src = ptr;
532 SkNx<N, uint64_t> px;
534 px = load(tail, ptr);
539 SkNh::Load4(src, &rh, &gh, &bh, &ah);
540 r = (1.0f / 65535.0f) * SkNx_cast<float>((rh << 8) | (rh >> 8));
541 g = (1.0f / 65535.0f) * SkNx_cast<float>((gh << 8) | (gh >> 8));
542 b = (1.0f / 65535.0f) * SkNx_cast<float>((bh << 8) | (bh >> 8));
543 a = (1.0f / 65535.0f) * SkNx_cast<float>((ah << 8) | (ah >> 8));
546 STAGE_CTX(load_rgb_u16_be, const uint16_t**) {
547 auto ptr = *ctx + 3*x;
548 const void* src = ptr;
549 uint16_t buf[N*3] = {0};
551 memcpy(buf, src, tail*3*sizeof(uint16_t));
556 SkNh::Load3(src, &rh, &gh, &bh);
557 r = (1.0f / 65535.0f) * SkNx_cast<float>((rh << 8) | (rh >> 8));
558 g = (1.0f / 65535.0f) * SkNx_cast<float>((gh << 8) | (gh >> 8));
559 b = (1.0f / 65535.0f) * SkNx_cast<float>((bh << 8) | (bh >> 8));
563 STAGE_CTX(store_u16_be, uint64_t**) {
564 auto to_u16_be = [](const SkNf& x) {
565 SkNh x16 = SkNx_cast<uint16_t>(65535.0f * x);
566 return (x16 << 8) | (x16 >> 8);
570 SkNx<N, uint64_t> px;
571 SkNh::Store4(tail ? (void*)&px : (void*)ptr, to_u16_be(r),
576 store(tail, px, ptr);
580 STAGE_CTX(load_tables, const LoadTablesContext*) {
581 auto ptr = (const uint32_t*)ctx->fSrc + x;
583 SkNu rgba = load(tail, ptr);
584 auto to_int = [](const SkNu& v) { return SkNi::Load(&v); };
585 r = gather(tail, ctx->fR, to_int((rgba >> 0) & 0xff));
586 g = gather(tail, ctx->fG, to_int((rgba >> 8) & 0xff));
587 b = gather(tail, ctx->fB, to_int((rgba >> 16) & 0xff));
588 a = SkNf_from_byte(rgba >> 24);
591 STAGE_CTX(load_tables_u16_be, const LoadTablesContext*) {
592 auto ptr = (const uint64_t*)ctx->fSrc + x;
593 const void* src = ptr;
594 SkNx<N, uint64_t> px;
596 px = load(tail, ptr);
601 SkNh::Load4(src, &rh, &gh, &bh, &ah);
603 // ctx->fSrc is big-endian, so "& 0xff" grabs the 8 most significant bits of each component.
604 r = gather(tail, ctx->fR, SkNx_cast<int>(rh & 0xff));
605 g = gather(tail, ctx->fG, SkNx_cast<int>(gh & 0xff));
606 b = gather(tail, ctx->fB, SkNx_cast<int>(bh & 0xff));
607 a = (1.0f / 65535.0f) * SkNx_cast<float>((ah << 8) | (ah >> 8));
610 STAGE_CTX(load_tables_rgb_u16_be, const LoadTablesContext*) {
611 auto ptr = (const uint16_t*)ctx->fSrc + 3*x;
612 const void* src = ptr;
613 uint16_t buf[N*3] = {0};
615 memcpy(buf, src, tail*3*sizeof(uint16_t));
620 SkNh::Load3(src, &rh, &gh, &bh);
622 // ctx->fSrc is big-endian, so "& 0xff" grabs the 8 most significant bits of each component.
623 r = gather(tail, ctx->fR, SkNx_cast<int>(rh & 0xff));
624 g = gather(tail, ctx->fG, SkNx_cast<int>(gh & 0xff));
625 b = gather(tail, ctx->fB, SkNx_cast<int>(bh & 0xff));
629 SI SkNf inv(const SkNf& x) { return 1.0f - x; }
631 RGBA_XFERMODE(clear) { return 0.0f; }
632 RGBA_XFERMODE(srcatop) { return s*da + d*inv(sa); }
633 RGBA_XFERMODE(srcin) { return s * da; }
634 RGBA_XFERMODE(srcout) { return s * inv(da); }
635 RGBA_XFERMODE(srcover) { return SkNf_fma(d, inv(sa), s); }
636 RGBA_XFERMODE(dstatop) { return srcatop_kernel(d,da,s,sa); }
637 RGBA_XFERMODE(dstin) { return srcin_kernel (d,da,s,sa); }
638 RGBA_XFERMODE(dstout) { return srcout_kernel (d,da,s,sa); }
639 RGBA_XFERMODE(dstover) { return srcover_kernel(d,da,s,sa); }
641 RGBA_XFERMODE(modulate) { return s*d; }
642 RGBA_XFERMODE(multiply) { return s*inv(da) + d*inv(sa) + s*d; }
643 RGBA_XFERMODE(plus_) { return s + d; }
644 RGBA_XFERMODE(screen) { return s + d - s*d; }
645 RGBA_XFERMODE(xor_) { return s*inv(da) + d*inv(sa); }
647 RGB_XFERMODE(colorburn) {
648 return (d == da ).thenElse(d + s*inv(da),
649 (s == 0.0f).thenElse(s + d*inv(sa),
650 sa*(da - SkNf::Min(da, (da-d)*sa/s)) + s*inv(da) + d*inv(sa)));
652 RGB_XFERMODE(colordodge) {
653 return (d == 0.0f).thenElse(d + s*inv(da),
654 (s == sa ).thenElse(s + d*inv(sa),
655 sa*SkNf::Min(da, (d*sa)/(sa - s)) + s*inv(da) + d*inv(sa)));
657 RGB_XFERMODE(darken) { return s + d - SkNf::Max(s*da, d*sa); }
658 RGB_XFERMODE(difference) { return s + d - 2.0f*SkNf::Min(s*da,d*sa); }
659 RGB_XFERMODE(exclusion) { return s + d - 2.0f*s*d; }
660 RGB_XFERMODE(hardlight) {
661 return s*inv(da) + d*inv(sa)
662 + (2.0f*s <= sa).thenElse(2.0f*s*d, sa*da - 2.0f*(da-d)*(sa-s));
664 RGB_XFERMODE(lighten) { return s + d - SkNf::Min(s*da, d*sa); }
665 RGB_XFERMODE(overlay) { return hardlight_kernel(d,da,s,sa); }
666 RGB_XFERMODE(softlight) {
667 SkNf m = (da > 0.0f).thenElse(d / da, 0.0f),
671 // The logic forks three ways:
673 // 2. light src, dark dst?
674 // 3. light src, light dst?
675 SkNf darkSrc = d*(sa + (s2 - sa)*(1.0f - m)), // Used in case 1.
676 darkDst = (m4*m4 + m4)*(m - 1.0f) + 7.0f*m, // Used in case 2.
677 liteDst = m.rsqrt().invert() - m, // Used in case 3.
678 liteSrc = d*sa + da*(s2 - sa) * (4.0f*d <= da).thenElse(darkDst, liteDst); // 2 or 3?
679 return s*inv(da) + d*inv(sa) + (s2 <= sa).thenElse(darkSrc, liteSrc); // 1 or (2 or 3)?
682 STAGE(luminance_to_alpha) {
683 a = SK_LUM_COEFF_R*r + SK_LUM_COEFF_G*g + SK_LUM_COEFF_B*b;
688 auto max = SkNf::Max(SkNf::Max(r, g), b);
689 auto min = SkNf::Min(SkNf::Min(r, g), b);
690 auto l = 0.5f * (max + min);
694 auto s = (max == min).thenElse(0.0f,
695 d/(l > 0.5f).thenElse(2.0f - max - min, max + min));
696 SkNf h = (max != r).thenElse(0.0f,
697 (g - b)*d_inv + (g < b).thenElse(6.0f, 0.0f));
698 h = (max == g).thenElse((b - r)*d_inv + 2.0f, h);
699 h = (max == b).thenElse((r - g)*d_inv + 4.0f, h);
702 h = (max == min).thenElse(0.0f, h);
713 auto q = (l < 0.5f).thenElse(l*(1.0f + s), l + s - l*s);
716 auto hue_to_rgb = [](const SkNf& p, const SkNf& q, const SkNf& t) {
717 auto t2 = (t < 0.0f).thenElse(t + 1.0f, (t > 1.0f).thenElse(t - 1.0f, t));
718 return (t2 < (1/6.0f)).thenElse(
719 p + (q - p)*6.0f*t, (t2 < (3/6.0f)).thenElse(
720 q, (t2 < (4/6.0f)).thenElse(
721 p + (q - p)*((4/6.0f) - t2)*6.0f, p)));
724 r = (s == 0.f).thenElse(l, hue_to_rgb(p, q, h + (1/3.0f)));
725 g = (s == 0.f).thenElse(l, hue_to_rgb(p, q, h));
726 b = (s == 0.f).thenElse(l, hue_to_rgb(p, q, h - (1/3.0f)));
729 STAGE_CTX(matrix_2x3, const float*) {
732 auto R = SkNf_fma(r,m[0], SkNf_fma(g,m[2], m[4])),
733 G = SkNf_fma(r,m[1], SkNf_fma(g,m[3], m[5]));
737 STAGE_CTX(matrix_3x4, const float*) {
740 auto R = SkNf_fma(r,m[0], SkNf_fma(g,m[3], SkNf_fma(b,m[6], m[ 9]))),
741 G = SkNf_fma(r,m[1], SkNf_fma(g,m[4], SkNf_fma(b,m[7], m[10]))),
742 B = SkNf_fma(r,m[2], SkNf_fma(g,m[5], SkNf_fma(b,m[8], m[11])));
747 STAGE_CTX(matrix_4x5, const float*) {
750 auto R = SkNf_fma(r,m[0], SkNf_fma(g,m[4], SkNf_fma(b,m[ 8], SkNf_fma(a,m[12], m[16])))),
751 G = SkNf_fma(r,m[1], SkNf_fma(g,m[5], SkNf_fma(b,m[ 9], SkNf_fma(a,m[13], m[17])))),
752 B = SkNf_fma(r,m[2], SkNf_fma(g,m[6], SkNf_fma(b,m[10], SkNf_fma(a,m[14], m[18])))),
753 A = SkNf_fma(r,m[3], SkNf_fma(g,m[7], SkNf_fma(b,m[11], SkNf_fma(a,m[15], m[19]))));
759 STAGE_CTX(matrix_perspective, const float*) {
760 // N.B. unlike the matrix_NxM stages, this takes a row-major matrix.
763 auto R = SkNf_fma(r,m[0], SkNf_fma(g,m[1], m[2])),
764 G = SkNf_fma(r,m[3], SkNf_fma(g,m[4], m[5])),
765 Z = SkNf_fma(r,m[6], SkNf_fma(g,m[7], m[8]));
770 SI SkNf parametric(const SkNf& v, const SkColorSpaceTransferFn& p) {
771 float result[N]; // Unconstrained powf() doesn't vectorize well...
772 for (int i = 0; i < N; i++) {
774 result[i] = (s <= p.fD) ? p.fC * s + p.fF
775 : powf(s * p.fA + p.fB, p.fG) + p.fE;
777 // Clamp the output to [0, 1].
778 // Max(NaN, 0) = 0, but Max(0, NaN) = NaN, so we want this exact order to ensure NaN => 0
779 return SkNf::Min(SkNf::Max(SkNf::Load(result), 0.0f), 1.0f);
781 STAGE_CTX(parametric_r, const SkColorSpaceTransferFn*) { r = parametric(r, *ctx); }
782 STAGE_CTX(parametric_g, const SkColorSpaceTransferFn*) { g = parametric(g, *ctx); }
783 STAGE_CTX(parametric_b, const SkColorSpaceTransferFn*) { b = parametric(b, *ctx); }
784 STAGE_CTX(parametric_a, const SkColorSpaceTransferFn*) { a = parametric(a, *ctx); }
786 SI SkNf table(const SkNf& v, const SkTableTransferFn& table) {
788 for (int i = 0; i < N; i++) {
789 result[i] = interp_lut(v[i], table.fData, table.fSize);
791 // no need to clamp - tables are by-design [0,1] -> [0,1]
792 return SkNf::Load(result);
794 STAGE_CTX(table_r, const SkTableTransferFn*) { r = table(r, *ctx); }
795 STAGE_CTX(table_g, const SkTableTransferFn*) { g = table(g, *ctx); }
796 STAGE_CTX(table_b, const SkTableTransferFn*) { b = table(b, *ctx); }
797 STAGE_CTX(table_a, const SkTableTransferFn*) { a = table(a, *ctx); }
799 STAGE_CTX(color_lookup_table, const SkColorLookUpTable*) {
800 const SkColorLookUpTable* colorLUT = ctx;
801 SkASSERT(3 == colorLUT->inputChannels() || 4 == colorLUT->inputChannels());
802 SkASSERT(3 == colorLUT->outputChannels());
804 for (int i = 0; i < N; ++i) {
805 const float in[4] = { r[i], g[i], b[i], a[i] };
807 colorLUT->interp(out, in);
808 for (int j = 0; j < colorLUT->outputChannels(); ++j) {
809 result[j][i] = out[j];
812 r = SkNf::Load(result[0]);
813 g = SkNf::Load(result[1]);
814 b = SkNf::Load(result[2]);
815 if (4 == colorLUT->inputChannels()) {
816 // we must set the pixel to opaque, as the alpha channel was used
817 // as input before this.
823 const auto lab_l = r * 100.0f;
824 const auto lab_a = g * 255.0f - 128.0f;
825 const auto lab_b = b * 255.0f - 128.0f;
826 auto Y = (lab_l + 16.0f) * (1/116.0f);
827 auto X = lab_a * (1/500.0f) + Y;
828 auto Z = Y - (lab_b * (1/200.0f));
830 const auto X3 = X*X*X;
831 X = (X3 > 0.008856f).thenElse(X3, (X - (16/116.0f)) * (1/7.787f));
832 const auto Y3 = Y*Y*Y;
833 Y = (Y3 > 0.008856f).thenElse(Y3, (Y - (16/116.0f)) * (1/7.787f));
834 const auto Z3 = Z*Z*Z;
835 Z = (Z3 > 0.008856f).thenElse(Z3, (Z - (16/116.0f)) * (1/7.787f));
837 // adjust to D50 illuminant
847 SI SkNf assert_in_tile(const SkNf& v, float limit) {
848 for (int i = 0; i < N; i++) {
849 SkASSERT(0 <= v[i] && v[i] < limit);
854 SI SkNf ulp_before(float v) {
857 SkNu uvs = SkNu::Load(&vs) - 1;
858 return SkNf::Load(&uvs);
861 SI SkNf clamp(const SkNf& v, float limit) {
862 SkNf result = SkNf::Max(0, SkNf::Min(v, ulp_before(limit)));
863 return assert_in_tile(result, limit);
865 SI SkNf repeat(const SkNf& v, float limit) {
866 SkNf result = v - (v/limit).floor()*limit;
867 // For small negative v, (v/limit).floor()*limit can dominate v in the subtraction,
868 // which leaves result == limit. We want result < limit, so clamp it one ULP.
869 result = SkNf::Min(result, ulp_before(limit));
870 return assert_in_tile(result, limit);
872 SI SkNf mirror(const SkNf& v, float l/*imit*/) {
873 SkNf result = ((v - l) - ((v - l) / (2*l)).floor()*(2*l) - l).abs();
874 // Same deal as repeat.
875 result = SkNf::Min(result, ulp_before(l));
876 return assert_in_tile(result, l);
878 STAGE_CTX( clamp_x, const float*) { r = clamp (r, *ctx); }
879 STAGE_CTX(repeat_x, const float*) { r = repeat(r, *ctx); }
880 STAGE_CTX(mirror_x, const float*) { r = mirror(r, *ctx); }
881 STAGE_CTX( clamp_y, const float*) { g = clamp (g, *ctx); }
882 STAGE_CTX(repeat_y, const float*) { g = repeat(g, *ctx); }
883 STAGE_CTX(mirror_y, const float*) { g = mirror(g, *ctx); }
885 STAGE_CTX(save_xy, SkJumper_SamplerCtx*) {
889 // Whether bilinear or bicubic, all sample points have the same fractional offset (fx,fy).
890 // They're either the 4 corners of a logical 1x1 pixel or the 16 corners of a 3x3 grid
891 // surrounding (x,y), all (0.5,0.5) off-center.
892 auto fract = [](const SkNf& v) { return v - v.floor(); };
893 fract(r + 0.5f).store(ctx->fx);
894 fract(g + 0.5f).store(ctx->fy);
897 STAGE_CTX(accumulate, const SkJumper_SamplerCtx*) {
898 // Bilinear and bicubic filtering are both separable, so we'll end up with independent
899 // scale contributions in x and y that we multiply together to get each pixel's scale factor.
900 auto scale = SkNf::Load(ctx->scalex) * SkNf::Load(ctx->scaley);
901 dr = SkNf_fma(scale, r, dr);
902 dg = SkNf_fma(scale, g, dg);
903 db = SkNf_fma(scale, b, db);
904 da = SkNf_fma(scale, a, da);
907 // In bilinear interpolation, the 4 pixels at +/- 0.5 offsets from the sample pixel center
908 // are combined in direct proportion to their area overlapping that logical query pixel.
909 // At positive offsets, the x-axis contribution to that rectangular area is fx; (1-fx)
910 // at negative x offsets. The y-axis is treated symmetrically.
912 SI void bilinear_x(SkJumper_SamplerCtx* ctx, SkNf* x) {
913 *x = SkNf::Load(ctx->x) + Scale*0.5f;
914 auto fx = SkNf::Load(ctx->fx);
915 (Scale > 0 ? fx : (1.0f - fx)).store(ctx->scalex);
918 SI void bilinear_y(SkJumper_SamplerCtx* ctx, SkNf* y) {
919 *y = SkNf::Load(ctx->y) + Scale*0.5f;
920 auto fy = SkNf::Load(ctx->fy);
921 (Scale > 0 ? fy : (1.0f - fy)).store(ctx->scaley);
923 STAGE_CTX(bilinear_nx, SkJumper_SamplerCtx*) { bilinear_x<-1>(ctx, &r); }
924 STAGE_CTX(bilinear_px, SkJumper_SamplerCtx*) { bilinear_x<+1>(ctx, &r); }
925 STAGE_CTX(bilinear_ny, SkJumper_SamplerCtx*) { bilinear_y<-1>(ctx, &g); }
926 STAGE_CTX(bilinear_py, SkJumper_SamplerCtx*) { bilinear_y<+1>(ctx, &g); }
929 // In bilinear interpolation, the 16 pixels at +/- 0.5 and +/- 1.5 offsets from the sample
930 // pixel center are combined with a non-uniform cubic filter, with high filter values near
931 // the center and lower values farther away.
933 // We break this filter function into two parts, one for near +/- 0.5 offsets,
934 // and one for far +/- 1.5 offsets.
936 // See GrBicubicEffect for details about this particular Mitchell-Netravali filter.
937 SI SkNf bicubic_near(const SkNf& t) {
938 // 1/18 + 9/18t + 27/18t^2 - 21/18t^3 == t ( t ( -21/18t + 27/18) + 9/18) + 1/18
939 return SkNf_fma(t, SkNf_fma(t, SkNf_fma(-21/18.0f, t, 27/18.0f), 9/18.0f), 1/18.0f);
941 SI SkNf bicubic_far(const SkNf& t) {
942 // 0/18 + 0/18*t - 6/18t^2 + 7/18t^3 == t^2 (7/18t - 6/18)
943 return (t*t)*SkNf_fma(7/18.0f, t, -6/18.0f);
947 SI void bicubic_x(SkJumper_SamplerCtx* ctx, SkNf* x) {
948 *x = SkNf::Load(ctx->x) + Scale*0.5f;
949 auto fx = SkNf::Load(ctx->fx);
950 if (Scale == -3) { return bicubic_far (1.0f - fx).store(ctx->scalex); }
951 if (Scale == -1) { return bicubic_near(1.0f - fx).store(ctx->scalex); }
952 if (Scale == +1) { return bicubic_near( fx).store(ctx->scalex); }
953 if (Scale == +3) { return bicubic_far ( fx).store(ctx->scalex); }
954 SkDEBUGFAIL("unreachable");
957 SI void bicubic_y(SkJumper_SamplerCtx* ctx, SkNf* y) {
958 *y = SkNf::Load(ctx->y) + Scale*0.5f;
959 auto fy = SkNf::Load(ctx->fy);
960 if (Scale == -3) { return bicubic_far (1.0f - fy).store(ctx->scaley); }
961 if (Scale == -1) { return bicubic_near(1.0f - fy).store(ctx->scaley); }
962 if (Scale == +1) { return bicubic_near( fy).store(ctx->scaley); }
963 if (Scale == +3) { return bicubic_far ( fy).store(ctx->scaley); }
964 SkDEBUGFAIL("unreachable");
966 STAGE_CTX(bicubic_n3x, SkJumper_SamplerCtx*) { bicubic_x<-3>(ctx, &r); }
967 STAGE_CTX(bicubic_n1x, SkJumper_SamplerCtx*) { bicubic_x<-1>(ctx, &r); }
968 STAGE_CTX(bicubic_p1x, SkJumper_SamplerCtx*) { bicubic_x<+1>(ctx, &r); }
969 STAGE_CTX(bicubic_p3x, SkJumper_SamplerCtx*) { bicubic_x<+3>(ctx, &r); }
971 STAGE_CTX(bicubic_n3y, SkJumper_SamplerCtx*) { bicubic_y<-3>(ctx, &g); }
972 STAGE_CTX(bicubic_n1y, SkJumper_SamplerCtx*) { bicubic_y<-1>(ctx, &g); }
973 STAGE_CTX(bicubic_p1y, SkJumper_SamplerCtx*) { bicubic_y<+1>(ctx, &g); }
974 STAGE_CTX(bicubic_p3y, SkJumper_SamplerCtx*) { bicubic_y<+3>(ctx, &g); }
977 template <typename T>
978 SI SkNi offset_and_ptr(T** ptr, const SkJumper_GatherCtx* ctx, const SkNf& x, const SkNf& y) {
979 SkNi ix = SkNx_cast<int>(x),
980 iy = SkNx_cast<int>(y);
981 SkNi offset = iy*ctx->stride + ix;
983 *ptr = (const T*)ctx->pixels;
987 STAGE_CTX(gather_a8, const SkJumper_GatherCtx*) {
989 SkNi offset = offset_and_ptr(&p, ctx, r, g);
992 a = SkNf_from_byte(gather(tail, p, offset));
994 STAGE_CTX(gather_i8, const SkJumper_GatherCtx*) {
996 SkNi offset = offset_and_ptr(&p, ctx, r, g);
998 SkNi ix = SkNx_cast<int>(gather(tail, p, offset));
999 from_8888(gather(tail, ctx->ctable, ix), &r, &g, &b, &a);
1001 STAGE_CTX(gather_g8, const SkJumper_GatherCtx*) {
1003 SkNi offset = offset_and_ptr(&p, ctx, r, g);
1005 r = g = b = SkNf_from_byte(gather(tail, p, offset));
1008 STAGE_CTX(gather_565, const SkJumper_GatherCtx*) {
1010 SkNi offset = offset_and_ptr(&p, ctx, r, g);
1012 from_565(gather(tail, p, offset), &r, &g, &b);
1015 STAGE_CTX(gather_4444, const SkJumper_GatherCtx*) {
1017 SkNi offset = offset_and_ptr(&p, ctx, r, g);
1019 from_4444(gather(tail, p, offset), &r, &g, &b, &a);
1021 STAGE_CTX(gather_8888, const SkJumper_GatherCtx*) {
1023 SkNi offset = offset_and_ptr(&p, ctx, r, g);
1025 from_8888(gather(tail, p, offset), &r, &g, &b, &a);
1027 STAGE_CTX(gather_f16, const SkJumper_GatherCtx*) {
1029 SkNi offset = offset_and_ptr(&p, ctx, r, g);
1031 auto px = gather(tail, p, offset);
1032 from_f16(&px, &r, &g, &b, &a);
1035 STAGE_CTX(linear_gradient, const SkPM4f*) {
1036 struct Stop { float pos; float f[4], b[4]; };
1037 struct Ctx { size_t n; Stop *stops; float start[4]; };
1039 auto c = (const Ctx*)ctx;
1040 SkNf fr = 0, fg = 0, fb = 0, fa = 0;
1041 SkNf br = c->start[0],
1046 for (size_t i = 0; i < c->n; i++) {
1047 fr = (t < c->stops[i].pos).thenElse(fr, c->stops[i].f[0]);
1048 fg = (t < c->stops[i].pos).thenElse(fg, c->stops[i].f[1]);
1049 fb = (t < c->stops[i].pos).thenElse(fb, c->stops[i].f[2]);
1050 fa = (t < c->stops[i].pos).thenElse(fa, c->stops[i].f[3]);
1051 br = (t < c->stops[i].pos).thenElse(br, c->stops[i].b[0]);
1052 bg = (t < c->stops[i].pos).thenElse(bg, c->stops[i].b[1]);
1053 bb = (t < c->stops[i].pos).thenElse(bb, c->stops[i].b[2]);
1054 ba = (t < c->stops[i].pos).thenElse(ba, c->stops[i].b[3]);
1057 r = SkNf_fma(t, fr, br);
1058 g = SkNf_fma(t, fg, bg);
1059 b = SkNf_fma(t, fb, bb);
1060 a = SkNf_fma(t, fa, ba);
1063 STAGE_CTX(linear_gradient_2stops, const SkPM4f*) {
1068 r = SkNf_fma(t, dc.r(), c0.r());
1069 g = SkNf_fma(t, dc.g(), c0.g());
1070 b = SkNf_fma(t, dc.b(), c0.b());
1071 a = SkNf_fma(t, dc.a(), c0.a());
1074 STAGE_CTX(byte_tables, const void*) {
1075 struct Tables { const uint8_t *r, *g, *b, *a; };
1076 auto tables = (const Tables*)ctx;
1078 r = SkNf_from_byte(gather(tail, tables->r, SkNf_round(255.0f, r)));
1079 g = SkNf_from_byte(gather(tail, tables->g, SkNf_round(255.0f, g)));
1080 b = SkNf_from_byte(gather(tail, tables->b, SkNf_round(255.0f, b)));
1081 a = SkNf_from_byte(gather(tail, tables->a, SkNf_round(255.0f, a)));
1084 STAGE_CTX(byte_tables_rgb, const void*) {
1085 struct Tables { const uint8_t *r, *g, *b; int n; };
1086 auto tables = (const Tables*)ctx;
1088 float scale = tables->n - 1;
1089 r = SkNf_from_byte(gather(tail, tables->r, SkNf_round(scale, r)));
1090 g = SkNf_from_byte(gather(tail, tables->g, SkNf_round(scale, g)));
1091 b = SkNf_from_byte(gather(tail, tables->b, SkNf_round(scale, b)));
1094 STAGE_CTX(shader_adapter, SkShader::Context*) {
1096 static_assert(sizeof(buf) == sizeof(r) + sizeof(g) + sizeof(b) + sizeof(a), "");
1097 ctx->shadeSpan4f(x, (int)g[0], buf, N);
1098 SkNf::Load4(buf, &r, &g, &b, &a);
1101 SI Fn enum_to_Fn(SkRasterPipeline::StockStage st) {
1103 #define M(stage) case SkRasterPipeline::stage: return stage;
1104 SK_RASTER_PIPELINE_STAGES(M)
1113 static void build_program(void** program, const SkRasterPipeline::Stage* stages, int nstages) {
1114 for (int i = 0; i < nstages; i++) {
1115 *program++ = (void*)enum_to_Fn(stages[i].stage);
1116 if (stages[i].ctx) {
1117 *program++ = stages[i].ctx;
1120 *program++ = (void*)just_return;
1123 static void run_program(void** program, size_t x, size_t n) {
1124 #if SK_BUILD_FOR_WINRT
1127 SkNf u; // fastest to start uninitialized.
1130 auto start = (Fn)load_and_increment(&program);
1132 start(x*N, program, u,u,u,u, u,u,u,u);
1137 start(x*N+n, program, u,u,u,u, u,u,u,u);
1141 // Compiled manages its memory manually because it's not safe to use
1142 // std::vector, SkTDArray, etc without setting us up for big ODR violations.
1144 Compiled(const SkRasterPipeline::Stage* stages, int nstages) {
1145 int slots = nstages + 1; // One extra for just_return.
1146 for (int i = 0; i < nstages; i++) {
1147 if (stages[i].ctx) {
1151 fProgram = (void**)sk_malloc_throw(slots * sizeof(void*));
1152 build_program(fProgram, stages, nstages);
1154 ~Compiled() { sk_free(fProgram); }
1156 Compiled(const Compiled& o) {
1158 while (o.fProgram[slots++] != (void*)just_return);
1160 fProgram = (void**)sk_malloc_throw(slots * sizeof(void*));
1161 memcpy(fProgram, o.fProgram, slots * sizeof(void*));
1164 void operator()(size_t x, size_t n) {
1165 run_program(fProgram, x, n);
1172 namespace SK_OPTS_NS {
1174 SI void run_pipeline(size_t x, size_t n,
1175 const SkRasterPipeline::Stage* stages, int nstages) {
1176 static const int kStackMax = 256;
1177 // Worst case is nstages stages with nstages context pointers, and just_return.
1178 if (2*nstages+1 <= kStackMax) {
1179 void* program[kStackMax];
1180 build_program(program, stages, nstages);
1181 run_program(program, x,n);
1183 Compiled{stages,nstages}(x,n);
1187 } // namespace SK_OPTS_NS
1192 #undef RGBA_XFERMODE
1195 #endif//SkRasterPipeline_opts_DEFINED