#include "SkNx.h"
#include "SkTypes.h"
+#if !defined(_MSC_VER) && SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_AVX2
+ #include <x86intrin.h>
+#endif
+
// 16-bit floating point value
// format is 1 bit sign, 5 bits exponent, 10 bits mantissa
// only used for storage
#endif
}
+static inline Sk8f SkHalfToFloat_finite_ftz(const Sk8h& hs) {
+#if !defined(SKNX_NO_SIMD) && SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_AVX2
+ return _mm256_cvtph_ps(hs.fVec);
+
+#else
+ uint64_t parts[2];
+ hs.store(parts);
+ return SkNx_join(SkHalfToFloat_finite_ftz(parts[0]),
+ SkHalfToFloat_finite_ftz(parts[1]));
+
+#endif
+}
+
+static inline Sk8h SkFloatToHalf_finite_ftz(const Sk8f& fs) {
+#if !defined(SKNX_NO_SIMD) && SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_AVX2
+ return _mm256_cvtps_ph(fs.fVec, _MM_FROUND_CUR_DIRECTION);
+
+#else
+ uint64_t parts[2];
+ SkFloatToHalf_finite_ftz(fs.fLo).store(parts+0);
+ SkFloatToHalf_finite_ftz(fs.fHi).store(parts+1);
+ return Sk8h::Load(parts);
+#endif
+}
+
#endif
return static_cast<Dst>(v.fVal);
}
+template <int N, typename T>
+SI SkNx<N,T> SkNx_fma(const SkNx<N,T>& f, const SkNx<N,T>& m, const SkNx<N,T>& a) {
+ return f*m+a;
+}
+
typedef SkNx<2, float> Sk2f;
typedef SkNx<4, float> Sk4f;
typedef SkNx<8, float> Sk8f;
typedef SkNx<16, uint16_t> Sk16h;
typedef SkNx<4, int32_t> Sk4i;
+typedef SkNx<8, int32_t> Sk8i;
typedef SkNx<4, uint32_t> Sk4u;
// Include platform specific specializations if available.
DEFINE_DEFAULT(srcover_srgb_srgb);
DEFINE_DEFAULT(hash_fn);
+
+ DEFINE_DEFAULT(run_pipeline);
#undef DEFINE_DEFAULT
- // TODO: might be nice to only create one instance of tail-insensitive stages.
-
- SkRasterPipeline::Fn stages_4[] = {
- stage_4<SK_OPTS_NS::store_565 , false>,
- stage_4<SK_OPTS_NS::store_srgb, false>,
- stage_4<SK_OPTS_NS::store_f16 , false>,
-
- stage_4<SK_OPTS_NS::load_s_565 , true>,
- stage_4<SK_OPTS_NS::load_s_srgb, true>,
- stage_4<SK_OPTS_NS::load_s_f16 , true>,
-
- stage_4<SK_OPTS_NS::load_d_565 , true>,
- stage_4<SK_OPTS_NS::load_d_srgb, true>,
- stage_4<SK_OPTS_NS::load_d_f16 , true>,
-
- stage_4<SK_OPTS_NS::scale_u8, true>,
-
- stage_4<SK_OPTS_NS::lerp_u8 , true>,
- stage_4<SK_OPTS_NS::lerp_565 , true>,
- stage_4<SK_OPTS_NS::lerp_constant_float, true>,
-
- stage_4<SK_OPTS_NS::constant_color, true>,
-
- SK_OPTS_NS::dst,
- SK_OPTS_NS::dstatop,
- SK_OPTS_NS::dstin,
- SK_OPTS_NS::dstout,
- SK_OPTS_NS::dstover,
- SK_OPTS_NS::srcatop,
- SK_OPTS_NS::srcin,
- SK_OPTS_NS::srcout,
- SK_OPTS_NS::srcover,
- SK_OPTS_NS::clear,
- SK_OPTS_NS::modulate,
- SK_OPTS_NS::multiply,
- SK_OPTS_NS::plus_,
- SK_OPTS_NS::screen,
- SK_OPTS_NS::xor_,
- SK_OPTS_NS::colorburn,
- SK_OPTS_NS::colordodge,
- SK_OPTS_NS::darken,
- SK_OPTS_NS::difference,
- SK_OPTS_NS::exclusion,
- SK_OPTS_NS::hardlight,
- SK_OPTS_NS::lighten,
- SK_OPTS_NS::overlay,
- SK_OPTS_NS::softlight,
+ SkOpts::VoidFn body[] = {
+ (SkOpts::VoidFn)SK_OPTS_NS::just_return,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::store_565,
+ (SkOpts::VoidFn)SK_OPTS_NS::store_srgb,
+ (SkOpts::VoidFn)SK_OPTS_NS::store_f16,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::load_s_565,
+ (SkOpts::VoidFn)SK_OPTS_NS::load_s_srgb,
+ (SkOpts::VoidFn)SK_OPTS_NS::load_s_f16,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::load_d_565,
+ (SkOpts::VoidFn)SK_OPTS_NS::load_d_srgb,
+ (SkOpts::VoidFn)SK_OPTS_NS::load_d_f16,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::scale_u8,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::lerp_u8,
+ (SkOpts::VoidFn)SK_OPTS_NS::lerp_565,
+ (SkOpts::VoidFn)SK_OPTS_NS::lerp_constant_float,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::constant_color,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::dst,
+ (SkOpts::VoidFn)SK_OPTS_NS::dstatop,
+ (SkOpts::VoidFn)SK_OPTS_NS::dstin,
+ (SkOpts::VoidFn)SK_OPTS_NS::dstout,
+ (SkOpts::VoidFn)SK_OPTS_NS::dstover,
+ (SkOpts::VoidFn)SK_OPTS_NS::srcatop,
+ (SkOpts::VoidFn)SK_OPTS_NS::srcin,
+ (SkOpts::VoidFn)SK_OPTS_NS::srcout,
+ (SkOpts::VoidFn)SK_OPTS_NS::srcover,
+ (SkOpts::VoidFn)SK_OPTS_NS::clear,
+ (SkOpts::VoidFn)SK_OPTS_NS::modulate,
+ (SkOpts::VoidFn)SK_OPTS_NS::multiply,
+ (SkOpts::VoidFn)SK_OPTS_NS::plus_,
+ (SkOpts::VoidFn)SK_OPTS_NS::screen,
+ (SkOpts::VoidFn)SK_OPTS_NS::xor_,
+ (SkOpts::VoidFn)SK_OPTS_NS::colorburn,
+ (SkOpts::VoidFn)SK_OPTS_NS::colordodge,
+ (SkOpts::VoidFn)SK_OPTS_NS::darken,
+ (SkOpts::VoidFn)SK_OPTS_NS::difference,
+ (SkOpts::VoidFn)SK_OPTS_NS::exclusion,
+ (SkOpts::VoidFn)SK_OPTS_NS::hardlight,
+ (SkOpts::VoidFn)SK_OPTS_NS::lighten,
+ (SkOpts::VoidFn)SK_OPTS_NS::overlay,
+ (SkOpts::VoidFn)SK_OPTS_NS::softlight,
};
- static_assert(SK_ARRAY_COUNT(stages_4) == SkRasterPipeline::kNumStockStages, "");
-
- SkRasterPipeline::Fn stages_1_3[] = {
- stage_1_3<SK_OPTS_NS::store_565 , false>,
- stage_1_3<SK_OPTS_NS::store_srgb, false>,
- stage_1_3<SK_OPTS_NS::store_f16 , false>,
-
- stage_1_3<SK_OPTS_NS::load_s_565 , true>,
- stage_1_3<SK_OPTS_NS::load_s_srgb, true>,
- stage_1_3<SK_OPTS_NS::load_s_f16 , true>,
-
- stage_1_3<SK_OPTS_NS::load_d_565 , true>,
- stage_1_3<SK_OPTS_NS::load_d_srgb, true>,
- stage_1_3<SK_OPTS_NS::load_d_f16 , true>,
-
- stage_1_3<SK_OPTS_NS::scale_u8, true>,
-
- stage_1_3<SK_OPTS_NS::lerp_u8 , true>,
- stage_1_3<SK_OPTS_NS::lerp_565 , true>,
- stage_1_3<SK_OPTS_NS::lerp_constant_float, true>,
-
- stage_1_3<SK_OPTS_NS::constant_color, true>,
-
- SK_OPTS_NS::dst,
- SK_OPTS_NS::dstatop,
- SK_OPTS_NS::dstin,
- SK_OPTS_NS::dstout,
- SK_OPTS_NS::dstover,
- SK_OPTS_NS::srcatop,
- SK_OPTS_NS::srcin,
- SK_OPTS_NS::srcout,
- SK_OPTS_NS::srcover,
- SK_OPTS_NS::clear,
- SK_OPTS_NS::modulate,
- SK_OPTS_NS::multiply,
- SK_OPTS_NS::plus_,
- SK_OPTS_NS::screen,
- SK_OPTS_NS::xor_,
- SK_OPTS_NS::colorburn,
- SK_OPTS_NS::colordodge,
- SK_OPTS_NS::darken,
- SK_OPTS_NS::difference,
- SK_OPTS_NS::exclusion,
- SK_OPTS_NS::hardlight,
- SK_OPTS_NS::lighten,
- SK_OPTS_NS::overlay,
- SK_OPTS_NS::softlight,
+ static_assert(SK_ARRAY_COUNT(body) == SkRasterPipeline::kNumStockStages, "");
+
+ SkOpts::VoidFn tail[] = {
+ (SkOpts::VoidFn)SK_OPTS_NS::just_return,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::store_565_tail,
+ (SkOpts::VoidFn)SK_OPTS_NS::store_srgb_tail,
+ (SkOpts::VoidFn)SK_OPTS_NS::store_f16_tail,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::load_s_565_tail,
+ (SkOpts::VoidFn)SK_OPTS_NS::load_s_srgb_tail,
+ (SkOpts::VoidFn)SK_OPTS_NS::load_s_f16_tail,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::load_d_565_tail,
+ (SkOpts::VoidFn)SK_OPTS_NS::load_d_srgb_tail,
+ (SkOpts::VoidFn)SK_OPTS_NS::load_d_f16_tail,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::scale_u8_tail,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::lerp_u8_tail,
+ (SkOpts::VoidFn)SK_OPTS_NS::lerp_565_tail,
+ (SkOpts::VoidFn)SK_OPTS_NS::lerp_constant_float,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::constant_color,
+
+ (SkOpts::VoidFn)SK_OPTS_NS::dst,
+ (SkOpts::VoidFn)SK_OPTS_NS::dstatop,
+ (SkOpts::VoidFn)SK_OPTS_NS::dstin,
+ (SkOpts::VoidFn)SK_OPTS_NS::dstout,
+ (SkOpts::VoidFn)SK_OPTS_NS::dstover,
+ (SkOpts::VoidFn)SK_OPTS_NS::srcatop,
+ (SkOpts::VoidFn)SK_OPTS_NS::srcin,
+ (SkOpts::VoidFn)SK_OPTS_NS::srcout,
+ (SkOpts::VoidFn)SK_OPTS_NS::srcover,
+ (SkOpts::VoidFn)SK_OPTS_NS::clear,
+ (SkOpts::VoidFn)SK_OPTS_NS::modulate,
+ (SkOpts::VoidFn)SK_OPTS_NS::multiply,
+ (SkOpts::VoidFn)SK_OPTS_NS::plus_,
+ (SkOpts::VoidFn)SK_OPTS_NS::screen,
+ (SkOpts::VoidFn)SK_OPTS_NS::xor_,
+ (SkOpts::VoidFn)SK_OPTS_NS::colorburn,
+ (SkOpts::VoidFn)SK_OPTS_NS::colordodge,
+ (SkOpts::VoidFn)SK_OPTS_NS::darken,
+ (SkOpts::VoidFn)SK_OPTS_NS::difference,
+ (SkOpts::VoidFn)SK_OPTS_NS::exclusion,
+ (SkOpts::VoidFn)SK_OPTS_NS::hardlight,
+ (SkOpts::VoidFn)SK_OPTS_NS::lighten,
+ (SkOpts::VoidFn)SK_OPTS_NS::overlay,
+ (SkOpts::VoidFn)SK_OPTS_NS::softlight,
};
- static_assert(SK_ARRAY_COUNT(stages_1_3) == SkRasterPipeline::kNumStockStages, "");
+ static_assert(SK_ARRAY_COUNT(tail) == SkRasterPipeline::kNumStockStages, "");
// Each Init_foo() is defined in src/opts/SkOpts_foo.cpp.
void Init_ssse3();
return hash_fn(data, bytes, seed);
}
- extern SkRasterPipeline::Fn stages_4 [SkRasterPipeline::kNumStockStages],
- stages_1_3[SkRasterPipeline::kNumStockStages];
+ // SkRasterPipeline::Fn has different types in different files (notably, in SkOpts_hsw.cpp
+ // they're all in terms of Sk8f.) We store them with a type everyone can agree on, void(*)().
+ using VoidFn = void(*)();
+ extern VoidFn body[SkRasterPipeline::kNumStockStages],
+ tail[SkRasterPipeline::kNumStockStages];
+ extern void (*run_pipeline)(size_t, size_t, void(*)(), SkRasterPipeline::Stage*,
+ void(*)(), SkRasterPipeline::Stage*);
}
#endif//SkOpts_DEFINED
#include "SkOpts.h"
#include "SkRasterPipeline.h"
-SkRasterPipeline::SkRasterPipeline() {}
+SkRasterPipeline::SkRasterPipeline() {
+ fBodyStart = SkOpts::body[just_return];
+ fTailStart = SkOpts::tail[just_return];
+}
-void SkRasterPipeline::append(SkRasterPipeline::Fn body,
- SkRasterPipeline::Fn tail,
- void* ctx) {
+void SkRasterPipeline::append(void (*body)(), void (*tail)(), void* ctx) {
// Each stage holds its own context and the next function to call.
// So the pipeline itself has to hold onto the first function that starts the pipeline.
(fBody.empty() ? fBodyStart : fBody.back().fNext) = body;
// Each last stage starts with its next function set to JustReturn as a safety net.
// It'll be overwritten by the next call to append().
- fBody.push_back({ &JustReturn, ctx });
- fTail.push_back({ &JustReturn, ctx });
+ fBody.push_back({ SkOpts::body[just_return], ctx });
+ fTail.push_back({ SkOpts::tail[just_return], ctx });
}
void SkRasterPipeline::append(StockStage stage, void* ctx) {
- this->append(SkOpts::stages_4[stage], SkOpts::stages_1_3[stage], ctx);
+ this->append(SkOpts::body[stage], SkOpts::tail[stage], ctx);
}
void SkRasterPipeline::extend(const SkRasterPipeline& src) {
SkASSERT(src.fBody.count() == src.fTail.count());
- Fn body = src.fBodyStart,
- tail = src.fTailStart;
+ auto body = src.fBodyStart,
+ tail = src.fTailStart;
for (int i = 0; i < src.fBody.count(); i++) {
SkASSERT(src.fBody[i].fCtx == src.fTail[i].fCtx);
this->append(body, tail, src.fBody[i].fCtx);
}
void SkRasterPipeline::run(size_t x, size_t n) {
- // It's fastest to start uninitialized if the compilers all let us. If not, next fastest is 0.
- Sk4f v;
-
- while (n >= 4) {
- fBodyStart(fBody.begin(), x,0, v,v,v,v, v,v,v,v);
- x += 4;
- n -= 4;
- }
- if (n > 0) {
- fTailStart(fTail.begin(), x,n, v,v,v,v, v,v,v,v);
- }
+ SkOpts::run_pipeline(x,n, fBodyStart,fBody.begin(), fTailStart,fTail.begin());
}
-
-void SK_VECTORCALL SkRasterPipeline::JustReturn(Stage*, size_t, size_t, Sk4f,Sk4f,Sk4f,Sk4f,
- Sk4f,Sk4f,Sk4f,Sk4f) {}
class SkRasterPipeline {
public:
struct Stage;
- using Fn = void(SK_VECTORCALL *)(Stage*, size_t, size_t, Sk4f,Sk4f,Sk4f,Sk4f,
- Sk4f,Sk4f,Sk4f,Sk4f);
+#if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_AVX2
+ using V = Sk8f;
+#else
+ using V = Sk4f;
+#endif
+ using Fn = void(SK_VECTORCALL *)(Stage*, size_t, size_t, V,V,V,V,
+ V,V,V,V);
+
struct Stage {
template <typename T>
T ctx() { return static_cast<T>(fCtx); }
- void SK_VECTORCALL next(size_t x, size_t tail, Sk4f v0, Sk4f v1, Sk4f v2, Sk4f v3,
- Sk4f v4, Sk4f v5, Sk4f v6, Sk4f v7) {
+ void SK_VECTORCALL next(size_t x, size_t tail, V v0, V v1, V v2, V v3,
+ V v4, V v5, V v6, V v7) {
// Stages are logically a pipeline, and physically are contiguous in an array.
// To get to the next stage, we just increment our pointer to the next array element.
- fNext(this+1, x,tail, v0,v1,v2,v3, v4,v5,v6,v7);
+ ((Fn)fNext)(this+1, x,tail, v0,v1,v2,v3, v4,v5,v6,v7);
}
// It makes next() a good bit cheaper if we hold the next function to call here,
// rather than logically simpler choice of the function implementing this stage.
- Fn fNext;
+ void (*fNext)();
void* fCtx;
};
void run(size_t n) { this->run(0, n); }
enum StockStage {
+ just_return,
+
store_565,
store_srgb,
store_f16,
void append(StockStage, void* = nullptr);
void append(StockStage stage, const void* ctx) { this->append(stage, const_cast<void*>(ctx)); }
-
// Append all stages to this pipeline.
void extend(const SkRasterPipeline&);
private:
using Stages = SkSTArray<10, Stage, /*MEM_COPY=*/true>;
- void append(Fn body, Fn tail, void*);
+ void append(void (*body)(), void (*tail)(), void*);
- // This no-op default makes fBodyStart and fTailStart unconditionally safe to call,
- // and is always the last stage's fNext as a sort of safety net to make sure even a
- // buggy pipeline can't walk off its own end.
- static void SK_VECTORCALL JustReturn(Stage*, size_t, size_t, Sk4f,Sk4f,Sk4f,Sk4f,
- Sk4f,Sk4f,Sk4f,Sk4f);
Stages fBody,
fTail;
- Fn fBodyStart = &JustReturn,
- fTailStart = &JustReturn;
+ void (*fBodyStart)() = nullptr;
+ void (*fTailStart)() = nullptr;
};
#endif//SkRasterPipeline_DEFINED
extern const float sk_linear_from_srgb[256];
-static inline Sk4f sk_clamp_0_255(const Sk4f& x) {
+template <typename V>
+static inline V sk_clamp_0_255(const V& x) {
// The order of the arguments is important here. We want to make sure that NaN
// clamps to zero. Note that max(NaN, 0) = 0, while max(0, NaN) = NaN.
- return Sk4f::Min(Sk4f::Max(x, 0.0f), 255.0f);
+ return V::Min(V::Max(x, 0.0f), 255.0f);
}
// This should probably only be called from sk_linear_to_srgb() or sk_linear_to_srgb_noclamp().
// It generally doesn't make sense to work with sRGB floats.
-static inline Sk4f sk_linear_to_srgb_needs_trunc(const Sk4f& x) {
+template <typename V>
+static inline V sk_linear_to_srgb_needs_trunc(const V& x) {
// Approximation of the sRGB gamma curve (within 1 when scaled to 8-bit pixels).
//
// Constants tuned by brute force to minimize (in order of importance) after truncation:
auto lo = (13.0471f * 255.0f) * x;
- auto hi = (-0.0974983f * 255.0f)
- + (+0.687999f * 255.0f) * sqrt
- + (+0.412999f * 255.0f) * ftrt;
+ auto hi = SkNx_fma(V{+0.412999f * 255.0f}, ftrt,
+ SkNx_fma(V{+0.687999f * 255.0f}, sqrt,
+ V{-0.0974983f * 255.0f}));
return (x < 0.0048f).thenElse(lo, hi);
}
-static inline Sk4i sk_linear_to_srgb(const Sk4f& x) {
- Sk4f f = sk_linear_to_srgb_needs_trunc(x);
+template <int N>
+static inline SkNx<N,int> sk_linear_to_srgb(const SkNx<N,float>& x) {
+ auto f = sk_linear_to_srgb_needs_trunc(x);
return SkNx_cast<int>(sk_clamp_0_255(f));
}
-static inline Sk4i sk_linear_to_srgb_noclamp(const Sk4f& x) {
- Sk4f f = sk_linear_to_srgb_needs_trunc(x);
+template <int N>
+static inline SkNx<N,int> sk_linear_to_srgb_noclamp(const SkNx<N,float>& x) {
+ auto f = sk_linear_to_srgb_needs_trunc(x);
for (int i = 0; i < 4; i++) {
SkASSERTF(0.0f <= f[i] && f[i] < 256.0f, "f[%d] was %g, outside [0,256)\n", i, f[i]);
}
}
// sRGB -> linear, using math instead of table lookups, scaling better to larger SIMD vectors.
-static inline Sk4f sk_linear_from_srgb_math(const Sk4i& s) {
+template <int N>
+static inline SkNx<N,float> sk_linear_from_srgb_math(const SkNx<N,int>& s) {
auto x = SkNx_cast<float>(s);
const float u = 1/255.0f; // x is [0,255], so x^n needs scaling by u^n.
// Non-linear segment of sRGB curve approximated by
// l = 0.0025 + 0.6975x^2 + 0.3x^3
- const float k0 = 0.0025f,
- k2 = 0.6975f * u*u,
- k3 = 0.3000f * u*u*u;
- auto hi = k0 + (k2 + k3*x) * (x*x);
+ const SkNx<N,float> k0 = 0.0025f,
+ k2 = 0.6975f * u*u,
+ k3 = 0.3000f * u*u*u;
+ auto hi = SkNx_fma(x*x, SkNx_fma(x, k3, k2), k0);
// Linear segment of sRGB curve: the normal slope, extended a little further than normal.
auto lo = x * (u/12.92f);
SkNx() {}
SkNx(uint16_t val) : fVec(_mm_set1_epi16(val)) {}
- static SkNx Load(const void* ptr) { return _mm_loadu_si128((const __m128i*)ptr); }
SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d,
uint16_t e, uint16_t f, uint16_t g, uint16_t h) : fVec(_mm_setr_epi16(a,b,c,d,e,f,g,h)) {}
+ static SkNx Load(const void* ptr) { return _mm_loadu_si128((const __m128i*)ptr); }
void store(void* ptr) const { _mm_storeu_si128((__m128i*)ptr, fVec); }
+ static void Load4(const void* ptr, SkNx* r, SkNx* g, SkNx* b, SkNx* a) {
+ // TODO: AVX2 version
+ __m128i _01 = _mm_loadu_si128(((__m128i*)ptr) + 0),
+ _23 = _mm_loadu_si128(((__m128i*)ptr) + 1),
+ _45 = _mm_loadu_si128(((__m128i*)ptr) + 2),
+ _67 = _mm_loadu_si128(((__m128i*)ptr) + 3);
+
+ __m128i _02 = _mm_unpacklo_epi16(_01, _23), // r0 r2 g0 g2 b0 b2 a0 a2
+ _13 = _mm_unpackhi_epi16(_01, _23), // r1 r3 g1 g3 b1 b3 a1 a3
+ _46 = _mm_unpacklo_epi16(_45, _67),
+ _57 = _mm_unpackhi_epi16(_45, _67);
+
+ __m128i rg0123 = _mm_unpacklo_epi16(_02, _13), // r0 r1 r2 r3 g0 g1 g2 g3
+ ba0123 = _mm_unpackhi_epi16(_02, _13), // b0 b1 b2 b3 a0 a1 a2 a3
+ rg4567 = _mm_unpacklo_epi16(_46, _57),
+ ba4567 = _mm_unpackhi_epi16(_46, _57);
+
+ *r = _mm_unpacklo_epi64(rg0123, rg4567);
+ *g = _mm_unpackhi_epi64(rg0123, rg4567);
+ *b = _mm_unpacklo_epi64(ba0123, ba4567);
+ *a = _mm_unpackhi_epi64(ba0123, ba4567);
+ }
+ static void Store4(void* ptr, const SkNx& r, const SkNx& g, const SkNx& b, const SkNx& a) {
+ // TODO: AVX2 version
+ __m128i rg0123 = _mm_unpacklo_epi16(r.fVec, g.fVec), // r0 g0 r1 g1 r2 g2 r3 g3
+ rg4567 = _mm_unpackhi_epi16(r.fVec, g.fVec), // r4 g4 r5 g5 r6 g6 r7 g7
+ ba0123 = _mm_unpacklo_epi16(b.fVec, a.fVec),
+ ba4567 = _mm_unpackhi_epi16(b.fVec, a.fVec);
+
+ _mm_storeu_si128((__m128i*)ptr + 0, _mm_unpacklo_epi32(rg0123, ba0123));
+ _mm_storeu_si128((__m128i*)ptr + 1, _mm_unpackhi_epi32(rg0123, ba0123));
+ _mm_storeu_si128((__m128i*)ptr + 2, _mm_unpacklo_epi32(rg4567, ba4567));
+ _mm_storeu_si128((__m128i*)ptr + 3, _mm_unpackhi_epi32(rg4567, ba4567));
+ }
+
SkNx operator + (const SkNx& o) const { return _mm_add_epi16(fVec, o.fVec); }
SkNx operator - (const SkNx& o) const { return _mm_sub_epi16(fVec, o.fVec); }
SkNx operator * (const SkNx& o) const { return _mm_mullo_epi16(fVec, o.fVec); }
__m128i fVec;
};
+#if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_AVX2
+
+// There are two different SkNx<8, uint8_t>, SkNx<8, int32_t>, SkNx<8, uint32_t>, SkNx<8, float>:
+// - the default paired SkNx<4, ...> versions used without AVX2
+// - the native AVX2 versions.
+// It is important that we don't call methods for one from the other.
+// Usually these methods inline, but they don't always in Debug builds.
+// For now, try to fix this by marking all the AVX2 versions as always-inline.
+// We may want or need to extend this strategy to all SkNx methods.
+#define I SK_ALWAYS_INLINE
+
+ template <>
+ class SkNx<8, uint8_t> {
+ public:
+ I SkNx(const __m128i& vec) : fVec(vec) {}
+
+ I SkNx() {}
+ I SkNx(uint8_t v) : fVec(_mm_set1_epi8(v)) {}
+ I SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d,
+ uint8_t e, uint8_t f, uint8_t g, uint8_t h)
+ : fVec(_mm_setr_epi8(a,b,c,d, e,f,g,h, 0,0,0,0, 0,0,0,0)) {}
+
+
+ I static SkNx Load(const void* ptr) { return _mm_loadl_epi64((const __m128i*)ptr); }
+ I void store(void* ptr) const { _mm_storel_epi64((__m128i*)ptr, fVec); }
+
+ I uint8_t operator[](int k) const {
+ SkASSERT(0 <= k && k < 8);
+ union { __m128i v; uint8_t us[16]; } pun = {fVec};
+ return pun.us[k&7];
+ }
+
+ __m128i fVec;
+ };
+
+ template <>
+ class SkNx<8, int32_t> {
+ public:
+ I SkNx(const __m256i& vec) : fVec(vec) {}
+
+ I SkNx() {}
+ I SkNx(int32_t v) : fVec(_mm256_set1_epi32(v)) {}
+ I SkNx(int32_t a, int32_t b, int32_t c, int32_t d,
+ int32_t e, int32_t f, int32_t g, int32_t h)
+ : fVec(_mm256_setr_epi32(a,b,c,d, e,f,g,h)) {}
+
+ I static SkNx Load(const void* ptr) { return _mm256_loadu_si256((const __m256i*)ptr); }
+ I void store(void* ptr) const { _mm256_storeu_si256((__m256i*)ptr, fVec); }
+
+ I SkNx operator + (const SkNx& o) const { return _mm256_add_epi32(fVec, o.fVec); }
+ I SkNx operator - (const SkNx& o) const { return _mm256_sub_epi32(fVec, o.fVec); }
+ I SkNx operator * (const SkNx& o) const { return _mm256_mullo_epi32(fVec, o.fVec); }
+
+ I SkNx operator & (const SkNx& o) const { return _mm256_and_si256(fVec, o.fVec); }
+ I SkNx operator | (const SkNx& o) const { return _mm256_or_si256(fVec, o.fVec); }
+ I SkNx operator ^ (const SkNx& o) const { return _mm256_xor_si256(fVec, o.fVec); }
+
+ I SkNx operator << (int bits) const { return _mm256_slli_epi32(fVec, bits); }
+ I SkNx operator >> (int bits) const { return _mm256_srai_epi32(fVec, bits); }
+
+ I int32_t operator[](int k) const {
+ SkASSERT(0 <= k && k < 8);
+ union { __m256i v; int32_t is[8]; } pun = {fVec};
+ return pun.is[k&7];
+ }
+
+ __m256i fVec;
+ };
+
+ template <>
+ class SkNx<8, uint32_t> {
+ public:
+ I SkNx(const __m256i& vec) : fVec(vec) {}
+
+ I SkNx() {}
+ I SkNx(uint32_t v) : fVec(_mm256_set1_epi32(v)) {}
+ I SkNx(uint32_t a, uint32_t b, uint32_t c, uint32_t d,
+ uint32_t e, uint32_t f, uint32_t g, uint32_t h)
+ : fVec(_mm256_setr_epi32(a,b,c,d, e,f,g,h)) {}
+
+ I static SkNx Load(const void* ptr) { return _mm256_loadu_si256((const __m256i*)ptr); }
+ I void store(void* ptr) const { _mm256_storeu_si256((__m256i*)ptr, fVec); }
+
+ I SkNx operator + (const SkNx& o) const { return _mm256_add_epi32(fVec, o.fVec); }
+ I SkNx operator - (const SkNx& o) const { return _mm256_sub_epi32(fVec, o.fVec); }
+ I SkNx operator * (const SkNx& o) const { return _mm256_mullo_epi32(fVec, o.fVec); }
+
+ I SkNx operator & (const SkNx& o) const { return _mm256_and_si256(fVec, o.fVec); }
+ I SkNx operator | (const SkNx& o) const { return _mm256_or_si256(fVec, o.fVec); }
+ I SkNx operator ^ (const SkNx& o) const { return _mm256_xor_si256(fVec, o.fVec); }
+
+ I SkNx operator << (int bits) const { return _mm256_slli_epi32(fVec, bits); }
+ I SkNx operator >> (int bits) const { return _mm256_srli_epi32(fVec, bits); }
+
+ I uint32_t operator[](int k) const {
+ SkASSERT(0 <= k && k < 8);
+ union { __m256i v; uint32_t us[8]; } pun = {fVec};
+ return pun.us[k&7];
+ }
+
+ __m256i fVec;
+ };
+
+ template <>
+ class SkNx<8, float> {
+ public:
+ I SkNx(const __m256& vec) : fVec(vec) {}
+
+ I SkNx() {}
+ I SkNx(float val) : fVec(_mm256_set1_ps(val)) {}
+ I SkNx(float a, float b, float c, float d,
+ float e, float f, float g, float h) : fVec(_mm256_setr_ps(a,b,c,d,e,f,g,h)) {}
+
+ I static SkNx Load(const void* ptr) { return _mm256_loadu_ps((const float*)ptr); }
+ I void store(void* ptr) const { _mm256_storeu_ps((float*)ptr, fVec); }
+
+ I SkNx operator+(const SkNx& o) const { return _mm256_add_ps(fVec, o.fVec); }
+ I SkNx operator-(const SkNx& o) const { return _mm256_sub_ps(fVec, o.fVec); }
+ I SkNx operator*(const SkNx& o) const { return _mm256_mul_ps(fVec, o.fVec); }
+ I SkNx operator/(const SkNx& o) const { return _mm256_div_ps(fVec, o.fVec); }
+
+ I SkNx operator==(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_EQ_OQ); }
+ I SkNx operator!=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_NEQ_OQ); }
+ I SkNx operator <(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_LT_OQ); }
+ I SkNx operator >(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_GT_OQ); }
+ I SkNx operator<=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_LE_OQ); }
+ I SkNx operator>=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_GE_OQ); }
+
+ I static SkNx Min(const SkNx& l, const SkNx& r) { return _mm256_min_ps(l.fVec, r.fVec); }
+ I static SkNx Max(const SkNx& l, const SkNx& r) { return _mm256_max_ps(l.fVec, r.fVec); }
+
+ I SkNx sqrt() const { return _mm256_sqrt_ps (fVec); }
+ I SkNx rsqrt() const { return _mm256_rsqrt_ps(fVec); }
+ I SkNx invert() const { return _mm256_rcp_ps (fVec); }
+
+ I float operator[](int k) const {
+ SkASSERT(0 <= k && k < 8);
+ union { __m256 v; float fs[8]; } pun = {fVec};
+ return pun.fs[k&7];
+ }
+
+ I SkNx thenElse(const SkNx& t, const SkNx& e) const {
+ return _mm256_blendv_ps(e.fVec, t.fVec, fVec);
+ }
+
+ __m256 fVec;
+ };
+
+ static I void SkNx_split(const Sk8f& v, Sk4f* lo, Sk4f* hi) {
+ *lo = _mm256_extractf128_ps(v.fVec, 0);
+ *hi = _mm256_extractf128_ps(v.fVec, 1);
+ }
+
+ static I Sk8f SkNx_join(const Sk4f& lo, const Sk4f& hi) {
+ return _mm256_insertf128_ps(_mm256_castps128_ps256(lo.fVec), hi.fVec, 1);
+ }
+
+ static I Sk8f SkNx_fma(const Sk8f& a, const Sk8f& b, const Sk8f& c) {
+ return _mm256_fmadd_ps(a.fVec, b.fVec, c.fVec);
+ }
+
+ template<> /*static*/ I Sk8f SkNx_cast<float>(const Sk8b& src) {
+ return _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(src.fVec));
+ }
+
+ template<> /*static*/ I Sk8f SkNx_cast<float>(const Sk8i& src) {
+ return _mm256_cvtepi32_ps(src.fVec);
+ }
+
+ template<> /*static*/ I Sk8i SkNx_cast<int>(const Sk8f& src) {
+ return _mm256_cvttps_epi32(src.fVec);
+ }
+
+ template<> /*static*/ I Sk8i SkNx_cast<int>(const Sk8h& src) {
+ return _mm256_cvtepu16_epi32(src.fVec);
+ }
+ template<> /*static*/ I Sk8h SkNx_cast<uint16_t>(const Sk8i& src) {
+ __m128i lo = _mm256_extractf128_si256(src.fVec, 0),
+ hi = _mm256_extractf128_si256(src.fVec, 1);
+ return _mm_packus_epi32(lo, hi);
+ }
+
+#undef I
+
+#endif
+
template<> /*static*/ inline Sk4f SkNx_cast<float, int32_t>(const Sk4i& src) {
return _mm_cvtepi32_ps(src.fVec);
}
#include "SkOpts.h"
+
#define SK_OPTS_NS hsw
+#include "SkRasterPipeline_opts.h"
namespace SkOpts {
- void Init_hsw() { }
+ void Init_hsw() {
+
+ run_pipeline = SK_OPTS_NS::run_pipeline;
+
+ #define STAGE(stage) \
+ body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \
+ tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage##_tail
+
+ STAGE(store_565);
+ STAGE(store_srgb);
+ STAGE(store_f16);
+
+ STAGE(load_s_565);
+ STAGE(load_s_srgb);
+ STAGE(load_s_f16);
+
+ STAGE(load_d_565);
+ STAGE(load_d_srgb);
+ STAGE(load_d_f16);
+
+ STAGE(scale_u8);
+
+ STAGE(lerp_u8);
+ STAGE(lerp_565);
+ #undef STAGE
+
+ #define STAGE(stage) \
+ body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \
+ tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage
+
+ STAGE(lerp_constant_float);
+ STAGE(constant_color);
+
+ STAGE(dst);
+ STAGE(dstatop);
+ STAGE(dstin);
+ STAGE(dstout);
+ STAGE(dstover);
+ STAGE(srcatop);
+ STAGE(srcin);
+ STAGE(srcout);
+ STAGE(srcover);
+ STAGE(clear);
+ STAGE(modulate);
+ STAGE(multiply);
+ STAGE(plus_);
+ STAGE(screen);
+ STAGE(xor_);
+ STAGE(colorburn);
+ STAGE(colordodge);
+ STAGE(darken);
+ STAGE(difference);
+ STAGE(exclusion);
+ STAGE(hardlight);
+ STAGE(lighten);
+ STAGE(overlay);
+ STAGE(softlight);
+ #undef STAGE
+
+ }
}
srcover_srgb_srgb = sse41::srcover_srgb_srgb;
blit_row_s32a_opaque = sse41::blit_row_s32a_opaque;
- #define STAGE(stage, kCallNext) \
- stages_4 [SkRasterPipeline::stage] = stage_4 <SK_OPTS_NS::stage, kCallNext>; \
- stages_1_3[SkRasterPipeline::stage] = stage_1_3<SK_OPTS_NS::stage, kCallNext>
+ #define STAGE(stage) \
+ body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \
+ tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage##_tail
- STAGE(store_565 , false);
- STAGE(store_srgb, false);
- STAGE(store_f16 , false);
+ STAGE(store_565);
+ STAGE(store_srgb);
+ STAGE(store_f16);
- STAGE(load_s_565 , true);
- STAGE(load_s_srgb, true);
- STAGE(load_s_f16 , true);
+ STAGE(load_s_565);
+ STAGE(load_s_srgb);
+ STAGE(load_s_f16);
- STAGE(load_d_565 , true);
- STAGE(load_d_srgb, true);
- STAGE(load_d_f16 , true);
+ STAGE(load_d_565);
+ STAGE(load_d_srgb);
+ STAGE(load_d_f16);
- STAGE(scale_u8, true);
+ STAGE(scale_u8);
- STAGE(lerp_u8 , true);
- STAGE(lerp_565 , true);
- STAGE(lerp_constant_float, true);
+ STAGE(lerp_u8);
+ STAGE(lerp_565);
+ #undef STAGE
- STAGE(constant_color, true);
+ #define STAGE(stage) \
+ body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \
+ tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage
- #undef STAGE
+ // The commented-out stages don't actually benefit from SSE 4.1.
+ // To cut down on code bloat we skip them here, using the identical SSE2 defaults.
- #define STAGE(stage) \
- stages_4 [SkRasterPipeline::stage] = SK_OPTS_NS::stage; \
- stages_1_3[SkRasterPipeline::stage] = SK_OPTS_NS::stage
+ //STAGE(lerp_constant_float);
+ //STAGE(constant_color);
- STAGE(dst);
- STAGE(dstatop);
- STAGE(dstin);
- STAGE(dstout);
- STAGE(dstover);
- STAGE(srcatop);
- STAGE(srcin);
- STAGE(srcout);
- STAGE(srcover);
- STAGE(clear);
- STAGE(modulate);
- STAGE(multiply);
- STAGE(plus_);
- STAGE(screen);
- STAGE(xor_);
+ //STAGE(dst);
+ //STAGE(dstatop);
+ //STAGE(dstin);
+ //STAGE(dstout);
+ //STAGE(dstover);
+ //STAGE(srcatop);
+ //STAGE(srcin);
+ //STAGE(srcout);
+ //STAGE(srcover);
+ //STAGE(clear);
+ //STAGE(modulate);
+ //STAGE(multiply);
+ //STAGE(plus_);
+ //STAGE(screen);
+ //STAGE(xor_);
STAGE(colorburn);
STAGE(colordodge);
- STAGE(darken);
- STAGE(difference);
- STAGE(exclusion);
+ //STAGE(darken);
+ //STAGE(difference);
+ //STAGE(exclusion);
STAGE(hardlight);
- STAGE(lighten);
+ //STAGE(lighten);
STAGE(overlay);
STAGE(softlight);
#undef STAGE
#include "SkRasterPipeline.h"
#include "SkSRGB.h"
-using Kernel_Sk4f = void(void*, size_t, size_t, Sk4f&, Sk4f&, Sk4f&, Sk4f&,
- Sk4f&, Sk4f&, Sk4f&, Sk4f&);
-
-// These are always static, and we _really_ want them to inline.
-// If you find yourself wanting a non-inline stage, write a SkRasterPipeline::Fn directly.
-#define KERNEL_Sk4f(name) \
- static SK_ALWAYS_INLINE void name(void* ctx, size_t x, size_t tail, \
- Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a, \
- Sk4f& dr, Sk4f& dg, Sk4f& db, Sk4f& da)
-
-
-template <Kernel_Sk4f kernel, bool kCallNext>
-static inline void SK_VECTORCALL stage_4(SkRasterPipeline::Stage* st, size_t x, size_t tail,
- Sk4f r, Sk4f g, Sk4f b, Sk4f a,
- Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) {
- // Passing 0 lets the optimizer completely drop any "if (tail) {...}" code in kernel.
- kernel(st->ctx<void*>(), x,0, r,g,b,a, dr,dg,db,da);
- if (kCallNext) {
- st->next(x,tail, r,g,b,a, dr,dg,db,da); // It's faster to pass t here than 0.
- }
-}
+using SkNf = SkRasterPipeline::V;
+static constexpr auto N = sizeof(SkNf) / sizeof(float);
+using SkNi = SkNx<N, int>;
+using SkNh = SkNx<N, uint16_t>;
+
+#define SI static inline
+
+#define STAGE(name, kCallNext) \
+ static SK_ALWAYS_INLINE void name##_kernel(void* ctx, size_t x, size_t tail, \
+ SkNf& r, SkNf& g, SkNf& b, SkNf& a, \
+ SkNf& dr, SkNf& dg, SkNf& db, SkNf& da); \
+ SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \
+ SkNf r, SkNf g, SkNf b, SkNf a, \
+ SkNf dr, SkNf dg, SkNf db, SkNf da) { \
+ name##_kernel(st->ctx<void*>(), x,0, r,g,b,a, dr,dg,db,da); \
+ if (kCallNext) { \
+ st->next(x,tail, r,g,b,a, dr,dg,db,da); \
+ } \
+ } \
+ SI void SK_VECTORCALL name##_tail(SkRasterPipeline::Stage* st, size_t x, size_t tail, \
+ SkNf r, SkNf g, SkNf b, SkNf a, \
+ SkNf dr, SkNf dg, SkNf db, SkNf da) { \
+ name##_kernel(st->ctx<void*>(), x,tail, r,g,b,a, dr,dg,db,da); \
+ if (kCallNext) { \
+ st->next(x,tail, r,g,b,a, dr,dg,db,da); \
+ } \
+ } \
+ static SK_ALWAYS_INLINE void name##_kernel(void* ctx, size_t x, size_t tail, \
+ SkNf& r, SkNf& g, SkNf& b, SkNf& a, \
+ SkNf& dr, SkNf& dg, SkNf& db, SkNf& da)
-template <Kernel_Sk4f kernel, bool kCallNext>
-static inline void SK_VECTORCALL stage_1_3(SkRasterPipeline::Stage* st, size_t x, size_t tail,
- Sk4f r, Sk4f g, Sk4f b, Sk4f a,
- Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) {
-#if defined(__clang__)
- __builtin_assume(tail > 0); // This flourish lets Clang compile away any tail==0 code.
-#endif
- kernel(st->ctx<void*>(), x,tail, r,g,b,a, dr,dg,db,da);
- if (kCallNext) {
- st->next(x,tail, r,g,b,a, dr,dg,db,da);
- }
-}
// Many xfermodes apply the same logic to each channel.
-#define RGBA_XFERMODE_Sk4f(name) \
- static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa, \
- const Sk4f& d, const Sk4f& da); \
- static void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \
- Sk4f r, Sk4f g, Sk4f b, Sk4f a, \
- Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) { \
- r = name##_kernel(r,a,dr,da); \
- g = name##_kernel(g,a,dg,da); \
- b = name##_kernel(b,a,db,da); \
- a = name##_kernel(a,a,da,da); \
- st->next(x,tail, r,g,b,a, dr,dg,db,da); \
- } \
- static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa, \
- const Sk4f& d, const Sk4f& da)
+#define RGBA_XFERMODE(name) \
+ static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \
+ const SkNf& d, const SkNf& da); \
+ SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \
+ SkNf r, SkNf g, SkNf b, SkNf a, \
+ SkNf dr, SkNf dg, SkNf db, SkNf da) { \
+ r = name##_kernel(r,a,dr,da); \
+ g = name##_kernel(g,a,dg,da); \
+ b = name##_kernel(b,a,db,da); \
+ a = name##_kernel(a,a,da,da); \
+ st->next(x,tail, r,g,b,a, dr,dg,db,da); \
+ } \
+ static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \
+ const SkNf& d, const SkNf& da)
// Most of the rest apply the same logic to color channels and use srcover's alpha logic.
-#define RGB_XFERMODE_Sk4f(name) \
- static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa, \
- const Sk4f& d, const Sk4f& da); \
- static void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \
- Sk4f r, Sk4f g, Sk4f b, Sk4f a, \
- Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) { \
- r = name##_kernel(r,a,dr,da); \
- g = name##_kernel(g,a,dg,da); \
- b = name##_kernel(b,a,db,da); \
- a = a + (da * (1.0f-a)); \
- st->next(x,tail, r,g,b,a, dr,dg,db,da); \
- } \
- static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa, \
- const Sk4f& d, const Sk4f& da)
+#define RGB_XFERMODE(name) \
+ static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \
+ const SkNf& d, const SkNf& da); \
+ SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \
+ SkNf r, SkNf g, SkNf b, SkNf a, \
+ SkNf dr, SkNf dg, SkNf db, SkNf da) { \
+ r = name##_kernel(r,a,dr,da); \
+ g = name##_kernel(g,a,dg,da); \
+ b = name##_kernel(b,a,db,da); \
+ a = a + (da * (1.0f-a)); \
+ st->next(x,tail, r,g,b,a, dr,dg,db,da); \
+ } \
+ static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \
+ const SkNf& d, const SkNf& da)
+
namespace SK_OPTS_NS {
+ SI void run_pipeline(size_t x, size_t n,
+ void (*vBodyStart)(), SkRasterPipeline::Stage* body,
+ void (*vTailStart)(), SkRasterPipeline::Stage* tail) {
+ auto bodyStart = (SkRasterPipeline::Fn)vBodyStart,
+ tailStart = (SkRasterPipeline::Fn)vTailStart;
+ SkNf v; // Fastest to start uninitialized.
+ while (n >= N) {
+ bodyStart(body, x,0, v,v,v,v, v,v,v,v);
+ x += N;
+ n -= N;
+ }
+ if (n > 0) {
+ tailStart(tail, x,n, v,v,v,v, v,v,v,v);
+ }
+ }
+
// Clamp colors into [0,1] premul (e.g. just before storing back to memory).
- static void clamp_01_premul(Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a) {
- a = Sk4f::Max(a, 0.0f);
- r = Sk4f::Max(r, 0.0f);
- g = Sk4f::Max(g, 0.0f);
- b = Sk4f::Max(b, 0.0f);
-
- a = Sk4f::Min(a, 1.0f);
- r = Sk4f::Min(r, a);
- g = Sk4f::Min(g, a);
- b = Sk4f::Min(b, a);
+ SI void clamp_01_premul(SkNf& r, SkNf& g, SkNf& b, SkNf& a) {
+ a = SkNf::Max(a, 0.0f);
+ r = SkNf::Max(r, 0.0f);
+ g = SkNf::Max(g, 0.0f);
+ b = SkNf::Max(b, 0.0f);
+
+ a = SkNf::Min(a, 1.0f);
+ r = SkNf::Min(r, a);
+ g = SkNf::Min(g, a);
+ b = SkNf::Min(b, a);
}
- static Sk4f inv(const Sk4f& x) { return 1.0f - x; }
+ SI SkNf inv(const SkNf& x) { return 1.0f - x; }
- static Sk4f lerp(const Sk4f& from, const Sk4f& to, const Sk4f& cov) {
- return from + (to-from)*cov;
+ SI SkNf lerp(const SkNf& from, const SkNf& to, const SkNf& cov) {
+ return SkNx_fma(to-from, cov, from);
}
template <typename T>
- static SkNx<4,T> load_tail(size_t tail, const T* src) {
+ SI SkNx<N,T> load_tail(size_t tail, const T* src) {
+ // TODO: better tail, maskload for 32- and 64-bit T
+ T buf[N] = {0};
if (tail) {
- return SkNx<4,T>(src[0], (tail>1 ? src[1] : 0), (tail>2 ? src[2] : 0), 0);
+ memcpy(buf, src, tail*sizeof(T));
+ src = buf;
}
- return SkNx<4,T>::Load(src);
+ return SkNx<N,T>::Load(src);
}
template <typename T>
- static void store_tail(size_t tail, const SkNx<4,T>& v, T* dst) {
- switch(tail) {
- case 0: return v.store(dst);
- case 3: dst[2] = v[2];
- case 2: dst[1] = v[1];
- case 1: dst[0] = v[0];
+ SI void store_tail(size_t tail, const SkNx<N,T>& v, T* dst) {
+ // TODO: better tail, maskstore for 32- and 64-bit T
+ T buf[N] = {0};
+ v.store(tail ? buf : dst);
+ if (tail) {
+ memcpy(dst, buf, tail*sizeof(T));
}
}
- static void from_565(const Sk4h& _565, Sk4f* r, Sk4f* g, Sk4f* b) {
- Sk4i _32_bit = SkNx_cast<int>(_565);
+ SI void from_565(const SkNh& _565, SkNf* r, SkNf* g, SkNf* b) {
+ auto _32_bit = SkNx_cast<int>(_565);
*r = SkNx_cast<float>(_32_bit & SK_R16_MASK_IN_PLACE) * (1.0f / SK_R16_MASK_IN_PLACE);
*g = SkNx_cast<float>(_32_bit & SK_G16_MASK_IN_PLACE) * (1.0f / SK_G16_MASK_IN_PLACE);
*b = SkNx_cast<float>(_32_bit & SK_B16_MASK_IN_PLACE) * (1.0f / SK_B16_MASK_IN_PLACE);
}
- static Sk4h to_565(const Sk4f& r, const Sk4f& g, const Sk4f& b) {
- return SkNx_cast<uint16_t>( Sk4f_round(r * SK_R16_MASK) << SK_R16_SHIFT
- | Sk4f_round(g * SK_G16_MASK) << SK_G16_SHIFT
- | Sk4f_round(b * SK_B16_MASK) << SK_B16_SHIFT);
+ SI SkNh to_565(const SkNf& r, const SkNf& g, const SkNf& b) {
+ return SkNx_cast<uint16_t>( SkNx_cast<int>(r * SK_R16_MASK + 0.5f) << SK_R16_SHIFT
+ | SkNx_cast<int>(g * SK_G16_MASK + 0.5f) << SK_G16_SHIFT
+ | SkNx_cast<int>(b * SK_B16_MASK + 0.5f) << SK_B16_SHIFT);
}
+ STAGE(just_return, false) { }
// The default shader produces a constant color (from the SkPaint).
- KERNEL_Sk4f(constant_color) {
+ STAGE(constant_color, true) {
auto color = (const SkPM4f*)ctx;
r = color->r();
g = color->g();
}
// s' = d(1-c) + sc, for a constant c.
- KERNEL_Sk4f(lerp_constant_float) {
- Sk4f c = *(const float*)ctx;
+ STAGE(lerp_constant_float, true) {
+ SkNf c = *(const float*)ctx;
r = lerp(dr, r, c);
g = lerp(dg, g, c);
}
// s' = sc for 8-bit c.
- KERNEL_Sk4f(scale_u8) {
+ STAGE(scale_u8, true) {
auto ptr = (const uint8_t*)ctx + x;
- Sk4f c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
+ SkNf c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
r = r*c;
g = g*c;
b = b*c;
}
// s' = d(1-c) + sc for 8-bit c.
- KERNEL_Sk4f(lerp_u8) {
+ STAGE(lerp_u8, true) {
auto ptr = (const uint8_t*)ctx + x;
- Sk4f c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
+ SkNf c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
r = lerp(dr, r, c);
g = lerp(dg, g, c);
b = lerp(db, b, c);
}
// s' = d(1-c) + sc for 565 c.
- KERNEL_Sk4f(lerp_565) {
+ STAGE(lerp_565, true) {
auto ptr = (const uint16_t*)ctx + x;
- Sk4f cr, cg, cb;
+ SkNf cr, cg, cb;
from_565(load_tail(tail, ptr), &cr, &cg, &cb);
r = lerp(dr, r, cr);
a = 1.0f;
}
- KERNEL_Sk4f(load_d_565) {
+ STAGE(load_d_565, true) {
auto ptr = (const uint16_t*)ctx + x;
from_565(load_tail(tail, ptr), &dr,&dg,&db);
da = 1.0f;
}
- KERNEL_Sk4f(load_s_565) {
+ STAGE(load_s_565, true) {
auto ptr = (const uint16_t*)ctx + x;
from_565(load_tail(tail, ptr), &r,&g,&b);
a = 1.0f;
}
- KERNEL_Sk4f(store_565) {
+ STAGE(store_565, false) {
clamp_01_premul(r,g,b,a);
auto ptr = (uint16_t*)ctx + x;
store_tail(tail, to_565(r,g,b), ptr);
}
- KERNEL_Sk4f(load_d_f16) {
+ STAGE(load_d_f16, true) {
auto ptr = (const uint64_t*)ctx + x;
+ uint64_t buf[N] = {0};
if (tail) {
- auto p0 = SkHalfToFloat_finite_ftz(ptr[0]) ,
- p1 = tail>1 ? SkHalfToFloat_finite_ftz(ptr[1]) : Sk4f{0},
- p2 = tail>2 ? SkHalfToFloat_finite_ftz(ptr[2]) : Sk4f{0};
- dr = { p0[0],p1[0],p2[0],0 };
- dg = { p0[1],p1[1],p2[1],0 };
- db = { p0[2],p1[2],p2[2],0 };
- da = { p0[3],p1[3],p2[3],0 };
- return;
+ memcpy(buf, ptr, tail*sizeof(uint64_t));
+ ptr = buf;
}
- Sk4h rh, gh, bh, ah;
- Sk4h::Load4(ptr, &rh, &gh, &bh, &ah);
+ SkNh rh, gh, bh, ah;
+ SkNh::Load4(ptr, &rh, &gh, &bh, &ah);
dr = SkHalfToFloat_finite_ftz(rh);
dg = SkHalfToFloat_finite_ftz(gh);
db = SkHalfToFloat_finite_ftz(bh);
da = SkHalfToFloat_finite_ftz(ah);
}
- KERNEL_Sk4f(load_s_f16) {
+ STAGE(load_s_f16, true) {
auto ptr = (const uint64_t*)ctx + x;
+ uint64_t buf[N] = {0};
if (tail) {
- auto p0 = SkHalfToFloat_finite_ftz(ptr[0]) ,
- p1 = tail>1 ? SkHalfToFloat_finite_ftz(ptr[1]) : Sk4f{0},
- p2 = tail>2 ? SkHalfToFloat_finite_ftz(ptr[2]) : Sk4f{0};
- r = { p0[0],p1[0],p2[0],0 };
- g = { p0[1],p1[1],p2[1],0 };
- b = { p0[2],p1[2],p2[2],0 };
- a = { p0[3],p1[3],p2[3],0 };
- return;
+ memcpy(buf, ptr, tail*sizeof(uint64_t));
+ ptr = buf;
}
- Sk4h rh, gh, bh, ah;
- Sk4h::Load4(ptr, &rh, &gh, &bh, &ah);
+ SkNh rh, gh, bh, ah;
+ SkNh::Load4(ptr, &rh, &gh, &bh, &ah);
r = SkHalfToFloat_finite_ftz(rh);
g = SkHalfToFloat_finite_ftz(gh);
b = SkHalfToFloat_finite_ftz(bh);
a = SkHalfToFloat_finite_ftz(ah);
}
- KERNEL_Sk4f(store_f16) {
+ STAGE(store_f16, false) {
clamp_01_premul(r,g,b,a);
auto ptr = (uint64_t*)ctx + x;
- switch (tail) {
- case 0: return Sk4h::Store4(ptr, SkFloatToHalf_finite_ftz(r),
- SkFloatToHalf_finite_ftz(g),
- SkFloatToHalf_finite_ftz(b),
- SkFloatToHalf_finite_ftz(a));
-
- case 3: SkFloatToHalf_finite_ftz({r[2], g[2], b[2], a[2]}).store(ptr+2);
- case 2: SkFloatToHalf_finite_ftz({r[1], g[1], b[1], a[1]}).store(ptr+1);
- case 1: SkFloatToHalf_finite_ftz({r[0], g[0], b[0], a[0]}).store(ptr+0);
+ uint64_t buf[N] = {0};
+ SkNh::Store4(tail ? buf : ptr, SkFloatToHalf_finite_ftz(r),
+ SkFloatToHalf_finite_ftz(g),
+ SkFloatToHalf_finite_ftz(b),
+ SkFloatToHalf_finite_ftz(a));
+ if (tail) {
+ memcpy(ptr, buf, tail*sizeof(uint64_t));
}
}
// Load 8-bit SkPMColor-order sRGB.
- KERNEL_Sk4f(load_d_srgb) {
+ STAGE(load_d_srgb, true) {
auto ptr = (const uint32_t*)ctx + x;
- auto px = load_tail(tail, (const int*)ptr);
- dr = sk_linear_from_srgb_math((px >> SK_R32_SHIFT) & 0xff);
- dg = sk_linear_from_srgb_math((px >> SK_G32_SHIFT) & 0xff);
- db = sk_linear_from_srgb_math((px >> SK_B32_SHIFT) & 0xff);
- da = (1/255.0f)*SkNx_cast<float>((px >> SK_A32_SHIFT) & 0xff);
+ auto px = load_tail(tail, ptr);
+ auto to_int = [](const SkNx<N, uint32_t>& v) { return SkNi::Load(&v); };
+ dr = sk_linear_from_srgb_math(to_int((px >> SK_R32_SHIFT) & 0xff));
+ dg = sk_linear_from_srgb_math(to_int((px >> SK_G32_SHIFT) & 0xff));
+ db = sk_linear_from_srgb_math(to_int((px >> SK_B32_SHIFT) & 0xff));
+ da = (1/255.0f)*SkNx_cast<float>(to_int( px >> SK_A32_SHIFT ));
}
- KERNEL_Sk4f(load_s_srgb) {
+ STAGE(load_s_srgb, true) {
auto ptr = (const uint32_t*)ctx + x;
- auto px = load_tail(tail, (const int*)ptr);
- r = sk_linear_from_srgb_math((px >> SK_R32_SHIFT) & 0xff);
- g = sk_linear_from_srgb_math((px >> SK_G32_SHIFT) & 0xff);
- b = sk_linear_from_srgb_math((px >> SK_B32_SHIFT) & 0xff);
- a = (1/255.0f)*SkNx_cast<float>((px >> SK_A32_SHIFT) & 0xff);
+ auto px = load_tail(tail, ptr);
+ auto to_int = [](const SkNx<N, uint32_t>& v) { return SkNi::Load(&v); };
+ r = sk_linear_from_srgb_math(to_int((px >> SK_R32_SHIFT) & 0xff));
+ g = sk_linear_from_srgb_math(to_int((px >> SK_G32_SHIFT) & 0xff));
+ b = sk_linear_from_srgb_math(to_int((px >> SK_B32_SHIFT) & 0xff));
+ a = (1/255.0f)*SkNx_cast<float>(to_int( px >> SK_A32_SHIFT ));
}
- KERNEL_Sk4f(store_srgb) {
+ STAGE(store_srgb, false) {
clamp_01_premul(r,g,b,a);
auto ptr = (uint32_t*)ctx + x;
- store_tail(tail, ( sk_linear_to_srgb_noclamp(r) << SK_R32_SHIFT
- | sk_linear_to_srgb_noclamp(g) << SK_G32_SHIFT
- | sk_linear_to_srgb_noclamp(b) << SK_B32_SHIFT
- | Sk4f_round(255.0f * a) << SK_A32_SHIFT), (int*)ptr);
+ store_tail(tail, ( sk_linear_to_srgb_noclamp(r) << SK_R32_SHIFT
+ | sk_linear_to_srgb_noclamp(g) << SK_G32_SHIFT
+ | sk_linear_to_srgb_noclamp(b) << SK_B32_SHIFT
+ | SkNx_cast<int>(255.0f * a + 0.5f) << SK_A32_SHIFT ), (int*)ptr);
}
- RGBA_XFERMODE_Sk4f(clear) { return 0.0f; }
- //RGBA_XFERMODE_Sk4f(src) { return s; } // This would be a no-op stage, so we just omit it.
- RGBA_XFERMODE_Sk4f(dst) { return d; }
-
- RGBA_XFERMODE_Sk4f(srcatop) { return s*da + d*inv(sa); }
- RGBA_XFERMODE_Sk4f(srcin) { return s * da; }
- RGBA_XFERMODE_Sk4f(srcout) { return s * inv(da); }
- RGBA_XFERMODE_Sk4f(srcover) { return s + inv(sa)*d; }
- RGBA_XFERMODE_Sk4f(dstatop) { return srcatop_kernel(d,da,s,sa); }
- RGBA_XFERMODE_Sk4f(dstin) { return srcin_kernel (d,da,s,sa); }
- RGBA_XFERMODE_Sk4f(dstout) { return srcout_kernel (d,da,s,sa); }
- RGBA_XFERMODE_Sk4f(dstover) { return srcover_kernel(d,da,s,sa); }
-
- RGBA_XFERMODE_Sk4f(modulate) { return s*d; }
- RGBA_XFERMODE_Sk4f(multiply) { return s*inv(da) + d*inv(sa) + s*d; }
- RGBA_XFERMODE_Sk4f(plus_) { return s + d; }
- RGBA_XFERMODE_Sk4f(screen) { return s + d - s*d; }
- RGBA_XFERMODE_Sk4f(xor_) { return s*inv(da) + d*inv(sa); }
-
- RGB_XFERMODE_Sk4f(colorburn) {
+ RGBA_XFERMODE(clear) { return 0.0f; }
+ //RGBA_XFERMODE(src) { return s; } // This would be a no-op stage, so we just omit it.
+ RGBA_XFERMODE(dst) { return d; }
+
+ RGBA_XFERMODE(srcatop) { return s*da + d*inv(sa); }
+ RGBA_XFERMODE(srcin) { return s * da; }
+ RGBA_XFERMODE(srcout) { return s * inv(da); }
+ RGBA_XFERMODE(srcover) { return SkNx_fma(d, inv(sa), s); }
+ RGBA_XFERMODE(dstatop) { return srcatop_kernel(d,da,s,sa); }
+ RGBA_XFERMODE(dstin) { return srcin_kernel (d,da,s,sa); }
+ RGBA_XFERMODE(dstout) { return srcout_kernel (d,da,s,sa); }
+ RGBA_XFERMODE(dstover) { return srcover_kernel(d,da,s,sa); }
+
+ RGBA_XFERMODE(modulate) { return s*d; }
+ RGBA_XFERMODE(multiply) { return s*inv(da) + d*inv(sa) + s*d; }
+ RGBA_XFERMODE(plus_) { return s + d; }
+ RGBA_XFERMODE(screen) { return s + d - s*d; }
+ RGBA_XFERMODE(xor_) { return s*inv(da) + d*inv(sa); }
+
+ RGB_XFERMODE(colorburn) {
return (d == da ).thenElse(d + s*inv(da),
(s == 0.0f).thenElse(s + d*inv(sa),
- sa*(da - Sk4f::Min(da, (da-d)*sa/s)) + s*inv(da) + d*inv(sa)));
+ sa*(da - SkNf::Min(da, (da-d)*sa/s)) + s*inv(da) + d*inv(sa)));
}
- RGB_XFERMODE_Sk4f(colordodge) {
+ RGB_XFERMODE(colordodge) {
return (d == 0.0f).thenElse(d + s*inv(da),
(s == sa ).thenElse(s + d*inv(sa),
- sa*Sk4f::Min(da, (d*sa)/(sa - s)) + s*inv(da) + d*inv(sa)));
+ sa*SkNf::Min(da, (d*sa)/(sa - s)) + s*inv(da) + d*inv(sa)));
}
- RGB_XFERMODE_Sk4f(darken) { return s + d - Sk4f::Max(s*da, d*sa); }
- RGB_XFERMODE_Sk4f(difference) { return s + d - 2.0f*Sk4f::Min(s*da,d*sa); }
- RGB_XFERMODE_Sk4f(exclusion) { return s + d - 2.0f*s*d; }
- RGB_XFERMODE_Sk4f(hardlight) {
+ RGB_XFERMODE(darken) { return s + d - SkNf::Max(s*da, d*sa); }
+ RGB_XFERMODE(difference) { return s + d - 2.0f*SkNf::Min(s*da,d*sa); }
+ RGB_XFERMODE(exclusion) { return s + d - 2.0f*s*d; }
+ RGB_XFERMODE(hardlight) {
return s*inv(da) + d*inv(sa)
+ (2.0f*s <= sa).thenElse(2.0f*s*d, sa*da - 2.0f*(da-d)*(sa-s));
}
- RGB_XFERMODE_Sk4f(lighten) { return s + d - Sk4f::Min(s*da, d*sa); }
- RGB_XFERMODE_Sk4f(overlay) { return hardlight_kernel(d,da,s,sa); }
- RGB_XFERMODE_Sk4f(softlight) {
- Sk4f m = (da > 0.0f).thenElse(d / da, 0.0f),
+ RGB_XFERMODE(lighten) { return s + d - SkNf::Min(s*da, d*sa); }
+ RGB_XFERMODE(overlay) { return hardlight_kernel(d,da,s,sa); }
+ RGB_XFERMODE(softlight) {
+ SkNf m = (da > 0.0f).thenElse(d / da, 0.0f),
s2 = 2.0f*s,
m4 = 4.0f*m;
// 1. dark src?
// 2. light src, dark dst?
// 3. light src, light dst?
- Sk4f darkSrc = d*(sa + (s2 - sa)*(1.0f - m)), // Used in case 1.
+ SkNf darkSrc = d*(sa + (s2 - sa)*(1.0f - m)), // Used in case 1.
darkDst = (m4*m4 + m4)*(m - 1.0f) + 7.0f*m, // Used in case 2.
liteDst = m.rsqrt().invert() - m, // Used in case 3.
liteSrc = d*sa + da*(s2 - sa) * (4.0f*d <= da).thenElse(darkDst, liteDst); // 2 or 3?
}
}
-#undef KERNEL_Sk4f
-#undef RGB_XFERMODE_Sk4f
-#undef RGB_XFERMODE_Sk4f
+#undef SI
+#undef STAGE
+#undef RGBA_XFERMODE
+#undef RGB_XFERMODE
#endif//SkRasterPipeline_opts_DEFINED
projects / platform / upstream / libSkiaSharp.git / commitdiff