src/core/SkColorSpaceXform.cpp

   1 /*
   2  * Copyright 2016 Google Inc.
   3  *
   4  * Use of this source code is governed by a BSD-style license that can be
   5  * found in the LICENSE file.
   6  */
   7
   8 #include "SkColorPriv.h"
   9 #include "SkColorSpace_A2B.h"
  10 #include "SkColorSpace_Base.h"
  11 #include "SkColorSpace_XYZ.h"
  12 #include "SkColorSpacePriv.h"
  13 #include "SkColorSpaceXform_A2B.h"
  14 #include "SkColorSpaceXform_Base.h"
  15 #include "SkColorSpaceXformPriv.h"
  16 #include "SkHalf.h"
  17 #include "SkOpts.h"
  18 #include "SkPM4fPriv.h"
  19 #include "SkRasterPipeline.h"
  20 #include "SkSRGB.h"
  21
  22 static constexpr float sk_linear_from_2dot2[256] = {
  23         0.000000000000000000f, 0.000005077051900662f, 0.000023328004666099f, 0.000056921765712193f,
  24         0.000107187362341244f, 0.000175123977503027f, 0.000261543754548491f, 0.000367136269815943f,
  25         0.000492503787191433f, 0.000638182842167022f, 0.000804658499513058f, 0.000992374304074325f,
  26         0.001201739522438400f, 0.001433134589671860f, 0.001686915316789280f, 0.001963416213396470f,
  27         0.002262953160706430f, 0.002585825596234170f, 0.002932318323938360f, 0.003302703032003640f,
  28         0.003697239578900130f, 0.004116177093282750f, 0.004559754922526020f, 0.005028203456855540f,
  29         0.005521744850239660f, 0.006040593654849810f, 0.006584957382581690f, 0.007155037004573030f,
  30         0.007751027397660610f, 0.008373117745148580f, 0.009021491898012130f, 0.009696328701658230f,
  31         0.010397802292555300f, 0.011126082368383200f, 0.011881334434813700f, 0.012663720031582100f,
  32         0.013473396940142600f, 0.014310519374884100f, 0.015175238159625200f, 0.016067700890886900f,
  33         0.016988052089250000f, 0.017936433339950200f, 0.018912983423721500f, 0.019917838438785700f,
  34         0.020951131914781100f, 0.022012994919336500f, 0.023103556157921400f, 0.024222942067534200f,
  35         0.025371276904734600f, 0.026548682828472900f, 0.027755279978126000f, 0.028991186547107800f,
  36         0.030256518852388700f, 0.031551391400226400f, 0.032875916948383800f, 0.034230206565082000f,
  37         0.035614369684918800f, 0.037028514161960200f, 0.038472746320194600f, 0.039947171001525600f,
  38         0.041451891611462500f, 0.042987010162657100f, 0.044552627316421400f, 0.046148842422351000f,
  39         0.047775753556170600f, 0.049433457555908000f, 0.051122050056493400f, 0.052841625522879000f,
  40         0.054592277281760300f, 0.056374097551979800f, 0.058187177473685400f, 0.060031607136313200f,
  41         0.061907475605455800f, 0.063814870948677200f, 0.065753880260330100f, 0.067724589685424300f,
  42         0.069727084442598800f, 0.071761448846239100f, 0.073827766327784600f, 0.075926119456264800f,
  43         0.078056589958101900f, 0.080219258736215100f, 0.082414205888459200f, 0.084641510725429500f,
  44         0.086901251787660300f, 0.089193506862247800f, 0.091518352998919500f, 0.093875866525577800f,
  45         0.096266123063339700f, 0.098689197541094500f, 0.101145164209600000f, 0.103634096655137000f,
  46         0.106156067812744000f, 0.108711149979039000f, 0.111299414824660000f, 0.113920933406333000f,
  47         0.116575776178572000f, 0.119264013005047000f, 0.121985713169619000f, 0.124740945387051000f,
  48         0.127529777813422000f, 0.130352278056244000f, 0.133208513184300000f, 0.136098549737202000f,
  49         0.139022453734703000f, 0.141980290685736000f, 0.144972125597231000f, 0.147998022982685000f,
  50         0.151058046870511000f, 0.154152260812165000f, 0.157280727890073000f, 0.160443510725344000f,
  51         0.163640671485290000f, 0.166872271890766000f, 0.170138373223312000f, 0.173439036332135000f,
  52         0.176774321640903000f, 0.180144289154390000f, 0.183548998464951000f, 0.186988508758844000f,
  53         0.190462878822409000f, 0.193972167048093000f, 0.197516431440340000f, 0.201095729621346000f,
  54         0.204710118836677000f, 0.208359655960767000f, 0.212044397502288000f, 0.215764399609395000f,
  55         0.219519718074868000f, 0.223310408341127000f, 0.227136525505149000f, 0.230998124323267000f,
  56         0.234895259215880000f, 0.238827984272048000f, 0.242796353254002000f, 0.246800419601550000f,
  57         0.250840236436400000f, 0.254915856566385000f, 0.259027332489606000f, 0.263174716398492000f,
  58         0.267358060183772000f, 0.271577415438375000f, 0.275832833461245000f, 0.280124365261085000f,
  59         0.284452061560024000f, 0.288815972797219000f, 0.293216149132375000f, 0.297652640449211000f,
  60         0.302125496358853000f, 0.306634766203158000f, 0.311180499057984000f, 0.315762743736397000f,
  61         0.320381548791810000f, 0.325036962521076000f, 0.329729032967515000f, 0.334457807923889000f,
  62         0.339223334935327000f, 0.344025661302187000f, 0.348864834082879000f, 0.353740900096629000f,
  63         0.358653905926199000f, 0.363603897920553000f, 0.368590922197487000f, 0.373615024646202000f,
  64         0.378676250929840000f, 0.383774646487975000f, 0.388910256539059000f, 0.394083126082829000f,
  65         0.399293299902674000f, 0.404540822567962000f, 0.409825738436323000f, 0.415148091655907000f,
  66         0.420507926167587000f, 0.425905285707146000f, 0.431340213807410000f, 0.436812753800359000f,
  67         0.442322948819202000f, 0.447870841800410000f, 0.453456475485731000f, 0.459079892424160000f,
  68         0.464741134973889000f, 0.470440245304218000f, 0.476177265397440000f, 0.481952237050698000f,
  69         0.487765201877811000f, 0.493616201311074000f, 0.499505276603030000f, 0.505432468828216000f,
  70         0.511397818884880000f, 0.517401367496673000f, 0.523443155214325000f, 0.529523222417277000f,
  71         0.535641609315311000f, 0.541798355950137000f, 0.547993502196972000f, 0.554227087766085000f,
  72         0.560499152204328000f, 0.566809734896638000f, 0.573158875067523000f, 0.579546611782525000f,
  73         0.585972983949661000f, 0.592438030320847000f, 0.598941789493296000f, 0.605484299910907000f,
  74         0.612065599865624000f, 0.618685727498780000f, 0.625344720802427000f, 0.632042617620641000f,
  75         0.638779455650817000f, 0.645555272444935000f, 0.652370105410821000f, 0.659223991813387000f,
  76         0.666116968775851000f, 0.673049073280942000f, 0.680020342172095000f, 0.687030812154625000f,
  77         0.694080519796882000f, 0.701169501531402000f, 0.708297793656032000f, 0.715465432335048000f,
  78         0.722672453600255000f, 0.729918893352071000f, 0.737204787360605000f, 0.744530171266715000f,
  79         0.751895080583051000f, 0.759299550695091000f, 0.766743616862161000f, 0.774227314218442000f,
  80         0.781750677773962000f, 0.789313742415586000f, 0.796916542907978000f, 0.804559113894567000f,
  81         0.812241489898490000f, 0.819963705323528000f, 0.827725794455034000f, 0.835527791460841000f,
  82         0.843369730392169000f, 0.851251645184515000f, 0.859173569658532000f, 0.867135537520905000f,
  83         0.875137582365205000f, 0.883179737672745000f, 0.891262036813419000f, 0.899384513046529000f,
  84         0.907547199521614000f, 0.915750129279253000f, 0.923993335251873000f, 0.932276850264543000f,
  85         0.940600707035753000f, 0.948964938178195000f, 0.957369576199527000f, 0.965814653503130000f,
  86         0.974300202388861000f, 0.982826255053791000f, 0.991392843592940000f, 1.000000000000000000f,
  87 };
  88
  89 ///////////////////////////////////////////////////////////////////////////////////////////////////
  90
  91 static void build_table_linear_from_gamma(float* outTable, float exponent) {
  92     for (float x = 0.0f; x <= 1.0f; x += (1.0f/255.0f)) {
  93         *outTable++ = powf(x, exponent);
  94     }
  95 }
  96
  97 // outTable is always 256 entries, inTable may be larger or smaller.
  98 static void build_table_linear_from_gamma(float* outTable, const float* inTable,
  99                                           int inTableSize) {
 100     if (256 == inTableSize) {
 101         memcpy(outTable, inTable, sizeof(float) * 256);
 102         return;
 103     }
 104
 105     for (float x = 0.0f; x <= 1.0f; x += (1.0f/255.0f)) {
 106         *outTable++ = interp_lut(x, inTable, inTableSize);
 107     }
 108 }
 109
 110
 111 static void build_table_linear_from_gamma(float* outTable, float g, float a, float b, float c,
 112                                           float d, float e, float f) {
 113     // Y = (aX + b)^g + e  for X >= d
 114     // Y = cX + f          otherwise
 115     for (float x = 0.0f; x <= 1.0f; x += (1.0f/255.0f)) {
 116         if (x >= d) {
 117             *outTable++ = clamp_0_1(powf(a * x + b, g) + e);
 118         } else {
 119             *outTable++ = clamp_0_1(c * x + f);
 120         }
 121     }
 122 }
 123
 124 ///////////////////////////////////////////////////////////////////////////////////////////////////
 125
 126 static const int kDstGammaTableSize = SkColorSpaceXform_Base::kDstGammaTableSize;
 127
 128 static void build_table_linear_to_gamma(uint8_t* outTable, float exponent) {
 129     float toGammaExp = 1.0f / exponent;
 130
 131     for (int i = 0; i < kDstGammaTableSize; i++) {
 132         float x = ((float) i) * (1.0f / ((float) (kDstGammaTableSize - 1)));
 133         outTable[i] = clamp_normalized_float_to_byte(powf(x, toGammaExp));
 134     }
 135 }
 136
 137 static void build_table_linear_to_gamma(uint8_t* outTable, const float* inTable,
 138                                         int inTableSize) {
 139     invert_table_gamma(nullptr, outTable, kDstGammaTableSize, inTable, inTableSize);
 140 }
 141
 142 static float inverse_parametric(float x, float g, float a, float b, float c, float d, float e,
 143                                 float f) {
 144     // We need to take the inverse of the following piecewise function.
 145     // Y = (aX + b)^g + c  for X >= d
 146     // Y = eX + f          otherwise
 147
 148     // Assume that the gamma function is continuous, or this won't make much sense anyway.
 149     // Plug in |d| to the first equation to calculate the new piecewise interval.
 150     // Then simply use the inverse of the original functions.
 151     float interval = c * d + f;
 152     if (x < interval) {
 153         // X = (Y - F) / C
 154         if (0.0f == c) {
 155             // The gamma curve for this segment is constant, so the inverse is undefined.
 156             // Since this is the lower segment, guess zero.
 157             return 0.0f;
 158         }
 159
 160         return (x - f) / c;
 161     }
 162
 163     // X = ((Y - E)^(1 / G) - B) / A
 164     if (0.0f == a || 0.0f == g) {
 165         // The gamma curve for this segment is constant, so the inverse is undefined.
 166         // Since this is the upper segment, guess one.
 167         return 1.0f;
 168     }
 169
 170     return (powf(x - e, 1.0f / g) - b) / a;
 171 }
 172
 173 static void build_table_linear_to_gamma(uint8_t* outTable, float g, float a,
 174                                         float b, float c, float d, float e, float f) {
 175     for (int i = 0; i < kDstGammaTableSize; i++) {
 176         float x = ((float) i) * (1.0f / ((float) (kDstGammaTableSize - 1)));
 177         float y = inverse_parametric(x, g, a, b, c, d, e, f);
 178         outTable[i] = clamp_normalized_float_to_byte(y);
 179     }
 180 }
 181
 182 ///////////////////////////////////////////////////////////////////////////////////////////////////
 183
 184 template <typename T>
 185 struct GammaFns {
 186     const T* fSRGBTable;
 187     const T* f2Dot2Table;
 188     void (*fBuildFromValue)(T*, float);
 189     void (*fBuildFromTable)(T*, const float*, int);
 190     void (*fBuildFromParam)(T*, float, float, float, float, float, float, float);
 191 };
 192
 193 static const GammaFns<float> kToLinear {
 194     sk_linear_from_srgb,
 195     sk_linear_from_2dot2,
 196     &build_table_linear_from_gamma,
 197     &build_table_linear_from_gamma,
 198     &build_table_linear_from_gamma,
 199 };
 200
 201 static const GammaFns<uint8_t> kFromLinear {
 202     nullptr,
 203     nullptr,
 204     &build_table_linear_to_gamma,
 205     &build_table_linear_to_gamma,
 206     &build_table_linear_to_gamma,
 207 };
 208
 209 // Build tables to transform src gamma to linear.
 210 template <typename T>
 211 static void build_gamma_tables(const T* outGammaTables[3], T* gammaTableStorage, int gammaTableSize,
 212                                const SkColorSpace_XYZ* space, const GammaFns<T>& fns,
 213                                bool gammasAreMatching)
 214 {
 215     switch (space->gammaNamed()) {
 216         case kSRGB_SkGammaNamed:
 217             outGammaTables[0] = outGammaTables[1] = outGammaTables[2] = fns.fSRGBTable;
 218             break;
 219         case k2Dot2Curve_SkGammaNamed:
 220             outGammaTables[0] = outGammaTables[1] = outGammaTables[2] = fns.f2Dot2Table;
 221             break;
 222         case kLinear_SkGammaNamed:
 223             outGammaTables[0] = outGammaTables[1] = outGammaTables[2] = nullptr;
 224             break;
 225         default: {
 226             const SkGammas* gammas = space->gammas();
 227             SkASSERT(gammas);
 228
 229             auto build_table = [=](int i) {
 230                 if (gammas->isNamed(i)) {
 231                     switch (gammas->data(i).fNamed) {
 232                         case kSRGB_SkGammaNamed:
 233                             (*fns.fBuildFromParam)(&gammaTableStorage[i * gammaTableSize], 2.4f,
 234                                                    (1.0f / 1.055f), (0.055f / 1.055f),
 235                                                    (1.0f / 12.92f), 0.04045f, 0.0f, 0.0f);
 236                             outGammaTables[i] = &gammaTableStorage[i * gammaTableSize];
 237                             break;
 238                         case k2Dot2Curve_SkGammaNamed:
 239                             (*fns.fBuildFromValue)(&gammaTableStorage[i * gammaTableSize], 2.2f);
 240                             outGammaTables[i] = &gammaTableStorage[i * gammaTableSize];
 241                             break;
 242                         case kLinear_SkGammaNamed:
 243                             (*fns.fBuildFromValue)(&gammaTableStorage[i * gammaTableSize], 1.0f);
 244                             outGammaTables[i] = &gammaTableStorage[i * gammaTableSize];
 245                             break;
 246                         default:
 247                             SkASSERT(false);
 248                             break;
 249                     }
 250                 } else if (gammas->isValue(i)) {
 251                     (*fns.fBuildFromValue)(&gammaTableStorage[i * gammaTableSize],
 252                                            gammas->data(i).fValue);
 253                     outGammaTables[i] = &gammaTableStorage[i * gammaTableSize];
 254                 } else if (gammas->isTable(i)) {
 255                     (*fns.fBuildFromTable)(&gammaTableStorage[i * gammaTableSize], gammas->table(i),
 256                                            gammas->data(i).fTable.fSize);
 257                     outGammaTables[i] = &gammaTableStorage[i * gammaTableSize];
 258                 } else {
 259                     SkASSERT(gammas->isParametric(i));
 260                     const SkColorSpaceTransferFn& params = gammas->params(i);
 261                     (*fns.fBuildFromParam)(&gammaTableStorage[i * gammaTableSize], params.fG,
 262                                            params.fA, params.fB, params.fC, params.fD, params.fE,
 263                                            params.fF);
 264                     outGammaTables[i] = &gammaTableStorage[i * gammaTableSize];
 265                 }
 266             };
 267
 268             if (gammasAreMatching) {
 269                 build_table(0);
 270                 outGammaTables[1] = outGammaTables[0];
 271                 outGammaTables[2] = outGammaTables[0];
 272             } else {
 273                 build_table(0);
 274                 build_table(1);
 275                 build_table(2);
 276             }
 277
 278             break;
 279         }
 280     }
 281 }
 282
 283 void SkColorSpaceXform_Base::BuildDstGammaTables(const uint8_t* dstGammaTables[3],
 284                                                  uint8_t* dstStorage,
 285                                                  const SkColorSpace_XYZ* space,
 286                                                  bool gammasAreMatching) {
 287     build_gamma_tables(dstGammaTables, dstStorage, kDstGammaTableSize, space, kFromLinear,
 288                        gammasAreMatching);
 289 }
 290
 291 ///////////////////////////////////////////////////////////////////////////////////////////////////
 292
 293 std::unique_ptr<SkColorSpaceXform> SkColorSpaceXform::New(SkColorSpace* srcSpace,
 294                                                           SkColorSpace* dstSpace) {
 295     return SkColorSpaceXform_Base::New(srcSpace, dstSpace, SkTransferFunctionBehavior::kRespect);
 296 }
 297
 298 std::unique_ptr<SkColorSpaceXform> SkColorSpaceXform_Base::New(SkColorSpace* srcSpace,
 299         SkColorSpace* dstSpace, SkTransferFunctionBehavior premulBehavior) {
 300
 301     if (!srcSpace || !dstSpace) {
 302         // Invalid input
 303         return nullptr;
 304     }
 305
 306     if (SkColorSpace_Base::Type::kA2B == as_CSB(dstSpace)->type()) {
 307         SkCSXformPrintf("A2B destinations not supported\n");
 308         return nullptr;
 309     }
 310
 311     if (SkColorSpace_Base::Type::kA2B == as_CSB(srcSpace)->type()) {
 312         SkColorSpace_A2B* src = static_cast<SkColorSpace_A2B*>(srcSpace);
 313         SkColorSpace_XYZ* dst = static_cast<SkColorSpace_XYZ*>(dstSpace);
 314         return std::unique_ptr<SkColorSpaceXform>(new SkColorSpaceXform_A2B(src, dst));
 315     }
 316     SkColorSpace_XYZ* srcSpaceXYZ = static_cast<SkColorSpace_XYZ*>(srcSpace);
 317     SkColorSpace_XYZ* dstSpaceXYZ = static_cast<SkColorSpace_XYZ*>(dstSpace);
 318
 319     ColorSpaceMatch csm = kNone_ColorSpaceMatch;
 320     SkMatrix44 srcToDst(SkMatrix44::kUninitialized_Constructor);
 321     if (SkColorSpace::Equals(srcSpace, dstSpace)) {
 322         srcToDst.setIdentity();
 323         csm = kFull_ColorSpaceMatch;
 324     } else {
 325         if (srcSpaceXYZ->toXYZD50Hash() == dstSpaceXYZ->toXYZD50Hash()) {
 326             SkASSERT(*srcSpaceXYZ->toXYZD50() == *dstSpaceXYZ->toXYZD50() && "Hash collision");
 327             srcToDst.setIdentity();
 328             csm = kGamut_ColorSpaceMatch;
 329         } else {
 330             srcToDst.setConcat(*dstSpaceXYZ->fromXYZD50(), *srcSpaceXYZ->toXYZD50());
 331         }
 332     }
 333
 334     switch (csm) {
 335         case kNone_ColorSpaceMatch:
 336             return std::unique_ptr<SkColorSpaceXform>(new SkColorSpaceXform_XYZ
 337                     <kNone_ColorSpaceMatch>(srcSpaceXYZ, srcToDst, dstSpaceXYZ, premulBehavior));
 338         case kGamut_ColorSpaceMatch:
 339             return std::unique_ptr<SkColorSpaceXform>(new SkColorSpaceXform_XYZ
 340                     <kGamut_ColorSpaceMatch>(srcSpaceXYZ, srcToDst, dstSpaceXYZ, premulBehavior));
 341         case kFull_ColorSpaceMatch:
 342             return std::unique_ptr<SkColorSpaceXform>(new SkColorSpaceXform_XYZ
 343                     <kFull_ColorSpaceMatch>(srcSpaceXYZ, srcToDst, dstSpaceXYZ, premulBehavior));
 344         default:
 345             SkASSERT(false);
 346             return nullptr;
 347     }
 348 }
 349
 350 ///////////////////////////////////////////////////////////////////////////////////////////////////
 351
 352 #define AI SK_ALWAYS_INLINE
 353
 354 static AI void load_matrix(const float matrix[13],
 355                            Sk4f& rXgXbX, Sk4f& rYgYbY, Sk4f& rZgZbZ, Sk4f& rTgTbT) {
 356     rXgXbX = Sk4f::Load(matrix + 0);
 357     rYgYbY = Sk4f::Load(matrix + 3);
 358     rZgZbZ = Sk4f::Load(matrix + 6);
 359     rTgTbT = Sk4f::Load(matrix + 9);
 360 }
 361
 362 enum Order {
 363     kRGBA_Order,
 364     kBGRA_Order,
 365 };
 366
 367 static AI void set_rb_shifts(Order kOrder, int* kRShift, int* kBShift) {
 368     if (kRGBA_Order == kOrder) {
 369         *kRShift = 0;
 370         *kBShift = 16;
 371     } else {
 372         *kRShift = 16;
 373         *kBShift = 0;
 374     }
 375 }
 376
 377 template <Order kOrder>
 378 static AI void load_rgb_from_tables(const uint32_t* src,
 379                                     Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a,
 380                                     const float* const srcTables[3]) {
 381     int kRShift, kGShift = 8, kBShift;
 382     set_rb_shifts(kOrder, &kRShift, &kBShift);
 383     r = { srcTables[0][(src[0] >> kRShift) & 0xFF],
 384           srcTables[0][(src[1] >> kRShift) & 0xFF],
 385           srcTables[0][(src[2] >> kRShift) & 0xFF],
 386           srcTables[0][(src[3] >> kRShift) & 0xFF], };
 387     g = { srcTables[1][(src[0] >> kGShift) & 0xFF],
 388           srcTables[1][(src[1] >> kGShift) & 0xFF],
 389           srcTables[1][(src[2] >> kGShift) & 0xFF],
 390           srcTables[1][(src[3] >> kGShift) & 0xFF], };
 391     b = { srcTables[2][(src[0] >> kBShift) & 0xFF],
 392           srcTables[2][(src[1] >> kBShift) & 0xFF],
 393           srcTables[2][(src[2] >> kBShift) & 0xFF],
 394           srcTables[2][(src[3] >> kBShift) & 0xFF], };
 395     a = 0.0f; // Don't let the compiler complain that |a| is uninitialized.
 396 }
 397
 398 template <Order kOrder>
 399 static AI void load_rgba_from_tables(const uint32_t* src,
 400                                      Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a,
 401                                      const float* const srcTables[3]) {
 402     int kRShift, kGShift = 8, kBShift;
 403     set_rb_shifts(kOrder, &kRShift, &kBShift);
 404     r = { srcTables[0][(src[0] >> kRShift) & 0xFF],
 405           srcTables[0][(src[1] >> kRShift) & 0xFF],
 406           srcTables[0][(src[2] >> kRShift) & 0xFF],
 407           srcTables[0][(src[3] >> kRShift) & 0xFF], };
 408     g = { srcTables[1][(src[0] >> kGShift) & 0xFF],
 409           srcTables[1][(src[1] >> kGShift) & 0xFF],
 410           srcTables[1][(src[2] >> kGShift) & 0xFF],
 411           srcTables[1][(src[3] >> kGShift) & 0xFF], };
 412     b = { srcTables[2][(src[0] >> kBShift) & 0xFF],
 413           srcTables[2][(src[1] >> kBShift) & 0xFF],
 414           srcTables[2][(src[2] >> kBShift) & 0xFF],
 415           srcTables[2][(src[3] >> kBShift) & 0xFF], };
 416     a = (1.0f / 255.0f) * SkNx_cast<float>(Sk4u::Load(src) >> 24);
 417 }
 418
 419 template <Order kOrder>
 420 static AI void load_rgb_linear(const uint32_t* src, Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a,
 421                                const float* const[3]) {
 422     int kRShift, kGShift = 8, kBShift;
 423     set_rb_shifts(kOrder, &kRShift, &kBShift);
 424     r = (1.0f / 255.0f) * SkNx_cast<float>((Sk4u::Load(src) >> kRShift) & 0xFF);
 425     g = (1.0f / 255.0f) * SkNx_cast<float>((Sk4u::Load(src) >> kGShift) & 0xFF);
 426     b = (1.0f / 255.0f) * SkNx_cast<float>((Sk4u::Load(src) >> kBShift) & 0xFF);
 427     a = 0.0f; // Don't let the compiler complain that |a| is uninitialized.
 428 }
 429
 430 template <Order kOrder>
 431 static AI void load_rgba_linear(const uint32_t* src, Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a,
 432                                 const float* const[3]) {
 433     int kRShift, kGShift = 8, kBShift;
 434     set_rb_shifts(kOrder, &kRShift, &kBShift);
 435     r = (1.0f / 255.0f) * SkNx_cast<float>((Sk4u::Load(src) >> kRShift) & 0xFF);
 436     g = (1.0f / 255.0f) * SkNx_cast<float>((Sk4u::Load(src) >> kGShift) & 0xFF);
 437     b = (1.0f / 255.0f) * SkNx_cast<float>((Sk4u::Load(src) >> kBShift) & 0xFF);
 438     a = (1.0f / 255.0f) * SkNx_cast<float>((Sk4u::Load(src) >> 24));
 439 }
 440
 441 template <Order kOrder>
 442 static AI void load_rgb_from_tables_1(const uint32_t* src,
 443                                       Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a,
 444                                       const float* const srcTables[3]) {
 445     int kRShift, kGShift = 8, kBShift;
 446     set_rb_shifts(kOrder, &kRShift, &kBShift);
 447     r = Sk4f(srcTables[0][(*src >> kRShift) & 0xFF]);
 448     g = Sk4f(srcTables[1][(*src >> kGShift) & 0xFF]);
 449     b = Sk4f(srcTables[2][(*src >> kBShift) & 0xFF]);
 450     a = 0.0f; // Don't let MSAN complain that |a| is uninitialized.
 451 }
 452
 453 template <Order kOrder>
 454 static AI void load_rgba_from_tables_1(const uint32_t* src,
 455                                        Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a,
 456                                        const float* const srcTables[3]) {
 457     int kRShift, kGShift = 8, kBShift;
 458     set_rb_shifts(kOrder, &kRShift, &kBShift);
 459     r = Sk4f(srcTables[0][(*src >> kRShift) & 0xFF]);
 460     g = Sk4f(srcTables[1][(*src >> kGShift) & 0xFF]);
 461     b = Sk4f(srcTables[2][(*src >> kBShift) & 0xFF]);
 462     a = (1.0f / 255.0f) * Sk4f(*src >> 24);
 463 }
 464
 465 template <Order kOrder>
 466 static AI void load_rgb_linear_1(const uint32_t* src,
 467                                  Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a,
 468                                  const float* const srcTables[3]) {
 469     int kRShift, kGShift = 8, kBShift;
 470     set_rb_shifts(kOrder, &kRShift, &kBShift);
 471     r = Sk4f((1.0f / 255.0f) * ((*src >> kRShift) & 0xFF));
 472     g = Sk4f((1.0f / 255.0f) * ((*src >> kGShift) & 0xFF));
 473     b = Sk4f((1.0f / 255.0f) * ((*src >> kBShift) & 0xFF));
 474     a = 0.0f; // Don't let MSAN complain that |a| is uninitialized.
 475 }
 476
 477 template <Order kOrder>
 478 static AI void load_rgba_linear_1(const uint32_t* src,
 479                                   Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a,
 480                                   const float* const srcTables[3]) {
 481     int kRShift, kGShift = 8, kBShift;
 482     set_rb_shifts(kOrder, &kRShift, &kBShift);
 483     r = Sk4f((1.0f / 255.0f) * ((*src >> kRShift) & 0xFF));
 484     g = Sk4f((1.0f / 255.0f) * ((*src >> kGShift) & 0xFF));
 485     b = Sk4f((1.0f / 255.0f) * ((*src >> kBShift) & 0xFF));
 486     a = Sk4f((1.0f / 255.0f) * ((*src >> 24)));
 487 }
 488
 489 static AI void transform_gamut(const Sk4f& r, const Sk4f& g, const Sk4f& b, const Sk4f& a,
 490                                const Sk4f& rXgXbX, const Sk4f& rYgYbY, const Sk4f& rZgZbZ,
 491                                Sk4f& dr, Sk4f& dg, Sk4f& db, Sk4f& da) {
 492     dr = rXgXbX[0]*r + rYgYbY[0]*g + rZgZbZ[0]*b;
 493     dg = rXgXbX[1]*r + rYgYbY[1]*g + rZgZbZ[1]*b;
 494     db = rXgXbX[2]*r + rYgYbY[2]*g + rZgZbZ[2]*b;
 495     da = a;
 496 }
 497
 498 static AI void transform_gamut_1(const Sk4f& r, const Sk4f& g, const Sk4f& b,
 499                                  const Sk4f& rXgXbX, const Sk4f& rYgYbY, const Sk4f& rZgZbZ,
 500                                  Sk4f& rgba) {
 501     rgba = rXgXbX*r + rYgYbY*g + rZgZbZ*b;
 502 }
 503
 504 static AI void translate_gamut(const Sk4f& rTgTbT, Sk4f& dr, Sk4f& dg, Sk4f& db) {
 505     dr = dr + rTgTbT[0];
 506     dg = dg + rTgTbT[1];
 507     db = db + rTgTbT[2];
 508 }
 509
 510 static AI void translate_gamut_1(const Sk4f& rTgTbT, Sk4f& rgba) {
 511     rgba = rgba + rTgTbT;
 512 }
 513
 514 template <Order kOrder>
 515 static AI void store_srgb(void* dst, const uint32_t* src, Sk4f& dr, Sk4f& dg, Sk4f& db, Sk4f&,
 516                           const uint8_t* const[3]) {
 517     int kRShift, kGShift = 8, kBShift;
 518     set_rb_shifts(kOrder, &kRShift, &kBShift);
 519     dr = sk_linear_to_srgb_needs_trunc(dr);
 520     dg = sk_linear_to_srgb_needs_trunc(dg);
 521     db = sk_linear_to_srgb_needs_trunc(db);
 522
 523     dr = sk_clamp_0_255(dr);
 524     dg = sk_clamp_0_255(dg);
 525     db = sk_clamp_0_255(db);
 526
 527     Sk4i da = Sk4i::Load(src) & 0xFF000000;
 528
 529     Sk4i rgba = (SkNx_cast<int>(dr) << kRShift)
 530               | (SkNx_cast<int>(dg) << kGShift)
 531               | (SkNx_cast<int>(db) << kBShift)
 532               | (da                           );
 533     rgba.store(dst);
 534 }
 535
 536 template <Order kOrder>
 537 static AI void store_srgb_1(void* dst, const uint32_t* src,
 538                             Sk4f& rgba, const Sk4f&,
 539                             const uint8_t* const[3]) {
 540     rgba = sk_clamp_0_255(sk_linear_to_srgb_needs_trunc(rgba));
 541
 542     uint32_t tmp;
 543     SkNx_cast<uint8_t>(SkNx_cast<int32_t>(rgba)).store(&tmp);
 544     tmp = (*src & 0xFF000000) | (tmp & 0x00FFFFFF);
 545     if (kBGRA_Order == kOrder) {
 546         tmp = SkSwizzle_RB(tmp);
 547     }
 548
 549     *(uint32_t*)dst = tmp;
 550 }
 551
 552 static AI Sk4f linear_to_2dot2(const Sk4f& x) {
 553     // x^(29/64) is a very good approximation of the true value, x^(1/2.2).
 554     auto x2  = x.rsqrt(),                            // x^(-1/2)
 555          x32 = x2.rsqrt().rsqrt().rsqrt().rsqrt(),   // x^(-1/32)
 556          x64 = x32.rsqrt();                          // x^(+1/64)
 557
 558     // 29 = 32 - 2 - 1
 559     return 255.0f * x2.invert() * x32 * x64.invert();
 560 }
 561
 562 template <Order kOrder>
 563 static AI void store_2dot2(void* dst, const uint32_t* src, Sk4f& dr, Sk4f& dg, Sk4f& db, Sk4f&,
 564                            const uint8_t* const[3]) {
 565     int kRShift, kGShift = 8, kBShift;
 566     set_rb_shifts(kOrder, &kRShift, &kBShift);
 567     dr = linear_to_2dot2(dr);
 568     dg = linear_to_2dot2(dg);
 569     db = linear_to_2dot2(db);
 570
 571     dr = sk_clamp_0_255(dr);
 572     dg = sk_clamp_0_255(dg);
 573     db = sk_clamp_0_255(db);
 574
 575     Sk4i da = Sk4i::Load(src) & 0xFF000000;
 576
 577     Sk4i rgba = (Sk4f_round(dr) << kRShift)
 578               | (Sk4f_round(dg) << kGShift)
 579               | (Sk4f_round(db) << kBShift)
 580               | (da                       );
 581     rgba.store(dst);
 582 }
 583
 584 template <Order kOrder>
 585 static AI void store_2dot2_1(void* dst, const uint32_t* src,
 586                              Sk4f& rgba, const Sk4f&,
 587                              const uint8_t* const[3]) {
 588     rgba = sk_clamp_0_255(linear_to_2dot2(rgba));
 589
 590     uint32_t tmp;
 591     SkNx_cast<uint8_t>(Sk4f_round(rgba)).store(&tmp);
 592     tmp = (*src & 0xFF000000) | (tmp & 0x00FFFFFF);
 593     if (kBGRA_Order == kOrder) {
 594         tmp = SkSwizzle_RB(tmp);
 595     }
 596
 597     *(uint32_t*)dst = tmp;
 598 }
 599
 600 template <Order kOrder>
 601 static AI void store_linear(void* dst, const uint32_t* src, Sk4f& dr, Sk4f& dg, Sk4f& db, Sk4f&,
 602                             const uint8_t* const[3]) {
 603     int kRShift, kGShift = 8, kBShift;
 604     set_rb_shifts(kOrder, &kRShift, &kBShift);
 605     dr = sk_clamp_0_255(255.0f * dr);
 606     dg = sk_clamp_0_255(255.0f * dg);
 607     db = sk_clamp_0_255(255.0f * db);
 608
 609     Sk4i da = Sk4i::Load(src) & 0xFF000000;
 610
 611     Sk4i rgba = (Sk4f_round(dr) << kRShift)
 612               | (Sk4f_round(dg) << kGShift)
 613               | (Sk4f_round(db) << kBShift)
 614               | (da                       );
 615     rgba.store(dst);
 616 }
 617
 618 template <Order kOrder>
 619 static AI void store_linear_1(void* dst, const uint32_t* src,
 620                               Sk4f& rgba, const Sk4f&,
 621                               const uint8_t* const[3]) {
 622     rgba = sk_clamp_0_255(255.0f * rgba);
 623
 624     uint32_t tmp;
 625     SkNx_cast<uint8_t>(Sk4f_round(rgba)).store(&tmp);
 626     tmp = (*src & 0xFF000000) | (tmp & 0x00FFFFFF);
 627     if (kBGRA_Order == kOrder) {
 628         tmp = SkSwizzle_RB(tmp);
 629     }
 630
 631     *(uint32_t*)dst = tmp;
 632 }
 633
 634 template <Order kOrder>
 635 static AI void store_f16(void* dst, const uint32_t* src, Sk4f& dr, Sk4f& dg, Sk4f& db, Sk4f& da,
 636                          const uint8_t* const[3]) {
 637     Sk4h::Store4(dst, SkFloatToHalf_finite_ftz(dr),
 638                       SkFloatToHalf_finite_ftz(dg),
 639                       SkFloatToHalf_finite_ftz(db),
 640                       SkFloatToHalf_finite_ftz(da));
 641 }
 642
 643 template <Order kOrder>
 644 static AI void store_f16_1(void* dst, const uint32_t* src,
 645                            Sk4f& rgba, const Sk4f& a,
 646                            const uint8_t* const[3]) {
 647     rgba = Sk4f(rgba[0], rgba[1], rgba[2], a[3]);
 648     SkFloatToHalf_finite_ftz(rgba).store((uint64_t*) dst);
 649 }
 650
 651 template <Order kOrder>
 652 static AI void store_f16_opaque(void* dst, const uint32_t* src, Sk4f& dr, Sk4f& dg, Sk4f& db,
 653                                 Sk4f&, const uint8_t* const[3]) {
 654     Sk4h::Store4(dst, SkFloatToHalf_finite_ftz(dr),
 655                       SkFloatToHalf_finite_ftz(dg),
 656                       SkFloatToHalf_finite_ftz(db),
 657                       SK_Half1);
 658 }
 659
 660 template <Order kOrder>
 661 static AI void store_f16_1_opaque(void* dst, const uint32_t* src,
 662                                   Sk4f& rgba, const Sk4f&,
 663                                   const uint8_t* const[3]) {
 664     uint64_t tmp;
 665     SkFloatToHalf_finite_ftz(rgba).store(&tmp);
 666     tmp &= 0x0000FFFFFFFFFFFF;
 667     tmp |= static_cast<uint64_t>(SK_Half1) << 48;
 668     *((uint64_t*) dst) = tmp;
 669 }
 670
 671 template <Order kOrder>
 672 static AI void store_generic(void* dst, const uint32_t* src, Sk4f& dr, Sk4f& dg, Sk4f& db, Sk4f&,
 673                              const uint8_t* const dstTables[3]) {
 674     int kRShift, kGShift = 8, kBShift;
 675     set_rb_shifts(kOrder, &kRShift, &kBShift);
 676     dr = Sk4f::Min(Sk4f::Max(1023.0f * dr, 0.0f), 1023.0f);
 677     dg = Sk4f::Min(Sk4f::Max(1023.0f * dg, 0.0f), 1023.0f);
 678     db = Sk4f::Min(Sk4f::Max(1023.0f * db, 0.0f), 1023.0f);
 679
 680     Sk4i ir = Sk4f_round(dr);
 681     Sk4i ig = Sk4f_round(dg);
 682     Sk4i ib = Sk4f_round(db);
 683
 684     Sk4i da = Sk4i::Load(src) & 0xFF000000;
 685
 686     uint32_t* dst32 = (uint32_t*) dst;
 687     dst32[0] = dstTables[0][ir[0]] << kRShift
 688              | dstTables[1][ig[0]] << kGShift
 689              | dstTables[2][ib[0]] << kBShift
 690              | da[0];
 691     dst32[1] = dstTables[0][ir[1]] << kRShift
 692              | dstTables[1][ig[1]] << kGShift
 693              | dstTables[2][ib[1]] << kBShift
 694              | da[1];
 695     dst32[2] = dstTables[0][ir[2]] << kRShift
 696              | dstTables[1][ig[2]] << kGShift
 697              | dstTables[2][ib[2]] << kBShift
 698              | da[2];
 699     dst32[3] = dstTables[0][ir[3]] << kRShift
 700              | dstTables[1][ig[3]] << kGShift
 701              | dstTables[2][ib[3]] << kBShift
 702              | da[3];
 703 }
 704
 705 template <Order kOrder>
 706 static AI void store_generic_1(void* dst, const uint32_t* src,
 707                                Sk4f& rgba, const Sk4f&,
 708                                const uint8_t* const dstTables[3]) {
 709     int kRShift, kGShift = 8, kBShift;
 710     set_rb_shifts(kOrder, &kRShift, &kBShift);
 711     rgba = Sk4f::Min(Sk4f::Max(1023.0f * rgba, 0.0f), 1023.0f);
 712
 713     Sk4i indices = Sk4f_round(rgba);
 714
 715     *((uint32_t*) dst) = dstTables[0][indices[0]] << kRShift
 716                        | dstTables[1][indices[1]] << kGShift
 717                        | dstTables[2][indices[2]] << kBShift
 718                        | (*src & 0xFF000000);
 719 }
 720
 721 typedef decltype(load_rgb_from_tables<kRGBA_Order>  )* LoadFn;
 722 typedef decltype(load_rgb_from_tables_1<kRGBA_Order>)* Load1Fn;
 723 typedef decltype(store_generic<kRGBA_Order>         )* StoreFn;
 724 typedef decltype(store_generic_1<kRGBA_Order>       )* Store1Fn;
 725
 726 enum SrcFormat {
 727     kRGBA_8888_Linear_SrcFormat,
 728     kRGBA_8888_Table_SrcFormat,
 729     kBGRA_8888_Linear_SrcFormat,
 730     kBGRA_8888_Table_SrcFormat,
 731 };
 732
 733 enum DstFormat {
 734     kRGBA_8888_Linear_DstFormat,
 735     kRGBA_8888_SRGB_DstFormat,
 736     kRGBA_8888_2Dot2_DstFormat,
 737     kRGBA_8888_Table_DstFormat,
 738     kBGRA_8888_Linear_DstFormat,
 739     kBGRA_8888_SRGB_DstFormat,
 740     kBGRA_8888_2Dot2_DstFormat,
 741     kBGRA_8888_Table_DstFormat,
 742     kF16_Linear_DstFormat,
 743 };
 744
 745 template <SrcFormat kSrc,
 746           DstFormat kDst,
 747           SkAlphaType kAlphaType,
 748           ColorSpaceMatch kCSM>
 749 static void color_xform_RGBA(void* dst, const void* vsrc, int len,
 750                              const float* const srcTables[3], const float matrix[13],
 751                              const uint8_t* const dstTables[3]) {
 752     LoadFn load;
 753     Load1Fn load_1;
 754     const bool kLoadAlpha = kF16_Linear_DstFormat == kDst && kOpaque_SkAlphaType != kAlphaType;
 755     switch (kSrc) {
 756         case kRGBA_8888_Linear_SrcFormat:
 757             if (kLoadAlpha) {
 758                 load = load_rgba_linear<kRGBA_Order>;
 759                 load_1 = load_rgba_linear_1<kRGBA_Order>;
 760             } else {
 761                 load = load_rgb_linear<kRGBA_Order>;
 762                 load_1 = load_rgb_linear_1<kRGBA_Order>;
 763             }
 764             break;
 765         case kRGBA_8888_Table_SrcFormat:
 766             if (kLoadAlpha) {
 767                 load = load_rgba_from_tables<kRGBA_Order>;
 768                 load_1 = load_rgba_from_tables_1<kRGBA_Order>;
 769             } else {
 770                 load = load_rgb_from_tables<kRGBA_Order>;
 771                 load_1 = load_rgb_from_tables_1<kRGBA_Order>;
 772             }
 773             break;
 774         case kBGRA_8888_Linear_SrcFormat:
 775             if (kLoadAlpha) {
 776                 load = load_rgba_linear<kBGRA_Order>;
 777                 load_1 = load_rgba_linear_1<kBGRA_Order>;
 778             } else {
 779                 load = load_rgb_linear<kBGRA_Order>;
 780                 load_1 = load_rgb_linear_1<kBGRA_Order>;
 781             }
 782             break;
 783         case kBGRA_8888_Table_SrcFormat:
 784             if (kLoadAlpha) {
 785                 load = load_rgba_from_tables<kBGRA_Order>;
 786                 load_1 = load_rgba_from_tables_1<kBGRA_Order>;
 787             } else {
 788                 load = load_rgb_from_tables<kBGRA_Order>;
 789                 load_1 = load_rgb_from_tables_1<kBGRA_Order>;
 790             }
 791             break;
 792     }
 793
 794     StoreFn store;
 795     Store1Fn store_1;
 796     size_t sizeOfDstPixel;
 797     switch (kDst) {
 798         case kRGBA_8888_Linear_DstFormat:
 799             store   = store_linear<kRGBA_Order>;
 800             store_1 = store_linear_1<kRGBA_Order>;
 801             sizeOfDstPixel = 4;
 802             break;
 803         case kRGBA_8888_SRGB_DstFormat:
 804             store   = store_srgb<kRGBA_Order>;
 805             store_1 = store_srgb_1<kRGBA_Order>;
 806             sizeOfDstPixel = 4;
 807             break;
 808         case kRGBA_8888_2Dot2_DstFormat:
 809             store   = store_2dot2<kRGBA_Order>;
 810             store_1 = store_2dot2_1<kRGBA_Order>;
 811             sizeOfDstPixel = 4;
 812             break;
 813         case kRGBA_8888_Table_DstFormat:
 814             store   = store_generic<kRGBA_Order>;
 815             store_1 = store_generic_1<kRGBA_Order>;
 816             sizeOfDstPixel = 4;
 817             break;
 818         case kBGRA_8888_Linear_DstFormat:
 819             store   = store_linear<kBGRA_Order>;
 820             store_1 = store_linear_1<kBGRA_Order>;
 821             sizeOfDstPixel = 4;
 822             break;
 823         case kBGRA_8888_SRGB_DstFormat:
 824             store   = store_srgb<kBGRA_Order>;
 825             store_1 = store_srgb_1<kBGRA_Order>;
 826             sizeOfDstPixel = 4;
 827             break;
 828         case kBGRA_8888_2Dot2_DstFormat:
 829             store   = store_2dot2<kBGRA_Order>;
 830             store_1 = store_2dot2_1<kBGRA_Order>;
 831             sizeOfDstPixel = 4;
 832             break;
 833         case kBGRA_8888_Table_DstFormat:
 834             store   = store_generic<kBGRA_Order>;
 835             store_1 = store_generic_1<kBGRA_Order>;
 836             sizeOfDstPixel = 4;
 837             break;
 838         case kF16_Linear_DstFormat:
 839             store   = (kOpaque_SkAlphaType == kAlphaType) ? store_f16_opaque<kRGBA_Order> :
 840                                                             store_f16<kRGBA_Order>;
 841             store_1 = (kOpaque_SkAlphaType == kAlphaType) ? store_f16_1_opaque<kRGBA_Order> :
 842                                                             store_f16_1<kRGBA_Order>;
 843             sizeOfDstPixel = 8;
 844             break;
 845     }
 846
 847     const uint32_t* src = (const uint32_t*) vsrc;
 848     Sk4f rXgXbX, rYgYbY, rZgZbZ, rTgTbT;
 849     load_matrix(matrix, rXgXbX, rYgYbY, rZgZbZ, rTgTbT);
 850
 851     if (len >= 4) {
 852         // Naively this would be a loop of load-transform-store, but we found it faster to
 853         // move the N+1th load ahead of the Nth store.  We don't bother doing this for N<4.
 854         Sk4f r, g, b, a;
 855         load(src, r, g, b, a, srcTables);
 856         src += 4;
 857         len -= 4;
 858
 859         Sk4f dr, dg, db, da;
 860         while (len >= 4) {
 861             if (kNone_ColorSpaceMatch == kCSM) {
 862                 transform_gamut(r, g, b, a, rXgXbX, rYgYbY, rZgZbZ, dr, dg, db, da);
 863                 translate_gamut(rTgTbT, dr, dg, db);
 864             } else {
 865                 dr = r;
 866                 dg = g;
 867                 db = b;
 868                 da = a;
 869             }
 870
 871             load(src, r, g, b, a, srcTables);
 872
 873             store(dst, src - 4, dr, dg, db, da, dstTables);
 874             dst = SkTAddOffset<void>(dst, 4 * sizeOfDstPixel);
 875             src += 4;
 876             len -= 4;
 877         }
 878
 879         if (kNone_ColorSpaceMatch == kCSM) {
 880             transform_gamut(r, g, b, a, rXgXbX, rYgYbY, rZgZbZ, dr, dg, db, da);
 881             translate_gamut(rTgTbT, dr, dg, db);
 882         } else {
 883             dr = r;
 884             dg = g;
 885             db = b;
 886             da = a;
 887         }
 888
 889         store(dst, src - 4, dr, dg, db, da, dstTables);
 890         dst = SkTAddOffset<void>(dst, 4 * sizeOfDstPixel);
 891     }
 892
 893     while (len > 0) {
 894         Sk4f r, g, b, a;
 895         load_1(src, r, g, b, a, srcTables);
 896
 897         Sk4f rgba;
 898         if (kNone_ColorSpaceMatch == kCSM) {
 899             transform_gamut_1(r, g, b, rXgXbX, rYgYbY, rZgZbZ, rgba);
 900             translate_gamut_1(rTgTbT, rgba);
 901         } else {
 902             rgba = Sk4f(r[0], g[0], b[0], a[0]);
 903         }
 904
 905         store_1(dst, src, rgba, a, dstTables);
 906
 907         src += 1;
 908         len -= 1;
 909         dst = SkTAddOffset<void>(dst, sizeOfDstPixel);
 910     }
 911 }
 912
 913 ///////////////////////////////////////////////////////////////////////////////////////////////////
 914
 915 static AI int num_tables(SkColorSpace_XYZ* space) {
 916     switch (space->gammaNamed()) {
 917         case kSRGB_SkGammaNamed:
 918         case k2Dot2Curve_SkGammaNamed:
 919         case kLinear_SkGammaNamed:
 920             return 0;
 921         default: {
 922             const SkGammas* gammas = space->gammas();
 923             SkASSERT(gammas);
 924
 925             bool gammasAreMatching = (gammas->type(0) == gammas->type(1)) &&
 926                                      (gammas->data(0) == gammas->data(1)) &&
 927                                      (gammas->type(0) == gammas->type(2)) &&
 928                                      (gammas->data(0) == gammas->data(2));
 929
 930             // It's likely that each component will have the same gamma.  In this case,
 931             // we only need to build one table.
 932             return gammasAreMatching ? 1 : 3;
 933         }
 934     }
 935 }
 936
 937 template <ColorSpaceMatch kCSM>
 938 SkColorSpaceXform_XYZ<kCSM>
 939 ::SkColorSpaceXform_XYZ(SkColorSpace_XYZ* srcSpace, const SkMatrix44& srcToDst,
 940                         SkColorSpace_XYZ* dstSpace, SkTransferFunctionBehavior premulBehavior)
 941     : fPremulBehavior(premulBehavior)
 942 {
 943     fSrcToDst[ 0] = srcToDst.get(0, 0);
 944     fSrcToDst[ 1] = srcToDst.get(1, 0);
 945     fSrcToDst[ 2] = srcToDst.get(2, 0);
 946     fSrcToDst[ 3] = srcToDst.get(0, 1);
 947     fSrcToDst[ 4] = srcToDst.get(1, 1);
 948     fSrcToDst[ 5] = srcToDst.get(2, 1);
 949     fSrcToDst[ 6] = srcToDst.get(0, 2);
 950     fSrcToDst[ 7] = srcToDst.get(1, 2);
 951     fSrcToDst[ 8] = srcToDst.get(2, 2);
 952     fSrcToDst[ 9] = srcToDst.get(0, 3);
 953     fSrcToDst[10] = srcToDst.get(1, 3);
 954     fSrcToDst[11] = srcToDst.get(2, 3);
 955     fSrcToDst[12] = 0.0f;
 956
 957     const int numSrcTables = num_tables(srcSpace);
 958     const size_t srcEntries = numSrcTables * 256;
 959     const bool srcGammasAreMatching = (1 >= numSrcTables);
 960     fSrcStorage.reset(srcEntries);
 961     build_gamma_tables(fSrcGammaTables, fSrcStorage.get(), 256, srcSpace, kToLinear,
 962                        srcGammasAreMatching);
 963
 964     const int numDstTables = num_tables(dstSpace);
 965     dstSpace->toDstGammaTables(fDstGammaTables, &fDstStorage, numDstTables);
 966
 967     if (srcSpace->gammaIsLinear()) {
 968         fSrcGamma = kLinear_SrcGamma;
 969     } else if (kSRGB_SkGammaNamed == srcSpace->gammaNamed()) {
 970         fSrcGamma = kSRGB_SrcGamma;
 971     } else {
 972         fSrcGamma = kTable_SrcGamma;
 973     }
 974
 975     switch (dstSpace->gammaNamed()) {
 976         case kSRGB_SkGammaNamed:
 977             fDstGamma = kSRGB_DstGamma;
 978             break;
 979         case k2Dot2Curve_SkGammaNamed:
 980             fDstGamma = k2Dot2_DstGamma;
 981             break;
 982         case kLinear_SkGammaNamed:
 983             fDstGamma = kLinear_DstGamma;
 984             break;
 985         default:
 986             fDstGamma = kTable_DstGamma;
 987             break;
 988     }
 989 }
 990
 991 ///////////////////////////////////////////////////////////////////////////////////////////////////
 992
 993 template <SrcFormat kSrc, DstFormat kDst, ColorSpaceMatch kCSM>
 994 static AI bool apply_set_alpha(void* dst, const void* src, int len, SkAlphaType alphaType,
 995                                const float* const srcTables[3], const float matrix[13],
 996                                const uint8_t* const dstTables[3]) {
 997     switch (alphaType) {
 998         case kOpaque_SkAlphaType:
 999             color_xform_RGBA<kSrc, kDst, kOpaque_SkAlphaType, kCSM>
1000                     (dst, src, len, srcTables, matrix, dstTables);
1001             return true;
1002         case kUnpremul_SkAlphaType:
1003             color_xform_RGBA<kSrc, kDst, kUnpremul_SkAlphaType, kCSM>
1004                     (dst, src, len, srcTables, matrix, dstTables);
1005             return true;
1006         default:
1007             return false;
1008     }
1009 }
1010
1011 template <DstFormat kDst, ColorSpaceMatch kCSM>
1012 static AI bool apply_set_src(void* dst, const void* src, int len, SkAlphaType alphaType,
1013                              const float* const srcTables[3], const float matrix[13],
1014                              const uint8_t* const dstTables[3],
1015                              SkColorSpaceXform::ColorFormat srcColorFormat,
1016                              SrcGamma srcGamma) {
1017     switch (srcColorFormat) {
1018         case SkColorSpaceXform::kRGBA_8888_ColorFormat:
1019             switch (srcGamma) {
1020                 case kLinear_SrcGamma:
1021                     return apply_set_alpha<kRGBA_8888_Linear_SrcFormat, kDst, kCSM>
1022                             (dst, src, len, alphaType, nullptr, matrix, dstTables);
1023                 default:
1024                     return apply_set_alpha<kRGBA_8888_Table_SrcFormat, kDst, kCSM>
1025                             (dst, src, len, alphaType, srcTables, matrix, dstTables);
1026             }
1027         case SkColorSpaceXform::kBGRA_8888_ColorFormat:
1028             switch (srcGamma) {
1029                 case kLinear_SrcGamma:
1030                     return apply_set_alpha<kBGRA_8888_Linear_SrcFormat, kDst, kCSM>
1031                             (dst, src, len, alphaType, nullptr, matrix, dstTables);
1032                 default:
1033                     return apply_set_alpha<kBGRA_8888_Table_SrcFormat, kDst, kCSM>
1034                             (dst, src, len, alphaType, srcTables, matrix, dstTables);
1035             }
1036         default:
1037             return false;
1038     }
1039 }
1040
1041 #undef AI
1042
1043 template <ColorSpaceMatch kCSM>
1044 bool SkColorSpaceXform_XYZ<kCSM>
1045 ::onApply(ColorFormat dstColorFormat, void* dst, ColorFormat srcColorFormat, const void* src,
1046           int len, SkAlphaType alphaType) const
1047 {
1048     if (kFull_ColorSpaceMatch == kCSM) {
1049         if (kPremul_SkAlphaType != alphaType) {
1050             if ((kRGBA_8888_ColorFormat == dstColorFormat &&
1051                  kRGBA_8888_ColorFormat == srcColorFormat) ||
1052                 (kBGRA_8888_ColorFormat == dstColorFormat &&
1053                  kBGRA_8888_ColorFormat == srcColorFormat))
1054             {
1055                 memcpy(dst, src, len * sizeof(uint32_t));
1056                 return true;
1057             }
1058             if ((kRGBA_8888_ColorFormat == dstColorFormat &&
1059                  kBGRA_8888_ColorFormat == srcColorFormat) ||
1060                 (kBGRA_8888_ColorFormat == dstColorFormat &&
1061                  kRGBA_8888_ColorFormat == srcColorFormat))
1062             {
1063                 SkOpts::RGBA_to_BGRA((uint32_t*) dst, src, len);
1064                 return true;
1065             }
1066         }
1067     }
1068
1069     if (kRGBA_F32_ColorFormat == dstColorFormat ||
1070         kBGR_565_ColorFormat == dstColorFormat ||
1071         kRGBA_F32_ColorFormat == srcColorFormat ||
1072         kRGBA_F16_ColorFormat == srcColorFormat ||
1073         kRGBA_U16_BE_ColorFormat == srcColorFormat ||
1074         kRGB_U16_BE_ColorFormat == srcColorFormat ||
1075         kPremul_SkAlphaType == alphaType)
1076     {
1077         return this->applyPipeline(dstColorFormat, dst, srcColorFormat, src, len, alphaType);
1078     }
1079
1080     switch (dstColorFormat) {
1081         case kRGBA_8888_ColorFormat:
1082             switch (fDstGamma) {
1083                 case kLinear_DstGamma:
1084                     return apply_set_src<kRGBA_8888_Linear_DstFormat, kCSM>
1085                             (dst, src, len, alphaType, fSrcGammaTables, fSrcToDst, nullptr,
1086                              srcColorFormat, fSrcGamma);
1087                 case kSRGB_DstGamma:
1088                     return apply_set_src<kRGBA_8888_SRGB_DstFormat, kCSM>
1089                             (dst, src, len, alphaType, fSrcGammaTables, fSrcToDst, nullptr,
1090                              srcColorFormat, fSrcGamma);
1091                 case k2Dot2_DstGamma:
1092                     return apply_set_src<kRGBA_8888_2Dot2_DstFormat, kCSM>
1093                             (dst, src, len, alphaType, fSrcGammaTables, fSrcToDst, nullptr,
1094                              srcColorFormat, fSrcGamma);
1095                 case kTable_DstGamma:
1096                     return apply_set_src<kRGBA_8888_Table_DstFormat, kCSM>
1097                             (dst, src, len, alphaType, fSrcGammaTables, fSrcToDst, fDstGammaTables,
1098                              srcColorFormat, fSrcGamma);
1099             }
1100         case kBGRA_8888_ColorFormat:
1101             switch (fDstGamma) {
1102                 case kLinear_DstGamma:
1103                     return apply_set_src<kBGRA_8888_Linear_DstFormat, kCSM>
1104                             (dst, src, len, alphaType, fSrcGammaTables, fSrcToDst, nullptr,
1105                              srcColorFormat, fSrcGamma);
1106                 case kSRGB_DstGamma:
1107                     return apply_set_src<kBGRA_8888_SRGB_DstFormat, kCSM>
1108                             (dst, src, len, alphaType, fSrcGammaTables, fSrcToDst, nullptr,
1109                              srcColorFormat, fSrcGamma);
1110                 case k2Dot2_DstGamma:
1111                     return apply_set_src<kBGRA_8888_2Dot2_DstFormat, kCSM>
1112                             (dst, src, len, alphaType, fSrcGammaTables, fSrcToDst, nullptr,
1113                              srcColorFormat, fSrcGamma);
1114                 case kTable_DstGamma:
1115                     return apply_set_src<kBGRA_8888_Table_DstFormat, kCSM>
1116                             (dst, src, len, alphaType, fSrcGammaTables, fSrcToDst, fDstGammaTables,
1117                              srcColorFormat, fSrcGamma);
1118             }
1119         case kRGBA_F16_ColorFormat:
1120             switch (fDstGamma) {
1121                 case kLinear_DstGamma:
1122                     return apply_set_src<kF16_Linear_DstFormat, kCSM>
1123                             (dst, src, len, alphaType, fSrcGammaTables, fSrcToDst, nullptr,
1124                              srcColorFormat, fSrcGamma);
1125                 default:
1126                     return false;
1127             }
1128         default:
1129             SkASSERT(false);
1130             return false;
1131     }
1132 }
1133
1134 bool SkColorSpaceXform::apply(ColorFormat dstColorFormat, void* dst, ColorFormat srcColorFormat,
1135                               const void* src, int len, SkAlphaType alphaType) const {
1136     return ((SkColorSpaceXform_Base*) this)->onApply(dstColorFormat, dst, srcColorFormat, src, len,
1137                                                      alphaType);
1138 }
1139
1140 bool SkColorSpaceXform::Apply(SkColorSpace* dstCS, ColorFormat dstFormat, void* dst,
1141                               SkColorSpace* srcCS, ColorFormat srcFormat, const void* src,
1142                               int count, AlphaOp op) {
1143     SkAlphaType at;
1144     switch (op) {
1145         case kPreserve_AlphaOp:    at = kUnpremul_SkAlphaType; break;
1146         case kPremul_AlphaOp:      at = kPremul_SkAlphaType;   break;
1147         case kSrcIsOpaque_AlphaOp: at = kOpaque_SkAlphaType;   break;
1148     }
1149     return New(srcCS, dstCS)->apply(dstFormat, dst, srcFormat, src, count, at);
1150 }
1151
1152 ///////////////////////////////////////////////////////////////////////////////////////////////////
1153
1154 template <ColorSpaceMatch kCSM>
1155 bool SkColorSpaceXform_XYZ<kCSM>
1156 ::applyPipeline(ColorFormat dstColorFormat, void* dst, ColorFormat srcColorFormat,
1157                 const void* src, int len, SkAlphaType alphaType) const {
1158     SkRasterPipeline_<256> pipeline;
1159
1160     LoadTablesContext loadTables;
1161     switch (srcColorFormat) {
1162         case kRGBA_8888_ColorFormat:
1163             if (kLinear_SrcGamma == fSrcGamma) {
1164                 pipeline.append(SkRasterPipeline::load_8888, &src);
1165             } else {
1166                 loadTables.fSrc = src;
1167                 loadTables.fR = fSrcGammaTables[0];
1168                 loadTables.fG = fSrcGammaTables[1];
1169                 loadTables.fB = fSrcGammaTables[2];
1170                 pipeline.append(SkRasterPipeline::load_tables, &loadTables);
1171             }
1172
1173             break;
1174         case kBGRA_8888_ColorFormat:
1175             if (kLinear_SrcGamma == fSrcGamma) {
1176                 pipeline.append(SkRasterPipeline::load_8888, &src);
1177             } else {
1178                 loadTables.fSrc = src;
1179                 loadTables.fR = fSrcGammaTables[2];
1180                 loadTables.fG = fSrcGammaTables[1];
1181                 loadTables.fB = fSrcGammaTables[0];
1182                 pipeline.append(SkRasterPipeline::load_tables, &loadTables);
1183             }
1184
1185             pipeline.append(SkRasterPipeline::swap_rb);
1186             break;
1187         case kRGBA_F16_ColorFormat:
1188             if (kLinear_SrcGamma != fSrcGamma) {
1189                 return false;
1190             }
1191             pipeline.append(SkRasterPipeline::load_f16, &src);
1192             break;
1193         case kRGBA_F32_ColorFormat:
1194             if (kLinear_SrcGamma != fSrcGamma) {
1195                 return false;
1196             }
1197             pipeline.append(SkRasterPipeline::load_f32, &src);
1198             break;
1199         case kRGBA_U16_BE_ColorFormat:
1200             switch (fSrcGamma) {
1201                 case kLinear_SrcGamma:
1202                     pipeline.append(SkRasterPipeline::load_u16_be, &src);
1203                     break;
1204                 case kSRGB_SrcGamma:
1205                     pipeline.append(SkRasterPipeline::load_u16_be, &src);
1206                     pipeline.append_from_srgb(kUnpremul_SkAlphaType);
1207                     break;
1208                 case kTable_SrcGamma:
1209                     loadTables.fSrc = src;
1210                     loadTables.fR = fSrcGammaTables[0];
1211                     loadTables.fG = fSrcGammaTables[1];
1212                     loadTables.fB = fSrcGammaTables[2];
1213                     pipeline.append(SkRasterPipeline::load_tables_u16_be, &loadTables);
1214                     break;
1215             }
1216             break;
1217         case kRGB_U16_BE_ColorFormat:
1218             switch (fSrcGamma) {
1219                 case kLinear_SrcGamma:
1220                     pipeline.append(SkRasterPipeline::load_rgb_u16_be, &src);
1221                     break;
1222                 case kSRGB_SrcGamma:
1223                     pipeline.append(SkRasterPipeline::load_rgb_u16_be, &src);
1224                     pipeline.append_from_srgb(kUnpremul_SkAlphaType);
1225                     break;
1226                 case kTable_SrcGamma:
1227                     loadTables.fSrc = src;
1228                     loadTables.fR = fSrcGammaTables[0];
1229                     loadTables.fG = fSrcGammaTables[1];
1230                     loadTables.fB = fSrcGammaTables[2];
1231                     pipeline.append(SkRasterPipeline::load_tables_rgb_u16_be, &loadTables);
1232                     break;
1233             }
1234             break;
1235         default:
1236             return false;
1237     }
1238
1239     if (kNone_ColorSpaceMatch == kCSM) {
1240         pipeline.append(SkRasterPipeline::matrix_3x4, fSrcToDst);
1241
1242         if (kRGBA_F16_ColorFormat != dstColorFormat &&
1243             kRGBA_F32_ColorFormat != dstColorFormat)
1244         {
1245             bool need_clamp_0, need_clamp_1;
1246             analyze_3x4_matrix(fSrcToDst, &need_clamp_0, &need_clamp_1);
1247
1248             if (need_clamp_0) { pipeline.append(SkRasterPipeline::clamp_0); }
1249             if (need_clamp_1) { pipeline.append(SkRasterPipeline::clamp_1); }
1250         }
1251     }
1252
1253     if (kPremul_SkAlphaType == alphaType && SkTransferFunctionBehavior::kRespect == fPremulBehavior)
1254     {
1255         pipeline.append(SkRasterPipeline::premul);
1256     }
1257
1258     TablesContext tables;
1259     SkColorSpaceTransferFn to_2dot2 = {0,0,0,0,0,0,0};
1260     to_2dot2.fG = 1/2.2f;
1261     to_2dot2.fA = 1;
1262     switch (fDstGamma) {
1263         case kSRGB_DstGamma:
1264             pipeline.append(SkRasterPipeline::to_srgb);
1265             break;
1266         case k2Dot2_DstGamma:
1267             pipeline.append(SkRasterPipeline::parametric_r, &to_2dot2);
1268             pipeline.append(SkRasterPipeline::parametric_g, &to_2dot2);
1269             pipeline.append(SkRasterPipeline::parametric_b, &to_2dot2);
1270             break;
1271         case kTable_DstGamma:
1272             tables.fR = fDstGammaTables[0];
1273             tables.fG = fDstGammaTables[1];
1274             tables.fB = fDstGammaTables[2];
1275             tables.fCount = SkColorSpaceXform_Base::kDstGammaTableSize;
1276             pipeline.append(SkRasterPipeline::byte_tables_rgb, &tables);
1277         default:
1278             break;
1279     }
1280
1281     if (kPremul_SkAlphaType == alphaType && SkTransferFunctionBehavior::kIgnore == fPremulBehavior)
1282     {
1283         pipeline.append(SkRasterPipeline::premul);
1284     }
1285
1286     switch (dstColorFormat) {
1287         case kRGBA_8888_ColorFormat:
1288              pipeline.append(SkRasterPipeline::store_8888, &dst);
1289             break;
1290         case kBGRA_8888_ColorFormat:
1291             pipeline.append(SkRasterPipeline::swap_rb);
1292             pipeline.append(SkRasterPipeline::store_8888, &dst);
1293             break;
1294         case kRGBA_F16_ColorFormat:
1295             if (kLinear_DstGamma != fDstGamma) {
1296                 return false;
1297             }
1298             pipeline.append(SkRasterPipeline::store_f16, &dst);
1299             break;
1300         case kRGBA_F32_ColorFormat:
1301             if (kLinear_DstGamma != fDstGamma) {
1302                 return false;
1303             }
1304             pipeline.append(SkRasterPipeline::store_f32, &dst);
1305             break;
1306         case kBGR_565_ColorFormat:
1307             if (kOpaque_SkAlphaType != alphaType) {
1308                 return false;
1309             }
1310             pipeline.append(SkRasterPipeline::store_565, &dst);
1311             break;
1312         default:
1313             return false;
1314     }
1315
1316     pipeline.run(0, len);
1317     return true;
1318 }
1319
1320 ///////////////////////////////////////////////////////////////////////////////////////////////////
1321
1322 std::unique_ptr<SkColorSpaceXform> SlowIdentityXform(SkColorSpace_XYZ* space) {
1323     return std::unique_ptr<SkColorSpaceXform>(new SkColorSpaceXform_XYZ<kNone_ColorSpaceMatch>
1324             (space, SkMatrix::I(), space, SkTransferFunctionBehavior::kRespect));
1325 }