src/third_party/libvpx/source/libvpx/vp9/encoder/vp9_quantize.c

   1 /*
   2  *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include <math.h>
  12
  13 #include "vpx_mem/vpx_mem.h"
  14
  15 #include "vp9/common/vp9_quant_common.h"
  16 #include "vp9/common/vp9_seg_common.h"
  17
  18 #include "vp9/encoder/vp9_encoder.h"
  19 #include "vp9/encoder/vp9_quantize.h"
  20 #include "vp9/encoder/vp9_rd.h"
  21
  22 void vp9_quantize_dc(const tran_low_t *coeff_ptr, int skip_block,
  23                      const int16_t *round_ptr, const int16_t quant,
  24                      tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
  25                      const int16_t dequant_ptr, uint16_t *eob_ptr) {
  26   const int rc = 0;
  27   const int coeff = coeff_ptr[rc];
  28   const int coeff_sign = (coeff >> 31);
  29   const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
  30   int tmp, eob = -1;
  31
  32   if (!skip_block) {
  33     tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX);
  34     tmp = (tmp * quant) >> 16;
  35     qcoeff_ptr[rc]  = (tmp ^ coeff_sign) - coeff_sign;
  36     dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr;
  37     if (tmp)
  38       eob = 0;
  39   }
  40   *eob_ptr = eob + 1;
  41 }
  42
  43 #if CONFIG_VP9_HIGHBITDEPTH
  44 void vp9_highbd_quantize_dc(const tran_low_t *coeff_ptr, int skip_block,
  45                             const int16_t *round_ptr, const int16_t quant,
  46                             tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
  47                             const int16_t dequant_ptr, uint16_t *eob_ptr) {
  48   int eob = -1;
  49
  50   if (!skip_block) {
  51     const int rc = 0;
  52     const int coeff = coeff_ptr[rc];
  53     const int coeff_sign = (coeff >> 31);
  54     const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
  55
  56     const int64_t tmp =
  57         (clamp(abs_coeff + round_ptr[rc != 0], INT32_MIN, INT32_MAX) *
  58          quant) >> 16;
  59     qcoeff_ptr[rc]  = (tmp ^ coeff_sign) - coeff_sign;
  60     dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr;
  61     if (tmp)
  62       eob = 0;
  63   }
  64   *eob_ptr = eob + 1;
  65 }
  66 #endif
  67
  68 void vp9_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block,
  69                            const int16_t *round_ptr, const int16_t quant,
  70                            tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
  71                            const int16_t dequant_ptr, uint16_t *eob_ptr) {
  72   const int rc = 0;
  73   const int coeff = coeff_ptr[rc];
  74   const int coeff_sign = (coeff >> 31);
  75   const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
  76   int tmp, eob = -1;
  77
  78   if (!skip_block) {
  79
  80     tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX);
  81     tmp = (tmp * quant) >> 15;
  82     qcoeff_ptr[rc]  = (tmp ^ coeff_sign) - coeff_sign;
  83     dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr / 2;
  84     if (tmp)
  85       eob = 0;
  86   }
  87   *eob_ptr = eob + 1;
  88 }
  89
  90 #if CONFIG_VP9_HIGHBITDEPTH
  91 void vp9_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr,
  92                                   int skip_block,
  93                                   const int16_t *round_ptr,
  94                                   const int16_t quant,
  95                                   tran_low_t *qcoeff_ptr,
  96                                   tran_low_t *dqcoeff_ptr,
  97                                   const int16_t dequant_ptr,
  98                                   uint16_t *eob_ptr) {
  99   int eob = -1;
 100
 101   if (!skip_block) {
 102     const int rc = 0;
 103     const int coeff = coeff_ptr[rc];
 104     const int coeff_sign = (coeff >> 31);
 105     const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 106
 107     const int64_t tmp =
 108         (clamp(abs_coeff + round_ptr[rc != 0], INT32_MIN, INT32_MAX) *
 109          quant) >> 15;
 110     qcoeff_ptr[rc]  = (tmp ^ coeff_sign) - coeff_sign;
 111     dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr / 2;
 112     if (tmp)
 113       eob = 0;
 114   }
 115   *eob_ptr = eob + 1;
 116 }
 117 #endif
 118
 119 void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
 120                        int skip_block,
 121                        const int16_t *zbin_ptr, const int16_t *round_ptr,
 122                        const int16_t *quant_ptr, const int16_t *quant_shift_ptr,
 123                        tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
 124                        const int16_t *dequant_ptr,
 125                        int zbin_oq_value, uint16_t *eob_ptr,
 126                        const int16_t *scan, const int16_t *iscan) {
 127   int i, eob = -1;
 128   // TODO(jingning) Decide the need of these arguments after the
 129   // quantization process is completed.
 130   (void)zbin_ptr;
 131   (void)quant_shift_ptr;
 132   (void)zbin_oq_value;
 133   (void)iscan;
 134
 135   vpx_memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 136   vpx_memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 137
 138   if (!skip_block) {
 139     // Quantization pass: All coefficients with index >= zero_flag are
 140     // skippable. Note: zero_flag can be zero.
 141     for (i = 0; i < n_coeffs; i++) {
 142       const int rc = scan[i];
 143       const int coeff = coeff_ptr[rc];
 144       const int coeff_sign = (coeff >> 31);
 145       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 146
 147       int tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX);
 148       tmp = (tmp * quant_ptr[rc != 0]) >> 16;
 149
 150       qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
 151       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
 152
 153       if (tmp)
 154         eob = i;
 155     }
 156   }
 157   *eob_ptr = eob + 1;
 158 }
 159
 160 #if CONFIG_VP9_HIGHBITDEPTH
 161 void vp9_highbd_quantize_fp_c(const tran_low_t *coeff_ptr,
 162                               intptr_t count,
 163                               int skip_block,
 164                               const int16_t *zbin_ptr,
 165                               const int16_t *round_ptr,
 166                               const int16_t *quant_ptr,
 167                               const int16_t *quant_shift_ptr,
 168                               tran_low_t *qcoeff_ptr,
 169                               tran_low_t *dqcoeff_ptr,
 170                               const int16_t *dequant_ptr,
 171                               int zbin_oq_value,
 172                               uint16_t *eob_ptr,
 173                               const int16_t *scan,
 174                               const int16_t *iscan) {
 175   int i;
 176   int eob = -1;
 177   // TODO(jingning) Decide the need of these arguments after the
 178   // quantization process is completed.
 179   (void)zbin_ptr;
 180   (void)quant_shift_ptr;
 181   (void)zbin_oq_value;
 182   (void)iscan;
 183
 184   vpx_memset(qcoeff_ptr, 0, count * sizeof(*qcoeff_ptr));
 185   vpx_memset(dqcoeff_ptr, 0, count * sizeof(*dqcoeff_ptr));
 186
 187   if (!skip_block) {
 188     // Quantization pass: All coefficients with index >= zero_flag are
 189     // skippable. Note: zero_flag can be zero.
 190     for (i = 0; i < count; i++) {
 191       const int rc = scan[i];
 192       const int coeff = coeff_ptr[rc];
 193       const int coeff_sign = (coeff >> 31);
 194       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 195
 196       const int64_t tmp =
 197           (clamp(abs_coeff + round_ptr[rc != 0], INT32_MIN, INT32_MAX) *
 198            quant_ptr[rc != 0]) >> 16;
 199
 200       qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
 201       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
 202
 203       if (tmp)
 204         eob = i;
 205     }
 206   }
 207   *eob_ptr = eob + 1;
 208 }
 209 #endif
 210
 211 // TODO(jingning) Refactor this file and combine functions with similar
 212 // operations.
 213 void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
 214                              int skip_block,
 215                              const int16_t *zbin_ptr, const int16_t *round_ptr,
 216                              const int16_t *quant_ptr,
 217                              const int16_t *quant_shift_ptr,
 218                              tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
 219                              const int16_t *dequant_ptr,
 220                              int zbin_oq_value, uint16_t *eob_ptr,
 221                              const int16_t *scan, const int16_t *iscan) {
 222   int i, eob = -1;
 223   (void)zbin_ptr;
 224   (void)quant_shift_ptr;
 225   (void)zbin_oq_value;
 226   (void)iscan;
 227
 228   vpx_memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 229   vpx_memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 230
 231   if (!skip_block) {
 232     for (i = 0; i < n_coeffs; i++) {
 233       const int rc = scan[i];
 234       const int coeff = coeff_ptr[rc];
 235       const int coeff_sign = (coeff >> 31);
 236       int tmp = 0;
 237       int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 238
 239       if (abs_coeff >= (dequant_ptr[rc != 0] >> 2)) {
 240         abs_coeff += ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1);
 241         abs_coeff = clamp(abs_coeff, INT16_MIN, INT16_MAX);
 242         tmp = (abs_coeff * quant_ptr[rc != 0]) >> 15;
 243         qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
 244         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
 245       }
 246
 247       if (tmp)
 248         eob = i;
 249     }
 250   }
 251   *eob_ptr = eob + 1;
 252 }
 253
 254 #if CONFIG_VP9_HIGHBITDEPTH
 255 void vp9_highbd_quantize_fp_32x32_c(const tran_low_t *coeff_ptr,
 256                                     intptr_t n_coeffs, int skip_block,
 257                                     const int16_t *zbin_ptr,
 258                                     const int16_t *round_ptr,
 259                                     const int16_t *quant_ptr,
 260                                     const int16_t *quant_shift_ptr,
 261                                     tran_low_t *qcoeff_ptr,
 262                                     tran_low_t *dqcoeff_ptr,
 263                                     const int16_t *dequant_ptr,
 264                                     int zbin_oq_value, uint16_t *eob_ptr,
 265                                     const int16_t *scan, const int16_t *iscan) {
 266   int i, eob = -1;
 267   (void)zbin_ptr;
 268   (void)quant_shift_ptr;
 269   (void)zbin_oq_value;
 270   (void)iscan;
 271
 272   vpx_memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 273   vpx_memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 274
 275   if (!skip_block) {
 276     for (i = 0; i < n_coeffs; i++) {
 277       const int rc = scan[i];
 278       const int coeff = coeff_ptr[rc];
 279       const int coeff_sign = (coeff >> 31);
 280       int64_t tmp = 0;
 281       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 282
 283       if (abs_coeff >= (dequant_ptr[rc != 0] >> 2)) {
 284         tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1),
 285                     INT32_MIN, INT32_MAX);
 286         tmp = (tmp * quant_ptr[rc != 0]) >> 15;
 287         qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
 288         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
 289       }
 290
 291       if (tmp)
 292         eob = i;
 293     }
 294   }
 295   *eob_ptr = eob + 1;
 296 }
 297 #endif
 298
 299 void vp9_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
 300                       int skip_block,
 301                       const int16_t *zbin_ptr, const int16_t *round_ptr,
 302                       const int16_t *quant_ptr, const int16_t *quant_shift_ptr,
 303                       tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
 304                       const int16_t *dequant_ptr,
 305                       int zbin_oq_value, uint16_t *eob_ptr,
 306                       const int16_t *scan, const int16_t *iscan) {
 307   int i, non_zero_count = (int)n_coeffs, eob = -1;
 308   const int zbins[2] = { zbin_ptr[0] + zbin_oq_value,
 309                          zbin_ptr[1] + zbin_oq_value };
 310   const int nzbins[2] = { zbins[0] * -1,
 311                           zbins[1] * -1 };
 312   (void)iscan;
 313
 314   vpx_memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 315   vpx_memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 316
 317   if (!skip_block) {
 318     // Pre-scan pass
 319     for (i = (int)n_coeffs - 1; i >= 0; i--) {
 320       const int rc = scan[i];
 321       const int coeff = coeff_ptr[rc];
 322
 323       if (coeff < zbins[rc != 0] && coeff > nzbins[rc != 0])
 324         non_zero_count--;
 325       else
 326         break;
 327     }
 328
 329     // Quantization pass: All coefficients with index >= zero_flag are
 330     // skippable. Note: zero_flag can be zero.
 331     for (i = 0; i < non_zero_count; i++) {
 332       const int rc = scan[i];
 333       const int coeff = coeff_ptr[rc];
 334       const int coeff_sign = (coeff >> 31);
 335       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 336
 337       if (abs_coeff >= zbins[rc != 0]) {
 338         int tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX);
 339         tmp = ((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) *
 340                   quant_shift_ptr[rc != 0]) >> 16;  // quantization
 341         qcoeff_ptr[rc]  = (tmp ^ coeff_sign) - coeff_sign;
 342         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
 343
 344         if (tmp)
 345           eob = i;
 346       }
 347     }
 348   }
 349   *eob_ptr = eob + 1;
 350 }
 351
 352 #if CONFIG_VP9_HIGHBITDEPTH
 353 void vp9_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
 354                              int skip_block, const int16_t *zbin_ptr,
 355                              const int16_t *round_ptr, const int16_t *quant_ptr,
 356                              const int16_t *quant_shift_ptr,
 357                              tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
 358                              const int16_t *dequant_ptr, int zbin_oq_value,
 359                              uint16_t *eob_ptr, const int16_t *scan,
 360                              const int16_t *iscan) {
 361   int i, non_zero_count = (int)n_coeffs, eob = -1;
 362   const int zbins[2] = { zbin_ptr[0] + zbin_oq_value,
 363                          zbin_ptr[1] + zbin_oq_value };
 364   const int nzbins[2] = { zbins[0] * -1,
 365                           zbins[1] * -1 };
 366   (void)iscan;
 367
 368   vpx_memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 369   vpx_memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 370
 371   if (!skip_block) {
 372     // Pre-scan pass
 373     for (i = (int)n_coeffs - 1; i >= 0; i--) {
 374       const int rc = scan[i];
 375       const int coeff = coeff_ptr[rc];
 376
 377       if (coeff < zbins[rc != 0] && coeff > nzbins[rc != 0])
 378         non_zero_count--;
 379       else
 380         break;
 381     }
 382
 383     // Quantization pass: All coefficients with index >= zero_flag are
 384     // skippable. Note: zero_flag can be zero.
 385     for (i = 0; i < non_zero_count; i++) {
 386       const int rc = scan[i];
 387       const int coeff = coeff_ptr[rc];
 388       const int coeff_sign = (coeff >> 31);
 389       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 390
 391       if (abs_coeff >= zbins[rc != 0]) {
 392         int64_t tmp = clamp(abs_coeff + round_ptr[rc != 0],
 393                             INT32_MIN, INT32_MAX);
 394         tmp = ((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) *
 395                   quant_shift_ptr[rc != 0]) >> 16;  // quantization
 396         qcoeff_ptr[rc]  = (tmp ^ coeff_sign) - coeff_sign;
 397         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
 398
 399         if (tmp)
 400           eob = i;
 401       }
 402     }
 403   }
 404   *eob_ptr = eob + 1;
 405 }
 406 #endif
 407
 408 void vp9_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
 409                             int skip_block,
 410                             const int16_t *zbin_ptr, const int16_t *round_ptr,
 411                             const int16_t *quant_ptr,
 412                             const int16_t *quant_shift_ptr,
 413                             tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
 414                             const int16_t *dequant_ptr,
 415                             int zbin_oq_value, uint16_t *eob_ptr,
 416                             const int16_t *scan, const int16_t *iscan) {
 417   const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0] + zbin_oq_value, 1),
 418                          ROUND_POWER_OF_TWO(zbin_ptr[1] + zbin_oq_value, 1) };
 419   const int nzbins[2] = {zbins[0] * -1, zbins[1] * -1};
 420
 421   int idx = 0;
 422   int idx_arr[1024];
 423   int i, eob = -1;
 424   (void)iscan;
 425
 426   vpx_memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 427   vpx_memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 428
 429   if (!skip_block) {
 430     // Pre-scan pass
 431     for (i = 0; i < n_coeffs; i++) {
 432       const int rc = scan[i];
 433       const int coeff = coeff_ptr[rc];
 434
 435       // If the coefficient is out of the base ZBIN range, keep it for
 436       // quantization.
 437       if (coeff >= zbins[rc != 0] || coeff <= nzbins[rc != 0])
 438         idx_arr[idx++] = i;
 439     }
 440
 441     // Quantization pass: only process the coefficients selected in
 442     // pre-scan pass. Note: idx can be zero.
 443     for (i = 0; i < idx; i++) {
 444       const int rc = scan[idx_arr[i]];
 445       const int coeff = coeff_ptr[rc];
 446       const int coeff_sign = (coeff >> 31);
 447       int tmp;
 448       int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 449       abs_coeff += ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1);
 450       abs_coeff = clamp(abs_coeff, INT16_MIN, INT16_MAX);
 451       tmp = ((((abs_coeff * quant_ptr[rc != 0]) >> 16) + abs_coeff) *
 452                quant_shift_ptr[rc != 0]) >> 15;
 453
 454       qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
 455       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
 456
 457       if (tmp)
 458         eob = idx_arr[i];
 459     }
 460   }
 461   *eob_ptr = eob + 1;
 462 }
 463
 464 #if CONFIG_VP9_HIGHBITDEPTH
 465 void vp9_highbd_quantize_b_32x32_c(const tran_low_t *coeff_ptr,
 466                                    intptr_t n_coeffs, int skip_block,
 467                                    const int16_t *zbin_ptr,
 468                                    const int16_t *round_ptr,
 469                                    const int16_t *quant_ptr,
 470                                    const int16_t *quant_shift_ptr,
 471                                    tran_low_t *qcoeff_ptr,
 472                                    tran_low_t *dqcoeff_ptr,
 473                                    const int16_t *dequant_ptr,
 474                                    int zbin_oq_value, uint16_t *eob_ptr,
 475                                    const int16_t *scan, const int16_t *iscan) {
 476   const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0] + zbin_oq_value, 1),
 477                          ROUND_POWER_OF_TWO(zbin_ptr[1] + zbin_oq_value, 1) };
 478   const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 };
 479
 480   int idx = 0;
 481   int idx_arr[1024];
 482   int i, eob = -1;
 483   (void)iscan;
 484
 485   vpx_memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 486   vpx_memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 487
 488   if (!skip_block) {
 489     // Pre-scan pass
 490     for (i = 0; i < n_coeffs; i++) {
 491       const int rc = scan[i];
 492       const int coeff = coeff_ptr[rc];
 493
 494       // If the coefficient is out of the base ZBIN range, keep it for
 495       // quantization.
 496       if (coeff >= zbins[rc != 0] || coeff <= nzbins[rc != 0])
 497         idx_arr[idx++] = i;
 498     }
 499
 500     // Quantization pass: only process the coefficients selected in
 501     // pre-scan pass. Note: idx can be zero.
 502     for (i = 0; i < idx; i++) {
 503       const int rc = scan[idx_arr[i]];
 504       const int coeff = coeff_ptr[rc];
 505       const int coeff_sign = (coeff >> 31);
 506       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 507       int64_t tmp = clamp(abs_coeff +
 508                           ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1),
 509                           INT32_MIN, INT32_MAX);
 510       tmp = ((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) *
 511                quant_shift_ptr[rc != 0]) >> 15;
 512
 513       qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
 514       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
 515
 516       if (tmp)
 517         eob = idx_arr[i];
 518     }
 519   }
 520   *eob_ptr = eob + 1;
 521 }
 522 #endif
 523
 524 void vp9_regular_quantize_b_4x4(MACROBLOCK *x, int plane, int block,
 525                                 const int16_t *scan, const int16_t *iscan) {
 526   MACROBLOCKD *const xd = &x->e_mbd;
 527   struct macroblock_plane *p = &x->plane[plane];
 528   struct macroblockd_plane *pd = &xd->plane[plane];
 529
 530 #if CONFIG_VP9_HIGHBITDEPTH
 531   if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
 532     vp9_highbd_quantize_b(BLOCK_OFFSET(p->coeff, block),
 533                         16, x->skip_block,
 534                         p->zbin, p->round, p->quant, p->quant_shift,
 535                         BLOCK_OFFSET(p->qcoeff, block),
 536                         BLOCK_OFFSET(pd->dqcoeff, block),
 537                         pd->dequant, p->zbin_extra, &p->eobs[block],
 538                         scan, iscan);
 539     return;
 540   }
 541 #endif
 542   vp9_quantize_b(BLOCK_OFFSET(p->coeff, block),
 543            16, x->skip_block,
 544            p->zbin, p->round, p->quant, p->quant_shift,
 545            BLOCK_OFFSET(p->qcoeff, block),
 546            BLOCK_OFFSET(pd->dqcoeff, block),
 547            pd->dequant, p->zbin_extra, &p->eobs[block], scan, iscan);
 548 }
 549
 550 static void invert_quant(int16_t *quant, int16_t *shift, int d) {
 551   unsigned t;
 552   int l;
 553   t = d;
 554   for (l = 0; t > 1; l++)
 555     t >>= 1;
 556   t = 1 + (1 << (16 + l)) / d;
 557   *quant = (int16_t)(t - (1 << 16));
 558   *shift = 1 << (16 - l);
 559 }
 560
 561 static int get_qzbin_factor(int q, vpx_bit_depth_t bit_depth) {
 562   const int quant = vp9_dc_quant(q, 0, bit_depth);
 563 #if CONFIG_VP9_HIGHBITDEPTH
 564   switch (bit_depth) {
 565     case VPX_BITS_8:
 566       return q == 0 ? 64 : (quant < 148 ? 84 : 80);
 567     case VPX_BITS_10:
 568       return q == 0 ? 64 : (quant < 592 ? 84 : 80);
 569     case VPX_BITS_12:
 570       return q == 0 ? 64 : (quant < 2368 ? 84 : 80);
 571     default:
 572       assert(0 && "bit_depth should be VPX_BITS_8, VPX_BITS_10 or VPX_BITS_12");
 573       return -1;
 574   }
 575 #else
 576   (void) bit_depth;
 577   return q == 0 ? 64 : (quant < 148 ? 84 : 80);
 578 #endif
 579 }
 580
 581 void vp9_init_quantizer(VP9_COMP *cpi) {
 582   VP9_COMMON *const cm = &cpi->common;
 583   QUANTS *const quants = &cpi->quants;
 584   int i, q, quant;
 585
 586   for (q = 0; q < QINDEX_RANGE; q++) {
 587     const int qzbin_factor = get_qzbin_factor(q, cm->bit_depth);
 588     const int qrounding_factor = q == 0 ? 64 : 48;
 589
 590     for (i = 0; i < 2; ++i) {
 591       int qrounding_factor_fp = i == 0 ? 48 : 42;
 592       if (q == 0)
 593         qrounding_factor_fp = 64;
 594
 595       // y
 596       quant = i == 0 ? vp9_dc_quant(q, cm->y_dc_delta_q, cm->bit_depth)
 597                      : vp9_ac_quant(q, 0, cm->bit_depth);
 598       invert_quant(&quants->y_quant[q][i], &quants->y_quant_shift[q][i], quant);
 599       quants->y_quant_fp[q][i] = (1 << 16) / quant;
 600       quants->y_round_fp[q][i] = (qrounding_factor_fp * quant) >> 7;
 601       quants->y_zbin[q][i] = ROUND_POWER_OF_TWO(qzbin_factor * quant, 7);
 602       quants->y_round[q][i] = (qrounding_factor * quant) >> 7;
 603       cm->y_dequant[q][i] = quant;
 604
 605       // uv
 606       quant = i == 0 ? vp9_dc_quant(q, cm->uv_dc_delta_q, cm->bit_depth)
 607                      : vp9_ac_quant(q, cm->uv_ac_delta_q, cm->bit_depth);
 608       invert_quant(&quants->uv_quant[q][i],
 609                    &quants->uv_quant_shift[q][i], quant);
 610       quants->uv_quant_fp[q][i] = (1 << 16) / quant;
 611       quants->uv_round_fp[q][i] = (qrounding_factor_fp * quant) >> 7;
 612       quants->uv_zbin[q][i] = ROUND_POWER_OF_TWO(qzbin_factor * quant, 7);
 613       quants->uv_round[q][i] = (qrounding_factor * quant) >> 7;
 614       cm->uv_dequant[q][i] = quant;
 615     }
 616
 617     for (i = 2; i < 8; i++) {
 618       quants->y_quant[q][i] = quants->y_quant[q][1];
 619       quants->y_quant_fp[q][i] = quants->y_quant_fp[q][1];
 620       quants->y_round_fp[q][i] = quants->y_round_fp[q][1];
 621       quants->y_quant_shift[q][i] = quants->y_quant_shift[q][1];
 622       quants->y_zbin[q][i] = quants->y_zbin[q][1];
 623       quants->y_round[q][i] = quants->y_round[q][1];
 624       cm->y_dequant[q][i] = cm->y_dequant[q][1];
 625
 626       quants->uv_quant[q][i] = quants->uv_quant[q][1];
 627       quants->uv_quant_fp[q][i] = quants->uv_quant_fp[q][1];
 628       quants->uv_round_fp[q][i] = quants->uv_round_fp[q][1];
 629       quants->uv_quant_shift[q][i] = quants->uv_quant_shift[q][1];
 630       quants->uv_zbin[q][i] = quants->uv_zbin[q][1];
 631       quants->uv_round[q][i] = quants->uv_round[q][1];
 632       cm->uv_dequant[q][i] = cm->uv_dequant[q][1];
 633     }
 634   }
 635 }
 636
 637 void vp9_init_plane_quantizers(VP9_COMP *cpi, MACROBLOCK *x) {
 638   const VP9_COMMON *const cm = &cpi->common;
 639   MACROBLOCKD *const xd = &x->e_mbd;
 640   QUANTS *const quants = &cpi->quants;
 641   const int segment_id = xd->mi[0].src_mi->mbmi.segment_id;
 642   const int qindex = vp9_get_qindex(&cm->seg, segment_id, cm->base_qindex);
 643   const int rdmult = vp9_compute_rd_mult(cpi, qindex + cm->y_dc_delta_q);
 644   const int zbin = cpi->zbin_mode_boost;
 645   int i;
 646
 647   // Y
 648   x->plane[0].quant = quants->y_quant[qindex];
 649   x->plane[0].quant_fp = quants->y_quant_fp[qindex];
 650   x->plane[0].round_fp = quants->y_round_fp[qindex];
 651   x->plane[0].quant_shift = quants->y_quant_shift[qindex];
 652   x->plane[0].zbin = quants->y_zbin[qindex];
 653   x->plane[0].round = quants->y_round[qindex];
 654   x->plane[0].zbin_extra = (int16_t)((cm->y_dequant[qindex][1] * zbin) >> 7);
 655   xd->plane[0].dequant = cm->y_dequant[qindex];
 656
 657   x->plane[0].quant_thred[0] = (x->plane[0].zbin[0] + x->plane[0].zbin_extra) *
 658       (x->plane[0].zbin[0] + x->plane[0].zbin_extra);
 659   x->plane[0].quant_thred[1] = (x->plane[0].zbin[1] + x->plane[0].zbin_extra) *
 660       (x->plane[0].zbin[1] + x->plane[0].zbin_extra);
 661
 662   // UV
 663   for (i = 1; i < 3; i++) {
 664     x->plane[i].quant = quants->uv_quant[qindex];
 665     x->plane[i].quant_fp = quants->uv_quant_fp[qindex];
 666     x->plane[i].round_fp = quants->uv_round_fp[qindex];
 667     x->plane[i].quant_shift = quants->uv_quant_shift[qindex];
 668     x->plane[i].zbin = quants->uv_zbin[qindex];
 669     x->plane[i].round = quants->uv_round[qindex];
 670     x->plane[i].zbin_extra = (int16_t)((cm->uv_dequant[qindex][1] * zbin) >> 7);
 671     xd->plane[i].dequant = cm->uv_dequant[qindex];
 672
 673     x->plane[i].quant_thred[0] =
 674         (x->plane[i].zbin[0] + x->plane[i].zbin_extra) *
 675         (x->plane[i].zbin[0] + x->plane[i].zbin_extra);
 676     x->plane[i].quant_thred[1] =
 677         (x->plane[i].zbin[1] + x->plane[i].zbin_extra) *
 678         (x->plane[i].zbin[1] + x->plane[i].zbin_extra);
 679   }
 680
 681   x->skip_block = vp9_segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP);
 682   x->q_index = qindex;
 683
 684   x->errorperbit = rdmult >> 6;
 685   x->errorperbit += (x->errorperbit == 0);
 686
 687   vp9_initialize_me_consts(cpi, x->q_index);
 688 }
 689
 690 void vp9_update_zbin_extra(VP9_COMP *cpi, MACROBLOCK *x) {
 691   const int qindex = x->q_index;
 692   const int y_zbin_extra = (cpi->common.y_dequant[qindex][1] *
 693                             cpi->zbin_mode_boost) >> 7;
 694   const int uv_zbin_extra = (cpi->common.uv_dequant[qindex][1] *
 695                              cpi->zbin_mode_boost) >> 7;
 696
 697   x->plane[0].zbin_extra = (int16_t)y_zbin_extra;
 698   x->plane[1].zbin_extra = (int16_t)uv_zbin_extra;
 699   x->plane[2].zbin_extra = (int16_t)uv_zbin_extra;
 700 }
 701
 702 void vp9_frame_init_quantizer(VP9_COMP *cpi) {
 703   cpi->zbin_mode_boost = 0;
 704   vp9_init_plane_quantizers(cpi, &cpi->mb);
 705 }
 706
 707 void vp9_set_quantizer(VP9_COMMON *cm, int q) {
 708   // quantizer has to be reinitialized with vp9_init_quantizer() if any
 709   // delta_q changes.
 710   cm->base_qindex = q;
 711   cm->y_dc_delta_q = 0;
 712   cm->uv_dc_delta_q = 0;
 713   cm->uv_ac_delta_q = 0;
 714 }
 715
 716 // Table that converts 0-63 Q-range values passed in outside to the Qindex
 717 // range used internally.
 718 static const int quantizer_to_qindex[] = {
 719   0,    4,   8,  12,  16,  20,  24,  28,
 720   32,   36,  40,  44,  48,  52,  56,  60,
 721   64,   68,  72,  76,  80,  84,  88,  92,
 722   96,  100, 104, 108, 112, 116, 120, 124,
 723   128, 132, 136, 140, 144, 148, 152, 156,
 724   160, 164, 168, 172, 176, 180, 184, 188,
 725   192, 196, 200, 204, 208, 212, 216, 220,
 726   224, 228, 232, 236, 240, 244, 249, 255,
 727 };
 728
 729 int vp9_quantizer_to_qindex(int quantizer) {
 730   return quantizer_to_qindex[quantizer];
 731 }
 732
 733 int vp9_qindex_to_quantizer(int qindex) {
 734   int quantizer;
 735
 736   for (quantizer = 0; quantizer < 64; ++quantizer)
 737     if (quantizer_to_qindex[quantizer] >= qindex)
 738       return quantizer;
 739
 740   return 63;
 741 }