rework LPC coefficient quantizer

diff --git a/src/libFLAC/include/private/lpc.h b/src/libFLAC/include/private/lpc.h
index f5b86e1..069d459 100644 (file)
--- a/src/libFLAC/include/private/lpc.h
+++ b/src/libFLAC/include/private/lpc.h
@@ -71,15 +71,15 @@ void FLAC__lpc_compute_lp_coefficients(const real autoc[], unsigned max_order, r
  *                               bit) of largest coefficient
  *     IN bits_per_sample > 0    bits per sample of the originial signal
  *     OUT qlp_coeff[0,order-1]  quantized coefficients
- *     OUT bits                  # of bits to shift right to get approximated
- *                               LP coefficients.  NOTE: could be negative,
- *                               but |*bits| will always be <= precision
+ *     OUT shift                 # of bits to shift right to get approximated
+ *                               LP coefficients.  NOTE: could be negative.
  *     RETURN 0 => quantization OK
- *            1 => coefficients vary too much to quantize to the desired
- *                 precision.  'bits' is unset
- *            2 => coefficients are all zero, which is bad.  'bits' is unset
+ *            1 => coefficients require too much shifting for *shift to
+ *              fit in the LPC subframe header.  'shift' is unset.
+ *         2 => coefficients are all zero, which is bad.  'shift' is
+ *              unset.
  */
-int FLAC__lpc_quantize_coefficients(const real lp_coeff[], unsigned order, unsigned precision, unsigned bits_per_sample, int32 qlp_coeff[], int *bits);
+int FLAC__lpc_quantize_coefficients(const real lp_coeff[], unsigned order, unsigned precision, unsigned bits_per_sample, int32 qlp_coeff[], int *shift);
 
 /*
  *     FLAC__lpc_compute_residual_from_qlp_coefficients()

diff --git a/src/libFLAC/lpc.c b/src/libFLAC/lpc.c
index 1e7a6a9..27cfc4b 100644 (file)
--- a/src/libFLAC/lpc.c
+++ b/src/libFLAC/lpc.c
@@ -80,7 +80,8 @@ void FLAC__lpc_compute_lp_coefficients(const real autoc[], unsigned max_order, r
        }
 }
 
-int FLAC__lpc_quantize_coefficients(const real lp_coeff[], unsigned order, unsigned precision, unsigned bits_per_sample, int32 qlp_coeff[], int *bits)
+#if 0
+int FLAC__lpc_quantize_coefficients(const real lp_coeff[], unsigned order, unsigned precision, unsigned bits_per_sample, int32 qlp_coeff[], int *shift)
 {
        unsigned i;
        real d, rprecision = (real)precision, maxlog = -1e99, minlog = 1e99;
@@ -109,15 +110,70 @@ int FLAC__lpc_quantize_coefficients(const real lp_coeff[], unsigned order, unsig
        else if((rprecision-1.0) - maxlog >= (real)(precision+1))
                rprecision = (real)precision + maxlog + 1.0;
 
-       *bits = (int)floor((rprecision-1.0) - maxlog); /* '-1' because bits can be negative and the sign bit costs 1 bit */
-       if(*bits > (int)precision || *bits <= -(int)precision) {
-               fprintf(stderr, "@@@ FLAC__lpc_quantize_coefficients(): ERROR: *bits=%d, maxlog=%f, minlog=%f, precision=%u, rprecision=%f\n", *bits, maxlog, minlog, precision, rprecision);
+       *shift = (int)floor((rprecision-1.0) - maxlog); /* '-1' because *shift can be negative and the sign bit costs 1 bit */
+       if(*shift > (int)precision || *shift <= -(int)precision) {
+               fprintf(stderr, "@@@ FLAC__lpc_quantize_coefficients(): ERROR: *shift=%d, maxlog=%f, minlog=%f, precision=%u, rprecision=%f\n", *shift, maxlog, minlog, precision, rprecision);
                return 1;
        }
 
-       if(*bits != 0) { /* just to avoid wasting time... */
+       if(*shift != 0) { /* just to avoid wasting time... */
                for(i = 0; i < order; i++)
-                       qlp_coeff[i] = (int32)floor(lp_coeff[i] * (real)(1 << *bits));
+                       qlp_coeff[i] = (int32)floor(lp_coeff[i] * (real)(1 << *shift));
+       }
+       return 0;
+}
+#endif
+
+int FLAC__lpc_quantize_coefficients(const real lp_coeff[], unsigned order, unsigned precision, unsigned bits_per_sample, int32 qlp_coeff[], int *shift)
+{
+       unsigned i;
+       real d, cmax = -1e99;//@@@, cmin = 1e99;
+
+       assert(bits_per_sample > 0);
+       assert(bits_per_sample <= sizeof(int32)*8);
+       assert(precision > 0);
+       assert(precision >= FLAC__MIN_QLP_COEFF_PRECISION);
+       assert(precision + bits_per_sample < sizeof(int32)*8);
+#ifdef NDEBUG
+       (void)bits_per_sample; /* silence compiler warning about unused parameter */
+#endif
+
+       /* drop one bit for the sign; from here on out we consider only |lp_coeff[i]| */
+       precision--;
+
+       for(i = 0; i < order; i++) {
+               if(lp_coeff[i] == 0.0)
+                       continue;
+               d = fabs(lp_coeff[i]);
+               if(d > cmax)
+                       cmax = d;
+//@@@          if(d < cmin)
+//@@@                  cmin = d;
+       }
+//@@@  if(cmax < cmin)
+       if(cmax < 0) {
+               /* => coeffients are all 0, which means our constant-detect didn't work */
+fprintf(stderr,"@@@ LPCQ ERROR, all lpc_coeffs are 0\n");
+               return 2;
+       }
+       else {
+//@@@          const int minshift = (int)precision - floor(log(cmin) / M_LN2) - 1;
+               const int maxshift = (int)precision - floor(log(cmax) / M_LN2) - 1;
+//@@@          assert(maxshift >= minshift);
+               const int max_shiftlimit = (1 << (FLAC__SUBFRAME_LPC_QLP_SHIFT_LEN-1)) - 1;
+               const int min_shiftlimit = -max_shiftlimit - 1;
+
+               *shift = maxshift;
+
+               if(*shift < min_shiftlimit || *shift > max_shiftlimit) {
+fprintf(stderr,"@@@ LPCQ ERROR, shift is outside shiftlimit\n");
+                       return 1;
+               }
+       }
+
+       if(*shift != 0) { /* just to avoid wasting time... */
+               for(i = 0; i < order; i++)
+                       qlp_coeff[i] = (int32)floor(lp_coeff[i] * (real)(1 << *shift));
        }
        return 0;
 }