avctx->bits_per_raw_sample is used in get_sbits_long, which only
supports up to 32 bits.
CC: libav-stable@libav.org
Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>
Signed-off-by: Anton Khirnov <anton@khirnov.net>
(cherry picked from commit
e191aaca44b986816695e3b7ecfae64697fd6631)
Signed-off-by: Anton Khirnov <anton@khirnov.net>
avctx->sample_fmt = sconf->resolution > 1
? AV_SAMPLE_FMT_S32 : AV_SAMPLE_FMT_S16;
avctx->bits_per_raw_sample = (sconf->resolution + 1) * 8;
+ if (avctx->bits_per_raw_sample > 32) {
+ av_log(avctx, AV_LOG_ERROR, "Bits per raw sample %d larger than 32.\n",
+ avctx->bits_per_raw_sample);
+ ret = AVERROR_INVALIDDATA;
+ goto fail;
+ }
}
// set maximum Rice parameter for progressive decoding based on resolution