From d49352c7cc22fd8928a761a373c3508be17c9f19 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Sun, 18 Dec 2011 08:27:43 -0800 Subject: [PATCH] swscale: fix overflows in vertical scaling at top/bottom edges. This fixes integer multiplication overflows in RGB48 output (vertical) scaling as detected by IOC. What happens is that for certain types of filters (lanczos, spline, bicubic), the intermediate sum of coefficients in the middle of a filter can be larger than the fixed-point equivalent of 1.0, even if the final sum is 1.0. This is fine and we support that. However, at frame edges, initFilter() will merge the coefficients for the off-screen pixels into the top or bottom pixel, such as to emulate edge extension. This means that suddenly, a single coefficient can be larger than the fixed-point equivalent of 1.0, which the vertical scaling routines do not support. Therefore, remove the merging of coefficients for edges for the vertical scaling filter, and instead add edge detection to the scaler itself so that it copies the pointers (not data) for the edges (i.e. it uses line[0] for line[-1] as well), so that a single coefficient is never larger than the fixed-point equivalent of 1.0. --- libswscale/swscale.c | 51 ++++++++++++++++++++++++++++++++++++++++++++--- libswscale/utils.c | 56 +++++++++++++++++++++++++++------------------------- 2 files changed, 77 insertions(+), 30 deletions(-) diff --git a/libswscale/swscale.c b/libswscale/swscale.c index c9dfc8d..f24561b 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -2487,9 +2487,11 @@ static int swScale(SwsContext *c, const uint8_t* src[], const int firstLumSrcY= vLumFilterPos[dstY]; //First line needed as input const int firstLumSrcY2= vLumFilterPos[FFMIN(dstY | ((1<chrDstVSubSample) - 1), dstH-1)]; const int firstChrSrcY= vChrFilterPos[chrDstY]; //First line needed as input - int lastLumSrcY= firstLumSrcY + vLumFilterSize -1; // Last line needed as input - int lastLumSrcY2=firstLumSrcY2+ vLumFilterSize -1; // Last line needed as input - int lastChrSrcY= firstChrSrcY + vChrFilterSize -1; // Last line needed as input + + // Last line needed as input + int lastLumSrcY = FFMIN(c->srcH, firstLumSrcY + vLumFilterSize) - 1; + int lastLumSrcY2 = FFMIN(c->srcH, firstLumSrcY2 + vLumFilterSize) - 1; + int lastChrSrcY = FFMIN(c->chrSrcH, firstChrSrcY + vChrFilterSize) - 1; int enough_lines; //handle holes (FAST_BILINEAR & weird filters) @@ -2585,6 +2587,49 @@ static int swScale(SwsContext *c, const uint8_t* src[], const int16_t **chrUSrcPtr= (const int16_t **) chrUPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize; const int16_t **chrVSrcPtr= (const int16_t **) chrVPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize; const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **) alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL; + + if (firstLumSrcY < 0 || firstLumSrcY + vLumFilterSize > c->srcH) { + const int16_t **tmpY = (const int16_t **) lumPixBuf + 2 * vLumBufSize; + int neg = -firstLumSrcY, i, end = FFMIN(c->srcH - firstLumSrcY, vLumFilterSize); + for (i = 0; i < neg; i++) + tmpY[i] = lumSrcPtr[neg]; + for ( ; i < end; i++) + tmpY[i] = lumSrcPtr[i]; + for ( ; i < vLumFilterSize; i++) + tmpY[i] = tmpY[i-1]; + lumSrcPtr = tmpY; + + if (alpSrcPtr) { + const int16_t **tmpA = (const int16_t **) alpPixBuf + 2 * vLumBufSize; + for (i = 0; i < neg; i++) + tmpA[i] = alpSrcPtr[neg]; + for ( ; i < end; i++) + tmpA[i] = alpSrcPtr[i]; + for ( ; i < vLumFilterSize; i++) + tmpA[i] = tmpA[i - 1]; + alpSrcPtr = tmpA; + } + } + if (firstChrSrcY < 0 || firstChrSrcY + vChrFilterSize > c->chrSrcH) { + const int16_t **tmpU = (const int16_t **) chrUPixBuf + 2 * vChrBufSize, + **tmpV = (const int16_t **) chrVPixBuf + 2 * vChrBufSize; + int neg = -firstChrSrcY, i, end = FFMIN(c->chrSrcH - firstChrSrcY, vChrFilterSize); + for (i = 0; i < neg; i++) { + tmpU[i] = chrUSrcPtr[neg]; + tmpV[i] = chrVSrcPtr[neg]; + } + for ( ; i < end; i++) { + tmpU[i] = chrUSrcPtr[i]; + tmpV[i] = chrVSrcPtr[i]; + } + for ( ; i < vChrFilterSize; i++) { + tmpU[i] = tmpU[i - 1]; + tmpV[i] = tmpV[i - 1]; + } + chrUSrcPtr = tmpU; + chrVSrcPtr = tmpV; + } + if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) { //YV12 like const int chrSkipMask= (1<chrDstVSubSample)-1; diff --git a/libswscale/utils.c b/libswscale/utils.c index b644ed9..12b3202 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -182,7 +182,7 @@ static double getSplineCoeff(double a, double b, double c, double d, double dist static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSize, int xInc, int srcW, int dstW, int filterAlign, int one, int flags, int cpu_flags, - SwsVector *srcFilter, SwsVector *dstFilter, double param[2]) + SwsVector *srcFilter, SwsVector *dstFilter, double param[2], int is_horizontal) { int i; int filterSize; @@ -459,27 +459,29 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi //FIXME try to align filterPos if possible //fix borders - for (i=0; i srcW) { - int shift= (*filterPos)[i] + filterSize - srcW; - // move filter coefficients right to compensate for filterPos - for (j=filterSize-2; j>=0; j--) { - int right= FFMIN(j + shift, filterSize-1); - filter[i*filterSize +right] += filter[i*filterSize +j]; - filter[i*filterSize +j]=0; + if ((*filterPos)[i] + filterSize > srcW) { + int shift = (*filterPos)[i] + filterSize - srcW; + // move filter coefficients right to compensate for filterPos + for (j = filterSize - 2; j >= 0; j--) { + int right = FFMIN(j + shift, filterSize - 1); + filter[i * filterSize + right] += filter[i * filterSize + j]; + filter[i * filterSize + j ] = 0; + } + (*filterPos)[i] = srcW - filterSize; } - (*filterPos)[i]= srcW - filterSize; } } @@ -958,12 +960,12 @@ int sws_init_context(SwsContext *c, SwsFilter *srcFilter, SwsFilter *dstFilter) if (initFilter(&c->hLumFilter, &c->hLumFilterPos, &c->hLumFilterSize, c->lumXInc, srcW , dstW, filterAlign, 1<<14, (flags&SWS_BICUBLIN) ? (flags|SWS_BICUBIC) : flags, cpu_flags, - srcFilter->lumH, dstFilter->lumH, c->param) < 0) + srcFilter->lumH, dstFilter->lumH, c->param, 1) < 0) goto fail; if (initFilter(&c->hChrFilter, &c->hChrFilterPos, &c->hChrFilterSize, c->chrXInc, c->chrSrcW, c->chrDstW, filterAlign, 1<<14, (flags&SWS_BICUBLIN) ? (flags|SWS_BILINEAR) : flags, cpu_flags, - srcFilter->chrH, dstFilter->chrH, c->param) < 0) + srcFilter->chrH, dstFilter->chrH, c->param, 1) < 0) goto fail; } } // initialize horizontal stuff @@ -978,12 +980,12 @@ int sws_init_context(SwsContext *c, SwsFilter *srcFilter, SwsFilter *dstFilter) if (initFilter(&c->vLumFilter, &c->vLumFilterPos, &c->vLumFilterSize, c->lumYInc, srcH , dstH, filterAlign, (1<<12), (flags&SWS_BICUBLIN) ? (flags|SWS_BICUBIC) : flags, cpu_flags, - srcFilter->lumV, dstFilter->lumV, c->param) < 0) + srcFilter->lumV, dstFilter->lumV, c->param, 0) < 0) goto fail; if (initFilter(&c->vChrFilter, &c->vChrFilterPos, &c->vChrFilterSize, c->chrYInc, c->chrSrcH, c->chrDstH, filterAlign, (1<<12), (flags&SWS_BICUBLIN) ? (flags|SWS_BILINEAR) : flags, cpu_flags, - srcFilter->chrV, dstFilter->chrV, c->param) < 0) + srcFilter->chrV, dstFilter->chrV, c->param, 0) < 0) goto fail; #if HAVE_ALTIVEC @@ -1024,11 +1026,11 @@ int sws_init_context(SwsContext *c, SwsFilter *srcFilter, SwsFilter *dstFilter) // allocate pixbufs (we use dynamic allocation because otherwise we would need to // allocate several megabytes to handle all possible cases) - FF_ALLOC_OR_GOTO(c, c->lumPixBuf, c->vLumBufSize*2*sizeof(int16_t*), fail); - FF_ALLOC_OR_GOTO(c, c->chrUPixBuf, c->vChrBufSize*2*sizeof(int16_t*), fail); - FF_ALLOC_OR_GOTO(c, c->chrVPixBuf, c->vChrBufSize*2*sizeof(int16_t*), fail); + FF_ALLOC_OR_GOTO(c, c->lumPixBuf, c->vLumBufSize*3*sizeof(int16_t*), fail); + FF_ALLOC_OR_GOTO(c, c->chrUPixBuf, c->vChrBufSize*3*sizeof(int16_t*), fail); + FF_ALLOC_OR_GOTO(c, c->chrVPixBuf, c->vChrBufSize*3*sizeof(int16_t*), fail); if (CONFIG_SWSCALE_ALPHA && isALPHA(c->srcFormat) && isALPHA(c->dstFormat)) - FF_ALLOCZ_OR_GOTO(c, c->alpPixBuf, c->vLumBufSize*2*sizeof(int16_t*), fail); + FF_ALLOCZ_OR_GOTO(c, c->alpPixBuf, c->vLumBufSize*3*sizeof(int16_t*), fail); //Note we need at least one pixel more at the end because of the MMX code (just in case someone wanna replace the 4000/8000) /* align at 16 bytes for AltiVec */ for (i=0; ivLumBufSize; i++) { -- 2.7.4