mathops: remove undefined behaviour from sign_extend()

This function intentionally overflows the signed range on
the left shift.  Using this type-punning avoids errors from
the overflow checker without disabling this test globally.

Signed-off-by: Mans Rullgard <mans@mansr.com>

diff --git a/libavcodec/mathops.h b/libavcodec/mathops.h
index ec76eaa..45b1ecf 100644 (file)
--- a/libavcodec/mathops.h
+++ b/libavcodec/mathops.h
@@ -116,7 +116,9 @@ static inline av_const int mid_pred(int a, int b, int c)
 #ifndef sign_extend
 static inline av_const int sign_extend(int val, unsigned bits)
 {
-    return (val << ((8 * sizeof(int)) - bits)) >> ((8 * sizeof(int)) - bits);
+    unsigned shift = 8 * sizeof(int) - bits;
+    union { unsigned u; int s; } v = { (unsigned) val << shift };
+    return v.s >> shift;
 }
 #endif