From 4ab10950b702d3352c12c94c5c910a0f5cc1ca97 Mon Sep 17 00:00:00 2001 From: Karl Williamson Date: Thu, 8 Nov 2012 12:04:34 -0700 Subject: [PATCH] utf8.h: Add macro that handled malformed 2-byte UTF-8 The macro used traditionally to see if there is a two-byte UTF-8 sequence doesn't make sure that there actually is a second byte available; it only checks if the first byte indicates that there is. This adds a macro that is safe in the face of malformed UTF-8. I inspected the existing calls in the core to the unsafe macro, and I believe that none of them need to be converted to the safe version. --- utf8.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utf8.h b/utf8.h index 7472de1..3fbc28a 100644 --- a/utf8.h +++ b/utf8.h @@ -171,8 +171,11 @@ Perl's extended UTF-8 means we can have start bytes up to FF. #define UTF8_IS_CONTINUATION(c) ((((U8)c) & 0xC0) == 0x80) #define UTF8_IS_CONTINUED(c) (((U8)c) & 0x80) -/* Masking with 0xfe allows low bit to be 0 or 1; thus this matches 0xc[23] */ +/* Use UTF8_IS_NEXT_CHAR_DOWNGRADEABLE() instead if the input isn't known to + * be well-formed. Masking with 0xfe allows low bit to be 0 or 1; thus this + * matches 0xc[23]. */ #define UTF8_IS_DOWNGRADEABLE_START(c) (((U8)c & 0xfe) == 0xc2) + #define UTF8_IS_ABOVE_LATIN1(c) ((U8)(c) >= 0xc4) #define UTF_START_MARK(len) (((len) > 7) ? 0xFF : (0xFE << (7-(len)))) @@ -243,6 +246,11 @@ Perl's extended UTF-8 means we can have start bytes up to FF. #define UTF8_ACCUMULATE(old, new) (((old) << UTF_ACCUMULATION_SHIFT) \ | (((U8)new) & UTF_CONTINUATION_MASK)) +/* This works in the face of malformed UTF-8. */ +#define UTF8_IS_NEXT_CHAR_DOWNGRADEABLE(s, e) (UTF8_IS_DOWNGRADEABLE_START(*s) \ + && ( (e) - (s) > 1) \ + && UTF8_IS_CONTINUATION(*((s)+1))) + /* Convert a two (not one) byte utf8 character to a unicode code point value. * Needs just one iteration of accumulate. Should not be used unless it is * known that the two bytes are legal: 1) two-byte start, and 2) continuation. -- 2.7.4