Add code support for ICU. diff --git a/third_party/libxml/encoding.c b/third_party/libxml/encoding.c index b86a547..0f41df9 100644 --- a/third_party/libxml/encoding.c +++ b/third_party/libxml/encoding.c @@ -58,7 +58,7 @@ static xmlCharEncodingAliasPtr xmlCharEncodingAliases = NULL; static int xmlCharEncodingAliasesNb = 0; static int xmlCharEncodingAliasesMax = 0; -#ifdef LIBXML_ICONV_ENABLED +#if defined(LIBXML_ICONV_ENABLED) || defined(LIBXML_ICU_ENABLED) #if 0 #define DEBUG_ENCODING /* Define this to get encoding traces */ #endif @@ -97,6 +97,54 @@ xmlEncodingErr(xmlParserErrors error, const char *msg, const char *val) NULL, 0, val, NULL, NULL, 0, 0, msg, val); } +#ifdef LIBXML_ICU_ENABLED +static uconv_t* +openIcuConverter(const char* name, int toUnicode) +{ + UErrorCode status = U_ZERO_ERROR; + uconv_t *conv = (uconv_t *) xmlMalloc(sizeof(uconv_t)); + if (conv == NULL) + return NULL; + + conv->uconv = ucnv_open(name, &status); + if (U_FAILURE(status)) + goto error; + + status = U_ZERO_ERROR; + if (toUnicode) { + ucnv_setToUCallBack(conv->uconv, UCNV_TO_U_CALLBACK_STOP, + NULL, NULL, NULL, &status); + } + else { + ucnv_setFromUCallBack(conv->uconv, UCNV_FROM_U_CALLBACK_STOP, + NULL, NULL, NULL, &status); + } + if (U_FAILURE(status)) + goto error; + + status = U_ZERO_ERROR; + conv->utf8 = ucnv_open("UTF-8", &status); + if (U_SUCCESS(status)) + return conv; + +error: + if (conv->uconv) + ucnv_close(conv->uconv); + xmlFree(conv); + return NULL; +} + +static void +closeIcuConverter(uconv_t *conv) +{ + if (conv != NULL) { + ucnv_close(conv->uconv); + ucnv_close(conv->utf8); + xmlFree(conv); + } +} +#endif /* LIBXML_ICU_ENABLED */ + /************************************************************************ * * * Conversions To/From UTF8 encoding * @@ -1306,7 +1354,11 @@ xmlNewCharEncodingHandler(const char *name, #ifdef LIBXML_ICONV_ENABLED handler->iconv_in = NULL; handler->iconv_out = NULL; -#endif /* LIBXML_ICONV_ENABLED */ +#endif +#ifdef LIBXML_ICU_ENABLED + handler->uconv_in = NULL; + handler->uconv_out = NULL; +#endif /* * registers and returns the handler. @@ -1371,7 +1423,7 @@ xmlInitCharEncodingHandlers(void) { xmlNewCharEncodingHandler("ASCII", asciiToUTF8, NULL); xmlNewCharEncodingHandler("US-ASCII", asciiToUTF8, NULL); #endif /* LIBXML_OUTPUT_ENABLED */ -#ifndef LIBXML_ICONV_ENABLED +#if !defined(LIBXML_ICONV_ENABLED) && !defined(LIBXML_ICU_ENABLED) #ifdef LIBXML_ISO8859X_ENABLED xmlRegisterCharEncodingHandlersISO8859x (); #endif @@ -1578,6 +1630,10 @@ xmlFindCharEncodingHandler(const char *name) { xmlCharEncodingHandlerPtr enc; iconv_t icv_in, icv_out; #endif /* LIBXML_ICONV_ENABLED */ +#ifdef LIBXML_ICU_ENABLED + xmlCharEncodingHandlerPtr enc; + uconv_t *ucv_in, *ucv_out; +#endif /* LIBXML_ICU_ENABLED */ char upper[100]; int i; @@ -1647,6 +1703,35 @@ xmlFindCharEncodingHandler(const char *name) { "iconv : problems with filters for '%s'\n", name); } #endif /* LIBXML_ICONV_ENABLED */ +#ifdef LIBXML_ICU_ENABLED + /* check whether icu can handle this */ + ucv_in = openIcuConverter(name, 1); + ucv_out = openIcuConverter(name, 0); + if (ucv_in != NULL && ucv_out != NULL) { + enc = (xmlCharEncodingHandlerPtr) + xmlMalloc(sizeof(xmlCharEncodingHandler)); + if (enc == NULL) { + closeIcuConverter(ucv_in); + closeIcuConverter(ucv_out); + return(NULL); + } + enc->name = xmlMemStrdup(name); + enc->input = NULL; + enc->output = NULL; + enc->uconv_in = ucv_in; + enc->uconv_out = ucv_out; +#ifdef DEBUG_ENCODING + xmlGenericError(xmlGenericErrorContext, + "Found ICU converter handler for encoding %s\n", name); +#endif + return enc; + } else if (ucv_in != NULL || ucv_out != NULL) { + closeIcuConverter(ucv_in); + closeIcuConverter(ucv_out); + xmlEncodingErr(XML_ERR_INTERNAL_ERROR, + "ICU converter : problems with filters for '%s'\n", name); + } +#endif /* LIBXML_ICU_ENABLED */ #ifdef DEBUG_ENCODING xmlGenericError(xmlGenericErrorContext, @@ -1737,6 +1822,75 @@ xmlIconvWrapper(iconv_t cd, unsigned char *out, int *outlen, /************************************************************************ * * + * ICU based generic conversion functions * + * * + ************************************************************************/ + +#ifdef LIBXML_ICU_ENABLED +/** + * xmlUconvWrapper: + * @cd: ICU uconverter data structure + * @toUnicode : non-zero if toUnicode. 0 otherwise. + * @out: a pointer to an array of bytes to store the result + * @outlen: the length of @out + * @in: a pointer to an array of ISO Latin 1 chars + * @inlen: the length of @in + * + * Returns 0 if success, or + * -1 by lack of space, or + * -2 if the transcoding fails (for *in is not valid utf8 string or + * the result of transformation can't fit into the encoding we want), or + * -3 if there the last byte can't form a single output char. + * + * The value of @inlen after return is the number of octets consumed + * as the return value is positive, else unpredictable. + * The value of @outlen after return is the number of ocetes consumed. + */ +static int +xmlUconvWrapper(uconv_t *cd, int toUnicode, unsigned char *out, int *outlen, + const unsigned char *in, int *inlen) { + const char *ucv_in = (const char *) in; + char *ucv_out = (char *) out; + UErrorCode err = U_ZERO_ERROR; + + if ((out == NULL) || (outlen == NULL) || (inlen == NULL) || (in == NULL)) { + if (outlen != NULL) *outlen = 0; + return(-1); + } + + /* + * TODO(jungshik) + * 1. is ucnv_convert(To|From)Algorithmic better? + * 2. had we better use an explicit pivot buffer? + * 3. error returned comes from 'fromUnicode' only even + * when toUnicode is true ! + */ + if (toUnicode) { + /* encoding => UTF-16 => UTF-8 */ + ucnv_convertEx(cd->utf8, cd->uconv, &ucv_out, ucv_out + *outlen, + &ucv_in, ucv_in + *inlen, NULL, NULL, NULL, NULL, + 0, TRUE, &err); + } else { + /* UTF-8 => UTF-16 => encoding */ + ucnv_convertEx(cd->uconv, cd->utf8, &ucv_out, ucv_out + *outlen, + &ucv_in, ucv_in + *inlen, NULL, NULL, NULL, NULL, + 0, TRUE, &err); + } + *inlen = ucv_in - (const char*) in; + *outlen = ucv_out - (char *) out; + if (U_SUCCESS(err)) + return 0; + if (err == U_BUFFER_OVERFLOW_ERROR) + return -1; + if (err == U_INVALID_CHAR_FOUND || err == U_ILLEGAL_CHAR_FOUND) + return -2; + /* if (err == U_TRUNCATED_CHAR_FOUND) */ + return -3; +} +#endif /* LIBXML_ICU_ENABLED */ + +/************************************************************************ + * * * The real API used by libxml for on-the-fly conversion * * * ************************************************************************/ @@ -1810,6 +1964,16 @@ xmlCharEncFirstLineInt(xmlCharEncodingHandler *handler, xmlBufferPtr out, if (ret == -1) ret = -3; } #endif /* LIBXML_ICONV_ENABLED */ +#ifdef LIBXML_ICU_ENABLED + else if (handler->uconv_in != NULL) { + ret = xmlUconvWrapper(handler->uconv_in, 1, &out->content[out->use], + &written, in->content, &toconv); + xmlBufferShrink(in, toconv); + out->use += written; + out->content[out->use] = 0; + if (ret == -1) ret = -3; + } +#endif /* LIBXML_ICU_ENABLED */ #ifdef DEBUG_ENCODING switch (ret) { case 0: @@ -1915,6 +2079,17 @@ xmlCharEncInFunc(xmlCharEncodingHandler * handler, xmlBufferPtr out, ret = -3; } #endif /* LIBXML_ICONV_ENABLED */ +#ifdef LIBXML_ICU_ENABLED + else if (handler->uconv_in != NULL) { + ret = xmlUconvWrapper(handler->uconv_in, 1, &out->content[out->use], + &written, in->content, &toconv); + xmlBufferShrink(in, toconv); + out->use += written; + out->content[out->use] = 0; + if (ret == -1) + ret = -3; + } +#endif /* LIBXML_ICU_ENABLED */ switch (ret) { case 0: #ifdef DEBUG_ENCODING @@ -2015,6 +2190,15 @@ retry: out->content[out->use] = 0; } #endif /* LIBXML_ICONV_ENABLED */ +#ifdef LIBXML_ICU_ENABLED + else if (handler->uconv_out != NULL) { + ret = xmlUconvWrapper(handler->uconv_out, 0, + &out->content[out->use], + &written, NULL, &toconv); + out->use += written; + out->content[out->use] = 0; + } +#endif /* LIBXML_ICU_ENABLED */ #ifdef DEBUG_ENCODING xmlGenericError(xmlGenericErrorContext, "initialized encoder\n"); @@ -2061,6 +2245,26 @@ retry: } } #endif /* LIBXML_ICONV_ENABLED */ +#ifdef LIBXML_ICU_ENABLED + else if (handler->uconv_out != NULL) { + ret = xmlUconvWrapper(handler->uconv_out, 0, + &out->content[out->use], + &written, in->content, &toconv); + xmlBufferShrink(in, toconv); + out->use += written; + writtentot += written; + out->content[out->use] = 0; + if (ret == -1) { + if (written > 0) { + /* + * Can be a limitation of iconv + */ + goto retry; + } + ret = -3; + } + } +#endif /* LIBXML_ICU_ENABLED */ else { xmlEncodingErr(XML_I18N_NO_OUTPUT, "xmlCharEncOutFunc: no output function !\n", NULL); @@ -2173,6 +2377,22 @@ xmlCharEncCloseFunc(xmlCharEncodingHandler *handler) { xmlFree(handler); } #endif /* LIBXML_ICONV_ENABLED */ +#ifdef LIBXML_ICU_ENABLED + if ((handler->uconv_out != NULL) || (handler->uconv_in != NULL)) { + if (handler->name != NULL) + xmlFree(handler->name); + handler->name = NULL; + if (handler->uconv_out != NULL) { + closeIcuConverter(handler->uconv_out); + handler->uconv_out = NULL; + } + if (handler->uconv_in != NULL) { + closeIcuConverter(handler->uconv_in); + handler->uconv_in = NULL; + } + xmlFree(handler); + } +#endif #ifdef DEBUG_ENCODING if (ret) xmlGenericError(xmlGenericErrorContext, @@ -2248,6 +2468,22 @@ xmlByteConsumed(xmlParserCtxtPtr ctxt) { cur += toconv; } while (ret == -2); #endif +#ifdef LIBXML_ICU_ENABLED + } else if (handler->uconv_out != NULL) { + do { + toconv = in->end - cur; + written = 32000; + ret = xmlUconvWrapper(handler->uconv_out, 0, &convbuf[0], + &written, cur, &toconv); + if (ret < 0) { + if (written > 0) + ret = -2; + else + return(-1); + } + unused += written; + cur += toconv; + } while (ret == -2); } else { /* could not find a converter */ return(-1); @@ -2259,8 +2495,9 @@ xmlByteConsumed(xmlParserCtxtPtr ctxt) { } return(in->consumed + (in->cur - in->base)); } +#endif -#ifndef LIBXML_ICONV_ENABLED +#if !defined(LIBXML_ICONV_ENABLED) && !defined(LIBXML_ICU_ENABLED) #ifdef LIBXML_ISO8859X_ENABLED /** diff --git a/third_party/libxml/include/libxml/encoding.h b/third_party/libxml/include/libxml/encoding.h index c74b25f..b5f8b48 100644 --- a/third_party/libxml/include/libxml/encoding.h +++ b/third_party/libxml/include/libxml/encoding.h @@ -26,6 +26,24 @@ #ifdef LIBXML_ICONV_ENABLED #include +#else +#ifdef LIBXML_ICU_ENABLED +#include +#if 0 +/* Forward-declare UConverter here rather than pulling in + * to prevent unwanted ICU symbols being exposed to users of libxml2. + * One particular case is Qt4 conflicting on UChar32. + */ +#include +struct UConverter; +typedef struct UConverter UConverter; +#ifdef _MSC_VER +typedef wchar_t UChar; +#else +typedef uint16_t UChar; +#endif +#endif +#endif #endif #ifdef __cplusplus extern "C" { @@ -125,6 +143,13 @@ typedef int (* xmlCharEncodingOutputFunc)(unsigned char *out, int *outlen, * Block defining the handlers for non UTF-8 encodings. * If iconv is supported, there are two extra fields. */ +#ifdef LIBXML_ICU_ENABLED +struct _uconv_t { + UConverter *uconv; /* for conversion between an encoding and UTF-16 */ + UConverter *utf8; /* for conversion between UTF-8 and UTF-16 */ +}; +typedef struct _uconv_t uconv_t; +#endif typedef struct _xmlCharEncodingHandler xmlCharEncodingHandler; typedef xmlCharEncodingHandler *xmlCharEncodingHandlerPtr; @@ -136,6 +161,10 @@ struct _xmlCharEncodingHandler { iconv_t iconv_in; iconv_t iconv_out; #endif /* LIBXML_ICONV_ENABLED */ +#ifdef LIBXML_ICU_ENABLED + uconv_t *uconv_in; + uconv_t *uconv_out; +#endif /* LIBXML_ICU_ENABLED */ }; #ifdef __cplusplus diff --git a/third_party/libxml/include/libxml/parser.h b/third_party/libxml/include/libxml/parser.h index dd79c42..3580b63 100644 --- a/third_party/libxml/include/libxml/parser.h +++ b/third_party/libxml/include/libxml/parser.h @@ -1222,6 +1222,7 @@ typedef enum { XML_WITH_DEBUG_MEM = 29, XML_WITH_DEBUG_RUN = 30, XML_WITH_ZLIB = 31, + XML_WITH_ICU = 32, XML_WITH_NONE = 99999 /* just to be sure of allocation size */ } xmlFeature; diff --git a/third_party/libxml/include/libxml/xmlversion.h.in b/third_party/libxml/include/libxml/xmlversion.h.in index 4739f3a..de310ab 100644 --- a/third_party/libxml/include/libxml/xmlversion.h.in +++ b/third_party/libxml/include/libxml/xmlversion.h.in @@ -269,6 +269,15 @@ XMLPUBFUN void XMLCALL xmlCheckVersion(int version); #endif /** + * LIBXML_ICU_ENABLED: + * + * Whether icu support is available + */ +#if @WITH_ICU@ +#define LIBXML_ICU_ENABLED +#endif + +/** * LIBXML_ISO8859X_ENABLED: * * Whether ISO-8859-* support is made available in case iconv is not diff --git a/third_party/libxml/parser.c b/third_party/libxml/parser.c index 85e7599..3ba2a06 100644 --- a/third_party/libxml/parser.c +++ b/third_party/libxml/parser.c @@ -954,6 +954,12 @@ xmlHasFeature(xmlFeature feature) #else return(0); #endif + case XML_WITH_ICU: +#ifdef LIBXML_ICU_ENABLED + return(1); +#else + return(0); +#endif default: break; }