src/third_party/libxml/patches/icu

   1 Add code support for ICU.
   2
   3 diff --git a/third_party/libxml/encoding.c b/third_party/libxml/encoding.c
   4 index b86a547..0f41df9 100644
   5 --- a/third_party/libxml/encoding.c
   6 +++ b/third_party/libxml/encoding.c
   7 @@ -58,7 +58,7 @@ static xmlCharEncodingAliasPtr xmlCharEncodingAliases = NULL;
   8  static int xmlCharEncodingAliasesNb = 0;
   9  static int xmlCharEncodingAliasesMax = 0;
  10
  11 -#ifdef LIBXML_ICONV_ENABLED
  12 +#if defined(LIBXML_ICONV_ENABLED) || defined(LIBXML_ICU_ENABLED)
  13  #if 0
  14  #define DEBUG_ENCODING  /* Define this to get encoding traces */
  15  #endif
  16 @@ -97,6 +97,54 @@ xmlEncodingErr(xmlParserErrors error, const char *msg, const char *val)
  17                      NULL, 0, val, NULL, NULL, 0, 0, msg, val);
  18  }
  19
  20 +#ifdef LIBXML_ICU_ENABLED
  21 +static uconv_t*
  22 +openIcuConverter(const char* name, int toUnicode)
  23 +{
  24 +  UErrorCode status = U_ZERO_ERROR;
  25 +  uconv_t *conv = (uconv_t *) xmlMalloc(sizeof(uconv_t));
  26 +  if (conv == NULL)
  27 +    return NULL;
  28 +
  29 +  conv->uconv = ucnv_open(name, &status);
  30 +  if (U_FAILURE(status))
  31 +    goto error;
  32 +
  33 +  status = U_ZERO_ERROR;
  34 +  if (toUnicode) {
  35 +    ucnv_setToUCallBack(conv->uconv, UCNV_TO_U_CALLBACK_STOP,
  36 +                        NULL, NULL, NULL, &status);
  37 +  }
  38 +  else {
  39 +    ucnv_setFromUCallBack(conv->uconv, UCNV_FROM_U_CALLBACK_STOP,
  40 +                        NULL, NULL, NULL, &status);
  41 +  }
  42 +  if (U_FAILURE(status))
  43 +    goto error;
  44 +
  45 +  status = U_ZERO_ERROR;
  46 +  conv->utf8 = ucnv_open("UTF-8", &status);
  47 +  if (U_SUCCESS(status))
  48 +    return conv;
  49 +
  50 +error:
  51 +  if (conv->uconv)
  52 +    ucnv_close(conv->uconv);
  53 +  xmlFree(conv);
  54 +  return NULL;
  55 +}
  56 +
  57 +static void
  58 +closeIcuConverter(uconv_t *conv)
  59 +{
  60 +  if (conv != NULL) {
  61 +    ucnv_close(conv->uconv);
  62 +    ucnv_close(conv->utf8);
  63 +    xmlFree(conv);
  64 +  }
  65 +}
  66 +#endif /* LIBXML_ICU_ENABLED */
  67 +
  68  /************************************************************************
  69   *                                                                     *
  70   *             Conversions To/From UTF8 encoding                       *
  71 @@ -1306,7 +1354,11 @@ xmlNewCharEncodingHandler(const char *name,
  72  #ifdef LIBXML_ICONV_ENABLED
  73      handler->iconv_in = NULL;
  74      handler->iconv_out = NULL;
  75 -#endif /* LIBXML_ICONV_ENABLED */
  76 +#endif
  77 +#ifdef LIBXML_ICU_ENABLED
  78 +    handler->uconv_in = NULL;
  79 +    handler->uconv_out = NULL;
  80 +#endif
  81
  82      /*
  83       * registers and returns the handler.
  84 @@ -1371,7 +1423,7 @@ xmlInitCharEncodingHandlers(void) {
  85      xmlNewCharEncodingHandler("ASCII", asciiToUTF8, NULL);
  86      xmlNewCharEncodingHandler("US-ASCII", asciiToUTF8, NULL);
  87  #endif /* LIBXML_OUTPUT_ENABLED */
  88 -#ifndef LIBXML_ICONV_ENABLED
  89 +#if !defined(LIBXML_ICONV_ENABLED) && !defined(LIBXML_ICU_ENABLED)
  90  #ifdef LIBXML_ISO8859X_ENABLED
  91      xmlRegisterCharEncodingHandlersISO8859x ();
  92  #endif
  93 @@ -1578,6 +1630,10 @@ xmlFindCharEncodingHandler(const char *name) {
  94      xmlCharEncodingHandlerPtr enc;
  95      iconv_t icv_in, icv_out;
  96  #endif /* LIBXML_ICONV_ENABLED */
  97 +#ifdef LIBXML_ICU_ENABLED
  98 +    xmlCharEncodingHandlerPtr enc;
  99 +    uconv_t *ucv_in, *ucv_out;
 100 +#endif /* LIBXML_ICU_ENABLED */
 101      char upper[100];
 102      int i;
 103
 104 @@ -1647,6 +1703,35 @@ xmlFindCharEncodingHandler(const char *name) {
 105                     "iconv : problems with filters for '%s'\n", name);
 106      }
 107  #endif /* LIBXML_ICONV_ENABLED */
 108 +#ifdef LIBXML_ICU_ENABLED
 109 +    /* check whether icu can handle this */
 110 +    ucv_in = openIcuConverter(name, 1);
 111 +    ucv_out = openIcuConverter(name, 0);
 112 +    if (ucv_in != NULL && ucv_out != NULL) {
 113 +           enc = (xmlCharEncodingHandlerPtr)
 114 +                 xmlMalloc(sizeof(xmlCharEncodingHandler));
 115 +           if (enc == NULL) {
 116 +                closeIcuConverter(ucv_in);
 117 +                closeIcuConverter(ucv_out);
 118 +               return(NULL);
 119 +           }
 120 +           enc->name = xmlMemStrdup(name);
 121 +           enc->input = NULL;
 122 +           enc->output = NULL;
 123 +           enc->uconv_in = ucv_in;
 124 +           enc->uconv_out = ucv_out;
 125 +#ifdef DEBUG_ENCODING
 126 +            xmlGenericError(xmlGenericErrorContext,
 127 +                   "Found ICU converter handler for encoding %s\n", name);
 128 +#endif
 129 +           return enc;
 130 +    } else if (ucv_in != NULL || ucv_out != NULL) {
 131 +            closeIcuConverter(ucv_in);
 132 +            closeIcuConverter(ucv_out);
 133 +           xmlEncodingErr(XML_ERR_INTERNAL_ERROR,
 134 +                   "ICU converter : problems with filters for '%s'\n", name);
 135 +    }
 136 +#endif /* LIBXML_ICU_ENABLED */
 137
 138  #ifdef DEBUG_ENCODING
 139      xmlGenericError(xmlGenericErrorContext,
 140 @@ -1737,6 +1822,75 @@ xmlIconvWrapper(iconv_t cd, unsigned char *out, int *outlen,
 141
 142  /************************************************************************
 143   *                                                                     *
 144 + *             ICU based generic conversion functions                  *
 145 + *                                                                     *
 146 + ************************************************************************/
 147 +
 148 +#ifdef LIBXML_ICU_ENABLED
 149 +/**
 150 + * xmlUconvWrapper:
 151 + * @cd: ICU uconverter data structure
 152 + * @toUnicode : non-zero if toUnicode. 0 otherwise.
 153 + * @out:  a pointer to an array of bytes to store the result
 154 + * @outlen:  the length of @out
 155 + * @in:  a pointer to an array of ISO Latin 1 chars
 156 + * @inlen:  the length of @in
 157 + *
 158 + * Returns 0 if success, or
 159 + *     -1 by lack of space, or
 160 + *     -2 if the transcoding fails (for *in is not valid utf8 string or
 161 + *        the result of transformation can't fit into the encoding we want), or
 162 + *     -3 if there the last byte can't form a single output char.
 163 + *
 164 + * The value of @inlen after return is the number of octets consumed
 165 + *     as the return value is positive, else unpredictable.
 166 + * The value of @outlen after return is the number of ocetes consumed.
 167 + */
 168 +static int
 169 +xmlUconvWrapper(uconv_t *cd, int toUnicode, unsigned char *out, int *outlen,
 170 +                const unsigned char *in, int *inlen) {
 171 +    const char *ucv_in = (const char *) in;
 172 +    char *ucv_out = (char *) out;
 173 +    UErrorCode err = U_ZERO_ERROR;
 174 +
 175 +    if ((out == NULL) || (outlen == NULL) || (inlen == NULL) || (in == NULL)) {
 176 +        if (outlen != NULL) *outlen = 0;
 177 +        return(-1);
 178 +    }
 179 +
 180 +    /*
 181 +     * TODO(jungshik)
 182 +     * 1. is ucnv_convert(To|From)Algorithmic better?
 183 +     * 2. had we better use an explicit pivot buffer?
 184 +     * 3. error returned comes from 'fromUnicode' only even
 185 +     *    when toUnicode is true !
 186 +     */
 187 +    if (toUnicode) {
 188 +        /* encoding => UTF-16 => UTF-8 */
 189 +        ucnv_convertEx(cd->utf8, cd->uconv, &ucv_out, ucv_out + *outlen,
 190 +                       &ucv_in, ucv_in + *inlen, NULL, NULL, NULL, NULL,
 191 +                       0, TRUE, &err);
 192 +    } else {
 193 +        /* UTF-8 => UTF-16 => encoding */
 194 +        ucnv_convertEx(cd->uconv, cd->utf8, &ucv_out, ucv_out + *outlen,
 195 +                       &ucv_in, ucv_in + *inlen, NULL, NULL, NULL, NULL,
 196 +                       0, TRUE, &err);
 197 +    }
 198 +    *inlen = ucv_in - (const char*) in;
 199 +    *outlen = ucv_out - (char *) out;
 200 +    if (U_SUCCESS(err))
 201 +        return 0;
 202 +    if (err == U_BUFFER_OVERFLOW_ERROR)
 203 +        return -1;
 204 +    if (err == U_INVALID_CHAR_FOUND || err == U_ILLEGAL_CHAR_FOUND)
 205 +        return -2;
 206 +    /* if (err == U_TRUNCATED_CHAR_FOUND) */
 207 +    return -3;
 208 +}
 209 +#endif /* LIBXML_ICU_ENABLED */
 210 +
 211 +/************************************************************************
 212 + *                                                                     *
 213   *             The real API used by libxml for on-the-fly conversion   *
 214   *                                                                     *
 215   ************************************************************************/
 216 @@ -1810,6 +1964,16 @@ xmlCharEncFirstLineInt(xmlCharEncodingHandler *handler, xmlBufferPtr out,
 217         if (ret == -1) ret = -3;
 218      }
 219  #endif /* LIBXML_ICONV_ENABLED */
 220 +#ifdef LIBXML_ICU_ENABLED
 221 +    else if (handler->uconv_in != NULL) {
 222 +       ret = xmlUconvWrapper(handler->uconv_in, 1, &out->content[out->use],
 223 +                             &written, in->content, &toconv);
 224 +       xmlBufferShrink(in, toconv);
 225 +       out->use += written;
 226 +       out->content[out->use] = 0;
 227 +       if (ret == -1) ret = -3;
 228 +    }
 229 +#endif /* LIBXML_ICU_ENABLED */
 230  #ifdef DEBUG_ENCODING
 231      switch (ret) {
 232          case 0:
 233 @@ -1915,6 +2079,17 @@ xmlCharEncInFunc(xmlCharEncodingHandler * handler, xmlBufferPtr out,
 234              ret = -3;
 235      }
 236  #endif /* LIBXML_ICONV_ENABLED */
 237 +#ifdef LIBXML_ICU_ENABLED
 238 +    else if (handler->uconv_in != NULL) {
 239 +        ret = xmlUconvWrapper(handler->uconv_in, 1, &out->content[out->use],
 240 +                              &written, in->content, &toconv);
 241 +        xmlBufferShrink(in, toconv);
 242 +        out->use += written;
 243 +        out->content[out->use] = 0;
 244 +        if (ret == -1)
 245 +            ret = -3;
 246 +    }
 247 +#endif /* LIBXML_ICU_ENABLED */
 248      switch (ret) {
 249          case 0:
 250  #ifdef DEBUG_ENCODING
 251 @@ -2015,6 +2190,15 @@ retry:
 252             out->content[out->use] = 0;
 253         }
 254  #endif /* LIBXML_ICONV_ENABLED */
 255 +#ifdef LIBXML_ICU_ENABLED
 256 +       else if (handler->uconv_out != NULL) {
 257 +           ret = xmlUconvWrapper(handler->uconv_out, 0,
 258 +                              &out->content[out->use],
 259 +                                             &written, NULL, &toconv);
 260 +           out->use += written;
 261 +           out->content[out->use] = 0;
 262 +       }
 263 +#endif /* LIBXML_ICU_ENABLED */
 264  #ifdef DEBUG_ENCODING
 265         xmlGenericError(xmlGenericErrorContext,
 266                 "initialized encoder\n");
 267 @@ -2061,6 +2245,26 @@ retry:
 268         }
 269      }
 270  #endif /* LIBXML_ICONV_ENABLED */
 271 +#ifdef LIBXML_ICU_ENABLED
 272 +    else if (handler->uconv_out != NULL) {
 273 +       ret = xmlUconvWrapper(handler->uconv_out, 0,
 274 +                              &out->content[out->use],
 275 +                             &written, in->content, &toconv);
 276 +       xmlBufferShrink(in, toconv);
 277 +       out->use += written;
 278 +       writtentot += written;
 279 +       out->content[out->use] = 0;
 280 +       if (ret == -1) {
 281 +           if (written > 0) {
 282 +               /*
 283 +                * Can be a limitation of iconv
 284 +                */
 285 +               goto retry;
 286 +           }
 287 +           ret = -3;
 288 +       }
 289 +    }
 290 +#endif /* LIBXML_ICU_ENABLED */
 291      else {
 292         xmlEncodingErr(XML_I18N_NO_OUTPUT,
 293                        "xmlCharEncOutFunc: no output function !\n", NULL);
 294 @@ -2173,6 +2377,22 @@ xmlCharEncCloseFunc(xmlCharEncodingHandler *handler) {
 295         xmlFree(handler);
 296      }
 297  #endif /* LIBXML_ICONV_ENABLED */
 298 +#ifdef LIBXML_ICU_ENABLED
 299 +    if ((handler->uconv_out != NULL) || (handler->uconv_in != NULL)) {
 300 +       if (handler->name != NULL)
 301 +           xmlFree(handler->name);
 302 +       handler->name = NULL;
 303 +       if (handler->uconv_out != NULL) {
 304 +           closeIcuConverter(handler->uconv_out);
 305 +           handler->uconv_out = NULL;
 306 +       }
 307 +       if (handler->uconv_in != NULL) {
 308 +           closeIcuConverter(handler->uconv_in);
 309 +           handler->uconv_in = NULL;
 310 +       }
 311 +       xmlFree(handler);
 312 +    }
 313 +#endif
 314  #ifdef DEBUG_ENCODING
 315      if (ret)
 316          xmlGenericError(xmlGenericErrorContext,
 317 @@ -2248,6 +2468,22 @@ xmlByteConsumed(xmlParserCtxtPtr ctxt) {
 318                     cur += toconv;
 319                 } while (ret == -2);
 320  #endif
 321 +#ifdef LIBXML_ICU_ENABLED
 322 +           } else if (handler->uconv_out != NULL) {
 323 +               do {
 324 +                   toconv = in->end - cur;
 325 +                   written = 32000;
 326 +                   ret = xmlUconvWrapper(handler->uconv_out, 0, &convbuf[0],
 327 +                             &written, cur, &toconv);
 328 +                   if (ret < 0) {
 329 +                       if (written > 0)
 330 +                           ret = -2;
 331 +                       else
 332 +                           return(-1);
 333 +                   }
 334 +                   unused += written;
 335 +                   cur += toconv;
 336 +               } while (ret == -2);
 337              } else {
 338                 /* could not find a converter */
 339                 return(-1);
 340 @@ -2259,8 +2495,9 @@ xmlByteConsumed(xmlParserCtxtPtr ctxt) {
 341      }
 342      return(in->consumed + (in->cur - in->base));
 343  }
 344 +#endif
 345
 346 -#ifndef LIBXML_ICONV_ENABLED
 347 +#if !defined(LIBXML_ICONV_ENABLED) && !defined(LIBXML_ICU_ENABLED)
 348  #ifdef LIBXML_ISO8859X_ENABLED
 349
 350  /**
 351 diff --git a/third_party/libxml/include/libxml/encoding.h b/third_party/libxml/include/libxml/encoding.h
 352 index c74b25f..b5f8b48 100644
 353 --- a/third_party/libxml/include/libxml/encoding.h
 354 +++ b/third_party/libxml/include/libxml/encoding.h
 355 @@ -26,6 +26,24 @@
 356
 357  #ifdef LIBXML_ICONV_ENABLED
 358  #include <iconv.h>
 359 +#else
 360 +#ifdef LIBXML_ICU_ENABLED
 361 +#include <unicode/ucnv.h>
 362 +#if 0
 363 +/* Forward-declare UConverter here rather than pulling in <unicode/ucnv.h>
 364 + * to prevent unwanted ICU symbols being exposed to users of libxml2.
 365 + * One particular case is Qt4 conflicting on UChar32.
 366 + */
 367 +#include <stdint.h>
 368 +struct UConverter;
 369 +typedef struct UConverter UConverter;
 370 +#ifdef _MSC_VER
 371 +typedef wchar_t UChar;
 372 +#else
 373 +typedef uint16_t UChar;
 374 +#endif
 375 +#endif
 376 +#endif
 377  #endif
 378  #ifdef __cplusplus
 379  extern "C" {
 380 @@ -125,6 +143,13 @@ typedef int (* xmlCharEncodingOutputFunc)(unsigned char *out, int *outlen,
 381   * Block defining the handlers for non UTF-8 encodings.
 382   * If iconv is supported, there are two extra fields.
 383   */
 384 +#ifdef LIBXML_ICU_ENABLED
 385 +struct _uconv_t {
 386 +  UConverter *uconv; /* for conversion between an encoding and UTF-16 */
 387 +  UConverter *utf8; /* for conversion between UTF-8 and UTF-16 */
 388 +};
 389 +typedef struct _uconv_t uconv_t;
 390 +#endif
 391
 392  typedef struct _xmlCharEncodingHandler xmlCharEncodingHandler;
 393  typedef xmlCharEncodingHandler *xmlCharEncodingHandlerPtr;
 394 @@ -136,6 +161,10 @@ struct _xmlCharEncodingHandler {
 395      iconv_t                    iconv_in;
 396      iconv_t                    iconv_out;
 397  #endif /* LIBXML_ICONV_ENABLED */
 398 +#ifdef LIBXML_ICU_ENABLED
 399 +    uconv_t                    *uconv_in;
 400 +    uconv_t                    *uconv_out;
 401 +#endif /* LIBXML_ICU_ENABLED */
 402  };
 403
 404  #ifdef __cplusplus
 405 diff --git a/third_party/libxml/include/libxml/parser.h b/third_party/libxml/include/libxml/parser.h
 406 index dd79c42..3580b63 100644
 407 --- a/third_party/libxml/include/libxml/parser.h
 408 +++ b/third_party/libxml/include/libxml/parser.h
 409 @@ -1222,6 +1222,7 @@ typedef enum {
 410      XML_WITH_DEBUG_MEM = 29,
 411      XML_WITH_DEBUG_RUN = 30,
 412      XML_WITH_ZLIB = 31,
 413 +    XML_WITH_ICU = 32,
 414      XML_WITH_NONE = 99999 /* just to be sure of allocation size */
 415  } xmlFeature;
 416
 417 diff --git a/third_party/libxml/include/libxml/xmlversion.h.in b/third_party/libxml/include/libxml/xmlversion.h.in
 418 index 4739f3a..de310ab 100644
 419 --- a/third_party/libxml/include/libxml/xmlversion.h.in
 420 +++ b/third_party/libxml/include/libxml/xmlversion.h.in
 421 @@ -269,6 +269,15 @@ XMLPUBFUN void XMLCALL xmlCheckVersion(int version);
 422  #endif
 423
 424  /**
 425 + * LIBXML_ICU_ENABLED:
 426 + *
 427 + * Whether icu support is available
 428 + */
 429 +#if @WITH_ICU@
 430 +#define LIBXML_ICU_ENABLED
 431 +#endif
 432 +
 433 +/**
 434   * LIBXML_ISO8859X_ENABLED:
 435   *
 436   * Whether ISO-8859-* support is made available in case iconv is not
 437 diff --git a/third_party/libxml/parser.c b/third_party/libxml/parser.c
 438 index 85e7599..3ba2a06 100644
 439 --- a/third_party/libxml/parser.c
 440 +++ b/third_party/libxml/parser.c
 441 @@ -954,6 +954,12 @@ xmlHasFeature(xmlFeature feature)
 442  #else
 443              return(0);
 444  #endif
 445 +        case XML_WITH_ICU:
 446 +#ifdef LIBXML_ICU_ENABLED
 447 +            return(1);
 448 +#else
 449 +            return(0);
 450 +#endif
 451          default:
 452             break;
 453       }