1 /* Character set conversion with error handling.
2 Copyright (C) 2001-2010 Free Software Foundation, Inc.
3 Written by Bruno Haible and Simon Josefsson.
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 3 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>. */
21 #include "striconveh.h"
33 #include "c-strcase.h"
34 #include "c-strcaseeq.h"
37 # define SIZE_MAX ((size_t) -1)
43 /* The caller must provide an iconveh_t, not just an iconv_t, because when a
44 conversion error occurs, we may have to determine the Unicode representation
45 of the inconvertible character. */
48 iconveh_open (const char *to_codeset, const char *from_codeset, iconveh_t *cdp)
54 /* Avoid glibc-2.1 bug with EUC-KR. */
55 # if (__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) && !defined _LIBICONV_VERSION
56 if (c_strcasecmp (from_codeset, "EUC-KR") == 0
57 || c_strcasecmp (to_codeset, "EUC-KR") == 0)
64 cd = iconv_open (to_codeset, from_codeset);
66 if (STRCASEEQ (from_codeset, "UTF-8", 'U','T','F','-','8',0,0,0,0))
70 cd1 = iconv_open ("UTF-8", from_codeset);
71 if (cd1 == (iconv_t)(-1))
73 int saved_errno = errno;
74 if (cd != (iconv_t)(-1))
75 iconv_close (cdp->cd);
81 if (STRCASEEQ (to_codeset, "UTF-8", 'U','T','F','-','8',0,0,0,0)
82 # if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 2) || __GLIBC__ > 2 || _LIBICONV_VERSION >= 0x0105
83 || c_strcasecmp (to_codeset, "UTF-8//TRANSLIT") == 0
89 cd2 = iconv_open (to_codeset, "UTF-8");
90 if (cd2 == (iconv_t)(-1))
92 int saved_errno = errno;
93 if (cd1 != (iconv_t)(-1))
95 if (cd != (iconv_t)(-1))
109 iconveh_close (const iconveh_t *cd)
111 if (cd->cd2 != (iconv_t)(-1) && iconv_close (cd->cd2) < 0)
113 /* Return -1, but preserve the errno from iconv_close. */
114 int saved_errno = errno;
115 if (cd->cd1 != (iconv_t)(-1))
116 iconv_close (cd->cd1);
117 if (cd->cd != (iconv_t)(-1))
118 iconv_close (cd->cd);
122 if (cd->cd1 != (iconv_t)(-1) && iconv_close (cd->cd1) < 0)
124 /* Return -1, but preserve the errno from iconv_close. */
125 int saved_errno = errno;
126 if (cd->cd != (iconv_t)(-1))
127 iconv_close (cd->cd);
131 if (cd->cd != (iconv_t)(-1) && iconv_close (cd->cd) < 0)
136 /* iconv_carefully is like iconv, except that it stops as soon as it encounters
137 a conversion error, and it returns in *INCREMENTED a boolean telling whether
138 it has incremented the input pointers past the error location. */
139 # if !defined _LIBICONV_VERSION && !defined __GLIBC__
140 /* Irix iconv() inserts a NUL byte if it cannot convert.
141 NetBSD iconv() inserts a question mark if it cannot convert.
142 Only GNU libiconv and GNU libc are known to prefer to fail rather
143 than doing a lossy conversion. */
145 iconv_carefully (iconv_t cd,
146 const char **inbuf, size_t *inbytesleft,
147 char **outbuf, size_t *outbytesleft,
150 const char *inptr = *inbuf;
151 const char *inptr_end = inptr + *inbytesleft;
152 char *outptr = *outbuf;
153 size_t outsize = *outbytesleft;
154 const char *inptr_before;
161 inptr_before = inptr;
164 for (insize = 1; inptr + insize <= inptr_end; insize++)
167 (ICONV_CONST char **) &inptr, &insize,
169 if (!(res == (size_t)(-1) && errno == EINVAL))
171 /* iconv can eat up a shift sequence but give EINVAL while attempting
172 to convert the first character. E.g. libiconv does this. */
173 if (inptr > inptr_before)
183 *outbytesleft = outsize;
186 while (res == 0 && inptr < inptr_end);
189 *inbytesleft = inptr_end - inptr;
190 if (res != (size_t)(-1) && res > 0)
192 /* iconv() has already incremented INPTR. We cannot go back to a
193 previous INPTR, otherwise the state inside CD would become invalid,
194 if FROM_CODESET is a stateful encoding. So, tell the caller that
195 *INBUF has already been incremented. */
196 *incremented = (inptr > inptr_before);
202 *incremented = false;
207 # define iconv_carefully(cd, inbuf, inbytesleft, outbuf, outbytesleft, incremented) \
208 (*(incremented) = false, \
209 iconv (cd, (ICONV_CONST char **) (inbuf), inbytesleft, outbuf, outbytesleft))
212 /* iconv_carefully_1 is like iconv_carefully, except that it stops after
213 converting one character or one shift sequence. */
215 iconv_carefully_1 (iconv_t cd,
216 const char **inbuf, size_t *inbytesleft,
217 char **outbuf, size_t *outbytesleft,
220 const char *inptr_before = *inbuf;
221 const char *inptr = inptr_before;
222 const char *inptr_end = inptr_before + *inbytesleft;
223 char *outptr = *outbuf;
224 size_t outsize = *outbytesleft;
225 size_t res = (size_t)(-1);
228 for (insize = 1; inptr_before + insize <= inptr_end; insize++)
230 inptr = inptr_before;
232 (ICONV_CONST char **) &inptr, &insize,
234 if (!(res == (size_t)(-1) && errno == EINVAL))
236 /* iconv can eat up a shift sequence but give EINVAL while attempting
237 to convert the first character. E.g. libiconv does this. */
238 if (inptr > inptr_before)
246 *inbytesleft = inptr_end - inptr;
247 # if !defined _LIBICONV_VERSION && !defined __GLIBC__
248 /* Irix iconv() inserts a NUL byte if it cannot convert.
249 NetBSD iconv() inserts a question mark if it cannot convert.
250 Only GNU libiconv and GNU libc are known to prefer to fail rather
251 than doing a lossy conversion. */
252 if (res != (size_t)(-1) && res > 0)
254 /* iconv() has already incremented INPTR. We cannot go back to a
255 previous INPTR, otherwise the state inside CD would become invalid,
256 if FROM_CODESET is a stateful encoding. So, tell the caller that
257 *INBUF has already been incremented. */
258 *incremented = (inptr > inptr_before);
264 if (res != (size_t)(-1))
267 *outbytesleft = outsize;
269 *incremented = false;
273 /* utf8conv_carefully is like iconv, except that
274 - it converts from UTF-8 to UTF-8,
275 - it stops as soon as it encounters a conversion error, and it returns
276 in *INCREMENTED a boolean telling whether it has incremented the input
277 pointers past the error location,
278 - if one_character_only is true, it stops after converting one
281 utf8conv_carefully (bool one_character_only,
282 const char **inbuf, size_t *inbytesleft,
283 char **outbuf, size_t *outbytesleft,
286 const char *inptr = *inbuf;
287 size_t insize = *inbytesleft;
288 char *outptr = *outbuf;
289 size_t outsize = *outbytesleft;
299 n = u8_mbtoucr (&uc, (const uint8_t *) inptr, insize);
302 errno = (n == -2 ? EINVAL : EILSEQ);
303 n = u8_mbtouc (&uc, (const uint8_t *) inptr, insize);
314 *incremented = false;
317 m = u8_uctomb ((uint8_t *) outptr, uc, outsize);
322 *incremented = false;
337 while (!one_character_only && insize > 0);
340 *inbytesleft = insize;
342 *outbytesleft = outsize;
347 mem_cd_iconveh_internal (const char *src, size_t srclen,
348 iconv_t cd, iconv_t cd1, iconv_t cd2,
349 enum iconv_ilseq_handler handler,
352 char **resultp, size_t *lengthp)
354 /* When a conversion error occurs, we cannot start using CD1 and CD2 at
355 this point: FROM_CODESET may be a stateful encoding like ISO-2022-KR.
356 Instead, we have to start afresh from the beginning of SRC. */
357 /* Use a temporary buffer, so that for small strings, a single malloc()
358 call will be sufficient. */
359 # define tmpbufsize 4096
360 /* The alignment is needed when converting e.g. to glibc's WCHAR_T or
361 libiconv's UCS-4-INTERNAL encoding. */
362 union { unsigned int align; char buf[tmpbufsize]; } tmp;
363 # define tmpbuf tmp.buf
365 char *initial_result;
369 size_t last_length = (size_t)(-1); /* only needed if offsets != NULL */
371 if (*resultp != NULL && *lengthp >= sizeof (tmpbuf))
373 initial_result = *resultp;
374 allocated = *lengthp;
378 initial_result = tmpbuf;
379 allocated = sizeof (tmpbuf);
381 result = initial_result;
383 /* Test whether a direct conversion is possible at all. */
384 if (cd == (iconv_t)(-1))
391 for (i = 0; i < srclen; i++)
392 offsets[i] = (size_t)(-1);
394 last_length = (size_t)(-1);
398 /* First, try a direct conversion, and see whether a conversion error
401 const char *inptr = src;
402 size_t insize = srclen;
404 /* Avoid glibc-2.1 bug and Solaris 2.7-2.9 bug. */
405 # if defined _LIBICONV_VERSION \
406 || !((__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) || defined __sun)
407 /* Set to the initial state. */
408 iconv (cd, NULL, NULL, NULL, NULL);
413 char *outptr = result + length;
414 size_t outsize = allocated - extra_alloc - length;
421 if (length != last_length) /* ensure that offset[] be increasing */
423 offsets[inptr - src] = length;
424 last_length = length;
426 res = iconv_carefully_1 (cd,
432 /* Use iconv_carefully instead of iconv here, because:
433 - If TO_CODESET is UTF-8, we can do the error handling in this
434 loop, no need for a second loop,
435 - With iconv() implementations other than GNU libiconv and GNU
436 libc, if we use iconv() in a big swoop, checking for an E2BIG
437 return, we lose the number of irreversible conversions. */
438 res = iconv_carefully (cd,
443 length = outptr - result;
444 grow = (length + extra_alloc > allocated / 2);
445 if (res == (size_t)(-1))
449 else if (errno == EINVAL)
451 else if (errno == EILSEQ && handler != iconveh_error)
453 if (cd2 == (iconv_t)(-1))
455 /* TO_CODESET is UTF-8. */
456 /* Error handling can produce up to 1 byte of output. */
457 if (length + 1 + extra_alloc > allocated)
461 allocated = 2 * allocated;
462 if (length + 1 + extra_alloc > allocated)
464 if (result == initial_result)
465 memory = (char *) malloc (allocated);
467 memory = (char *) realloc (result, allocated);
470 if (result != initial_result)
475 if (result == initial_result)
476 memcpy (memory, initial_result, length);
480 /* The input is invalid in FROM_CODESET. Eat up one byte
481 and emit a question mark. */
489 result[length] = '?';
497 if (result != initial_result)
499 int saved_errno = errno;
512 allocated = 2 * allocated;
513 if (result == initial_result)
514 memory = (char *) malloc (allocated);
516 memory = (char *) realloc (result, allocated);
519 if (result != initial_result)
524 if (result == initial_result)
525 memcpy (memory, initial_result, length);
531 /* Now get the conversion state back to the initial state.
532 But avoid glibc-2.1 bug and Solaris 2.7 bug. */
533 #if defined _LIBICONV_VERSION \
534 || !((__GLIBC__ == 2 && __GLIBC_MINOR__ <= 1) || defined __sun)
537 char *outptr = result + length;
538 size_t outsize = allocated - extra_alloc - length;
541 res = iconv (cd, NULL, NULL, &outptr, &outsize);
542 length = outptr - result;
543 if (res == (size_t)(-1))
549 allocated = 2 * allocated;
550 if (result == initial_result)
551 memory = (char *) malloc (allocated);
553 memory = (char *) realloc (result, allocated);
556 if (result != initial_result)
561 if (result == initial_result)
562 memcpy (memory, initial_result, length);
567 if (result != initial_result)
569 int saved_errno = errno;
581 /* The direct conversion succeeded. */
585 /* The direct conversion failed.
586 Use a conversion through UTF-8. */
591 for (i = 0; i < srclen; i++)
592 offsets[i] = (size_t)(-1);
594 last_length = (size_t)(-1);
598 const bool slowly = (offsets != NULL || handler == iconveh_error);
599 # define utf8bufsize 4096 /* may also be smaller or larger than tmpbufsize */
600 char utf8buf[utf8bufsize + 1];
602 const char *in1ptr = src;
603 size_t in1size = srclen;
604 bool do_final_flush1 = true;
605 bool do_final_flush2 = true;
607 /* Avoid glibc-2.1 bug and Solaris 2.7-2.9 bug. */
608 # if defined _LIBICONV_VERSION \
609 || !((__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) || defined __sun)
610 /* Set to the initial state. */
611 if (cd1 != (iconv_t)(-1))
612 iconv (cd1, NULL, NULL, NULL, NULL);
613 if (cd2 != (iconv_t)(-1))
614 iconv (cd2, NULL, NULL, NULL, NULL);
617 while (in1size > 0 || do_final_flush1 || utf8len > 0 || do_final_flush2)
619 char *out1ptr = utf8buf + utf8len;
620 size_t out1size = utf8bufsize - utf8len;
625 /* Conversion step 1: from FROM_CODESET to UTF-8. */
629 && length != last_length) /* ensure that offset[] be increasing */
631 offsets[in1ptr - src] = length;
632 last_length = length;
634 if (cd1 != (iconv_t)(-1))
637 res1 = iconv_carefully_1 (cd1,
642 res1 = iconv_carefully (cd1,
649 /* FROM_CODESET is UTF-8. */
650 res1 = utf8conv_carefully (slowly,
656 else if (do_final_flush1)
658 /* Now get the conversion state of CD1 back to the initial state.
659 But avoid glibc-2.1 bug and Solaris 2.7 bug. */
660 # if defined _LIBICONV_VERSION \
661 || !((__GLIBC__ == 2 && __GLIBC_MINOR__ <= 1) || defined __sun)
662 if (cd1 != (iconv_t)(-1))
663 res1 = iconv (cd1, NULL, NULL, &out1ptr, &out1size);
667 do_final_flush1 = false;
675 if (res1 == (size_t)(-1)
676 && !(errno == E2BIG || errno == EINVAL || errno == EILSEQ))
678 if (result != initial_result)
680 int saved_errno = errno;
686 if (res1 == (size_t)(-1)
687 && errno == EILSEQ && handler != iconveh_error)
689 /* The input is invalid in FROM_CODESET. Eat up one byte and
690 emit a question mark. Room for the question mark was allocated
691 at the end of utf8buf. */
703 utf8len = out1ptr - utf8buf;
707 || utf8len > utf8bufsize / 2
708 || (res1 == (size_t)(-1) && errno1 == E2BIG))
710 /* Conversion step 2: from UTF-8 to TO_CODESET. */
711 const char *in2ptr = utf8buf;
712 size_t in2size = utf8len;
715 || (in1size == 0 && !do_final_flush1 && do_final_flush2))
717 char *out2ptr = result + length;
718 size_t out2size = allocated - extra_alloc - length;
725 if (cd2 != (iconv_t)(-1))
726 res2 = iconv_carefully (cd2,
731 /* TO_CODESET is UTF-8. */
732 res2 = utf8conv_carefully (false,
737 else /* in1size == 0 && !do_final_flush1
738 && in2size == 0 && do_final_flush2 */
740 /* Now get the conversion state of CD1 back to the initial
741 state. But avoid glibc-2.1 bug and Solaris 2.7 bug. */
742 # if defined _LIBICONV_VERSION \
743 || !((__GLIBC__ == 2 && __GLIBC_MINOR__ <= 1) || defined __sun)
744 if (cd2 != (iconv_t)(-1))
745 res2 = iconv (cd2, NULL, NULL, &out2ptr, &out2size);
749 do_final_flush2 = false;
753 length = out2ptr - result;
754 grow = (length + extra_alloc > allocated / 2);
755 if (res2 == (size_t)(-1))
759 else if (errno == EINVAL)
761 else if (errno == EILSEQ && handler != iconveh_error)
763 /* Error handling can produce up to 10 bytes of ASCII
764 output. But TO_CODESET may be UCS-2, UTF-16 or
765 UCS-4, so use CD2 here as well. */
775 if (u8_prev (&uc, (const uint8_t *) in2ptr,
776 (const uint8_t *) utf8buf)
785 n = u8_mbtouc_unsafe (&uc, (const uint8_t *) in2ptr,
791 if (handler == iconveh_escape_sequence)
793 static char hex[16] = "0123456789ABCDEF";
795 scratchbuf[scratchlen++] = '\\';
797 scratchbuf[scratchlen++] = 'u';
800 scratchbuf[scratchlen++] = 'U';
801 scratchbuf[scratchlen++] = hex[(uc>>28) & 15];
802 scratchbuf[scratchlen++] = hex[(uc>>24) & 15];
803 scratchbuf[scratchlen++] = hex[(uc>>20) & 15];
804 scratchbuf[scratchlen++] = hex[(uc>>16) & 15];
806 scratchbuf[scratchlen++] = hex[(uc>>12) & 15];
807 scratchbuf[scratchlen++] = hex[(uc>>8) & 15];
808 scratchbuf[scratchlen++] = hex[(uc>>4) & 15];
809 scratchbuf[scratchlen++] = hex[uc & 15];
819 if (cd2 != (iconv_t)(-1))
821 (ICONV_CONST char **) &inptr, &insize,
822 &out2ptr, &out2size);
825 /* TO_CODESET is UTF-8. */
826 if (out2size >= insize)
828 memcpy (out2ptr, inptr, insize);
841 length = out2ptr - result;
842 if (res == (size_t)(-1) && errno == E2BIG)
846 allocated = 2 * allocated;
847 if (length + 1 + extra_alloc > allocated)
849 if (result == initial_result)
850 memory = (char *) malloc (allocated);
852 memory = (char *) realloc (result, allocated);
855 if (result != initial_result)
860 if (result == initial_result)
861 memcpy (memory, initial_result, length);
865 out2ptr = result + length;
866 out2size = allocated - extra_alloc - length;
867 if (cd2 != (iconv_t)(-1))
869 (ICONV_CONST char **) &inptr,
871 &out2ptr, &out2size);
874 /* TO_CODESET is UTF-8. */
875 if (!(out2size >= insize))
877 memcpy (out2ptr, inptr, insize);
884 length = out2ptr - result;
886 # if !defined _LIBICONV_VERSION && !defined __GLIBC__
887 /* Irix iconv() inserts a NUL byte if it cannot convert.
888 NetBSD iconv() inserts a question mark if it cannot
890 Only GNU libiconv and GNU libc are known to prefer
891 to fail rather than doing a lossy conversion. */
892 if (res != (size_t)(-1) && res > 0)
898 if (res == (size_t)(-1))
900 /* Failure converting the ASCII replacement. */
901 if (result != initial_result)
903 int saved_errno = errno;
912 if (result != initial_result)
914 int saved_errno = errno;
922 || (in1size == 0 && !do_final_flush1 && do_final_flush2)))
928 allocated = 2 * allocated;
929 if (result == initial_result)
930 memory = (char *) malloc (allocated);
932 memory = (char *) realloc (result, allocated);
935 if (result != initial_result)
940 if (result == initial_result)
941 memcpy (memory, initial_result, length);
946 /* Move the remaining bytes to the beginning of utf8buf. */
948 memmove (utf8buf, in2ptr, in2size);
952 if (res1 == (size_t)(-1))
954 if (errno1 == EINVAL)
956 else if (errno1 == EILSEQ)
958 if (result != initial_result)
969 /* Now the final memory allocation. */
970 if (result == tmpbuf)
972 size_t memsize = length + extra_alloc;
975 memory = (char *) malloc (memsize > 0 ? memsize : 1);
978 memcpy (memory, tmpbuf, length);
987 else if (result != *resultp && length + extra_alloc < allocated)
989 /* Shrink the allocated memory if possible. */
990 size_t memsize = length + extra_alloc;
993 memory = (char *) realloc (result, memsize > 0 ? memsize : 1);
1005 mem_cd_iconveh (const char *src, size_t srclen,
1006 const iconveh_t *cd,
1007 enum iconv_ilseq_handler handler,
1009 char **resultp, size_t *lengthp)
1011 return mem_cd_iconveh_internal (src, srclen, cd->cd, cd->cd1, cd->cd2,
1012 handler, 0, offsets, resultp, lengthp);
1016 str_cd_iconveh (const char *src,
1017 const iconveh_t *cd,
1018 enum iconv_ilseq_handler handler)
1020 /* For most encodings, a trailing NUL byte in the input will be converted
1021 to a trailing NUL byte in the output. But not for UTF-7. So that this
1022 function is usable for UTF-7, we have to exclude the NUL byte from the
1023 conversion and add it by hand afterwards. */
1024 char *result = NULL;
1026 int retval = mem_cd_iconveh_internal (src, strlen (src),
1027 cd->cd, cd->cd1, cd->cd2, handler, 1,
1028 NULL, &result, &length);
1034 int saved_errno = errno;
1036 errno = saved_errno;
1041 /* Add the terminating NUL byte. */
1042 result[length] = '\0';
1050 mem_iconveh (const char *src, size_t srclen,
1051 const char *from_codeset, const char *to_codeset,
1052 enum iconv_ilseq_handler handler,
1054 char **resultp, size_t *lengthp)
1058 /* Nothing to convert. */
1062 else if (offsets == NULL && c_strcasecmp (from_codeset, to_codeset) == 0)
1066 if (*resultp != NULL && *lengthp >= srclen)
1070 result = (char *) malloc (srclen);
1077 memcpy (result, src, srclen);
1090 if (iconveh_open (to_codeset, from_codeset, &cd) < 0)
1095 retval = mem_cd_iconveh (src, srclen, &cd, handler, offsets,
1100 /* Close cd, but preserve the errno from str_cd_iconv. */
1101 int saved_errno = errno;
1102 iconveh_close (&cd);
1103 errno = saved_errno;
1107 if (iconveh_close (&cd) < 0)
1109 /* Return -1, but free the allocated memory, and while doing
1110 that, preserve the errno from iconveh_close. */
1111 int saved_errno = errno;
1112 if (result != *resultp && result != NULL)
1114 errno = saved_errno;
1122 /* This is a different error code than if iconv_open existed but didn't
1123 support from_codeset and to_codeset, so that the caller can emit
1124 an error message such as
1125 "iconv() is not supported. Installing GNU libiconv and
1126 then reinstalling this package would fix this." */
1134 str_iconveh (const char *src,
1135 const char *from_codeset, const char *to_codeset,
1136 enum iconv_ilseq_handler handler)
1138 if (*src == '\0' || c_strcasecmp (from_codeset, to_codeset) == 0)
1140 char *result = strdup (src);
1152 if (iconveh_open (to_codeset, from_codeset, &cd) < 0)
1155 result = str_cd_iconveh (src, &cd, handler);
1159 /* Close cd, but preserve the errno from str_cd_iconv. */
1160 int saved_errno = errno;
1161 iconveh_close (&cd);
1162 errno = saved_errno;
1166 if (iconveh_close (&cd) < 0)
1168 /* Return NULL, but free the allocated memory, and while doing
1169 that, preserve the errno from iconveh_close. */
1170 int saved_errno = errno;
1172 errno = saved_errno;
1178 /* This is a different error code than if iconv_open existed but didn't
1179 support from_codeset and to_codeset, so that the caller can emit
1180 an error message such as
1181 "iconv() is not supported. Installing GNU libiconv and
1182 then reinstalling this package would fix this." */