1 /* gutf8.c - Operations on UTF-8 strings.
3 * Copyright (C) 1999 Tom Tromey
4 * Copyright (C) 2000 Red Hat, Inc.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 02111-1307, USA.
32 #ifdef G_PLATFORM_WIN32
39 #include "libcharset/libcharset.h"
43 #define UTF8_COMPUTE(Char, Mask, Len) \
49 else if ((Char & 0xe0) == 0xc0) \
54 else if ((Char & 0xf0) == 0xe0) \
59 else if ((Char & 0xf8) == 0xf0) \
64 else if ((Char & 0xfc) == 0xf8) \
69 else if ((Char & 0xfe) == 0xfc) \
77 #define UTF8_LENGTH(Char) \
78 ((Char) < 0x80 ? 1 : \
79 ((Char) < 0x800 ? 2 : \
80 ((Char) < 0x10000 ? 3 : \
81 ((Char) < 0x200000 ? 4 : \
82 ((Char) < 0x4000000 ? 5 : 6)))))
85 #define UTF8_GET(Result, Chars, Count, Mask, Len) \
86 (Result) = (Chars)[0] & (Mask); \
87 for ((Count) = 1; (Count) < (Len); ++(Count)) \
89 if (((Chars)[(Count)] & 0xc0) != 0x80) \
95 (Result) |= ((Chars)[(Count)] & 0x3f); \
98 #define UNICODE_VALID(Char) \
99 ((Char) < 0x110000 && \
100 ((Char) < 0xD800 || (Char) >= 0xE000) && \
101 (Char) != 0xFFFE && (Char) != 0xFFFF)
104 static const gchar utf8_skip_data[256] = {
105 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
106 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
107 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
108 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
109 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
110 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
111 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
112 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1
115 const gchar * const g_utf8_skip = utf8_skip_data;
118 * g_utf8_find_prev_char:
119 * @str: pointer to the beginning of a UTF-8 encoded string
120 * @p: pointer to some position within @str
122 * Given a position @p with a UTF-8 encoded string @str, find the start
123 * of the previous UTF-8 character starting before @p. Returns %NULL if no
124 * UTF-8 characters are present in @p before @str.
126 * @p does not have to be at the beginning of a UTF-8 character. No check
127 * is made to see if the character found is actually valid other than
128 * it starts with an appropriate byte.
130 * Return value: a pointer to the found character or %NULL.
133 g_utf8_find_prev_char (const char *str,
136 for (--p; p >= str; --p)
138 if ((*p & 0xc0) != 0x80)
145 * g_utf8_find_next_char:
146 * @p: a pointer to a position within a UTF-8 encoded string
147 * @end: a pointer to the end of the string, or %NULL to indicate
148 * that the string is nul-terminated, in which case
149 * the returned value will be
151 * Finds the start of the next UTF-8 character in the string after @p.
153 * @p does not have to be at the beginning of a UTF-8 character. No check
154 * is made to see if the character found is actually valid other than
155 * it starts with an appropriate byte.
157 * Return value: a pointer to the found character or %NULL
160 g_utf8_find_next_char (const gchar *p,
166 for (++p; p < end && (*p & 0xc0) == 0x80; ++p)
169 for (++p; (*p & 0xc0) == 0x80; ++p)
172 return (p == end) ? NULL : (gchar *)p;
177 * @p: a pointer to a position within a UTF-8 encoded string
179 * Finds the previous UTF-8 character in the string before @p.
181 * @p does not have to be at the beginning of a UTF-8 character. No check
182 * is made to see if the character found is actually valid other than
183 * it starts with an appropriate byte. If @p might be the first
184 * character of the string, you must use g_utf8_find_prev_char() instead.
186 * Return value: a pointer to the found character.
189 g_utf8_prev_char (const gchar *p)
194 if ((*p & 0xc0) != 0x80)
201 * @p: pointer to the start of a UTF-8 encoded string.
202 * @max: the maximum number of bytes to examine. If @max
203 * is less than 0, then the string is assumed to be
206 * Returns the length of the string in characters.
208 * Return value: the length of the string in characters
211 g_utf8_strlen (const gchar *p,
215 const gchar *start = p;
221 p = g_utf8_next_char (p);
230 p = g_utf8_next_char (p);
232 while (p - start < max && *p)
235 p = g_utf8_next_char (p);
238 /* only do the last len increment if we got a complete
239 * char (don't count partial chars)
241 if (p - start == max)
250 * @p: a pointer to Unicode character encoded as UTF-8
252 * Converts a sequence of bytes encoded as UTF-8 to a Unicode character.
253 * If @p does not point to a valid UTF-8 encoded character, results are
254 * undefined. If you are not sure that the bytes are complete
255 * valid Unicode characters, you should use g_utf8_get_char_validated()
258 * Return value: the resulting character
261 g_utf8_get_char (const gchar *p)
263 int i, mask = 0, len;
265 unsigned char c = (unsigned char) *p;
267 UTF8_COMPUTE (c, mask, len);
270 UTF8_GET (result, p, i, mask, len);
276 * g_utf8_offset_to_pointer:
277 * @str: a UTF-8 encoded string
278 * @offset: a character offset within @str
280 * Converts from an integer character offset to a pointer to a position
283 * Return value: the resulting pointer
286 g_utf8_offset_to_pointer (const gchar *str,
289 const gchar *s = str;
291 s = g_utf8_next_char (s);
297 * g_utf8_pointer_to_offset:
298 * @str: a UTF-8 encoded string
299 * @pos: a pointer to a position within @str
301 * Converts from a pointer to position within a string to a integer
304 * Return value: the resulting character offset
307 g_utf8_pointer_to_offset (const gchar *str,
310 const gchar *s = str;
315 s = g_utf8_next_char (s);
325 * @dest: buffer to fill with characters from @src
326 * @src: UTF-8 encoded string
327 * @n: character count
329 * Like the standard C <function>strncpy()</function> function, but
330 * copies a given number of characters instead of a given number of
331 * bytes. The @src string must be valid UTF-8 encoded text.
332 * (Use g_utf8_validate() on all text before trying to use UTF-8
333 * utility functions with it.)
335 * Return value: @dest
338 g_utf8_strncpy (gchar *dest,
342 const gchar *s = src;
345 s = g_utf8_next_char(s);
348 strncpy(dest, src, s - src);
353 G_LOCK_DEFINE_STATIC (aliases);
356 get_alias_hash (void)
358 static GHashTable *alias_hash = NULL;
365 alias_hash = g_hash_table_new (g_str_hash, g_str_equal);
367 aliases = _g_locale_get_charset_aliases ();
368 while (*aliases != '\0')
370 const char *canonical;
372 const char **alias_array;
376 aliases += strlen (aliases) + 1;
378 aliases += strlen (aliases) + 1;
380 alias_array = g_hash_table_lookup (alias_hash, canonical);
383 while (alias_array[count])
387 alias_array = g_renew (const char *, alias_array, count + 2);
388 alias_array[count] = alias;
389 alias_array[count + 1] = NULL;
391 g_hash_table_insert (alias_hash, (char *)canonical, alias_array);
400 /* As an abuse of the alias table, the following routines gets
401 * the charsets that are aliases for the canonical name.
404 _g_charset_get_aliases (const char *canonical_name)
406 GHashTable *alias_hash = get_alias_hash ();
408 return g_hash_table_lookup (alias_hash, canonical_name);
412 g_utf8_get_charset_internal (const char **a)
414 const char *charset = getenv("CHARSET");
416 if (charset && *charset)
420 if (charset && strstr (charset, "UTF-8"))
426 /* The libcharset code tries to be thread-safe without
427 * a lock, but has a memory leak and a missing memory
428 * barrier, so we lock for it
431 charset = _g_locale_charset ();
434 if (charset && *charset)
438 if (charset && strstr (charset, "UTF-8"))
444 /* Assume this for compatibility at present. */
450 static int utf8_locale_cache = -1;
451 static const char *utf8_charset_cache = NULL;
455 * @charset: return location for character set name
457 * Obtains the character set for the current locale; you might use
458 * this character set as an argument to g_convert(), to convert from
459 * the current locale's encoding to some other encoding. (Frequently
460 * g_locale_to_utf8() and g_locale_from_utf8() are nice shortcuts,
463 * The return value is %TRUE if the locale's encoding is UTF-8, in that
464 * case you can perhaps avoid calling g_convert().
466 * The string returned in @charset is not allocated, and should not be
469 * Return value: %TRUE if the returned charset is UTF-8
472 g_get_charset (G_CONST_RETURN char **charset)
474 if (utf8_locale_cache != -1)
477 *charset = utf8_charset_cache;
478 return utf8_locale_cache;
480 utf8_locale_cache = g_utf8_get_charset_internal (&utf8_charset_cache);
482 *charset = utf8_charset_cache;
483 return utf8_locale_cache;
490 * @c: a ISO10646 character code
491 * @outbuf: output buffer, must have at least 6 bytes of space.
492 * If %NULL, the length will be computed and returned
493 * and nothing will be written to @outbuf.
495 * Converts a single character to UTF-8.
497 * Return value: number of bytes written
500 g_unichar_to_utf8 (gunichar c,
517 else if (c < 0x10000)
522 else if (c < 0x200000)
527 else if (c < 0x4000000)
540 for (i = len - 1; i > 0; --i)
542 outbuf[i] = (c & 0x3f) | 0x80;
545 outbuf[0] = c | first;
553 * @p: a nul-terminated UTF-8 encoded string
554 * @len: the maximum length of @p
555 * @c: a ISO10646 character
557 * Finds the leftmost occurrence of the given ISO10646 character
558 * in a UTF-8 encoded string, while limiting the search to @len bytes.
559 * If @len is -1, allow unbounded search.
561 * Return value: %NULL if the string does not contain the character,
562 * otherwise, a pointer to the start of the leftmost occurrence of
563 * the character in the string.
566 g_utf8_strchr (const char *p,
572 gint charlen = g_unichar_to_utf8 (c, ch);
575 return g_strstr_len (p, len, ch);
581 * @p: a nul-terminated UTF-8 encoded string
582 * @len: the maximum length of @p
583 * @c: a ISO10646 character
585 * Find the rightmost occurrence of the given ISO10646 character
586 * in a UTF-8 encoded string, while limiting the search to @len bytes.
587 * If @len is -1, allow unbounded search.
589 * Return value: %NULL if the string does not contain the character,
590 * otherwise, a pointer to the start of the rightmost occurrence of the
591 * character in the string.
594 g_utf8_strrchr (const char *p,
600 gint charlen = g_unichar_to_utf8 (c, ch);
603 return g_strrstr_len (p, len, ch);
607 /* Like g_utf8_get_char, but take a maximum length
608 * and return (gunichar)-2 on incomplete trailing character
610 static inline gunichar
611 g_utf8_get_char_extended (const gchar *p,
615 gunichar wc = (guchar) *p;
655 if (max_len >= 0 && len > max_len)
657 for (i = 1; i < max_len; i++)
659 if ((((guchar *)p)[i] & 0xc0) != 0x80)
665 for (i = 1; i < len; ++i)
667 gunichar ch = ((guchar *)p)[i];
669 if ((ch & 0xc0) != 0x80)
681 if (UTF8_LENGTH(wc) != len)
688 * g_utf8_get_char_validated:
689 * @p: a pointer to Unicode character encoded as UTF-8
690 * @max_len: the maximum number of bytes to read, or -1, for no maximum.
692 * Convert a sequence of bytes encoded as UTF-8 to a Unicode character.
693 * This function checks for incomplete characters, for invalid characters
694 * such as characters that are out of the range of Unicode, and for
695 * overlong encodings of valid characters.
697 * Return value: the resulting character. If @p points to a partial
698 * sequence at the end of a string that could begin a valid character,
699 * returns (gunichar)-2; otherwise, if @p does not point to a valid
700 * UTF-8 encoded Unicode character, returns (gunichar)-1.
703 g_utf8_get_char_validated (const gchar *p,
706 gunichar result = g_utf8_get_char_extended (p, max_len);
708 if (result & 0x80000000)
710 else if (!UNICODE_VALID (result))
717 * g_utf8_to_ucs4_fast:
718 * @str: a UTF-8 encoded string
719 * @len: the maximum length of @str to use. If @len < 0, then
720 * the string is nul-terminated.
721 * @items_written: location to store the number of characters in the
724 * Convert a string from UTF-8 to a 32-bit fixed width
725 * representation as UCS-4, assuming valid UTF-8 input.
726 * This function is roughly twice as fast as g_utf8_to_ucs4()
727 * but does no error checking on the input.
729 * Return value: a pointer to a newly allocated UCS-4 string.
730 * This value must be freed with g_free().
733 g_utf8_to_ucs4_fast (const gchar *str,
735 glong *items_written)
742 g_return_val_if_fail (str != NULL, NULL);
750 p = g_utf8_next_char (p);
756 while (p < str + len && *p)
758 p = g_utf8_next_char (p);
763 result = g_new (gunichar, n_chars + 1);
766 for (i=0; i < n_chars; i++)
768 gunichar wc = ((unsigned char *)p)[0];
803 for (j = 1; j < charlen; j++)
806 wc |= ((unsigned char *)p)[j] & 0x3f;
823 * @str: a UTF-8 encoded string
824 * @len: the maximum length of @str to use. If @len < 0, then
825 * the string is nul-terminated.
826 * @items_read: location to store number of bytes read, or %NULL.
827 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
828 * returned in case @str contains a trailing partial
829 * character. If an error occurs then the index of the
830 * invalid input is stored here.
831 * @items_written: location to store number of characters written or %NULL.
832 * The value here stored does not include the trailing 0
834 * @error: location to store the error occuring, or %NULL to ignore
835 * errors. Any of the errors in #GConvertError other than
836 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
838 * Convert a string from UTF-8 to a 32-bit fixed width
839 * representation as UCS-4. A trailing 0 will be added to the
840 * string after the converted text.
842 * Return value: a pointer to a newly allocated UCS-4 string.
843 * This value must be freed with g_free(). If an
844 * error occurs, %NULL will be returned and
848 g_utf8_to_ucs4 (const gchar *str,
851 glong *items_written,
854 gunichar *result = NULL;
860 while ((len < 0 || str + len - in > 0) && *in)
862 gunichar wc = g_utf8_get_char_extended (in, str + len - in);
865 if (wc == (gunichar)-2)
870 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
871 _("Partial character sequence at end of input"));
874 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
875 _("Invalid byte sequence in conversion input"));
882 in = g_utf8_next_char (in);
885 result = g_new (gunichar, n_chars + 1);
888 for (i=0; i < n_chars; i++)
890 result[i] = g_utf8_get_char (in);
891 in = g_utf8_next_char (in);
896 *items_written = n_chars;
900 *items_read = in - str;
907 * @str: a UCS-4 encoded string
908 * @len: the maximum length of @str to use. If @len < 0, then
909 * the string is terminated with a 0 character.
910 * @items_read: location to store number of characters read read, or %NULL.
911 * @items_written: location to store number of bytes written or %NULL.
912 * The value here stored does not include the trailing 0
914 * @error: location to store the error occuring, or %NULL to ignore
915 * errors. Any of the errors in #GConvertError other than
916 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
918 * Convert a string from a 32-bit fixed width representation as UCS-4.
919 * to UTF-8. The result will be terminated with a 0 byte.
921 * Return value: a pointer to a newly allocated UTF-8 string.
922 * This value must be freed with g_free(). If an
923 * error occurs, %NULL will be returned and
927 g_ucs4_to_utf8 (const gunichar *str,
930 glong *items_written,
934 gchar *result = NULL;
939 for (i = 0; len < 0 || i < len ; i++)
944 if (str[i] >= 0x80000000)
949 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
950 _("Character out of range for UTF-8"));
954 result_length += UTF8_LENGTH (str[i]);
957 result = g_malloc (result_length + 1);
961 while (p < result + result_length)
962 p += g_unichar_to_utf8 (str[i++], p);
967 *items_written = p - result;
976 #define SURROGATE_VALUE(h,l) (((h) - 0xd800) * 0x400 + (l) - 0xdc00 + 0x10000)
980 * @str: a UTF-16 encoded string
981 * @len: the maximum length of @str to use. If @len < 0, then
982 * the string is terminated with a 0 character.
983 * @items_read: location to store number of words read, or %NULL.
984 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
985 * returned in case @str contains a trailing partial
986 * character. If an error occurs then the index of the
987 * invalid input is stored here.
988 * @items_written: location to store number of bytes written, or %NULL.
989 * The value stored here does not include the trailing
991 * @error: location to store the error occuring, or %NULL to ignore
992 * errors. Any of the errors in #GConvertError other than
993 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
995 * Convert a string from UTF-16 to UTF-8. The result will be
996 * terminated with a 0 byte.
998 * Return value: a pointer to a newly allocated UTF-8 string.
999 * This value must be freed with g_free(). If an
1000 * error occurs, %NULL will be returned and
1004 g_utf16_to_utf8 (const gunichar2 *str,
1007 glong *items_written,
1010 /* This function and g_utf16_to_ucs4 are almost exactly identical - The lines that differ
1013 const gunichar2 *in;
1015 gchar *result = NULL;
1017 gunichar high_surrogate;
1019 g_return_val_if_fail (str != 0, NULL);
1024 while ((len < 0 || in - str < len) && *in)
1029 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1033 wc = SURROGATE_VALUE (high_surrogate, c);
1038 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1039 _("Invalid sequence in conversion input"));
1047 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1048 _("Invalid sequence in conversion input"));
1052 if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1061 /********** DIFFERENT for UTF8/UCS4 **********/
1062 n_bytes += UTF8_LENGTH (wc);
1068 if (high_surrogate && !items_read)
1070 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1071 _("Partial character sequence at end of input"));
1075 /* At this point, everything is valid, and we just need to convert
1077 /********** DIFFERENT for UTF8/UCS4 **********/
1078 result = g_malloc (n_bytes + 1);
1083 while (out < result + n_bytes)
1088 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1090 wc = SURROGATE_VALUE (high_surrogate, c);
1093 else if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1101 /********** DIFFERENT for UTF8/UCS4 **********/
1102 out += g_unichar_to_utf8 (wc, out);
1108 /********** DIFFERENT for UTF8/UCS4 **********/
1112 /********** DIFFERENT for UTF8/UCS4 **********/
1113 *items_written = out - result;
1117 *items_read = in - str;
1124 * @str: a UTF-16 encoded string
1125 * @len: the maximum length of @str to use. If @len < 0, then
1126 * the string is terminated with a 0 character.
1127 * @items_read: location to store number of words read, or %NULL.
1128 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1129 * returned in case @str contains a trailing partial
1130 * character. If an error occurs then the index of the
1131 * invalid input is stored here.
1132 * @items_written: location to store number of characters written, or %NULL.
1133 * The value stored here does not include the trailing
1135 * @error: location to store the error occuring, or %NULL to ignore
1136 * errors. Any of the errors in #GConvertError other than
1137 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1139 * Convert a string from UTF-16 to UCS-4. The result will be
1140 * terminated with a 0 character.
1142 * Return value: a pointer to a newly allocated UCS-4 string.
1143 * This value must be freed with g_free(). If an
1144 * error occurs, %NULL will be returned and
1148 g_utf16_to_ucs4 (const gunichar2 *str,
1151 glong *items_written,
1154 const gunichar2 *in;
1156 gchar *result = NULL;
1158 gunichar high_surrogate;
1160 g_return_val_if_fail (str != 0, NULL);
1165 while ((len < 0 || in - str < len) && *in)
1170 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1174 wc = SURROGATE_VALUE (high_surrogate, c);
1179 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1180 _("Invalid sequence in conversion input"));
1188 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1189 _("Invalid sequence in conversion input"));
1193 if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1202 /********** DIFFERENT for UTF8/UCS4 **********/
1203 n_bytes += sizeof (gunichar);
1209 if (high_surrogate && !items_read)
1211 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1212 _("Partial character sequence at end of input"));
1216 /* At this point, everything is valid, and we just need to convert
1218 /********** DIFFERENT for UTF8/UCS4 **********/
1219 result = g_malloc (n_bytes + 4);
1224 while (out < result + n_bytes)
1229 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1231 wc = SURROGATE_VALUE (high_surrogate, c);
1234 else if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1242 /********** DIFFERENT for UTF8/UCS4 **********/
1243 *(gunichar *)out = wc;
1244 out += sizeof (gunichar);
1250 /********** DIFFERENT for UTF8/UCS4 **********/
1251 *(gunichar *)out = 0;
1254 /********** DIFFERENT for UTF8/UCS4 **********/
1255 *items_written = (out - result) / sizeof (gunichar);
1259 *items_read = in - str;
1261 return (gunichar *)result;
1266 * @str: a UTF-8 encoded string
1267 * @len: the maximum length of @str to use. If @len < 0, then
1268 * the string is nul-terminated.
1270 * @items_read: location to store number of bytes read, or %NULL.
1271 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1272 * returned in case @str contains a trailing partial
1273 * character. If an error occurs then the index of the
1274 * invalid input is stored here.
1275 * @items_written: location to store number of words written, or %NULL.
1276 * The value stored here does not include the trailing
1278 * @error: location to store the error occuring, or %NULL to ignore
1279 * errors. Any of the errors in #GConvertError other than
1280 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1282 * Convert a string from UTF-8 to UTF-16. A 0 word will be
1283 * added to the result after the converted text.
1285 * Return value: a pointer to a newly allocated UTF-16 string.
1286 * This value must be freed with g_free(). If an
1287 * error occurs, %NULL will be returned and
1291 g_utf8_to_utf16 (const gchar *str,
1294 glong *items_written,
1297 gunichar2 *result = NULL;
1302 g_return_val_if_fail (str != NULL, NULL);
1306 while ((len < 0 || str + len - in > 0) && *in)
1308 gunichar wc = g_utf8_get_char_extended (in, str + len - in);
1309 if (wc & 0x80000000)
1311 if (wc == (gunichar)-2)
1316 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1317 _("Partial character sequence at end of input"));
1320 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1321 _("Invalid byte sequence in conversion input"));
1328 else if (wc < 0xe000)
1330 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1331 _("Invalid sequence in conversion input"));
1335 else if (wc < 0x10000)
1337 else if (wc < 0x110000)
1341 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1342 _("Character out of range for UTF-16"));
1347 in = g_utf8_next_char (in);
1350 result = g_new (gunichar2, n16 + 1);
1353 for (i = 0; i < n16;)
1355 gunichar wc = g_utf8_get_char (in);
1363 result[i++] = (wc - 0x10000) / 0x400 + 0xd800;
1364 result[i++] = (wc - 0x10000) % 0x400 + 0xdc00;
1367 in = g_utf8_next_char (in);
1373 *items_written = n16;
1377 *items_read = in - str;
1384 * @str: a UCS-4 encoded string
1385 * @len: the maximum length of @str to use. If @len < 0, then
1386 * the string is terminated with a 0 character.
1387 * @items_read: location to store number of bytes read, or %NULL.
1388 * If an error occurs then the index of the invalid input
1390 * @items_written: location to store number of words written, or %NULL.
1391 * The value stored here does not include the trailing
1393 * @error: location to store the error occuring, or %NULL to ignore
1394 * errors. Any of the errors in #GConvertError other than
1395 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1397 * Convert a string from UCS-4 to UTF-16. A 0 word will be
1398 * added to the result after the converted text.
1400 * Return value: a pointer to a newly allocated UTF-16 string.
1401 * This value must be freed with g_free(). If an
1402 * error occurs, %NULL will be returned and
1406 g_ucs4_to_utf16 (const gunichar *str,
1409 glong *items_written,
1412 gunichar2 *result = NULL;
1418 while ((len < 0 || i < len) && str[i])
1420 gunichar wc = str[i];
1424 else if (wc < 0xe000)
1426 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1427 _("Invalid sequence in conversion input"));
1431 else if (wc < 0x10000)
1433 else if (wc < 0x110000)
1437 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1438 _("Character out of range for UTF-16"));
1446 result = g_new (gunichar2, n16 + 1);
1448 for (i = 0, j = 0; j < n16; i++)
1450 gunichar wc = str[i];
1458 result[j++] = (wc - 0x10000) / 0x400 + 0xd800;
1459 result[j++] = (wc - 0x10000) % 0x400 + 0xdc00;
1465 *items_written = n16;
1476 * @str: a pointer to character data
1477 * @max_len: max bytes to validate, or -1 to go until nul
1478 * @end: return location for end of valid data
1480 * Validates UTF-8 encoded text. @str is the text to validate;
1481 * if @str is nul-terminated, then @max_len can be -1, otherwise
1482 * @max_len should be the number of bytes to validate.
1483 * If @end is non-%NULL, then the end of the valid range
1484 * will be stored there (i.e. the address of the first invalid byte
1485 * if some bytes were invalid, or the end of the text being validated
1488 * Returns %TRUE if all of @str was valid. Many GLib and GTK+
1489 * routines <emphasis>require</emphasis> valid UTF-8 as input;
1490 * so data read from a file or the network should be checked
1491 * with g_utf8_validate() before doing anything else with it.
1493 * Return value: %TRUE if the text was valid UTF-8
1496 g_utf8_validate (const gchar *str,
1503 g_return_val_if_fail (str != NULL, FALSE);
1510 while ((max_len < 0 || (p - str) < max_len) && *p)
1512 int i, mask = 0, len;
1514 unsigned char c = (unsigned char) *p;
1516 UTF8_COMPUTE (c, mask, len);
1521 /* check that the expected number of bytes exists in str */
1523 ((max_len - (p - str)) < len))
1526 UTF8_GET (result, p, i, mask, len);
1528 if (UTF8_LENGTH (result) != len) /* Check for overlong UTF-8 */
1531 if (result == (gunichar)-1)
1534 if (!UNICODE_VALID (result))
1543 /* See that we covered the entire length if a length was
1544 * passed in, or that we ended on a nul if not
1547 p != (str + max_len))
1549 else if (max_len < 0 &&
1557 * g_unichar_validate:
1558 * @ch: a Unicode character
1560 * Checks whether @ch is a valid Unicode character. Some possible
1561 * integer values of @ch will not be valid. 0 is considered a valid
1562 * character, though it's normally a string terminator.
1564 * Return value: %TRUE if @ch is a valid Unicode character
1567 g_unichar_validate (gunichar ch)
1569 return UNICODE_VALID (ch);