1 /* gutf8.c - Operations on UTF-8 strings.
3 * Copyright (C) 1999 Tom Tromey
4 * Copyright (C) 2000 Red Hat, Inc.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 02111-1307, USA.
32 #ifdef G_PLATFORM_WIN32
39 #include "libcharset/libcharset.h"
43 #define UTF8_COMPUTE(Char, Mask, Len) \
49 else if ((Char & 0xe0) == 0xc0) \
54 else if ((Char & 0xf0) == 0xe0) \
59 else if ((Char & 0xf8) == 0xf0) \
64 else if ((Char & 0xfc) == 0xf8) \
69 else if ((Char & 0xfe) == 0xfc) \
77 #define UTF8_LENGTH(Char) \
78 ((Char) < 0x80 ? 1 : \
79 ((Char) < 0x800 ? 2 : \
80 ((Char) < 0x10000 ? 3 : \
81 ((Char) < 0x200000 ? 4 : \
82 ((Char) < 0x4000000 ? 5 : 6)))))
85 #define UTF8_GET(Result, Chars, Count, Mask, Len) \
86 (Result) = (Chars)[0] & (Mask); \
87 for ((Count) = 1; (Count) < (Len); ++(Count)) \
89 if (((Chars)[(Count)] & 0xc0) != 0x80) \
95 (Result) |= ((Chars)[(Count)] & 0x3f); \
98 #define UNICODE_VALID(Char) \
99 ((Char) < 0x110000 && \
100 (((Char) & 0xFFFFF800) != 0xD800) && \
101 ((Char) < 0xFDD0 || (Char) > 0xFDEF) && \
102 ((Char) & 0xFFFF) != 0xFFFF)
105 static const gchar utf8_skip_data[256] = {
106 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
107 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
108 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
109 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
110 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
111 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
112 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
113 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1
116 const gchar * const g_utf8_skip = utf8_skip_data;
119 * g_utf8_find_prev_char:
120 * @str: pointer to the beginning of a UTF-8 encoded string
121 * @p: pointer to some position within @str
123 * Given a position @p with a UTF-8 encoded string @str, find the start
124 * of the previous UTF-8 character starting before @p. Returns %NULL if no
125 * UTF-8 characters are present in @p before @str.
127 * @p does not have to be at the beginning of a UTF-8 character. No check
128 * is made to see if the character found is actually valid other than
129 * it starts with an appropriate byte.
131 * Return value: a pointer to the found character or %NULL.
134 g_utf8_find_prev_char (const char *str,
137 for (--p; p >= str; --p)
139 if ((*p & 0xc0) != 0x80)
146 * g_utf8_find_next_char:
147 * @p: a pointer to a position within a UTF-8 encoded string
148 * @end: a pointer to the end of the string, or %NULL to indicate
149 * that the string is nul-terminated, in which case
150 * the returned value will be
152 * Finds the start of the next UTF-8 character in the string after @p.
154 * @p does not have to be at the beginning of a UTF-8 character. No check
155 * is made to see if the character found is actually valid other than
156 * it starts with an appropriate byte.
158 * Return value: a pointer to the found character or %NULL
161 g_utf8_find_next_char (const gchar *p,
167 for (++p; p < end && (*p & 0xc0) == 0x80; ++p)
170 for (++p; (*p & 0xc0) == 0x80; ++p)
173 return (p == end) ? NULL : (gchar *)p;
178 * @p: a pointer to a position within a UTF-8 encoded string
180 * Finds the previous UTF-8 character in the string before @p.
182 * @p does not have to be at the beginning of a UTF-8 character. No check
183 * is made to see if the character found is actually valid other than
184 * it starts with an appropriate byte. If @p might be the first
185 * character of the string, you must use g_utf8_find_prev_char() instead.
187 * Return value: a pointer to the found character.
190 g_utf8_prev_char (const gchar *p)
195 if ((*p & 0xc0) != 0x80)
202 * @p: pointer to the start of a UTF-8 encoded string.
203 * @max: the maximum number of bytes to examine. If @max
204 * is less than 0, then the string is assumed to be
207 * Returns the length of the string in characters.
209 * Return value: the length of the string in characters
212 g_utf8_strlen (const gchar *p,
216 const gchar *start = p;
222 p = g_utf8_next_char (p);
231 p = g_utf8_next_char (p);
233 while (p - start < max && *p)
236 p = g_utf8_next_char (p);
239 /* only do the last len increment if we got a complete
240 * char (don't count partial chars)
242 if (p - start == max)
251 * @p: a pointer to Unicode character encoded as UTF-8
253 * Converts a sequence of bytes encoded as UTF-8 to a Unicode character.
254 * If @p does not point to a valid UTF-8 encoded character, results are
255 * undefined. If you are not sure that the bytes are complete
256 * valid Unicode characters, you should use g_utf8_get_char_validated()
259 * Return value: the resulting character
262 g_utf8_get_char (const gchar *p)
264 int i, mask = 0, len;
266 unsigned char c = (unsigned char) *p;
268 UTF8_COMPUTE (c, mask, len);
271 UTF8_GET (result, p, i, mask, len);
277 * g_utf8_offset_to_pointer:
278 * @str: a UTF-8 encoded string
279 * @offset: a character offset within @str
281 * Converts from an integer character offset to a pointer to a position
284 * Return value: the resulting pointer
287 g_utf8_offset_to_pointer (const gchar *str,
290 const gchar *s = str;
292 s = g_utf8_next_char (s);
298 * g_utf8_pointer_to_offset:
299 * @str: a UTF-8 encoded string
300 * @pos: a pointer to a position within @str
302 * Converts from a pointer to position within a string to a integer
305 * Return value: the resulting character offset
308 g_utf8_pointer_to_offset (const gchar *str,
311 const gchar *s = str;
316 s = g_utf8_next_char (s);
326 * @dest: buffer to fill with characters from @src
327 * @src: UTF-8 encoded string
328 * @n: character count
330 * Like the standard C <function>strncpy()</function> function, but
331 * copies a given number of characters instead of a given number of
332 * bytes. The @src string must be valid UTF-8 encoded text.
333 * (Use g_utf8_validate() on all text before trying to use UTF-8
334 * utility functions with it.)
336 * Return value: @dest
339 g_utf8_strncpy (gchar *dest,
343 const gchar *s = src;
346 s = g_utf8_next_char(s);
349 strncpy(dest, src, s - src);
354 G_LOCK_DEFINE_STATIC (aliases);
357 get_alias_hash (void)
359 static GHashTable *alias_hash = NULL;
366 alias_hash = g_hash_table_new (g_str_hash, g_str_equal);
368 aliases = _g_locale_get_charset_aliases ();
369 while (*aliases != '\0')
371 const char *canonical;
373 const char **alias_array;
377 aliases += strlen (aliases) + 1;
379 aliases += strlen (aliases) + 1;
381 alias_array = g_hash_table_lookup (alias_hash, canonical);
384 while (alias_array[count])
388 alias_array = g_renew (const char *, alias_array, count + 2);
389 alias_array[count] = alias;
390 alias_array[count + 1] = NULL;
392 g_hash_table_insert (alias_hash, (char *)canonical, alias_array);
401 /* As an abuse of the alias table, the following routines gets
402 * the charsets that are aliases for the canonical name.
405 _g_charset_get_aliases (const char *canonical_name)
407 GHashTable *alias_hash = get_alias_hash ();
409 return g_hash_table_lookup (alias_hash, canonical_name);
413 g_utf8_get_charset_internal (const char *raw_data,
416 const char *charset = getenv("CHARSET");
418 if (charset && *charset)
422 if (charset && strstr (charset, "UTF-8"))
428 /* The libcharset code tries to be thread-safe without
429 * a lock, but has a memory leak and a missing memory
430 * barrier, so we lock for it
433 charset = _g_locale_charset_unalias (raw_data);
436 if (charset && *charset)
440 if (charset && strstr (charset, "UTF-8"))
446 /* Assume this for compatibility at present. */
452 typedef struct _GCharsetCache GCharsetCache;
454 struct _GCharsetCache {
461 charset_cache_free (gpointer data)
463 GCharsetCache *cache = data;
465 g_free (cache->charset);
471 * @charset: return location for character set name
473 * Obtains the character set for the current locale; you might use
474 * this character set as an argument to g_convert(), to convert from
475 * the current locale's encoding to some other encoding. (Frequently
476 * g_locale_to_utf8() and g_locale_from_utf8() are nice shortcuts,
479 * The return value is %TRUE if the locale's encoding is UTF-8, in that
480 * case you can perhaps avoid calling g_convert().
482 * The string returned in @charset is not allocated, and should not be
485 * Return value: %TRUE if the returned charset is UTF-8
488 g_get_charset (G_CONST_RETURN char **charset)
490 static GStaticPrivate cache_private = G_STATIC_PRIVATE_INIT;
491 GCharsetCache *cache = g_static_private_get (&cache_private);
496 cache = g_new0 (GCharsetCache, 1);
497 g_static_private_set (&cache_private, cache, charset_cache_free);
500 raw = _g_locale_charset_raw ();
502 if (!(cache->raw && strcmp (cache->raw, raw) == 0))
504 const gchar *new_charset;
507 g_free (cache->charset);
508 cache->raw = g_strdup (raw);
509 cache->is_utf8 = g_utf8_get_charset_internal (raw, &new_charset);
510 cache->charset = g_strdup (new_charset);
514 *charset = cache->charset;
516 return cache->is_utf8;
523 * @c: a ISO10646 character code
524 * @outbuf: output buffer, must have at least 6 bytes of space.
525 * If %NULL, the length will be computed and returned
526 * and nothing will be written to @outbuf.
528 * Converts a single character to UTF-8.
530 * Return value: number of bytes written
533 g_unichar_to_utf8 (gunichar c,
550 else if (c < 0x10000)
555 else if (c < 0x200000)
560 else if (c < 0x4000000)
573 for (i = len - 1; i > 0; --i)
575 outbuf[i] = (c & 0x3f) | 0x80;
578 outbuf[0] = c | first;
586 * @p: a nul-terminated UTF-8 encoded string
587 * @len: the maximum length of @p
588 * @c: a ISO10646 character
590 * Finds the leftmost occurrence of the given ISO10646 character
591 * in a UTF-8 encoded string, while limiting the search to @len bytes.
592 * If @len is -1, allow unbounded search.
594 * Return value: %NULL if the string does not contain the character,
595 * otherwise, a pointer to the start of the leftmost occurrence of
596 * the character in the string.
599 g_utf8_strchr (const char *p,
605 gint charlen = g_unichar_to_utf8 (c, ch);
608 return g_strstr_len (p, len, ch);
614 * @p: a nul-terminated UTF-8 encoded string
615 * @len: the maximum length of @p
616 * @c: a ISO10646 character
618 * Find the rightmost occurrence of the given ISO10646 character
619 * in a UTF-8 encoded string, while limiting the search to @len bytes.
620 * If @len is -1, allow unbounded search.
622 * Return value: %NULL if the string does not contain the character,
623 * otherwise, a pointer to the start of the rightmost occurrence of the
624 * character in the string.
627 g_utf8_strrchr (const char *p,
633 gint charlen = g_unichar_to_utf8 (c, ch);
636 return g_strrstr_len (p, len, ch);
640 /* Like g_utf8_get_char, but take a maximum length
641 * and return (gunichar)-2 on incomplete trailing character
643 static inline gunichar
644 g_utf8_get_char_extended (const gchar *p,
648 gunichar wc = (guchar) *p;
688 if (max_len >= 0 && len > max_len)
690 for (i = 1; i < max_len; i++)
692 if ((((guchar *)p)[i] & 0xc0) != 0x80)
698 for (i = 1; i < len; ++i)
700 gunichar ch = ((guchar *)p)[i];
702 if ((ch & 0xc0) != 0x80)
714 if (UTF8_LENGTH(wc) != len)
721 * g_utf8_get_char_validated:
722 * @p: a pointer to Unicode character encoded as UTF-8
723 * @max_len: the maximum number of bytes to read, or -1, for no maximum.
725 * Convert a sequence of bytes encoded as UTF-8 to a Unicode character.
726 * This function checks for incomplete characters, for invalid characters
727 * such as characters that are out of the range of Unicode, and for
728 * overlong encodings of valid characters.
730 * Return value: the resulting character. If @p points to a partial
731 * sequence at the end of a string that could begin a valid character,
732 * returns (gunichar)-2; otherwise, if @p does not point to a valid
733 * UTF-8 encoded Unicode character, returns (gunichar)-1.
736 g_utf8_get_char_validated (const gchar *p,
739 gunichar result = g_utf8_get_char_extended (p, max_len);
741 if (result & 0x80000000)
743 else if (!UNICODE_VALID (result))
750 * g_utf8_to_ucs4_fast:
751 * @str: a UTF-8 encoded string
752 * @len: the maximum length of @str to use. If @len < 0, then
753 * the string is nul-terminated.
754 * @items_written: location to store the number of characters in the
757 * Convert a string from UTF-8 to a 32-bit fixed width
758 * representation as UCS-4, assuming valid UTF-8 input.
759 * This function is roughly twice as fast as g_utf8_to_ucs4()
760 * but does no error checking on the input.
762 * Return value: a pointer to a newly allocated UCS-4 string.
763 * This value must be freed with g_free().
766 g_utf8_to_ucs4_fast (const gchar *str,
768 glong *items_written)
775 g_return_val_if_fail (str != NULL, NULL);
783 p = g_utf8_next_char (p);
789 while (p < str + len && *p)
791 p = g_utf8_next_char (p);
796 result = g_new (gunichar, n_chars + 1);
799 for (i=0; i < n_chars; i++)
801 gunichar wc = ((unsigned char *)p)[0];
836 for (j = 1; j < charlen; j++)
839 wc |= ((unsigned char *)p)[j] & 0x3f;
856 * @str: a UTF-8 encoded string
857 * @len: the maximum length of @str to use. If @len < 0, then
858 * the string is nul-terminated.
859 * @items_read: location to store number of bytes read, or %NULL.
860 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
861 * returned in case @str contains a trailing partial
862 * character. If an error occurs then the index of the
863 * invalid input is stored here.
864 * @items_written: location to store number of characters written or %NULL.
865 * The value here stored does not include the trailing 0
867 * @error: location to store the error occuring, or %NULL to ignore
868 * errors. Any of the errors in #GConvertError other than
869 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
871 * Convert a string from UTF-8 to a 32-bit fixed width
872 * representation as UCS-4. A trailing 0 will be added to the
873 * string after the converted text.
875 * Return value: a pointer to a newly allocated UCS-4 string.
876 * This value must be freed with g_free(). If an
877 * error occurs, %NULL will be returned and
881 g_utf8_to_ucs4 (const gchar *str,
884 glong *items_written,
887 gunichar *result = NULL;
893 while ((len < 0 || str + len - in > 0) && *in)
895 gunichar wc = g_utf8_get_char_extended (in, str + len - in);
898 if (wc == (gunichar)-2)
903 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
904 _("Partial character sequence at end of input"));
907 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
908 _("Invalid byte sequence in conversion input"));
915 in = g_utf8_next_char (in);
918 result = g_new (gunichar, n_chars + 1);
921 for (i=0; i < n_chars; i++)
923 result[i] = g_utf8_get_char (in);
924 in = g_utf8_next_char (in);
929 *items_written = n_chars;
933 *items_read = in - str;
940 * @str: a UCS-4 encoded string
941 * @len: the maximum length of @str to use. If @len < 0, then
942 * the string is terminated with a 0 character.
943 * @items_read: location to store number of characters read read, or %NULL.
944 * @items_written: location to store number of bytes written or %NULL.
945 * The value here stored does not include the trailing 0
947 * @error: location to store the error occuring, or %NULL to ignore
948 * errors. Any of the errors in #GConvertError other than
949 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
951 * Convert a string from a 32-bit fixed width representation as UCS-4.
952 * to UTF-8. The result will be terminated with a 0 byte.
954 * Return value: a pointer to a newly allocated UTF-8 string.
955 * This value must be freed with g_free(). If an
956 * error occurs, %NULL will be returned and
960 g_ucs4_to_utf8 (const gunichar *str,
963 glong *items_written,
967 gchar *result = NULL;
972 for (i = 0; len < 0 || i < len ; i++)
977 if (str[i] >= 0x80000000)
982 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
983 _("Character out of range for UTF-8"));
987 result_length += UTF8_LENGTH (str[i]);
990 result = g_malloc (result_length + 1);
994 while (p < result + result_length)
995 p += g_unichar_to_utf8 (str[i++], p);
1000 *items_written = p - result;
1009 #define SURROGATE_VALUE(h,l) (((h) - 0xd800) * 0x400 + (l) - 0xdc00 + 0x10000)
1013 * @str: a UTF-16 encoded string
1014 * @len: the maximum length of @str to use. If @len < 0, then
1015 * the string is terminated with a 0 character.
1016 * @items_read: location to store number of words read, or %NULL.
1017 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1018 * returned in case @str contains a trailing partial
1019 * character. If an error occurs then the index of the
1020 * invalid input is stored here.
1021 * @items_written: location to store number of bytes written, or %NULL.
1022 * The value stored here does not include the trailing
1024 * @error: location to store the error occuring, or %NULL to ignore
1025 * errors. Any of the errors in #GConvertError other than
1026 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1028 * Convert a string from UTF-16 to UTF-8. The result will be
1029 * terminated with a 0 byte.
1031 * Return value: a pointer to a newly allocated UTF-8 string.
1032 * This value must be freed with g_free(). If an
1033 * error occurs, %NULL will be returned and
1037 g_utf16_to_utf8 (const gunichar2 *str,
1040 glong *items_written,
1043 /* This function and g_utf16_to_ucs4 are almost exactly identical - The lines that differ
1046 const gunichar2 *in;
1048 gchar *result = NULL;
1050 gunichar high_surrogate;
1052 g_return_val_if_fail (str != 0, NULL);
1057 while ((len < 0 || in - str < len) && *in)
1062 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1066 wc = SURROGATE_VALUE (high_surrogate, c);
1071 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1072 _("Invalid sequence in conversion input"));
1080 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1081 _("Invalid sequence in conversion input"));
1085 if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1094 /********** DIFFERENT for UTF8/UCS4 **********/
1095 n_bytes += UTF8_LENGTH (wc);
1101 if (high_surrogate && !items_read)
1103 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1104 _("Partial character sequence at end of input"));
1108 /* At this point, everything is valid, and we just need to convert
1110 /********** DIFFERENT for UTF8/UCS4 **********/
1111 result = g_malloc (n_bytes + 1);
1116 while (out < result + n_bytes)
1121 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1123 wc = SURROGATE_VALUE (high_surrogate, c);
1126 else if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1134 /********** DIFFERENT for UTF8/UCS4 **********/
1135 out += g_unichar_to_utf8 (wc, out);
1141 /********** DIFFERENT for UTF8/UCS4 **********/
1145 /********** DIFFERENT for UTF8/UCS4 **********/
1146 *items_written = out - result;
1150 *items_read = in - str;
1157 * @str: a UTF-16 encoded string
1158 * @len: the maximum length of @str to use. If @len < 0, then
1159 * the string is terminated with a 0 character.
1160 * @items_read: location to store number of words read, or %NULL.
1161 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1162 * returned in case @str contains a trailing partial
1163 * character. If an error occurs then the index of the
1164 * invalid input is stored here.
1165 * @items_written: location to store number of characters written, or %NULL.
1166 * The value stored here does not include the trailing
1168 * @error: location to store the error occuring, or %NULL to ignore
1169 * errors. Any of the errors in #GConvertError other than
1170 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1172 * Convert a string from UTF-16 to UCS-4. The result will be
1173 * terminated with a 0 character.
1175 * Return value: a pointer to a newly allocated UCS-4 string.
1176 * This value must be freed with g_free(). If an
1177 * error occurs, %NULL will be returned and
1181 g_utf16_to_ucs4 (const gunichar2 *str,
1184 glong *items_written,
1187 const gunichar2 *in;
1189 gchar *result = NULL;
1191 gunichar high_surrogate;
1193 g_return_val_if_fail (str != 0, NULL);
1198 while ((len < 0 || in - str < len) && *in)
1203 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1207 wc = SURROGATE_VALUE (high_surrogate, c);
1212 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1213 _("Invalid sequence in conversion input"));
1221 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1222 _("Invalid sequence in conversion input"));
1226 if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1235 /********** DIFFERENT for UTF8/UCS4 **********/
1236 n_bytes += sizeof (gunichar);
1242 if (high_surrogate && !items_read)
1244 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1245 _("Partial character sequence at end of input"));
1249 /* At this point, everything is valid, and we just need to convert
1251 /********** DIFFERENT for UTF8/UCS4 **********/
1252 result = g_malloc (n_bytes + 4);
1257 while (out < result + n_bytes)
1262 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1264 wc = SURROGATE_VALUE (high_surrogate, c);
1267 else if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1275 /********** DIFFERENT for UTF8/UCS4 **********/
1276 *(gunichar *)out = wc;
1277 out += sizeof (gunichar);
1283 /********** DIFFERENT for UTF8/UCS4 **********/
1284 *(gunichar *)out = 0;
1287 /********** DIFFERENT for UTF8/UCS4 **********/
1288 *items_written = (out - result) / sizeof (gunichar);
1292 *items_read = in - str;
1294 return (gunichar *)result;
1299 * @str: a UTF-8 encoded string
1300 * @len: the maximum length of @str to use. If @len < 0, then
1301 * the string is nul-terminated.
1302 * @items_read: location to store number of bytes read, or %NULL.
1303 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1304 * returned in case @str contains a trailing partial
1305 * character. If an error occurs then the index of the
1306 * invalid input is stored here.
1307 * @items_written: location to store number of words written, or %NULL.
1308 * The value stored here does not include the trailing
1310 * @error: location to store the error occuring, or %NULL to ignore
1311 * errors. Any of the errors in #GConvertError other than
1312 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1314 * Convert a string from UTF-8 to UTF-16. A 0 word will be
1315 * added to the result after the converted text.
1317 * Return value: a pointer to a newly allocated UTF-16 string.
1318 * This value must be freed with g_free(). If an
1319 * error occurs, %NULL will be returned and
1323 g_utf8_to_utf16 (const gchar *str,
1326 glong *items_written,
1329 gunichar2 *result = NULL;
1334 g_return_val_if_fail (str != NULL, NULL);
1338 while ((len < 0 || str + len - in > 0) && *in)
1340 gunichar wc = g_utf8_get_char_extended (in, str + len - in);
1341 if (wc & 0x80000000)
1343 if (wc == (gunichar)-2)
1348 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1349 _("Partial character sequence at end of input"));
1352 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1353 _("Invalid byte sequence in conversion input"));
1360 else if (wc < 0xe000)
1362 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1363 _("Invalid sequence in conversion input"));
1367 else if (wc < 0x10000)
1369 else if (wc < 0x110000)
1373 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1374 _("Character out of range for UTF-16"));
1379 in = g_utf8_next_char (in);
1382 result = g_new (gunichar2, n16 + 1);
1385 for (i = 0; i < n16;)
1387 gunichar wc = g_utf8_get_char (in);
1395 result[i++] = (wc - 0x10000) / 0x400 + 0xd800;
1396 result[i++] = (wc - 0x10000) % 0x400 + 0xdc00;
1399 in = g_utf8_next_char (in);
1405 *items_written = n16;
1409 *items_read = in - str;
1416 * @str: a UCS-4 encoded string
1417 * @len: the maximum length of @str to use. If @len < 0, then
1418 * the string is terminated with a 0 character.
1419 * @items_read: location to store number of bytes read, or %NULL.
1420 * If an error occurs then the index of the invalid input
1422 * @items_written: location to store number of words written, or %NULL.
1423 * The value stored here does not include the trailing
1425 * @error: location to store the error occuring, or %NULL to ignore
1426 * errors. Any of the errors in #GConvertError other than
1427 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1429 * Convert a string from UCS-4 to UTF-16. A 0 word will be
1430 * added to the result after the converted text.
1432 * Return value: a pointer to a newly allocated UTF-16 string.
1433 * This value must be freed with g_free(). If an
1434 * error occurs, %NULL will be returned and
1438 g_ucs4_to_utf16 (const gunichar *str,
1441 glong *items_written,
1444 gunichar2 *result = NULL;
1450 while ((len < 0 || i < len) && str[i])
1452 gunichar wc = str[i];
1456 else if (wc < 0xe000)
1458 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1459 _("Invalid sequence in conversion input"));
1463 else if (wc < 0x10000)
1465 else if (wc < 0x110000)
1469 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1470 _("Character out of range for UTF-16"));
1478 result = g_new (gunichar2, n16 + 1);
1480 for (i = 0, j = 0; j < n16; i++)
1482 gunichar wc = str[i];
1490 result[j++] = (wc - 0x10000) / 0x400 + 0xd800;
1491 result[j++] = (wc - 0x10000) % 0x400 + 0xdc00;
1497 *items_written = n16;
1508 * @str: a pointer to character data
1509 * @max_len: max bytes to validate, or -1 to go until nul
1510 * @end: return location for end of valid data
1512 * Validates UTF-8 encoded text. @str is the text to validate;
1513 * if @str is nul-terminated, then @max_len can be -1, otherwise
1514 * @max_len should be the number of bytes to validate.
1515 * If @end is non-%NULL, then the end of the valid range
1516 * will be stored there (i.e. the address of the first invalid byte
1517 * if some bytes were invalid, or the end of the text being validated
1520 * Returns %TRUE if all of @str was valid. Many GLib and GTK+
1521 * routines <emphasis>require</emphasis> valid UTF-8 as input;
1522 * so data read from a file or the network should be checked
1523 * with g_utf8_validate() before doing anything else with it.
1525 * Return value: %TRUE if the text was valid UTF-8
1528 g_utf8_validate (const gchar *str,
1535 g_return_val_if_fail (str != NULL, FALSE);
1542 while ((max_len < 0 || (p - str) < max_len) && *p)
1544 int i, mask = 0, len;
1546 unsigned char c = (unsigned char) *p;
1548 UTF8_COMPUTE (c, mask, len);
1553 /* check that the expected number of bytes exists in str */
1555 ((max_len - (p - str)) < len))
1558 UTF8_GET (result, p, i, mask, len);
1560 if (UTF8_LENGTH (result) != len) /* Check for overlong UTF-8 */
1563 if (result == (gunichar)-1)
1566 if (!UNICODE_VALID (result))
1575 /* See that we covered the entire length if a length was
1576 * passed in, or that we ended on a nul if not
1579 p != (str + max_len))
1581 else if (max_len < 0 &&
1589 * g_unichar_validate:
1590 * @ch: a Unicode character
1592 * Checks whether @ch is a valid Unicode character. Some possible
1593 * integer values of @ch will not be valid. 0 is considered a valid
1594 * character, though it's normally a string terminator.
1596 * Return value: %TRUE if @ch is a valid Unicode character
1599 g_unichar_validate (gunichar ch)
1601 return UNICODE_VALID (ch);
1605 * g_utf8_strreverse:
1606 * @str: a UTF-8 encoded string
1607 * @len: the maximum length of @str to use. If @len < 0, then
1608 * the string is nul-terminated.
1610 * Reverses a UTF-8 string. @str must be valid UTF-8 encoded text.
1611 * (Use g_utf8_validate() on all text before trying to use UTF-8
1612 * utility functions with it.)
1614 * Note that unlike g_strreverse(), this function returns
1615 * newly-allocated memory, which should be freed with g_free() when
1618 * Returns: a newly-allocated string which is the reverse of @str.
1623 g_utf8_strreverse (const gchar *str,
1633 result = g_new (gchar, len + 1);
1638 skip = g_utf8_skip[*(guchar*)p];
1640 for (m = r; skip; skip--)