1 /* gutf8.c - Operations on UTF-8 strings.
3 * Copyright (C) 1999 Tom Tromey
4 * Copyright (C) 2000 Red Hat, Inc.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 02111-1307, USA.
32 #ifdef G_PLATFORM_WIN32
39 #include "libcharset/libcharset.h"
43 #define UTF8_COMPUTE(Char, Mask, Len) \
49 else if ((Char & 0xe0) == 0xc0) \
54 else if ((Char & 0xf0) == 0xe0) \
59 else if ((Char & 0xf8) == 0xf0) \
64 else if ((Char & 0xfc) == 0xf8) \
69 else if ((Char & 0xfe) == 0xfc) \
77 #define UTF8_LENGTH(Char) \
78 ((Char) < 0x80 ? 1 : \
79 ((Char) < 0x800 ? 2 : \
80 ((Char) < 0x10000 ? 3 : \
81 ((Char) < 0x200000 ? 4 : \
82 ((Char) < 0x4000000 ? 5 : 6)))))
85 #define UTF8_GET(Result, Chars, Count, Mask, Len) \
86 (Result) = (Chars)[0] & (Mask); \
87 for ((Count) = 1; (Count) < (Len); ++(Count)) \
89 if (((Chars)[(Count)] & 0xc0) != 0x80) \
95 (Result) |= ((Chars)[(Count)] & 0x3f); \
99 * Check whether a Unicode (5.2) char is in a valid range.
101 * The first check comes from the Unicode guarantee to never encode
102 * a point above 0x0010ffff, since UTF-16 couldn't represent it.
104 * The second check covers surrogate pairs (category Cs).
106 * The last two checks cover "Noncharacter": defined as:
107 * "A code point that is permanently reserved for
108 * internal use, and that should never be interchanged. In
109 * Unicode 3.1, these consist of the values U+nFFFE and U+nFFFF
110 * (where n is from 0 to 10_16) and the values U+FDD0..U+FDEF."
112 * @param Char the character
114 #define UNICODE_VALID(Char) \
115 ((Char) < 0x110000 && \
116 (((Char) & 0xFFFFF800) != 0xD800) && \
117 ((Char) < 0xFDD0 || (Char) > 0xFDEF) && \
118 ((Char) & 0xFFFE) != 0xFFFE)
121 static const gchar utf8_skip_data[256] = {
122 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
123 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
124 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
125 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
126 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
127 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
128 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
129 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1
132 const gchar * const g_utf8_skip = utf8_skip_data;
135 * g_utf8_find_prev_char:
136 * @str: pointer to the beginning of a UTF-8 encoded string
137 * @p: pointer to some position within @str
139 * Given a position @p with a UTF-8 encoded string @str, find the start
140 * of the previous UTF-8 character starting before @p. Returns %NULL if no
141 * UTF-8 characters are present in @str before @p.
143 * @p does not have to be at the beginning of a UTF-8 character. No check
144 * is made to see if the character found is actually valid other than
145 * it starts with an appropriate byte.
147 * Return value: a pointer to the found character or %NULL.
150 g_utf8_find_prev_char (const char *str,
153 for (--p; p >= str; --p)
155 if ((*p & 0xc0) != 0x80)
162 * g_utf8_find_next_char:
163 * @p: a pointer to a position within a UTF-8 encoded string
164 * @end: a pointer to the byte following the end of the string,
165 * or %NULL to indicate that the string is nul-terminated.
167 * Finds the start of the next UTF-8 character in the string after @p.
169 * @p does not have to be at the beginning of a UTF-8 character. No check
170 * is made to see if the character found is actually valid other than
171 * it starts with an appropriate byte.
173 * Return value: a pointer to the found character or %NULL
176 g_utf8_find_next_char (const gchar *p,
182 for (++p; p < end && (*p & 0xc0) == 0x80; ++p)
185 for (++p; (*p & 0xc0) == 0x80; ++p)
188 return (p == end) ? NULL : (gchar *)p;
193 * @p: a pointer to a position within a UTF-8 encoded string
195 * Finds the previous UTF-8 character in the string before @p.
197 * @p does not have to be at the beginning of a UTF-8 character. No check
198 * is made to see if the character found is actually valid other than
199 * it starts with an appropriate byte. If @p might be the first
200 * character of the string, you must use g_utf8_find_prev_char() instead.
202 * Return value: a pointer to the found character.
205 g_utf8_prev_char (const gchar *p)
210 if ((*p & 0xc0) != 0x80)
217 * @p: pointer to the start of a UTF-8 encoded string
218 * @max: the maximum number of bytes to examine. If @max
219 * is less than 0, then the string is assumed to be
220 * nul-terminated. If @max is 0, @p will not be examined and
223 * Computes the length of the string in characters, not including
224 * the terminating nul character.
226 * Return value: the length of the string in characters
229 g_utf8_strlen (const gchar *p,
233 const gchar *start = p;
234 g_return_val_if_fail (p != NULL || max == 0, 0);
240 p = g_utf8_next_char (p);
249 p = g_utf8_next_char (p);
251 while (p - start < max && *p)
254 p = g_utf8_next_char (p);
257 /* only do the last len increment if we got a complete
258 * char (don't count partial chars)
260 if (p - start <= max)
269 * @p: a pointer to Unicode character encoded as UTF-8
271 * Converts a sequence of bytes encoded as UTF-8 to a Unicode character.
272 * If @p does not point to a valid UTF-8 encoded character, results are
273 * undefined. If you are not sure that the bytes are complete
274 * valid Unicode characters, you should use g_utf8_get_char_validated()
277 * Return value: the resulting character
280 g_utf8_get_char (const gchar *p)
282 int i, mask = 0, len;
284 unsigned char c = (unsigned char) *p;
286 UTF8_COMPUTE (c, mask, len);
289 UTF8_GET (result, p, i, mask, len);
295 * g_utf8_offset_to_pointer:
296 * @str: a UTF-8 encoded string
297 * @offset: a character offset within @str
299 * Converts from an integer character offset to a pointer to a position
302 * Since 2.10, this function allows to pass a negative @offset to
303 * step backwards. It is usually worth stepping backwards from the end
304 * instead of forwards if @offset is in the last fourth of the string,
305 * since moving forward is about 3 times faster than moving backward.
308 * This function doesn't abort when reaching the end of @str. Therefore
309 * you should be sure that @offset is within string boundaries before
310 * calling that function. Call g_utf8_strlen() when unsure.
312 * This limitation exists as this function is called frequently during
313 * text rendering and therefore has to be as fast as possible.
316 * Return value: the resulting pointer
319 g_utf8_offset_to_pointer (const gchar *str,
322 const gchar *s = str;
326 s = g_utf8_next_char (s);
331 /* This nice technique for fast backwards stepping
332 * through a UTF-8 string was dubbed "stutter stepping"
333 * by its inventor, Larry Ewing.
339 while ((*s & 0xc0) == 0x80)
342 offset += g_utf8_pointer_to_offset (s, s1);
350 * g_utf8_pointer_to_offset:
351 * @str: a UTF-8 encoded string
352 * @pos: a pointer to a position within @str
354 * Converts from a pointer to position within a string to a integer
357 * Since 2.10, this function allows @pos to be before @str, and returns
358 * a negative offset in this case.
360 * Return value: the resulting character offset
363 g_utf8_pointer_to_offset (const gchar *str,
366 const gchar *s = str;
370 offset = - g_utf8_pointer_to_offset (pos, str);
374 s = g_utf8_next_char (s);
384 * @dest: buffer to fill with characters from @src
385 * @src: UTF-8 encoded string
386 * @n: character count
388 * Like the standard C strncpy() function, but
389 * copies a given number of characters instead of a given number of
390 * bytes. The @src string must be valid UTF-8 encoded text.
391 * (Use g_utf8_validate() on all text before trying to use UTF-8
392 * utility functions with it.)
394 * Return value: @dest
397 g_utf8_strncpy (gchar *dest,
401 const gchar *s = src;
404 s = g_utf8_next_char(s);
407 strncpy(dest, src, s - src);
412 G_LOCK_DEFINE_STATIC (aliases);
415 get_alias_hash (void)
417 static GHashTable *alias_hash = NULL;
424 alias_hash = g_hash_table_new (g_str_hash, g_str_equal);
426 aliases = _g_locale_get_charset_aliases ();
427 while (*aliases != '\0')
429 const char *canonical;
431 const char **alias_array;
435 aliases += strlen (aliases) + 1;
437 aliases += strlen (aliases) + 1;
439 alias_array = g_hash_table_lookup (alias_hash, canonical);
442 while (alias_array[count])
446 alias_array = g_renew (const char *, alias_array, count + 2);
447 alias_array[count] = alias;
448 alias_array[count + 1] = NULL;
450 g_hash_table_insert (alias_hash, (char *)canonical, alias_array);
459 /* As an abuse of the alias table, the following routines gets
460 * the charsets that are aliases for the canonical name.
462 G_GNUC_INTERNAL const char **
463 _g_charset_get_aliases (const char *canonical_name)
465 GHashTable *alias_hash = get_alias_hash ();
467 return g_hash_table_lookup (alias_hash, canonical_name);
471 g_utf8_get_charset_internal (const char *raw_data,
474 const char *charset = getenv("CHARSET");
476 if (charset && *charset)
480 if (charset && strstr (charset, "UTF-8"))
486 /* The libcharset code tries to be thread-safe without
487 * a lock, but has a memory leak and a missing memory
488 * barrier, so we lock for it
491 charset = _g_locale_charset_unalias (raw_data);
494 if (charset && *charset)
498 if (charset && strstr (charset, "UTF-8"))
504 /* Assume this for compatibility at present. */
510 typedef struct _GCharsetCache GCharsetCache;
512 struct _GCharsetCache {
519 charset_cache_free (gpointer data)
521 GCharsetCache *cache = data;
523 g_free (cache->charset);
529 * @charset: return location for character set name
531 * Obtains the character set for the <link linkend="setlocale">current
532 * locale</link>; you might use this character set as an argument to
533 * g_convert(), to convert from the current locale's encoding to some
534 * other encoding. (Frequently g_locale_to_utf8() and g_locale_from_utf8()
535 * are nice shortcuts, though.)
537 * On Windows the character set returned by this function is the
538 * so-called system default ANSI code-page. That is the character set
539 * used by the "narrow" versions of C library and Win32 functions that
540 * handle file names. It might be different from the character set
541 * used by the C library's current locale.
543 * The return value is %TRUE if the locale's encoding is UTF-8, in that
544 * case you can perhaps avoid calling g_convert().
546 * The string returned in @charset is not allocated, and should not be
549 * Return value: %TRUE if the returned charset is UTF-8
552 g_get_charset (G_CONST_RETURN char **charset)
554 static GStaticPrivate cache_private = G_STATIC_PRIVATE_INIT;
555 GCharsetCache *cache = g_static_private_get (&cache_private);
560 cache = g_new0 (GCharsetCache, 1);
561 g_static_private_set (&cache_private, cache, charset_cache_free);
564 raw = _g_locale_charset_raw ();
566 if (!(cache->raw && strcmp (cache->raw, raw) == 0))
568 const gchar *new_charset;
571 g_free (cache->charset);
572 cache->raw = g_strdup (raw);
573 cache->is_utf8 = g_utf8_get_charset_internal (raw, &new_charset);
574 cache->charset = g_strdup (new_charset);
578 *charset = cache->charset;
580 return cache->is_utf8;
587 * @c: a Unicode character code
588 * @outbuf: output buffer, must have at least 6 bytes of space.
589 * If %NULL, the length will be computed and returned
590 * and nothing will be written to @outbuf.
592 * Converts a single character to UTF-8.
594 * Return value: number of bytes written
597 g_unichar_to_utf8 (gunichar c,
600 /* If this gets modified, also update the copy in g_string_insert_unichar() */
615 else if (c < 0x10000)
620 else if (c < 0x200000)
625 else if (c < 0x4000000)
638 for (i = len - 1; i > 0; --i)
640 outbuf[i] = (c & 0x3f) | 0x80;
643 outbuf[0] = c | first;
651 * @p: a nul-terminated UTF-8 encoded string
652 * @len: the maximum length of @p
653 * @c: a Unicode character
655 * Finds the leftmost occurrence of the given Unicode character
656 * in a UTF-8 encoded string, while limiting the search to @len bytes.
657 * If @len is -1, allow unbounded search.
659 * Return value: %NULL if the string does not contain the character,
660 * otherwise, a pointer to the start of the leftmost occurrence of
661 * the character in the string.
664 g_utf8_strchr (const char *p,
670 gint charlen = g_unichar_to_utf8 (c, ch);
673 return g_strstr_len (p, len, ch);
679 * @p: a nul-terminated UTF-8 encoded string
680 * @len: the maximum length of @p
681 * @c: a Unicode character
683 * Find the rightmost occurrence of the given Unicode character
684 * in a UTF-8 encoded string, while limiting the search to @len bytes.
685 * If @len is -1, allow unbounded search.
687 * Return value: %NULL if the string does not contain the character,
688 * otherwise, a pointer to the start of the rightmost occurrence of the
689 * character in the string.
692 g_utf8_strrchr (const char *p,
698 gint charlen = g_unichar_to_utf8 (c, ch);
701 return g_strrstr_len (p, len, ch);
705 /* Like g_utf8_get_char, but take a maximum length
706 * and return (gunichar)-2 on incomplete trailing character;
707 * also check for malformed or overlong sequences
708 * and return (gunichar)-1 in this case.
710 static inline gunichar
711 g_utf8_get_char_extended (const gchar *p,
716 gunichar wc = (guchar) *p;
722 else if (G_UNLIKELY (wc < 0xc0))
761 if (G_UNLIKELY (max_len >= 0 && len > max_len))
763 for (i = 1; i < max_len; i++)
765 if ((((guchar *)p)[i] & 0xc0) != 0x80)
771 for (i = 1; i < len; ++i)
773 gunichar ch = ((guchar *)p)[i];
775 if (G_UNLIKELY ((ch & 0xc0) != 0x80))
787 if (G_UNLIKELY (wc < min_code))
794 * g_utf8_get_char_validated:
795 * @p: a pointer to Unicode character encoded as UTF-8
796 * @max_len: the maximum number of bytes to read, or -1, for no maximum or
797 * if @p is nul-terminated
799 * Convert a sequence of bytes encoded as UTF-8 to a Unicode character.
800 * This function checks for incomplete characters, for invalid characters
801 * such as characters that are out of the range of Unicode, and for
802 * overlong encodings of valid characters.
804 * Return value: the resulting character. If @p points to a partial
805 * sequence at the end of a string that could begin a valid
806 * character (or if @max_len is zero), returns (gunichar)-2;
807 * otherwise, if @p does not point to a valid UTF-8 encoded
808 * Unicode character, returns (gunichar)-1.
811 g_utf8_get_char_validated (const gchar *p,
819 result = g_utf8_get_char_extended (p, max_len);
821 if (result & 0x80000000)
823 else if (!UNICODE_VALID (result))
830 * g_utf8_to_ucs4_fast:
831 * @str: a UTF-8 encoded string
832 * @len: the maximum length of @str to use, in bytes. If @len < 0,
833 * then the string is nul-terminated.
834 * @items_written: location to store the number of characters in the
837 * Convert a string from UTF-8 to a 32-bit fixed width
838 * representation as UCS-4, assuming valid UTF-8 input.
839 * This function is roughly twice as fast as g_utf8_to_ucs4()
840 * but does no error checking on the input.
842 * Return value: a pointer to a newly allocated UCS-4 string.
843 * This value must be freed with g_free().
846 g_utf8_to_ucs4_fast (const gchar *str,
848 glong *items_written)
855 g_return_val_if_fail (str != NULL, NULL);
863 p = g_utf8_next_char (p);
869 while (p < str + len && *p)
871 p = g_utf8_next_char (p);
876 result = g_new (gunichar, n_chars + 1);
879 for (i=0; i < n_chars; i++)
881 gunichar wc = ((unsigned char *)p)[0];
916 for (j = 1; j < charlen; j++)
919 wc |= ((unsigned char *)p)[j] & 0x3f;
936 * @str: a UTF-8 encoded string
937 * @len: the maximum length of @str to use, in bytes. If @len < 0,
938 * then the string is nul-terminated.
939 * @items_read: location to store number of bytes read, or %NULL.
940 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
941 * returned in case @str contains a trailing partial
942 * character. If an error occurs then the index of the
943 * invalid input is stored here.
944 * @items_written: location to store number of characters written or %NULL.
945 * The value here stored does not include the trailing 0
947 * @error: location to store the error occuring, or %NULL to ignore
948 * errors. Any of the errors in #GConvertError other than
949 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
951 * Convert a string from UTF-8 to a 32-bit fixed width
952 * representation as UCS-4. A trailing 0 will be added to the
953 * string after the converted text.
955 * Return value: a pointer to a newly allocated UCS-4 string.
956 * This value must be freed with g_free(). If an
957 * error occurs, %NULL will be returned and
961 g_utf8_to_ucs4 (const gchar *str,
964 glong *items_written,
967 gunichar *result = NULL;
973 while ((len < 0 || str + len - in > 0) && *in)
975 gunichar wc = g_utf8_get_char_extended (in, len < 0 ? 6 : str + len - in);
978 if (wc == (gunichar)-2)
983 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
984 _("Partial character sequence at end of input"));
987 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
988 _("Invalid byte sequence in conversion input"));
995 in = g_utf8_next_char (in);
998 result = g_new (gunichar, n_chars + 1);
1001 for (i=0; i < n_chars; i++)
1003 result[i] = g_utf8_get_char (in);
1004 in = g_utf8_next_char (in);
1009 *items_written = n_chars;
1013 *items_read = in - str;
1020 * @str: a UCS-4 encoded string
1021 * @len: the maximum length (number of characters) of @str to use.
1022 * If @len < 0, then the string is nul-terminated.
1023 * @items_read: location to store number of characters read, or %NULL.
1024 * @items_written: location to store number of bytes written or %NULL.
1025 * The value here stored does not include the trailing 0
1027 * @error: location to store the error occuring, or %NULL to ignore
1028 * errors. Any of the errors in #GConvertError other than
1029 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1031 * Convert a string from a 32-bit fixed width representation as UCS-4.
1032 * to UTF-8. The result will be terminated with a 0 byte.
1034 * Return value: a pointer to a newly allocated UTF-8 string.
1035 * This value must be freed with g_free(). If an
1036 * error occurs, %NULL will be returned and
1037 * @error set. In that case, @items_read will be
1038 * set to the position of the first invalid input
1042 g_ucs4_to_utf8 (const gunichar *str,
1045 glong *items_written,
1049 gchar *result = NULL;
1054 for (i = 0; len < 0 || i < len ; i++)
1059 if (str[i] >= 0x80000000)
1061 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1062 _("Character out of range for UTF-8"));
1066 result_length += UTF8_LENGTH (str[i]);
1069 result = g_malloc (result_length + 1);
1073 while (p < result + result_length)
1074 p += g_unichar_to_utf8 (str[i++], p);
1079 *items_written = p - result;
1088 #define SURROGATE_VALUE(h,l) (((h) - 0xd800) * 0x400 + (l) - 0xdc00 + 0x10000)
1092 * @str: a UTF-16 encoded string
1093 * @len: the maximum length (number of <type>gunichar2</type>) of @str to use.
1094 * If @len < 0, then the string is nul-terminated.
1095 * @items_read: location to store number of words read, or %NULL.
1096 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1097 * returned in case @str contains a trailing partial
1098 * character. If an error occurs then the index of the
1099 * invalid input is stored here.
1100 * @items_written: location to store number of bytes written, or %NULL.
1101 * The value stored here does not include the trailing
1103 * @error: location to store the error occuring, or %NULL to ignore
1104 * errors. Any of the errors in #GConvertError other than
1105 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1107 * Convert a string from UTF-16 to UTF-8. The result will be
1108 * terminated with a 0 byte.
1110 * Note that the input is expected to be already in native endianness,
1111 * an initial byte-order-mark character is not handled specially.
1112 * g_convert() can be used to convert a byte buffer of UTF-16 data of
1113 * ambiguous endianess.
1115 * Further note that this function does not validate the result
1116 * string; it may e.g. include embedded NUL characters. The only
1117 * validation done by this function is to ensure that the input can
1118 * be correctly interpreted as UTF-16, i.e. it doesn't contain
1119 * things unpaired surrogates.
1121 * Return value: a pointer to a newly allocated UTF-8 string.
1122 * This value must be freed with g_free(). If an
1123 * error occurs, %NULL will be returned and
1127 g_utf16_to_utf8 (const gunichar2 *str,
1130 glong *items_written,
1133 /* This function and g_utf16_to_ucs4 are almost exactly identical - The lines that differ
1136 const gunichar2 *in;
1138 gchar *result = NULL;
1140 gunichar high_surrogate;
1142 g_return_val_if_fail (str != NULL, NULL);
1147 while ((len < 0 || in - str < len) && *in)
1152 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1156 wc = SURROGATE_VALUE (high_surrogate, c);
1161 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1162 _("Invalid sequence in conversion input"));
1170 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1171 _("Invalid sequence in conversion input"));
1175 if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1184 /********** DIFFERENT for UTF8/UCS4 **********/
1185 n_bytes += UTF8_LENGTH (wc);
1191 if (high_surrogate && !items_read)
1193 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1194 _("Partial character sequence at end of input"));
1198 /* At this point, everything is valid, and we just need to convert
1200 /********** DIFFERENT for UTF8/UCS4 **********/
1201 result = g_malloc (n_bytes + 1);
1206 while (out < result + n_bytes)
1211 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1213 wc = SURROGATE_VALUE (high_surrogate, c);
1216 else if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1224 /********** DIFFERENT for UTF8/UCS4 **********/
1225 out += g_unichar_to_utf8 (wc, out);
1231 /********** DIFFERENT for UTF8/UCS4 **********/
1235 /********** DIFFERENT for UTF8/UCS4 **********/
1236 *items_written = out - result;
1240 *items_read = in - str;
1247 * @str: a UTF-16 encoded string
1248 * @len: the maximum length (number of <type>gunichar2</type>) of @str to use.
1249 * If @len < 0, then the string is nul-terminated.
1250 * @items_read: location to store number of words read, or %NULL.
1251 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1252 * returned in case @str contains a trailing partial
1253 * character. If an error occurs then the index of the
1254 * invalid input is stored here.
1255 * @items_written: location to store number of characters written, or %NULL.
1256 * The value stored here does not include the trailing
1258 * @error: location to store the error occuring, or %NULL to ignore
1259 * errors. Any of the errors in #GConvertError other than
1260 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1262 * Convert a string from UTF-16 to UCS-4. The result will be
1265 * Return value: a pointer to a newly allocated UCS-4 string.
1266 * This value must be freed with g_free(). If an
1267 * error occurs, %NULL will be returned and
1271 g_utf16_to_ucs4 (const gunichar2 *str,
1274 glong *items_written,
1277 const gunichar2 *in;
1279 gchar *result = NULL;
1281 gunichar high_surrogate;
1283 g_return_val_if_fail (str != NULL, NULL);
1288 while ((len < 0 || in - str < len) && *in)
1293 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1297 wc = SURROGATE_VALUE (high_surrogate, c);
1302 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1303 _("Invalid sequence in conversion input"));
1311 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1312 _("Invalid sequence in conversion input"));
1316 if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1325 /********** DIFFERENT for UTF8/UCS4 **********/
1326 n_bytes += sizeof (gunichar);
1332 if (high_surrogate && !items_read)
1334 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1335 _("Partial character sequence at end of input"));
1339 /* At this point, everything is valid, and we just need to convert
1341 /********** DIFFERENT for UTF8/UCS4 **********/
1342 result = g_malloc (n_bytes + 4);
1347 while (out < result + n_bytes)
1352 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1354 wc = SURROGATE_VALUE (high_surrogate, c);
1357 else if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1365 /********** DIFFERENT for UTF8/UCS4 **********/
1366 *(gunichar *)out = wc;
1367 out += sizeof (gunichar);
1373 /********** DIFFERENT for UTF8/UCS4 **********/
1374 *(gunichar *)out = 0;
1377 /********** DIFFERENT for UTF8/UCS4 **********/
1378 *items_written = (out - result) / sizeof (gunichar);
1382 *items_read = in - str;
1384 return (gunichar *)result;
1389 * @str: a UTF-8 encoded string
1390 * @len: the maximum length (number of bytes) of @str to use.
1391 * If @len < 0, then the string is nul-terminated.
1392 * @items_read: location to store number of bytes read, or %NULL.
1393 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1394 * returned in case @str contains a trailing partial
1395 * character. If an error occurs then the index of the
1396 * invalid input is stored here.
1397 * @items_written: location to store number of <type>gunichar2</type> written,
1399 * The value stored here does not include the trailing 0.
1400 * @error: location to store the error occuring, or %NULL to ignore
1401 * errors. Any of the errors in #GConvertError other than
1402 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1404 * Convert a string from UTF-8 to UTF-16. A 0 character will be
1405 * added to the result after the converted text.
1407 * Return value: a pointer to a newly allocated UTF-16 string.
1408 * This value must be freed with g_free(). If an
1409 * error occurs, %NULL will be returned and
1413 g_utf8_to_utf16 (const gchar *str,
1416 glong *items_written,
1419 gunichar2 *result = NULL;
1424 g_return_val_if_fail (str != NULL, NULL);
1428 while ((len < 0 || str + len - in > 0) && *in)
1430 gunichar wc = g_utf8_get_char_extended (in, len < 0 ? 6 : str + len - in);
1431 if (wc & 0x80000000)
1433 if (wc == (gunichar)-2)
1438 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1439 _("Partial character sequence at end of input"));
1442 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1443 _("Invalid byte sequence in conversion input"));
1450 else if (wc < 0xe000)
1452 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1453 _("Invalid sequence in conversion input"));
1457 else if (wc < 0x10000)
1459 else if (wc < 0x110000)
1463 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1464 _("Character out of range for UTF-16"));
1469 in = g_utf8_next_char (in);
1472 result = g_new (gunichar2, n16 + 1);
1475 for (i = 0; i < n16;)
1477 gunichar wc = g_utf8_get_char (in);
1485 result[i++] = (wc - 0x10000) / 0x400 + 0xd800;
1486 result[i++] = (wc - 0x10000) % 0x400 + 0xdc00;
1489 in = g_utf8_next_char (in);
1495 *items_written = n16;
1499 *items_read = in - str;
1506 * @str: a UCS-4 encoded string
1507 * @len: the maximum length (number of characters) of @str to use.
1508 * If @len < 0, then the string is nul-terminated.
1509 * @items_read: location to store number of bytes read, or %NULL.
1510 * If an error occurs then the index of the invalid input
1512 * @items_written: location to store number of <type>gunichar2</type>
1513 * written, or %NULL. The value stored here does not
1514 * include the trailing 0.
1515 * @error: location to store the error occuring, or %NULL to ignore
1516 * errors. Any of the errors in #GConvertError other than
1517 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1519 * Convert a string from UCS-4 to UTF-16. A 0 character will be
1520 * added to the result after the converted text.
1522 * Return value: a pointer to a newly allocated UTF-16 string.
1523 * This value must be freed with g_free(). If an
1524 * error occurs, %NULL will be returned and
1528 g_ucs4_to_utf16 (const gunichar *str,
1531 glong *items_written,
1534 gunichar2 *result = NULL;
1540 while ((len < 0 || i < len) && str[i])
1542 gunichar wc = str[i];
1546 else if (wc < 0xe000)
1548 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1549 _("Invalid sequence in conversion input"));
1553 else if (wc < 0x10000)
1555 else if (wc < 0x110000)
1559 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1560 _("Character out of range for UTF-16"));
1568 result = g_new (gunichar2, n16 + 1);
1570 for (i = 0, j = 0; j < n16; i++)
1572 gunichar wc = str[i];
1580 result[j++] = (wc - 0x10000) / 0x400 + 0xd800;
1581 result[j++] = (wc - 0x10000) % 0x400 + 0xdc00;
1587 *items_written = n16;
1596 #define CONTINUATION_CHAR \
1598 if ((*(guchar *)p & 0xc0) != 0x80) /* 10xxxxxx */ \
1601 val |= (*(guchar *)p) & 0x3f; \
1604 static const gchar *
1605 fast_validate (const char *str)
1612 for (p = str; *p; p++)
1614 if (*(guchar *)p < 128)
1621 if ((*(guchar *)p & 0xe0) == 0xc0) /* 110xxxxx */
1623 if (G_UNLIKELY ((*(guchar *)p & 0x1e) == 0))
1626 if (G_UNLIKELY ((*(guchar *)p & 0xc0) != 0x80)) /* 10xxxxxx */
1631 if ((*(guchar *)p & 0xf0) == 0xe0) /* 1110xxxx */
1634 val = *(guchar *)p & 0x0f;
1637 else if ((*(guchar *)p & 0xf8) == 0xf0) /* 11110xxx */
1640 val = *(guchar *)p & 0x07;
1653 if (G_UNLIKELY (val < min))
1656 if (G_UNLIKELY (!UNICODE_VALID(val)))
1670 static const gchar *
1671 fast_validate_len (const char *str,
1679 g_assert (max_len >= 0);
1681 for (p = str; ((p - str) < max_len) && *p; p++)
1683 if (*(guchar *)p < 128)
1690 if ((*(guchar *)p & 0xe0) == 0xc0) /* 110xxxxx */
1692 if (G_UNLIKELY (max_len - (p - str) < 2))
1695 if (G_UNLIKELY ((*(guchar *)p & 0x1e) == 0))
1698 if (G_UNLIKELY ((*(guchar *)p & 0xc0) != 0x80)) /* 10xxxxxx */
1703 if ((*(guchar *)p & 0xf0) == 0xe0) /* 1110xxxx */
1705 if (G_UNLIKELY (max_len - (p - str) < 3))
1709 val = *(guchar *)p & 0x0f;
1712 else if ((*(guchar *)p & 0xf8) == 0xf0) /* 11110xxx */
1714 if (G_UNLIKELY (max_len - (p - str) < 4))
1718 val = *(guchar *)p & 0x07;
1731 if (G_UNLIKELY (val < min))
1733 if (G_UNLIKELY (!UNICODE_VALID(val)))
1749 * @str: a pointer to character data
1750 * @max_len: max bytes to validate, or -1 to go until NUL
1751 * @end: return location for end of valid data
1753 * Validates UTF-8 encoded text. @str is the text to validate;
1754 * if @str is nul-terminated, then @max_len can be -1, otherwise
1755 * @max_len should be the number of bytes to validate.
1756 * If @end is non-%NULL, then the end of the valid range
1757 * will be stored there (i.e. the start of the first invalid
1758 * character if some bytes were invalid, or the end of the text
1759 * being validated otherwise).
1761 * Note that g_utf8_validate() returns %FALSE if @max_len is
1762 * positive and NUL is met before @max_len bytes have been read.
1764 * Returns %TRUE if all of @str was valid. Many GLib and GTK+
1765 * routines <emphasis>require</emphasis> valid UTF-8 as input;
1766 * so data read from a file or the network should be checked
1767 * with g_utf8_validate() before doing anything else with it.
1769 * Return value: %TRUE if the text was valid UTF-8
1772 g_utf8_validate (const char *str,
1780 p = fast_validate (str);
1782 p = fast_validate_len (str, max_len);
1787 if ((max_len >= 0 && p != str + max_len) ||
1788 (max_len < 0 && *p != '\0'))
1795 * g_unichar_validate:
1796 * @ch: a Unicode character
1798 * Checks whether @ch is a valid Unicode character. Some possible
1799 * integer values of @ch will not be valid. 0 is considered a valid
1800 * character, though it's normally a string terminator.
1802 * Return value: %TRUE if @ch is a valid Unicode character
1805 g_unichar_validate (gunichar ch)
1807 return UNICODE_VALID (ch);
1811 * g_utf8_strreverse:
1812 * @str: a UTF-8 encoded string
1813 * @len: the maximum length of @str to use, in bytes. If @len < 0,
1814 * then the string is nul-terminated.
1816 * Reverses a UTF-8 string. @str must be valid UTF-8 encoded text.
1817 * (Use g_utf8_validate() on all text before trying to use UTF-8
1818 * utility functions with it.)
1820 * This function is intended for programmatic uses of reversed strings.
1821 * It pays no attention to decomposed characters, combining marks, byte
1822 * order marks, directional indicators (LRM, LRO, etc) and similar
1823 * characters which might need special handling when reversing a string
1824 * for display purposes.
1826 * Note that unlike g_strreverse(), this function returns
1827 * newly-allocated memory, which should be freed with g_free() when
1830 * Returns: a newly-allocated string which is the reverse of @str.
1835 g_utf8_strreverse (const gchar *str,
1844 result = g_new (gchar, len + 1);
1849 gchar *m, skip = g_utf8_skip[*(guchar*) p];
1851 for (m = r; skip; skip--)
1861 _g_utf8_make_valid (const gchar *name)
1864 const gchar *remainder, *invalid;
1865 gint remaining_bytes, valid_bytes;
1867 g_return_val_if_fail (name != NULL, NULL);
1871 remaining_bytes = strlen (name);
1873 while (remaining_bytes != 0)
1875 if (g_utf8_validate (remainder, remaining_bytes, &invalid))
1877 valid_bytes = invalid - remainder;
1880 string = g_string_sized_new (remaining_bytes);
1882 g_string_append_len (string, remainder, valid_bytes);
1883 /* append U+FFFD REPLACEMENT CHARACTER */
1884 g_string_append (string, "\357\277\275");
1886 remaining_bytes -= valid_bytes + 1;
1887 remainder = invalid + 1;
1891 return g_strdup (name);
1893 g_string_append (string, remainder);
1895 g_assert (g_utf8_validate (string->str, -1, NULL));
1897 return g_string_free (string, FALSE);