1 /* gutf8.c - Operations on UTF-8 strings.
3 * Copyright (C) 1999 Tom Tromey
4 * Copyright (C) 2000 Red Hat, Inc.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 02111-1307, USA.
32 #ifdef G_PLATFORM_WIN32
39 #include "libcharset/libcharset.h"
44 #define UTF8_COMPUTE(Char, Mask, Len) \
50 else if ((Char & 0xe0) == 0xc0) \
55 else if ((Char & 0xf0) == 0xe0) \
60 else if ((Char & 0xf8) == 0xf0) \
65 else if ((Char & 0xfc) == 0xf8) \
70 else if ((Char & 0xfe) == 0xfc) \
78 #define UTF8_LENGTH(Char) \
79 ((Char) < 0x80 ? 1 : \
80 ((Char) < 0x800 ? 2 : \
81 ((Char) < 0x10000 ? 3 : \
82 ((Char) < 0x200000 ? 4 : \
83 ((Char) < 0x4000000 ? 5 : 6)))))
86 #define UTF8_GET(Result, Chars, Count, Mask, Len) \
87 (Result) = (Chars)[0] & (Mask); \
88 for ((Count) = 1; (Count) < (Len); ++(Count)) \
90 if (((Chars)[(Count)] & 0xc0) != 0x80) \
96 (Result) |= ((Chars)[(Count)] & 0x3f); \
100 * Check whether a Unicode (5.2) char is in a valid range.
102 * The first check comes from the Unicode guarantee to never encode
103 * a point above 0x0010ffff, since UTF-16 couldn't represent it.
105 * The second check covers surrogate pairs (category Cs).
107 * The last two checks cover "Noncharacter": defined as:
108 * "A code point that is permanently reserved for
109 * internal use, and that should never be interchanged. In
110 * Unicode 3.1, these consist of the values U+nFFFE and U+nFFFF
111 * (where n is from 0 to 10_16) and the values U+FDD0..U+FDEF."
113 * @param Char the character
115 #define UNICODE_VALID(Char) \
116 ((Char) < 0x110000 && \
117 (((Char) & 0xFFFFF800) != 0xD800) && \
118 ((Char) < 0xFDD0 || (Char) > 0xFDEF) && \
119 ((Char) & 0xFFFE) != 0xFFFE)
122 static const gchar utf8_skip_data[256] = {
123 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
124 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
125 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
126 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
127 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
128 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
129 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
130 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1
133 const gchar * const g_utf8_skip = utf8_skip_data;
136 * g_utf8_find_prev_char:
137 * @str: pointer to the beginning of a UTF-8 encoded string
138 * @p: pointer to some position within @str
140 * Given a position @p with a UTF-8 encoded string @str, find the start
141 * of the previous UTF-8 character starting before @p. Returns %NULL if no
142 * UTF-8 characters are present in @str before @p.
144 * @p does not have to be at the beginning of a UTF-8 character. No check
145 * is made to see if the character found is actually valid other than
146 * it starts with an appropriate byte.
148 * Return value: a pointer to the found character or %NULL.
151 g_utf8_find_prev_char (const char *str,
154 for (--p; p >= str; --p)
156 if ((*p & 0xc0) != 0x80)
163 * g_utf8_find_next_char:
164 * @p: a pointer to a position within a UTF-8 encoded string
165 * @end: a pointer to the byte following the end of the string,
166 * or %NULL to indicate that the string is nul-terminated.
168 * Finds the start of the next UTF-8 character in the string after @p.
170 * @p does not have to be at the beginning of a UTF-8 character. No check
171 * is made to see if the character found is actually valid other than
172 * it starts with an appropriate byte.
174 * Return value: a pointer to the found character or %NULL
177 g_utf8_find_next_char (const gchar *p,
183 for (++p; p < end && (*p & 0xc0) == 0x80; ++p)
186 for (++p; (*p & 0xc0) == 0x80; ++p)
189 return (p == end) ? NULL : (gchar *)p;
194 * @p: a pointer to a position within a UTF-8 encoded string
196 * Finds the previous UTF-8 character in the string before @p.
198 * @p does not have to be at the beginning of a UTF-8 character. No check
199 * is made to see if the character found is actually valid other than
200 * it starts with an appropriate byte. If @p might be the first
201 * character of the string, you must use g_utf8_find_prev_char() instead.
203 * Return value: a pointer to the found character.
206 g_utf8_prev_char (const gchar *p)
211 if ((*p & 0xc0) != 0x80)
218 * @p: pointer to the start of a UTF-8 encoded string
219 * @max: the maximum number of bytes to examine. If @max
220 * is less than 0, then the string is assumed to be
221 * nul-terminated. If @max is 0, @p will not be examined and
224 * Computes the length of the string in characters, not including
225 * the terminating nul character.
227 * Return value: the length of the string in characters
230 g_utf8_strlen (const gchar *p,
234 const gchar *start = p;
235 g_return_val_if_fail (p != NULL || max == 0, 0);
241 p = g_utf8_next_char (p);
250 p = g_utf8_next_char (p);
252 while (p - start < max && *p)
255 p = g_utf8_next_char (p);
258 /* only do the last len increment if we got a complete
259 * char (don't count partial chars)
261 if (p - start <= max)
270 * @p: a pointer to Unicode character encoded as UTF-8
272 * Converts a sequence of bytes encoded as UTF-8 to a Unicode character.
273 * If @p does not point to a valid UTF-8 encoded character, results are
274 * undefined. If you are not sure that the bytes are complete
275 * valid Unicode characters, you should use g_utf8_get_char_validated()
278 * Return value: the resulting character
281 g_utf8_get_char (const gchar *p)
283 int i, mask = 0, len;
285 unsigned char c = (unsigned char) *p;
287 UTF8_COMPUTE (c, mask, len);
290 UTF8_GET (result, p, i, mask, len);
296 * g_utf8_offset_to_pointer:
297 * @str: a UTF-8 encoded string
298 * @offset: a character offset within @str
300 * Converts from an integer character offset to a pointer to a position
303 * Since 2.10, this function allows to pass a negative @offset to
304 * step backwards. It is usually worth stepping backwards from the end
305 * instead of forwards if @offset is in the last fourth of the string,
306 * since moving forward is about 3 times faster than moving backward.
309 * This function doesn't abort when reaching the end of @str. Therefore
310 * you should be sure that @offset is within string boundaries before
311 * calling that function. Call g_utf8_strlen() when unsure.
313 * This limitation exists as this function is called frequently during
314 * text rendering and therefore has to be as fast as possible.
317 * Return value: the resulting pointer
320 g_utf8_offset_to_pointer (const gchar *str,
323 const gchar *s = str;
327 s = g_utf8_next_char (s);
332 /* This nice technique for fast backwards stepping
333 * through a UTF-8 string was dubbed "stutter stepping"
334 * by its inventor, Larry Ewing.
340 while ((*s & 0xc0) == 0x80)
343 offset += g_utf8_pointer_to_offset (s, s1);
351 * g_utf8_pointer_to_offset:
352 * @str: a UTF-8 encoded string
353 * @pos: a pointer to a position within @str
355 * Converts from a pointer to position within a string to a integer
358 * Since 2.10, this function allows @pos to be before @str, and returns
359 * a negative offset in this case.
361 * Return value: the resulting character offset
364 g_utf8_pointer_to_offset (const gchar *str,
367 const gchar *s = str;
371 offset = - g_utf8_pointer_to_offset (pos, str);
375 s = g_utf8_next_char (s);
385 * @dest: buffer to fill with characters from @src
386 * @src: UTF-8 encoded string
387 * @n: character count
389 * Like the standard C strncpy() function, but
390 * copies a given number of characters instead of a given number of
391 * bytes. The @src string must be valid UTF-8 encoded text.
392 * (Use g_utf8_validate() on all text before trying to use UTF-8
393 * utility functions with it.)
395 * Return value: @dest
398 g_utf8_strncpy (gchar *dest,
402 const gchar *s = src;
405 s = g_utf8_next_char(s);
408 strncpy(dest, src, s - src);
413 G_LOCK_DEFINE_STATIC (aliases);
416 get_alias_hash (void)
418 static GHashTable *alias_hash = NULL;
425 alias_hash = g_hash_table_new (g_str_hash, g_str_equal);
427 aliases = _g_locale_get_charset_aliases ();
428 while (*aliases != '\0')
430 const char *canonical;
432 const char **alias_array;
436 aliases += strlen (aliases) + 1;
438 aliases += strlen (aliases) + 1;
440 alias_array = g_hash_table_lookup (alias_hash, canonical);
443 while (alias_array[count])
447 alias_array = g_renew (const char *, alias_array, count + 2);
448 alias_array[count] = alias;
449 alias_array[count + 1] = NULL;
451 g_hash_table_insert (alias_hash, (char *)canonical, alias_array);
460 /* As an abuse of the alias table, the following routines gets
461 * the charsets that are aliases for the canonical name.
463 G_GNUC_INTERNAL const char **
464 _g_charset_get_aliases (const char *canonical_name)
466 GHashTable *alias_hash = get_alias_hash ();
468 return g_hash_table_lookup (alias_hash, canonical_name);
472 g_utf8_get_charset_internal (const char *raw_data,
475 const char *charset = getenv("CHARSET");
477 if (charset && *charset)
481 if (charset && strstr (charset, "UTF-8"))
487 /* The libcharset code tries to be thread-safe without
488 * a lock, but has a memory leak and a missing memory
489 * barrier, so we lock for it
492 charset = _g_locale_charset_unalias (raw_data);
495 if (charset && *charset)
499 if (charset && strstr (charset, "UTF-8"))
505 /* Assume this for compatibility at present. */
511 typedef struct _GCharsetCache GCharsetCache;
513 struct _GCharsetCache {
520 charset_cache_free (gpointer data)
522 GCharsetCache *cache = data;
524 g_free (cache->charset);
530 * @charset: return location for character set name
532 * Obtains the character set for the <link linkend="setlocale">current
533 * locale</link>; you might use this character set as an argument to
534 * g_convert(), to convert from the current locale's encoding to some
535 * other encoding. (Frequently g_locale_to_utf8() and g_locale_from_utf8()
536 * are nice shortcuts, though.)
538 * On Windows the character set returned by this function is the
539 * so-called system default ANSI code-page. That is the character set
540 * used by the "narrow" versions of C library and Win32 functions that
541 * handle file names. It might be different from the character set
542 * used by the C library's current locale.
544 * The return value is %TRUE if the locale's encoding is UTF-8, in that
545 * case you can perhaps avoid calling g_convert().
547 * The string returned in @charset is not allocated, and should not be
550 * Return value: %TRUE if the returned charset is UTF-8
553 g_get_charset (G_CONST_RETURN char **charset)
555 static GStaticPrivate cache_private = G_STATIC_PRIVATE_INIT;
556 GCharsetCache *cache = g_static_private_get (&cache_private);
561 cache = g_new0 (GCharsetCache, 1);
562 g_static_private_set (&cache_private, cache, charset_cache_free);
565 raw = _g_locale_charset_raw ();
567 if (!(cache->raw && strcmp (cache->raw, raw) == 0))
569 const gchar *new_charset;
572 g_free (cache->charset);
573 cache->raw = g_strdup (raw);
574 cache->is_utf8 = g_utf8_get_charset_internal (raw, &new_charset);
575 cache->charset = g_strdup (new_charset);
579 *charset = cache->charset;
581 return cache->is_utf8;
588 * @c: a Unicode character code
589 * @outbuf: output buffer, must have at least 6 bytes of space.
590 * If %NULL, the length will be computed and returned
591 * and nothing will be written to @outbuf.
593 * Converts a single character to UTF-8.
595 * Return value: number of bytes written
598 g_unichar_to_utf8 (gunichar c,
601 /* If this gets modified, also update the copy in g_string_insert_unichar() */
616 else if (c < 0x10000)
621 else if (c < 0x200000)
626 else if (c < 0x4000000)
639 for (i = len - 1; i > 0; --i)
641 outbuf[i] = (c & 0x3f) | 0x80;
644 outbuf[0] = c | first;
652 * @p: a nul-terminated UTF-8 encoded string
653 * @len: the maximum length of @p
654 * @c: a Unicode character
656 * Finds the leftmost occurrence of the given Unicode character
657 * in a UTF-8 encoded string, while limiting the search to @len bytes.
658 * If @len is -1, allow unbounded search.
660 * Return value: %NULL if the string does not contain the character,
661 * otherwise, a pointer to the start of the leftmost occurrence of
662 * the character in the string.
665 g_utf8_strchr (const char *p,
671 gint charlen = g_unichar_to_utf8 (c, ch);
674 return g_strstr_len (p, len, ch);
680 * @p: a nul-terminated UTF-8 encoded string
681 * @len: the maximum length of @p
682 * @c: a Unicode character
684 * Find the rightmost occurrence of the given Unicode character
685 * in a UTF-8 encoded string, while limiting the search to @len bytes.
686 * If @len is -1, allow unbounded search.
688 * Return value: %NULL if the string does not contain the character,
689 * otherwise, a pointer to the start of the rightmost occurrence of the
690 * character in the string.
693 g_utf8_strrchr (const char *p,
699 gint charlen = g_unichar_to_utf8 (c, ch);
702 return g_strrstr_len (p, len, ch);
706 /* Like g_utf8_get_char, but take a maximum length
707 * and return (gunichar)-2 on incomplete trailing character;
708 * also check for malformed or overlong sequences
709 * and return (gunichar)-1 in this case.
711 static inline gunichar
712 g_utf8_get_char_extended (const gchar *p,
717 gunichar wc = (guchar) *p;
723 else if (G_UNLIKELY (wc < 0xc0))
762 if (G_UNLIKELY (max_len >= 0 && len > max_len))
764 for (i = 1; i < max_len; i++)
766 if ((((guchar *)p)[i] & 0xc0) != 0x80)
772 for (i = 1; i < len; ++i)
774 gunichar ch = ((guchar *)p)[i];
776 if (G_UNLIKELY ((ch & 0xc0) != 0x80))
788 if (G_UNLIKELY (wc < min_code))
795 * g_utf8_get_char_validated:
796 * @p: a pointer to Unicode character encoded as UTF-8
797 * @max_len: the maximum number of bytes to read, or -1, for no maximum or
798 * if @p is nul-terminated
800 * Convert a sequence of bytes encoded as UTF-8 to a Unicode character.
801 * This function checks for incomplete characters, for invalid characters
802 * such as characters that are out of the range of Unicode, and for
803 * overlong encodings of valid characters.
805 * Return value: the resulting character. If @p points to a partial
806 * sequence at the end of a string that could begin a valid
807 * character (or if @max_len is zero), returns (gunichar)-2;
808 * otherwise, if @p does not point to a valid UTF-8 encoded
809 * Unicode character, returns (gunichar)-1.
812 g_utf8_get_char_validated (const gchar *p,
820 result = g_utf8_get_char_extended (p, max_len);
822 if (result & 0x80000000)
824 else if (!UNICODE_VALID (result))
831 * g_utf8_to_ucs4_fast:
832 * @str: a UTF-8 encoded string
833 * @len: the maximum length of @str to use, in bytes. If @len < 0,
834 * then the string is nul-terminated.
835 * @items_written: location to store the number of characters in the
838 * Convert a string from UTF-8 to a 32-bit fixed width
839 * representation as UCS-4, assuming valid UTF-8 input.
840 * This function is roughly twice as fast as g_utf8_to_ucs4()
841 * but does no error checking on the input.
843 * Return value: a pointer to a newly allocated UCS-4 string.
844 * This value must be freed with g_free().
847 g_utf8_to_ucs4_fast (const gchar *str,
849 glong *items_written)
856 g_return_val_if_fail (str != NULL, NULL);
864 p = g_utf8_next_char (p);
870 while (p < str + len && *p)
872 p = g_utf8_next_char (p);
877 result = g_new (gunichar, n_chars + 1);
880 for (i=0; i < n_chars; i++)
882 gunichar wc = ((unsigned char *)p)[0];
917 for (j = 1; j < charlen; j++)
920 wc |= ((unsigned char *)p)[j] & 0x3f;
937 * @str: a UTF-8 encoded string
938 * @len: the maximum length of @str to use, in bytes. If @len < 0,
939 * then the string is nul-terminated.
940 * @items_read: location to store number of bytes read, or %NULL.
941 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
942 * returned in case @str contains a trailing partial
943 * character. If an error occurs then the index of the
944 * invalid input is stored here.
945 * @items_written: location to store number of characters written or %NULL.
946 * The value here stored does not include the trailing 0
948 * @error: location to store the error occuring, or %NULL to ignore
949 * errors. Any of the errors in #GConvertError other than
950 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
952 * Convert a string from UTF-8 to a 32-bit fixed width
953 * representation as UCS-4. A trailing 0 will be added to the
954 * string after the converted text.
956 * Return value: a pointer to a newly allocated UCS-4 string.
957 * This value must be freed with g_free(). If an
958 * error occurs, %NULL will be returned and
962 g_utf8_to_ucs4 (const gchar *str,
965 glong *items_written,
968 gunichar *result = NULL;
974 while ((len < 0 || str + len - in > 0) && *in)
976 gunichar wc = g_utf8_get_char_extended (in, len < 0 ? 6 : str + len - in);
979 if (wc == (gunichar)-2)
984 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
985 _("Partial character sequence at end of input"));
988 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
989 _("Invalid byte sequence in conversion input"));
996 in = g_utf8_next_char (in);
999 result = g_new (gunichar, n_chars + 1);
1002 for (i=0; i < n_chars; i++)
1004 result[i] = g_utf8_get_char (in);
1005 in = g_utf8_next_char (in);
1010 *items_written = n_chars;
1014 *items_read = in - str;
1021 * @str: a UCS-4 encoded string
1022 * @len: the maximum length (number of characters) of @str to use.
1023 * If @len < 0, then the string is nul-terminated.
1024 * @items_read: location to store number of characters read, or %NULL.
1025 * @items_written: location to store number of bytes written or %NULL.
1026 * The value here stored does not include the trailing 0
1028 * @error: location to store the error occuring, or %NULL to ignore
1029 * errors. Any of the errors in #GConvertError other than
1030 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1032 * Convert a string from a 32-bit fixed width representation as UCS-4.
1033 * to UTF-8. The result will be terminated with a 0 byte.
1035 * Return value: a pointer to a newly allocated UTF-8 string.
1036 * This value must be freed with g_free(). If an
1037 * error occurs, %NULL will be returned and
1038 * @error set. In that case, @items_read will be
1039 * set to the position of the first invalid input
1043 g_ucs4_to_utf8 (const gunichar *str,
1046 glong *items_written,
1050 gchar *result = NULL;
1055 for (i = 0; len < 0 || i < len ; i++)
1060 if (str[i] >= 0x80000000)
1062 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1063 _("Character out of range for UTF-8"));
1067 result_length += UTF8_LENGTH (str[i]);
1070 result = g_malloc (result_length + 1);
1074 while (p < result + result_length)
1075 p += g_unichar_to_utf8 (str[i++], p);
1080 *items_written = p - result;
1089 #define SURROGATE_VALUE(h,l) (((h) - 0xd800) * 0x400 + (l) - 0xdc00 + 0x10000)
1093 * @str: a UTF-16 encoded string
1094 * @len: the maximum length (number of <type>gunichar2</type>) of @str to use.
1095 * If @len < 0, then the string is nul-terminated.
1096 * @items_read: location to store number of words read, or %NULL.
1097 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1098 * returned in case @str contains a trailing partial
1099 * character. If an error occurs then the index of the
1100 * invalid input is stored here.
1101 * @items_written: location to store number of bytes written, or %NULL.
1102 * The value stored here does not include the trailing
1104 * @error: location to store the error occuring, or %NULL to ignore
1105 * errors. Any of the errors in #GConvertError other than
1106 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1108 * Convert a string from UTF-16 to UTF-8. The result will be
1109 * terminated with a 0 byte.
1111 * Note that the input is expected to be already in native endianness,
1112 * an initial byte-order-mark character is not handled specially.
1113 * g_convert() can be used to convert a byte buffer of UTF-16 data of
1114 * ambiguous endianess.
1116 * Further note that this function does not validate the result
1117 * string; it may e.g. include embedded NUL characters. The only
1118 * validation done by this function is to ensure that the input can
1119 * be correctly interpreted as UTF-16, i.e. it doesn't contain
1120 * things unpaired surrogates.
1122 * Return value: a pointer to a newly allocated UTF-8 string.
1123 * This value must be freed with g_free(). If an
1124 * error occurs, %NULL will be returned and
1128 g_utf16_to_utf8 (const gunichar2 *str,
1131 glong *items_written,
1134 /* This function and g_utf16_to_ucs4 are almost exactly identical - The lines that differ
1137 const gunichar2 *in;
1139 gchar *result = NULL;
1141 gunichar high_surrogate;
1143 g_return_val_if_fail (str != NULL, NULL);
1148 while ((len < 0 || in - str < len) && *in)
1153 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1157 wc = SURROGATE_VALUE (high_surrogate, c);
1162 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1163 _("Invalid sequence in conversion input"));
1171 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1172 _("Invalid sequence in conversion input"));
1176 if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1185 /********** DIFFERENT for UTF8/UCS4 **********/
1186 n_bytes += UTF8_LENGTH (wc);
1192 if (high_surrogate && !items_read)
1194 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1195 _("Partial character sequence at end of input"));
1199 /* At this point, everything is valid, and we just need to convert
1201 /********** DIFFERENT for UTF8/UCS4 **********/
1202 result = g_malloc (n_bytes + 1);
1207 while (out < result + n_bytes)
1212 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1214 wc = SURROGATE_VALUE (high_surrogate, c);
1217 else if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1225 /********** DIFFERENT for UTF8/UCS4 **********/
1226 out += g_unichar_to_utf8 (wc, out);
1232 /********** DIFFERENT for UTF8/UCS4 **********/
1236 /********** DIFFERENT for UTF8/UCS4 **********/
1237 *items_written = out - result;
1241 *items_read = in - str;
1248 * @str: a UTF-16 encoded string
1249 * @len: the maximum length (number of <type>gunichar2</type>) of @str to use.
1250 * If @len < 0, then the string is nul-terminated.
1251 * @items_read: location to store number of words read, or %NULL.
1252 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1253 * returned in case @str contains a trailing partial
1254 * character. If an error occurs then the index of the
1255 * invalid input is stored here.
1256 * @items_written: location to store number of characters written, or %NULL.
1257 * The value stored here does not include the trailing
1259 * @error: location to store the error occuring, or %NULL to ignore
1260 * errors. Any of the errors in #GConvertError other than
1261 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1263 * Convert a string from UTF-16 to UCS-4. The result will be
1266 * Return value: a pointer to a newly allocated UCS-4 string.
1267 * This value must be freed with g_free(). If an
1268 * error occurs, %NULL will be returned and
1272 g_utf16_to_ucs4 (const gunichar2 *str,
1275 glong *items_written,
1278 const gunichar2 *in;
1280 gchar *result = NULL;
1282 gunichar high_surrogate;
1284 g_return_val_if_fail (str != NULL, NULL);
1289 while ((len < 0 || in - str < len) && *in)
1294 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1298 wc = SURROGATE_VALUE (high_surrogate, c);
1303 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1304 _("Invalid sequence in conversion input"));
1312 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1313 _("Invalid sequence in conversion input"));
1317 if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1326 /********** DIFFERENT for UTF8/UCS4 **********/
1327 n_bytes += sizeof (gunichar);
1333 if (high_surrogate && !items_read)
1335 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1336 _("Partial character sequence at end of input"));
1340 /* At this point, everything is valid, and we just need to convert
1342 /********** DIFFERENT for UTF8/UCS4 **********/
1343 result = g_malloc (n_bytes + 4);
1348 while (out < result + n_bytes)
1353 if (c >= 0xdc00 && c < 0xe000) /* low surrogate */
1355 wc = SURROGATE_VALUE (high_surrogate, c);
1358 else if (c >= 0xd800 && c < 0xdc00) /* high surrogate */
1366 /********** DIFFERENT for UTF8/UCS4 **********/
1367 *(gunichar *)out = wc;
1368 out += sizeof (gunichar);
1374 /********** DIFFERENT for UTF8/UCS4 **********/
1375 *(gunichar *)out = 0;
1378 /********** DIFFERENT for UTF8/UCS4 **********/
1379 *items_written = (out - result) / sizeof (gunichar);
1383 *items_read = in - str;
1385 return (gunichar *)result;
1390 * @str: a UTF-8 encoded string
1391 * @len: the maximum length (number of bytes) of @str to use.
1392 * If @len < 0, then the string is nul-terminated.
1393 * @items_read: location to store number of bytes read, or %NULL.
1394 * If %NULL, then %G_CONVERT_ERROR_PARTIAL_INPUT will be
1395 * returned in case @str contains a trailing partial
1396 * character. If an error occurs then the index of the
1397 * invalid input is stored here.
1398 * @items_written: location to store number of <type>gunichar2</type> written,
1400 * The value stored here does not include the trailing 0.
1401 * @error: location to store the error occuring, or %NULL to ignore
1402 * errors. Any of the errors in #GConvertError other than
1403 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1405 * Convert a string from UTF-8 to UTF-16. A 0 character will be
1406 * added to the result after the converted text.
1408 * Return value: a pointer to a newly allocated UTF-16 string.
1409 * This value must be freed with g_free(). If an
1410 * error occurs, %NULL will be returned and
1414 g_utf8_to_utf16 (const gchar *str,
1417 glong *items_written,
1420 gunichar2 *result = NULL;
1425 g_return_val_if_fail (str != NULL, NULL);
1429 while ((len < 0 || str + len - in > 0) && *in)
1431 gunichar wc = g_utf8_get_char_extended (in, len < 0 ? 6 : str + len - in);
1432 if (wc & 0x80000000)
1434 if (wc == (gunichar)-2)
1439 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
1440 _("Partial character sequence at end of input"));
1443 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1444 _("Invalid byte sequence in conversion input"));
1451 else if (wc < 0xe000)
1453 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1454 _("Invalid sequence in conversion input"));
1458 else if (wc < 0x10000)
1460 else if (wc < 0x110000)
1464 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1465 _("Character out of range for UTF-16"));
1470 in = g_utf8_next_char (in);
1473 result = g_new (gunichar2, n16 + 1);
1476 for (i = 0; i < n16;)
1478 gunichar wc = g_utf8_get_char (in);
1486 result[i++] = (wc - 0x10000) / 0x400 + 0xd800;
1487 result[i++] = (wc - 0x10000) % 0x400 + 0xdc00;
1490 in = g_utf8_next_char (in);
1496 *items_written = n16;
1500 *items_read = in - str;
1507 * @str: a UCS-4 encoded string
1508 * @len: the maximum length (number of characters) of @str to use.
1509 * If @len < 0, then the string is nul-terminated.
1510 * @items_read: location to store number of bytes read, or %NULL.
1511 * If an error occurs then the index of the invalid input
1513 * @items_written: location to store number of <type>gunichar2</type>
1514 * written, or %NULL. The value stored here does not
1515 * include the trailing 0.
1516 * @error: location to store the error occuring, or %NULL to ignore
1517 * errors. Any of the errors in #GConvertError other than
1518 * %G_CONVERT_ERROR_NO_CONVERSION may occur.
1520 * Convert a string from UCS-4 to UTF-16. A 0 character will be
1521 * added to the result after the converted text.
1523 * Return value: a pointer to a newly allocated UTF-16 string.
1524 * This value must be freed with g_free(). If an
1525 * error occurs, %NULL will be returned and
1529 g_ucs4_to_utf16 (const gunichar *str,
1532 glong *items_written,
1535 gunichar2 *result = NULL;
1541 while ((len < 0 || i < len) && str[i])
1543 gunichar wc = str[i];
1547 else if (wc < 0xe000)
1549 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1550 _("Invalid sequence in conversion input"));
1554 else if (wc < 0x10000)
1556 else if (wc < 0x110000)
1560 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1561 _("Character out of range for UTF-16"));
1569 result = g_new (gunichar2, n16 + 1);
1571 for (i = 0, j = 0; j < n16; i++)
1573 gunichar wc = str[i];
1581 result[j++] = (wc - 0x10000) / 0x400 + 0xd800;
1582 result[j++] = (wc - 0x10000) % 0x400 + 0xdc00;
1588 *items_written = n16;
1597 #define CONTINUATION_CHAR \
1599 if ((*(guchar *)p & 0xc0) != 0x80) /* 10xxxxxx */ \
1602 val |= (*(guchar *)p) & 0x3f; \
1605 static const gchar *
1606 fast_validate (const char *str)
1613 for (p = str; *p; p++)
1615 if (*(guchar *)p < 128)
1622 if ((*(guchar *)p & 0xe0) == 0xc0) /* 110xxxxx */
1624 if (G_UNLIKELY ((*(guchar *)p & 0x1e) == 0))
1627 if (G_UNLIKELY ((*(guchar *)p & 0xc0) != 0x80)) /* 10xxxxxx */
1632 if ((*(guchar *)p & 0xf0) == 0xe0) /* 1110xxxx */
1635 val = *(guchar *)p & 0x0f;
1638 else if ((*(guchar *)p & 0xf8) == 0xf0) /* 11110xxx */
1641 val = *(guchar *)p & 0x07;
1654 if (G_UNLIKELY (val < min))
1657 if (G_UNLIKELY (!UNICODE_VALID(val)))
1671 static const gchar *
1672 fast_validate_len (const char *str,
1680 g_assert (max_len >= 0);
1682 for (p = str; ((p - str) < max_len) && *p; p++)
1684 if (*(guchar *)p < 128)
1691 if ((*(guchar *)p & 0xe0) == 0xc0) /* 110xxxxx */
1693 if (G_UNLIKELY (max_len - (p - str) < 2))
1696 if (G_UNLIKELY ((*(guchar *)p & 0x1e) == 0))
1699 if (G_UNLIKELY ((*(guchar *)p & 0xc0) != 0x80)) /* 10xxxxxx */
1704 if ((*(guchar *)p & 0xf0) == 0xe0) /* 1110xxxx */
1706 if (G_UNLIKELY (max_len - (p - str) < 3))
1710 val = *(guchar *)p & 0x0f;
1713 else if ((*(guchar *)p & 0xf8) == 0xf0) /* 11110xxx */
1715 if (G_UNLIKELY (max_len - (p - str) < 4))
1719 val = *(guchar *)p & 0x07;
1732 if (G_UNLIKELY (val < min))
1734 if (G_UNLIKELY (!UNICODE_VALID(val)))
1750 * @str: a pointer to character data
1751 * @max_len: max bytes to validate, or -1 to go until NUL
1752 * @end: return location for end of valid data
1754 * Validates UTF-8 encoded text. @str is the text to validate;
1755 * if @str is nul-terminated, then @max_len can be -1, otherwise
1756 * @max_len should be the number of bytes to validate.
1757 * If @end is non-%NULL, then the end of the valid range
1758 * will be stored there (i.e. the start of the first invalid
1759 * character if some bytes were invalid, or the end of the text
1760 * being validated otherwise).
1762 * Note that g_utf8_validate() returns %FALSE if @max_len is
1763 * positive and NUL is met before @max_len bytes have been read.
1765 * Returns %TRUE if all of @str was valid. Many GLib and GTK+
1766 * routines <emphasis>require</emphasis> valid UTF-8 as input;
1767 * so data read from a file or the network should be checked
1768 * with g_utf8_validate() before doing anything else with it.
1770 * Return value: %TRUE if the text was valid UTF-8
1773 g_utf8_validate (const char *str,
1781 p = fast_validate (str);
1783 p = fast_validate_len (str, max_len);
1788 if ((max_len >= 0 && p != str + max_len) ||
1789 (max_len < 0 && *p != '\0'))
1796 * g_unichar_validate:
1797 * @ch: a Unicode character
1799 * Checks whether @ch is a valid Unicode character. Some possible
1800 * integer values of @ch will not be valid. 0 is considered a valid
1801 * character, though it's normally a string terminator.
1803 * Return value: %TRUE if @ch is a valid Unicode character
1806 g_unichar_validate (gunichar ch)
1808 return UNICODE_VALID (ch);
1812 * g_utf8_strreverse:
1813 * @str: a UTF-8 encoded string
1814 * @len: the maximum length of @str to use, in bytes. If @len < 0,
1815 * then the string is nul-terminated.
1817 * Reverses a UTF-8 string. @str must be valid UTF-8 encoded text.
1818 * (Use g_utf8_validate() on all text before trying to use UTF-8
1819 * utility functions with it.)
1821 * This function is intended for programmatic uses of reversed strings.
1822 * It pays no attention to decomposed characters, combining marks, byte
1823 * order marks, directional indicators (LRM, LRO, etc) and similar
1824 * characters which might need special handling when reversing a string
1825 * for display purposes.
1827 * Note that unlike g_strreverse(), this function returns
1828 * newly-allocated memory, which should be freed with g_free() when
1831 * Returns: a newly-allocated string which is the reverse of @str.
1836 g_utf8_strreverse (const gchar *str,
1845 result = g_new (gchar, len + 1);
1850 gchar *m, skip = g_utf8_skip[*(guchar*) p];
1852 for (m = r; skip; skip--)
1862 _g_utf8_make_valid (const gchar *name)
1865 const gchar *remainder, *invalid;
1866 gint remaining_bytes, valid_bytes;
1868 g_return_val_if_fail (name != NULL, NULL);
1872 remaining_bytes = strlen (name);
1874 while (remaining_bytes != 0)
1876 if (g_utf8_validate (remainder, remaining_bytes, &invalid))
1878 valid_bytes = invalid - remainder;
1881 string = g_string_sized_new (remaining_bytes);
1883 g_string_append_len (string, remainder, valid_bytes);
1884 /* append U+FFFD REPLACEMENT CHARACTER */
1885 g_string_append (string, "\357\277\275");
1887 remaining_bytes -= valid_bytes + 1;
1888 remainder = invalid + 1;
1892 return g_strdup (name);
1894 g_string_append (string, remainder);
1896 g_assert (g_utf8_validate (string->str, -1, NULL));
1898 return g_string_free (string, FALSE);
1902 #define __G_UTF8_C__
1903 #include "galiasdef.c"