1 /* GLIB - Library of useful routines for C programming
3 * gconvert.c: Convert between character sets using iconv
4 * Copyright Red Hat Inc., 2000
5 * Authors: Havoc Pennington <hp@redhat.com>, Owen Taylor <otaylor@redhat.com>
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
22 #include "glibconfig.h"
33 #include "win_iconv.c"
36 #ifdef G_PLATFORM_WIN32
44 #include "gcharsetprivate.h"
46 #include "gstrfuncs.h"
47 #include "gtestutils.h"
50 #include "gfileutils.h"
54 #if defined(USE_LIBICONV_GNU) && !defined (_LIBICONV_H)
55 #error GNU libiconv in use but included iconv.h not from libiconv
57 #if !defined(USE_LIBICONV_GNU) && defined (_LIBICONV_H) \
58 && !defined (__APPLE_CC__) && !defined (__LP_64__)
59 #error GNU libiconv not in use but included iconv.h is from libiconv
65 * @title: Character Set Conversion
66 * @short_description: convert strings between different character sets
68 * The g_convert() family of function wraps the functionality of iconv().
69 * In addition to pure character set conversions, GLib has functions to
70 * deal with the extra complications of encodings for file names.
72 * ## File Name Encodings
74 * Historically, UNIX has not had a defined encoding for file names:
75 * a file name is valid as long as it does not have path separators
76 * in it ("/"). However, displaying file names may require conversion:
77 * from the character set in which they were created, to the character
78 * set in which the application operates. Consider the Spanish file name
79 * "Presentación.sxi". If the application which created it uses
80 * ISO-8859-1 for its encoding,
82 * Character: P r e s e n t a c i ó n . s x i
83 * Hex code: 50 72 65 73 65 6e 74 61 63 69 f3 6e 2e 73 78 69
85 * However, if the application use UTF-8, the actual file name on
86 * disk would look like this:
88 * Character: P r e s e n t a c i ó n . s x i
89 * Hex code: 50 72 65 73 65 6e 74 61 63 69 c3 b3 6e 2e 73 78 69
91 * Glib uses UTF-8 for its strings, and GUI toolkits like GTK+ that use
92 * Glib do the same thing. If you get a file name from the file system,
93 * for example, from readdir() or from g_dir_read_name(), and you wish
94 * to display the file name to the user, you will need to convert it
95 * into UTF-8. The opposite case is when the user types the name of a
96 * file he wishes to save: the toolkit will give you that string in
97 * UTF-8 encoding, and you will need to convert it to the character
98 * set used for file names before you can create the file with open()
101 * By default, Glib assumes that file names on disk are in UTF-8
102 * encoding. This is a valid assumption for file systems which
103 * were created relatively recently: most applications use UTF-8
104 * encoding for their strings, and that is also what they use for
105 * the file names they create. However, older file systems may
106 * still contain file names created in "older" encodings, such as
107 * ISO-8859-1. In this case, for compatibility reasons, you may want
108 * to instruct Glib to use that particular encoding for file names
109 * rather than UTF-8. You can do this by specifying the encoding for
110 * file names in the [`G_FILENAME_ENCODING`][G_FILENAME_ENCODING]
111 * environment variable. For example, if your installation uses
112 * ISO-8859-1 for file names, you can put this in your `~/.profile`
114 * export G_FILENAME_ENCODING=ISO-8859-1
116 * Glib provides the functions g_filename_to_utf8() and
117 * g_filename_from_utf8() to perform the necessary conversions.
118 * These functions convert file names from the encoding specified
119 * in `G_FILENAME_ENCODING` to UTF-8 and vice-versa. This
120 * [diagram][file-name-encodings-diagram] illustrates how
121 * these functions are used to convert between UTF-8 and the
122 * encoding for file names in the file system.
124 * ## Conversion between file name encodings # {#file-name-encodings-diagram)
126 * ![](file-name-encodings.png)
128 * ## Checklist for Application Writers
130 * This section is a practical summary of the detailed
132 * things to do to make sure your applications process file
133 * name encodings correctly.
135 * 1. If you get a file name from the file system from a function
136 * such as readdir() or gtk_file_chooser_get_filename(), you do
137 * not need to do any conversion to pass that file name to
138 * functions like open(), rename(), or fopen() -- those are "raw"
139 * file names which the file system understands.
141 * 2. If you need to display a file name, convert it to UTF-8 first
142 * by using g_filename_to_utf8(). If conversion fails, display a
143 * string like "Unknown file name". Do not convert this string back
144 * into the encoding used for file names if you wish to pass it to
145 * the file system; use the original file name instead.
147 * For example, the document window of a word processor could display
148 * "Unknown file name" in its title bar but still let the user save
149 * the file, as it would keep the raw file name internally. This
150 * can happen if the user has not set the `G_FILENAME_ENCODING`
151 * environment variable even though he has files whose names are
152 * not encoded in UTF-8.
154 * 3. If your user interface lets the user type a file name for saving
155 * or renaming, convert it to the encoding used for file names in
156 * the file system by using g_filename_from_utf8(). Pass the converted
157 * file name to functions like fopen(). If conversion fails, ask the
158 * user to enter a different file name. This can happen if the user
159 * types Japanese characters when `G_FILENAME_ENCODING` is set to
160 * `ISO-8859-1`, for example.
163 /* We try to terminate strings in unknown charsets with this many zero bytes
164 * to ensure that multibyte strings really are nul-terminated when we return
165 * them from g_convert() and friends.
167 #define NUL_TERMINATOR_LENGTH 4
169 G_DEFINE_QUARK (g_convert_error, g_convert_error)
172 try_conversion (const char *to_codeset,
173 const char *from_codeset,
176 *cd = iconv_open (to_codeset, from_codeset);
178 if (*cd == (iconv_t)-1 && errno == EINVAL)
185 try_to_aliases (const char **to_aliases,
186 const char *from_codeset,
191 const char **p = to_aliases;
194 if (try_conversion (*p, from_codeset, cd))
206 * @to_codeset: destination codeset
207 * @from_codeset: source codeset
209 * Same as the standard UNIX routine iconv_open(), but
210 * may be implemented via libiconv on UNIX flavors that lack
211 * a native implementation.
213 * GLib provides g_convert() and g_locale_to_utf8() which are likely
214 * more convenient than the raw iconv wrappers.
216 * Return value: a "conversion descriptor", or (GIConv)-1 if
217 * opening the converter failed.
220 g_iconv_open (const gchar *to_codeset,
221 const gchar *from_codeset)
225 if (!try_conversion (to_codeset, from_codeset, &cd))
227 const char **to_aliases = _g_charset_get_aliases (to_codeset);
228 const char **from_aliases = _g_charset_get_aliases (from_codeset);
232 const char **p = from_aliases;
235 if (try_conversion (to_codeset, *p, &cd))
238 if (try_to_aliases (to_aliases, *p, &cd))
245 if (try_to_aliases (to_aliases, from_codeset, &cd))
250 return (cd == (iconv_t)-1) ? (GIConv)-1 : (GIConv)cd;
255 * @converter: conversion descriptor from g_iconv_open()
256 * @inbuf: bytes to convert
257 * @inbytes_left: inout parameter, bytes remaining to convert in @inbuf
258 * @outbuf: converted output bytes
259 * @outbytes_left: inout parameter, bytes available to fill in @outbuf
261 * Same as the standard UNIX routine iconv(), but
262 * may be implemented via libiconv on UNIX flavors that lack
263 * a native implementation.
265 * GLib provides g_convert() and g_locale_to_utf8() which are likely
266 * more convenient than the raw iconv wrappers.
268 * Return value: count of non-reversible conversions, or -1 on error
271 g_iconv (GIConv converter,
275 gsize *outbytes_left)
277 iconv_t cd = (iconv_t)converter;
279 return iconv (cd, inbuf, inbytes_left, outbuf, outbytes_left);
284 * @converter: a conversion descriptor from g_iconv_open()
286 * Same as the standard UNIX routine iconv_close(), but
287 * may be implemented via libiconv on UNIX flavors that lack
288 * a native implementation. Should be called to clean up
289 * the conversion descriptor from g_iconv_open() when
290 * you are done converting things.
292 * GLib provides g_convert() and g_locale_to_utf8() which are likely
293 * more convenient than the raw iconv wrappers.
295 * Return value: -1 on error, 0 on success
298 g_iconv_close (GIConv converter)
300 iconv_t cd = (iconv_t)converter;
302 return iconv_close (cd);
306 open_converter (const gchar *to_codeset,
307 const gchar *from_codeset,
312 cd = g_iconv_open (to_codeset, from_codeset);
314 if (cd == (GIConv) -1)
316 /* Something went wrong. */
320 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_NO_CONVERSION,
321 _("Conversion from character set '%s' to '%s' is not supported"),
322 from_codeset, to_codeset);
324 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
325 _("Could not open converter from '%s' to '%s'"),
326 from_codeset, to_codeset);
334 close_converter (GIConv cd)
336 if (cd == (GIConv) -1)
339 return g_iconv_close (cd);
343 * g_convert_with_iconv:
344 * @str: the string to convert
345 * @len: the length of the string, or -1 if the string is
346 * nul-terminated (Note that some encodings may allow nul
347 * bytes to occur inside strings. In that case, using -1
348 * for the @len parameter is unsafe)
349 * @converter: conversion descriptor from g_iconv_open()
350 * @bytes_read: location to store the number of bytes in the
351 * input string that were successfully converted, or %NULL.
352 * Even if the conversion was successful, this may be
353 * less than @len if there were partial characters
354 * at the end of the input. If the error
355 * #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
356 * stored will the byte offset after the last valid
358 * @bytes_written: the number of bytes stored in the output buffer (not
359 * including the terminating nul).
360 * @error: location to store the error occurring, or %NULL to ignore
361 * errors. Any of the errors in #GConvertError may occur.
363 * Converts a string from one character set to another.
365 * Note that you should use g_iconv() for streaming conversions.
366 * Despite the fact that @byes_read can return information about partial
367 * characters, the g_convert_... functions are not generally suitable
368 * for streaming. If the underlying converter maintains internal state,
369 * then this won't be preserved across successive calls to g_convert(),
370 * g_convert_with_iconv() or g_convert_with_fallback(). (An example of
371 * this is the GNU C converter for CP1255 which does not emit a base
372 * character until it knows that the next character is not a mark that
373 * could combine with the base character.)
375 * Return value: If the conversion was successful, a newly allocated
376 * nul-terminated string, which must be freed with
377 * g_free(). Otherwise %NULL and @error will be set.
380 g_convert_with_iconv (const gchar *str,
384 gsize *bytes_written,
390 gsize inbytes_remaining;
391 gsize outbytes_remaining;
394 gboolean have_error = FALSE;
395 gboolean done = FALSE;
396 gboolean reset = FALSE;
398 g_return_val_if_fail (converter != (GIConv) -1, NULL);
404 inbytes_remaining = len;
405 outbuf_size = len + NUL_TERMINATOR_LENGTH;
407 outbytes_remaining = outbuf_size - NUL_TERMINATOR_LENGTH;
408 outp = dest = g_malloc (outbuf_size);
410 while (!done && !have_error)
413 err = g_iconv (converter, NULL, &inbytes_remaining, &outp, &outbytes_remaining);
415 err = g_iconv (converter, (char **)&p, &inbytes_remaining, &outp, &outbytes_remaining);
417 if (err == (gsize) -1)
422 /* Incomplete text, do not report an error */
427 gsize used = outp - dest;
430 dest = g_realloc (dest, outbuf_size);
433 outbytes_remaining = outbuf_size - used - NUL_TERMINATOR_LENGTH;
437 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
438 _("Invalid byte sequence in conversion input"));
445 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
446 _("Error during conversion: %s"),
457 /* call g_iconv with NULL inbuf to cleanup shift state */
459 inbytes_remaining = 0;
466 memset (outp, 0, NUL_TERMINATOR_LENGTH);
469 *bytes_read = p - str;
472 if ((p - str) != len)
476 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
477 _("Partial character sequence at end of input"));
484 *bytes_written = outp - dest; /* Doesn't include '\0' */
497 * @str: the string to convert
498 * @len: the length of the string, or -1 if the string is
499 * nul-terminated (Note that some encodings may allow nul
500 * bytes to occur inside strings. In that case, using -1
501 * for the @len parameter is unsafe)
502 * @to_codeset: name of character set into which to convert @str
503 * @from_codeset: character set of @str.
504 * @bytes_read: (out): location to store the number of bytes in the
505 * input string that were successfully converted, or %NULL.
506 * Even if the conversion was successful, this may be
507 * less than @len if there were partial characters
508 * at the end of the input. If the error
509 * #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
510 * stored will the byte offset after the last valid
512 * @bytes_written: (out): the number of bytes stored in the output buffer (not
513 * including the terminating nul).
514 * @error: location to store the error occurring, or %NULL to ignore
515 * errors. Any of the errors in #GConvertError may occur.
517 * Converts a string from one character set to another.
519 * Note that you should use g_iconv() for streaming conversions.
520 * Despite the fact that @byes_read can return information about partial
521 * characters, the g_convert_... functions are not generally suitable
522 * for streaming. If the underlying converter maintains internal state,
523 * then this won't be preserved across successive calls to g_convert(),
524 * g_convert_with_iconv() or g_convert_with_fallback(). (An example of
525 * this is the GNU C converter for CP1255 which does not emit a base
526 * character until it knows that the next character is not a mark that
527 * could combine with the base character.)
529 * Return value: If the conversion was successful, a newly allocated
530 * nul-terminated string, which must be freed with
531 * g_free(). Otherwise %NULL and @error will be set.
534 g_convert (const gchar *str,
536 const gchar *to_codeset,
537 const gchar *from_codeset,
539 gsize *bytes_written,
545 g_return_val_if_fail (str != NULL, NULL);
546 g_return_val_if_fail (to_codeset != NULL, NULL);
547 g_return_val_if_fail (from_codeset != NULL, NULL);
549 cd = open_converter (to_codeset, from_codeset, error);
551 if (cd == (GIConv) -1)
562 res = g_convert_with_iconv (str, len, cd,
563 bytes_read, bytes_written,
566 close_converter (cd);
572 * g_convert_with_fallback:
573 * @str: the string to convert
574 * @len: the length of the string, or -1 if the string is
575 * nul-terminated (Note that some encodings may allow nul
576 * bytes to occur inside strings. In that case, using -1
577 * for the @len parameter is unsafe)
578 * @to_codeset: name of character set into which to convert @str
579 * @from_codeset: character set of @str.
580 * @fallback: UTF-8 string to use in place of character not
581 * present in the target encoding. (The string must be
582 * representable in the target encoding).
583 If %NULL, characters not in the target encoding will
584 be represented as Unicode escapes \uxxxx or \Uxxxxyyyy.
585 * @bytes_read: location to store the number of bytes in the
586 * input string that were successfully converted, or %NULL.
587 * Even if the conversion was successful, this may be
588 * less than @len if there were partial characters
589 * at the end of the input.
590 * @bytes_written: the number of bytes stored in the output buffer (not
591 * including the terminating nul).
592 * @error: location to store the error occurring, or %NULL to ignore
593 * errors. Any of the errors in #GConvertError may occur.
595 * Converts a string from one character set to another, possibly
596 * including fallback sequences for characters not representable
597 * in the output. Note that it is not guaranteed that the specification
598 * for the fallback sequences in @fallback will be honored. Some
599 * systems may do an approximate conversion from @from_codeset
600 * to @to_codeset in their iconv() functions,
601 * in which case GLib will simply return that approximate conversion.
603 * Note that you should use g_iconv() for streaming conversions.
604 * Despite the fact that @byes_read can return information about partial
605 * characters, the g_convert_... functions are not generally suitable
606 * for streaming. If the underlying converter maintains internal state,
607 * then this won't be preserved across successive calls to g_convert(),
608 * g_convert_with_iconv() or g_convert_with_fallback(). (An example of
609 * this is the GNU C converter for CP1255 which does not emit a base
610 * character until it knows that the next character is not a mark that
611 * could combine with the base character.)
613 * Return value: If the conversion was successful, a newly allocated
614 * nul-terminated string, which must be freed with
615 * g_free(). Otherwise %NULL and @error will be set.
618 g_convert_with_fallback (const gchar *str,
620 const gchar *to_codeset,
621 const gchar *from_codeset,
622 const gchar *fallback,
624 gsize *bytes_written,
630 const gchar *insert_str = NULL;
632 gsize inbytes_remaining;
633 const gchar *save_p = NULL;
634 gsize save_inbytes = 0;
635 gsize outbytes_remaining;
639 gboolean have_error = FALSE;
640 gboolean done = FALSE;
642 GError *local_error = NULL;
644 g_return_val_if_fail (str != NULL, NULL);
645 g_return_val_if_fail (to_codeset != NULL, NULL);
646 g_return_val_if_fail (from_codeset != NULL, NULL);
651 /* Try an exact conversion; we only proceed if this fails
652 * due to an illegal sequence in the input string.
654 dest = g_convert (str, len, to_codeset, from_codeset,
655 bytes_read, bytes_written, &local_error);
659 if (!g_error_matches (local_error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE))
661 g_propagate_error (error, local_error);
665 g_error_free (local_error);
669 /* No go; to proceed, we need a converter from "UTF-8" to
670 * to_codeset, and the string as UTF-8.
672 cd = open_converter (to_codeset, "UTF-8", error);
673 if (cd == (GIConv) -1)
684 utf8 = g_convert (str, len, "UTF-8", from_codeset,
685 bytes_read, &inbytes_remaining, error);
688 close_converter (cd);
694 /* Now the heart of the code. We loop through the UTF-8 string, and
695 * whenever we hit an offending character, we form fallback, convert
696 * the fallback to the target codeset, and then go back to
697 * converting the original string after finishing with the fallback.
699 * The variables save_p and save_inbytes store the input state
700 * for the original string while we are converting the fallback
704 outbuf_size = len + NUL_TERMINATOR_LENGTH;
705 outbytes_remaining = outbuf_size - NUL_TERMINATOR_LENGTH;
706 outp = dest = g_malloc (outbuf_size);
708 while (!done && !have_error)
710 gsize inbytes_tmp = inbytes_remaining;
711 err = g_iconv (cd, (char **)&p, &inbytes_tmp, &outp, &outbytes_remaining);
712 inbytes_remaining = inbytes_tmp;
714 if (err == (gsize) -1)
719 g_assert_not_reached();
723 gsize used = outp - dest;
726 dest = g_realloc (dest, outbuf_size);
729 outbytes_remaining = outbuf_size - used - NUL_TERMINATOR_LENGTH;
736 /* Error converting fallback string - fatal
738 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
739 _("Cannot convert fallback '%s' to codeset '%s'"),
740 insert_str, to_codeset);
748 gunichar ch = g_utf8_get_char (p);
749 insert_str = g_strdup_printf (ch < 0x10000 ? "\\u%04x" : "\\U%08x",
753 insert_str = fallback;
755 save_p = g_utf8_next_char (p);
756 save_inbytes = inbytes_remaining - (save_p - p);
758 inbytes_remaining = strlen (p);
761 /* fall thru if p is NULL */
766 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
767 _("Error during conversion: %s"),
780 g_free ((gchar *)insert_str);
782 inbytes_remaining = save_inbytes;
787 /* call g_iconv with NULL inbuf to cleanup shift state */
789 inbytes_remaining = 0;
798 memset (outp, 0, NUL_TERMINATOR_LENGTH);
800 close_converter (cd);
803 *bytes_written = outp - dest; /* Doesn't include '\0' */
809 if (save_p && !fallback)
810 g_free ((gchar *)insert_str);
825 strdup_len (const gchar *string,
827 gsize *bytes_written,
834 if (!g_utf8_validate (string, len, NULL))
841 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
842 _("Invalid byte sequence in conversion input"));
847 real_len = strlen (string);
852 while (real_len < len && string[real_len])
857 *bytes_read = real_len;
859 *bytes_written = real_len;
861 return g_strndup (string, real_len);
866 * @opsysstring: a string in the encoding of the current locale. On Windows
867 * this means the system codepage.
868 * @len: the length of the string, or -1 if the string is
869 * nul-terminated (Note that some encodings may allow nul
870 * bytes to occur inside strings. In that case, using -1
871 * for the @len parameter is unsafe)
872 * @bytes_read: location to store the number of bytes in the
873 * input string that were successfully converted, or %NULL.
874 * Even if the conversion was successful, this may be
875 * less than @len if there were partial characters
876 * at the end of the input. If the error
877 * #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
878 * stored will the byte offset after the last valid
880 * @bytes_written: the number of bytes stored in the output buffer (not
881 * including the terminating nul).
882 * @error: location to store the error occurring, or %NULL to ignore
883 * errors. Any of the errors in #GConvertError may occur.
885 * Converts a string which is in the encoding used for strings by
886 * the C runtime (usually the same as that used by the operating
887 * system) in the [current locale][setlocale] into a UTF-8 string.
889 * Return value: A newly-allocated buffer containing the converted string,
890 * or %NULL on an error, and error will be set.
893 g_locale_to_utf8 (const gchar *opsysstring,
896 gsize *bytes_written,
901 if (g_get_charset (&charset))
902 return strdup_len (opsysstring, len, bytes_read, bytes_written, error);
904 return g_convert (opsysstring, len,
905 "UTF-8", charset, bytes_read, bytes_written, error);
909 * g_locale_from_utf8:
910 * @utf8string: a UTF-8 encoded string
911 * @len: the length of the string, or -1 if the string is
912 * nul-terminated (Note that some encodings may allow nul
913 * bytes to occur inside strings. In that case, using -1
914 * for the @len parameter is unsafe)
915 * @bytes_read: location to store the number of bytes in the
916 * input string that were successfully converted, or %NULL.
917 * Even if the conversion was successful, this may be
918 * less than @len if there were partial characters
919 * at the end of the input. If the error
920 * #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
921 * stored will the byte offset after the last valid
923 * @bytes_written: the number of bytes stored in the output buffer (not
924 * including the terminating nul).
925 * @error: location to store the error occurring, or %NULL to ignore
926 * errors. Any of the errors in #GConvertError may occur.
928 * Converts a string from UTF-8 to the encoding used for strings by
929 * the C runtime (usually the same as that used by the operating
930 * system) in the [current locale][setlocale]. On Windows this means
931 * the system codepage.
933 * Return value: A newly-allocated buffer containing the converted string,
934 * or %NULL on an error, and error will be set.
937 g_locale_from_utf8 (const gchar *utf8string,
940 gsize *bytes_written,
943 const gchar *charset;
945 if (g_get_charset (&charset))
946 return strdup_len (utf8string, len, bytes_read, bytes_written, error);
948 return g_convert (utf8string, len,
949 charset, "UTF-8", bytes_read, bytes_written, error);
952 #ifndef G_PLATFORM_WIN32
954 typedef struct _GFilenameCharsetCache GFilenameCharsetCache;
956 struct _GFilenameCharsetCache {
959 gchar **filename_charsets;
963 filename_charset_cache_free (gpointer data)
965 GFilenameCharsetCache *cache = data;
966 g_free (cache->charset);
967 g_strfreev (cache->filename_charsets);
972 * g_get_filename_charsets:
973 * @charsets: return location for the %NULL-terminated list of encoding names
975 * Determines the preferred character sets used for filenames.
976 * The first character set from the @charsets is the filename encoding, the
977 * subsequent character sets are used when trying to generate a displayable
978 * representation of a filename, see g_filename_display_name().
980 * On Unix, the character sets are determined by consulting the
981 * environment variables `G_FILENAME_ENCODING` and `G_BROKEN_FILENAMES`.
982 * On Windows, the character set used in the GLib API is always UTF-8
983 * and said environment variables have no effect.
985 * `G_FILENAME_ENCODING` may be set to a comma-separated list of
986 * character set names. The special token "@locale" is taken
987 * to mean the character set for the [current locale][setlocale].
988 * If `G_FILENAME_ENCODING` is not set, but `G_BROKEN_FILENAMES` is,
989 * the character set of the current locale is taken as the filename
990 * encoding. If neither environment variable is set, UTF-8 is taken
991 * as the filename encoding, but the character set of the current locale
992 * is also put in the list of encodings.
994 * The returned @charsets belong to GLib and must not be freed.
996 * Note that on Unix, regardless of the locale character set or
997 * `G_FILENAME_ENCODING` value, the actual file names present
998 * on a system might be in any random encoding or just gibberish.
1000 * Return value: %TRUE if the filename encoding is UTF-8.
1005 g_get_filename_charsets (const gchar ***filename_charsets)
1007 static GPrivate cache_private = G_PRIVATE_INIT (filename_charset_cache_free);
1008 GFilenameCharsetCache *cache = g_private_get (&cache_private);
1009 const gchar *charset;
1013 cache = g_new0 (GFilenameCharsetCache, 1);
1014 g_private_set (&cache_private, cache);
1017 g_get_charset (&charset);
1019 if (!(cache->charset && strcmp (cache->charset, charset) == 0))
1021 const gchar *new_charset;
1025 g_free (cache->charset);
1026 g_strfreev (cache->filename_charsets);
1027 cache->charset = g_strdup (charset);
1029 p = getenv ("G_FILENAME_ENCODING");
1030 if (p != NULL && p[0] != '\0')
1032 cache->filename_charsets = g_strsplit (p, ",", 0);
1033 cache->is_utf8 = (strcmp (cache->filename_charsets[0], "UTF-8") == 0);
1035 for (i = 0; cache->filename_charsets[i]; i++)
1037 if (strcmp ("@locale", cache->filename_charsets[i]) == 0)
1039 g_get_charset (&new_charset);
1040 g_free (cache->filename_charsets[i]);
1041 cache->filename_charsets[i] = g_strdup (new_charset);
1045 else if (getenv ("G_BROKEN_FILENAMES") != NULL)
1047 cache->filename_charsets = g_new0 (gchar *, 2);
1048 cache->is_utf8 = g_get_charset (&new_charset);
1049 cache->filename_charsets[0] = g_strdup (new_charset);
1053 cache->filename_charsets = g_new0 (gchar *, 3);
1054 cache->is_utf8 = TRUE;
1055 cache->filename_charsets[0] = g_strdup ("UTF-8");
1056 if (!g_get_charset (&new_charset))
1057 cache->filename_charsets[1] = g_strdup (new_charset);
1061 if (filename_charsets)
1062 *filename_charsets = (const gchar **)cache->filename_charsets;
1064 return cache->is_utf8;
1067 #else /* G_PLATFORM_WIN32 */
1070 g_get_filename_charsets (const gchar ***filename_charsets)
1072 static const gchar *charsets[] = {
1078 /* On Windows GLib pretends that the filename charset is UTF-8 */
1079 if (filename_charsets)
1080 *filename_charsets = charsets;
1086 /* Cygwin works like before */
1087 result = g_get_charset (&(charsets[0]));
1089 if (filename_charsets)
1090 *filename_charsets = charsets;
1096 #endif /* G_PLATFORM_WIN32 */
1099 get_filename_charset (const gchar **filename_charset)
1101 const gchar **charsets;
1104 is_utf8 = g_get_filename_charsets (&charsets);
1106 if (filename_charset)
1107 *filename_charset = charsets[0];
1113 * g_filename_to_utf8:
1114 * @opsysstring: a string in the encoding for filenames
1115 * @len: the length of the string, or -1 if the string is
1116 * nul-terminated (Note that some encodings may allow nul
1117 * bytes to occur inside strings. In that case, using -1
1118 * for the @len parameter is unsafe)
1119 * @bytes_read: location to store the number of bytes in the
1120 * input string that were successfully converted, or %NULL.
1121 * Even if the conversion was successful, this may be
1122 * less than @len if there were partial characters
1123 * at the end of the input. If the error
1124 * #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
1125 * stored will the byte offset after the last valid
1127 * @bytes_written: the number of bytes stored in the output buffer (not
1128 * including the terminating nul).
1129 * @error: location to store the error occurring, or %NULL to ignore
1130 * errors. Any of the errors in #GConvertError may occur.
1132 * Converts a string which is in the encoding used by GLib for
1133 * filenames into a UTF-8 string. Note that on Windows GLib uses UTF-8
1134 * for filenames; on other platforms, this function indirectly depends on
1135 * the [current locale][setlocale].
1137 * Return value: The converted string, or %NULL on an error.
1140 g_filename_to_utf8 (const gchar *opsysstring,
1143 gsize *bytes_written,
1146 const gchar *charset;
1148 g_return_val_if_fail (opsysstring != NULL, NULL);
1150 if (get_filename_charset (&charset))
1151 return strdup_len (opsysstring, len, bytes_read, bytes_written, error);
1153 return g_convert (opsysstring, len,
1154 "UTF-8", charset, bytes_read, bytes_written, error);
1157 #if defined (G_OS_WIN32) && !defined (_WIN64)
1159 #undef g_filename_to_utf8
1161 /* Binary compatibility version. Not for newly compiled code. Also not needed for
1162 * 64-bit versions as there should be no old deployed binaries that would use
1167 g_filename_to_utf8 (const gchar *opsysstring,
1170 gsize *bytes_written,
1173 const gchar *charset;
1175 g_return_val_if_fail (opsysstring != NULL, NULL);
1177 if (g_get_charset (&charset))
1178 return strdup_len (opsysstring, len, bytes_read, bytes_written, error);
1180 return g_convert (opsysstring, len,
1181 "UTF-8", charset, bytes_read, bytes_written, error);
1187 * g_filename_from_utf8:
1188 * @utf8string: a UTF-8 encoded string.
1189 * @len: the length of the string, or -1 if the string is
1191 * @bytes_read: (out) (allow-none): location to store the number of bytes in
1192 * the input string that were successfully converted, or %NULL.
1193 * Even if the conversion was successful, this may be
1194 * less than @len if there were partial characters
1195 * at the end of the input. If the error
1196 * #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
1197 * stored will the byte offset after the last valid
1199 * @bytes_written: (out): the number of bytes stored in the output buffer (not
1200 * including the terminating nul).
1201 * @error: location to store the error occurring, or %NULL to ignore
1202 * errors. Any of the errors in #GConvertError may occur.
1204 * Converts a string from UTF-8 to the encoding GLib uses for
1205 * filenames. Note that on Windows GLib uses UTF-8 for filenames;
1206 * on other platforms, this function indirectly depends on the
1207 * [current locale][setlocale].
1209 * Return value: (array length=bytes_written) (element-type guint8) (transfer full):
1210 * The converted string, or %NULL on an error.
1213 g_filename_from_utf8 (const gchar *utf8string,
1216 gsize *bytes_written,
1219 const gchar *charset;
1221 if (get_filename_charset (&charset))
1222 return strdup_len (utf8string, len, bytes_read, bytes_written, error);
1224 return g_convert (utf8string, len,
1225 charset, "UTF-8", bytes_read, bytes_written, error);
1228 #if defined (G_OS_WIN32) && !defined (_WIN64)
1230 #undef g_filename_from_utf8
1232 /* Binary compatibility version. Not for newly compiled code. */
1235 g_filename_from_utf8 (const gchar *utf8string,
1238 gsize *bytes_written,
1241 const gchar *charset;
1243 if (g_get_charset (&charset))
1244 return strdup_len (utf8string, len, bytes_read, bytes_written, error);
1246 return g_convert (utf8string, len,
1247 charset, "UTF-8", bytes_read, bytes_written, error);
1252 /* Test of haystack has the needle prefix, comparing case
1253 * insensitive. haystack may be UTF-8, but needle must
1254 * contain only ascii. */
1256 has_case_prefix (const gchar *haystack, const gchar *needle)
1260 /* Eat one character at a time. */
1265 g_ascii_tolower (*n) == g_ascii_tolower (*h))
1275 UNSAFE_ALL = 0x1, /* Escape all unsafe characters */
1276 UNSAFE_ALLOW_PLUS = 0x2, /* Allows '+' */
1277 UNSAFE_PATH = 0x8, /* Allows '/', '&', '=', ':', '@', '+', '$' and ',' */
1278 UNSAFE_HOST = 0x10, /* Allows '/' and ':' and '@' */
1279 UNSAFE_SLASHES = 0x20 /* Allows all characters except for '/' and '%' */
1280 } UnsafeCharacterSet;
1282 static const guchar acceptable[96] = {
1283 /* A table of the ASCII chars from space (32) to DEL (127) */
1284 /* ! " # $ % & ' ( ) * + , - . / */
1285 0x00,0x3F,0x20,0x20,0x28,0x00,0x2C,0x3F,0x3F,0x3F,0x3F,0x2A,0x28,0x3F,0x3F,0x1C,
1286 /* 0 1 2 3 4 5 6 7 8 9 : ; < = > ? */
1287 0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x38,0x20,0x20,0x2C,0x20,0x20,
1288 /* @ A B C D E F G H I J K L M N O */
1289 0x38,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,
1290 /* P Q R S T U V W X Y Z [ \ ] ^ _ */
1291 0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x20,0x20,0x20,0x20,0x3F,
1292 /* ` a b c d e f g h i j k l m n o */
1293 0x20,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,
1294 /* p q r s t u v w x y z { | } ~ DEL */
1295 0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x3F,0x20,0x20,0x20,0x3F,0x20
1298 static const gchar hex[16] = "0123456789ABCDEF";
1300 /* Note: This escape function works on file: URIs, but if you want to
1301 * escape something else, please read RFC-2396 */
1303 g_escape_uri_string (const gchar *string,
1304 UnsafeCharacterSet mask)
1306 #define ACCEPTABLE(a) ((a)>=32 && (a)<128 && (acceptable[(a)-32] & use_mask))
1313 UnsafeCharacterSet use_mask;
1315 g_return_val_if_fail (mask == UNSAFE_ALL
1316 || mask == UNSAFE_ALLOW_PLUS
1317 || mask == UNSAFE_PATH
1318 || mask == UNSAFE_HOST
1319 || mask == UNSAFE_SLASHES, NULL);
1323 for (p = string; *p != '\0'; p++)
1326 if (!ACCEPTABLE (c))
1330 result = g_malloc (p - string + unacceptable * 2 + 1);
1333 for (q = result, p = string; *p != '\0'; p++)
1337 if (!ACCEPTABLE (c))
1339 *q++ = '%'; /* means hex coming */
1354 g_escape_file_uri (const gchar *hostname,
1355 const gchar *pathname)
1357 char *escaped_hostname = NULL;
1362 char *p, *backslash;
1364 /* Turn backslashes into forward slashes. That's what Netscape
1365 * does, and they are actually more or less equivalent in Windows.
1368 pathname = g_strdup (pathname);
1369 p = (char *) pathname;
1371 while ((backslash = strchr (p, '\\')) != NULL)
1378 if (hostname && *hostname != '\0')
1380 escaped_hostname = g_escape_uri_string (hostname, UNSAFE_HOST);
1383 escaped_path = g_escape_uri_string (pathname, UNSAFE_PATH);
1385 res = g_strconcat ("file://",
1386 (escaped_hostname) ? escaped_hostname : "",
1387 (*escaped_path != '/') ? "/" : "",
1392 g_free ((char *) pathname);
1395 g_free (escaped_hostname);
1396 g_free (escaped_path);
1402 unescape_character (const char *scanner)
1407 first_digit = g_ascii_xdigit_value (scanner[0]);
1408 if (first_digit < 0)
1411 second_digit = g_ascii_xdigit_value (scanner[1]);
1412 if (second_digit < 0)
1415 return (first_digit << 4) | second_digit;
1419 g_unescape_uri_string (const char *escaped,
1421 const char *illegal_escaped_characters,
1422 gboolean ascii_must_not_be_escaped)
1424 const gchar *in, *in_end;
1425 gchar *out, *result;
1428 if (escaped == NULL)
1432 len = strlen (escaped);
1434 result = g_malloc (len + 1);
1437 for (in = escaped, in_end = escaped + len; in < in_end; in++)
1443 /* catch partial escape sequences past the end of the substring */
1444 if (in + 3 > in_end)
1447 c = unescape_character (in + 1);
1449 /* catch bad escape sequences and NUL characters */
1453 /* catch escaped ASCII */
1454 if (ascii_must_not_be_escaped && c <= 0x7F)
1457 /* catch other illegal escaped characters */
1458 if (strchr (illegal_escaped_characters, c) != NULL)
1467 g_assert (out - result <= len);
1480 is_asciialphanum (gunichar c)
1482 return c <= 0x7F && g_ascii_isalnum (c);
1486 is_asciialpha (gunichar c)
1488 return c <= 0x7F && g_ascii_isalpha (c);
1491 /* allows an empty string */
1493 hostname_validate (const char *hostname)
1496 gunichar c, first_char, last_char;
1503 /* read in a label */
1504 c = g_utf8_get_char (p);
1505 p = g_utf8_next_char (p);
1506 if (!is_asciialphanum (c))
1512 c = g_utf8_get_char (p);
1513 p = g_utf8_next_char (p);
1515 while (is_asciialphanum (c) || c == '-');
1516 if (last_char == '-')
1519 /* if that was the last label, check that it was a toplabel */
1520 if (c == '\0' || (c == '.' && *p == '\0'))
1521 return is_asciialpha (first_char);
1528 * g_filename_from_uri:
1529 * @uri: a uri describing a filename (escaped, encoded in ASCII).
1530 * @hostname: (out) (allow-none): Location to store hostname for the URI, or %NULL.
1531 * If there is no hostname in the URI, %NULL will be
1532 * stored in this location.
1533 * @error: location to store the error occurring, or %NULL to ignore
1534 * errors. Any of the errors in #GConvertError may occur.
1536 * Converts an escaped ASCII-encoded URI to a local filename in the
1537 * encoding used for filenames.
1539 * Return value: (type filename): a newly-allocated string holding
1540 * the resulting filename, or %NULL on an error.
1543 g_filename_from_uri (const gchar *uri,
1547 const char *path_part;
1548 const char *host_part;
1549 char *unescaped_hostname;
1560 if (!has_case_prefix (uri, "file:/"))
1562 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_BAD_URI,
1563 _("The URI '%s' is not an absolute URI using the \"file\" scheme"),
1568 path_part = uri + strlen ("file:");
1570 if (strchr (path_part, '#') != NULL)
1572 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_BAD_URI,
1573 _("The local file URI '%s' may not include a '#'"),
1578 if (has_case_prefix (path_part, "///"))
1580 else if (has_case_prefix (path_part, "//"))
1583 host_part = path_part;
1585 path_part = strchr (path_part, '/');
1587 if (path_part == NULL)
1589 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_BAD_URI,
1590 _("The URI '%s' is invalid"),
1595 unescaped_hostname = g_unescape_uri_string (host_part, path_part - host_part, "", TRUE);
1597 if (unescaped_hostname == NULL ||
1598 !hostname_validate (unescaped_hostname))
1600 g_free (unescaped_hostname);
1601 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_BAD_URI,
1602 _("The hostname of the URI '%s' is invalid"),
1608 *hostname = unescaped_hostname;
1610 g_free (unescaped_hostname);
1613 filename = g_unescape_uri_string (path_part, -1, "/", FALSE);
1615 if (filename == NULL)
1617 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_BAD_URI,
1618 _("The URI '%s' contains invalidly escaped characters"),
1625 /* Drop localhost */
1626 if (hostname && *hostname != NULL &&
1627 g_ascii_strcasecmp (*hostname, "localhost") == 0)
1633 /* Turn slashes into backslashes, because that's the canonical spelling */
1635 while ((slash = strchr (p, '/')) != NULL)
1641 /* Windows URIs with a drive letter can be like "file://host/c:/foo"
1642 * or "file://host/c|/foo" (some Netscape versions). In those cases, start
1643 * the filename from the drive letter.
1645 if (g_ascii_isalpha (filename[1]))
1647 if (filename[2] == ':')
1649 else if (filename[2] == '|')
1657 result = g_strdup (filename + offs);
1663 #if defined (G_OS_WIN32) && !defined (_WIN64)
1665 #undef g_filename_from_uri
1668 g_filename_from_uri (const gchar *uri,
1672 gchar *utf8_filename;
1673 gchar *retval = NULL;
1675 utf8_filename = g_filename_from_uri_utf8 (uri, hostname, error);
1678 retval = g_locale_from_utf8 (utf8_filename, -1, NULL, NULL, error);
1679 g_free (utf8_filename);
1687 * g_filename_to_uri:
1688 * @filename: an absolute filename specified in the GLib file name encoding,
1689 * which is the on-disk file name bytes on Unix, and UTF-8 on
1691 * @hostname: (allow-none): A UTF-8 encoded hostname, or %NULL for none.
1692 * @error: location to store the error occurring, or %NULL to ignore
1693 * errors. Any of the errors in #GConvertError may occur.
1695 * Converts an absolute filename to an escaped ASCII-encoded URI, with the path
1696 * component following Section 3.3. of RFC 2396.
1698 * Return value: a newly-allocated string holding the resulting
1699 * URI, or %NULL on an error.
1702 g_filename_to_uri (const gchar *filename,
1703 const gchar *hostname,
1708 g_return_val_if_fail (filename != NULL, NULL);
1710 if (!g_path_is_absolute (filename))
1712 g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_NOT_ABSOLUTE_PATH,
1713 _("The pathname '%s' is not an absolute path"),
1719 !(g_utf8_validate (hostname, -1, NULL)
1720 && hostname_validate (hostname)))
1722 g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
1723 _("Invalid hostname"));
1728 /* Don't use localhost unnecessarily */
1729 if (hostname && g_ascii_strcasecmp (hostname, "localhost") == 0)
1733 escaped_uri = g_escape_file_uri (hostname, filename);
1738 #if defined (G_OS_WIN32) && !defined (_WIN64)
1740 #undef g_filename_to_uri
1743 g_filename_to_uri (const gchar *filename,
1744 const gchar *hostname,
1747 gchar *utf8_filename;
1748 gchar *retval = NULL;
1750 utf8_filename = g_locale_to_utf8 (filename, -1, NULL, NULL, error);
1754 retval = g_filename_to_uri_utf8 (utf8_filename, hostname, error);
1755 g_free (utf8_filename);
1764 * g_uri_list_extract_uris:
1765 * @uri_list: an URI list
1767 * Splits an URI list conforming to the text/uri-list
1768 * mime type defined in RFC 2483 into individual URIs,
1769 * discarding any comments. The URIs are not validated.
1771 * Returns: (transfer full): a newly allocated %NULL-terminated list
1772 * of strings holding the individual URIs. The array should be freed
1773 * with g_strfreev().
1778 g_uri_list_extract_uris (const gchar *uri_list)
1789 /* We don't actually try to validate the URI according to RFC
1790 * 2396, or even check for allowed characters - we just ignore
1791 * comments and trim whitespace off the ends. We also
1792 * allow LF delimination as well as the specified CRLF.
1794 * We do allow comments like specified in RFC 2483.
1800 while (g_ascii_isspace (*p))
1804 while (*q && (*q != '\n') && (*q != '\r'))
1810 while (q > p && g_ascii_isspace (*q))
1815 uris = g_slist_prepend (uris, g_strndup (p, q - p + 1));
1820 p = strchr (p, '\n');
1825 result = g_new (gchar *, n_uris + 1);
1827 result[n_uris--] = NULL;
1828 for (u = uris; u; u = u->next)
1829 result[n_uris--] = u->data;
1831 g_slist_free (uris);
1837 * g_filename_display_basename:
1838 * @filename: an absolute pathname in the GLib file name encoding
1840 * Returns the display basename for the particular filename, guaranteed
1841 * to be valid UTF-8. The display name might not be identical to the filename,
1842 * for instance there might be problems converting it to UTF-8, and some files
1843 * can be translated in the display.
1845 * If GLib cannot make sense of the encoding of @filename, as a last resort it
1846 * replaces unknown characters with U+FFFD, the Unicode replacement character.
1847 * You can search the result for the UTF-8 encoding of this character (which is
1848 * "\357\277\275" in octal notation) to find out if @filename was in an invalid
1851 * You must pass the whole absolute pathname to this functions so that
1852 * translation of well known locations can be done.
1854 * This function is preferred over g_filename_display_name() if you know the
1855 * whole path, as it allows translation.
1857 * Return value: a newly allocated string containing
1858 * a rendition of the basename of the filename in valid UTF-8
1863 g_filename_display_basename (const gchar *filename)
1868 g_return_val_if_fail (filename != NULL, NULL);
1870 basename = g_path_get_basename (filename);
1871 display_name = g_filename_display_name (basename);
1873 return display_name;
1877 * g_filename_display_name:
1878 * @filename: a pathname hopefully in the GLib file name encoding
1880 * Converts a filename into a valid UTF-8 string. The conversion is
1881 * not necessarily reversible, so you should keep the original around
1882 * and use the return value of this function only for display purposes.
1883 * Unlike g_filename_to_utf8(), the result is guaranteed to be non-%NULL
1884 * even if the filename actually isn't in the GLib file name encoding.
1886 * If GLib cannot make sense of the encoding of @filename, as a last resort it
1887 * replaces unknown characters with U+FFFD, the Unicode replacement character.
1888 * You can search the result for the UTF-8 encoding of this character (which is
1889 * "\357\277\275" in octal notation) to find out if @filename was in an invalid
1892 * If you know the whole pathname of the file you should use
1893 * g_filename_display_basename(), since that allows location-based
1894 * translation of filenames.
1896 * Return value: a newly allocated string containing
1897 * a rendition of the filename in valid UTF-8
1902 g_filename_display_name (const gchar *filename)
1905 const gchar **charsets;
1906 gchar *display_name = NULL;
1909 is_utf8 = g_get_filename_charsets (&charsets);
1913 if (g_utf8_validate (filename, -1, NULL))
1914 display_name = g_strdup (filename);
1919 /* Try to convert from the filename charsets to UTF-8.
1920 * Skip the first charset if it is UTF-8.
1922 for (i = is_utf8 ? 1 : 0; charsets[i]; i++)
1924 display_name = g_convert (filename, -1, "UTF-8", charsets[i],
1932 /* if all conversions failed, we replace invalid UTF-8
1933 * by a question mark
1936 display_name = _g_utf8_make_valid (filename);
1938 return display_name;