X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=glib%2Fgconvert.c;h=368bef57c09139b7e709275bbdc15f438e717552;hb=4c2a6595889eff44fa5f610e6c69016702100e95;hp=4566f88d7f5718fcef127fda6c9c4eb11908811e;hpb=286d84fcb015da87985c6d2d996d76960322b36d;p=platform%2Fupstream%2Fglib.git diff --git a/glib/gconvert.c b/glib/gconvert.c index 4566f88..368bef5 100644 --- a/glib/gconvert.c +++ b/glib/gconvert.c @@ -2,7 +2,7 @@ * * gconvert.c: Convert between character sets using iconv * Copyright Red Hat Inc., 2000 - * Authors: Havoc Pennington , Owen Taylor , Owen Taylor * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -21,17 +21,19 @@ */ #include "config.h" +#include "glibconfig.h" +#ifndef G_OS_WIN32 #include +#endif #include #include #include #include -#include "galias.h" -#include "glib.h" -#include "gprintfint.h" -#include "gthreadinit.h" +#ifdef G_OS_WIN32 +#include "win_iconv.c" +#endif #ifdef G_PLATFORM_WIN32 #define STRICT @@ -39,24 +41,160 @@ #undef STRICT #endif +#include "gconvert.h" + +#include "gcharsetprivate.h" +#include "gslist.h" +#include "gstrfuncs.h" +#include "gtestutils.h" +#include "gthread.h" +#include "gunicode.h" +#include "gfileutils.h" + +#ifdef NEED_ICONV_CACHE +#include "glist.h" +#include "ghash.h" +#endif + #include "glibintl.h" #if defined(USE_LIBICONV_GNU) && !defined (_LIBICONV_H) #error GNU libiconv in use but included iconv.h not from libiconv #endif -#if !defined(USE_LIBICONV_GNU) && defined (_LIBICONV_H) +#if !defined(USE_LIBICONV_GNU) && defined (_LIBICONV_H) \ + && !defined (__APPLE_CC__) && !defined (__LP_64__) #error GNU libiconv not in use but included iconv.h is from libiconv #endif -GQuark -g_convert_error_quark (void) -{ - static GQuark quark; - if (!quark) - quark = g_quark_from_static_string ("g_convert_error"); - return quark; -} +/** + * SECTION:conversions + * @title: Character Set Conversion + * @short_description: convert strings between different character sets + * + * The g_convert() family of function wraps the functionality of iconv(). In + * addition to pure character set conversions, GLib has functions to deal + * with the extra complications of encodings for file names. + * + * + * File Name Encodings + * + * Historically, Unix has not had a defined encoding for file + * names: a file name is valid as long as it does not have path + * separators in it ("/"). However, displaying file names may + * require conversion: from the character set in which they were + * created, to the character set in which the application + * operates. Consider the Spanish file name + * "Presentación.sxi". If the + * application which created it uses ISO-8859-1 for its encoding, + * + * + * Character: P r e s e n t a c i ó n . s x i + * Hex code: 50 72 65 73 65 6e 74 61 63 69 f3 6e 2e 73 78 69 + * + * + * However, if the application use UTF-8, the actual file name on + * disk would look like this: + * + * + * Character: P r e s e n t a c i ó n . s x i + * Hex code: 50 72 65 73 65 6e 74 61 63 69 c3 b3 6e 2e 73 78 69 + * + * + * Glib uses UTF-8 for its strings, and GUI toolkits like GTK+ + * that use Glib do the same thing. If you get a file name from + * the file system, for example, from readdir(3) or from g_dir_read_name(), + * and you wish to display the file name to the user, you + * will need to convert it into UTF-8. The + * opposite case is when the user types the name of a file he + * wishes to save: the toolkit will give you that string in + * UTF-8 encoding, and you will need to convert it to the + * character set used for file names before you can create the + * file with open(2) or fopen(3). + * + * + * By default, Glib assumes that file names on disk are in UTF-8 + * encoding. This is a valid assumption for file systems which + * were created relatively recently: most applications use UTF-8 + * encoding for their strings, and that is also what they use for + * the file names they create. However, older file systems may + * still contain file names created in "older" encodings, such as + * ISO-8859-1. In this case, for compatibility reasons, you may + * want to instruct Glib to use that particular encoding for file + * names rather than UTF-8. You can do this by specifying the + * encoding for file names in the G_FILENAME_ENCODING + * environment variable. For example, if your installation uses + * ISO-8859-1 for file names, you can put this in your + * ~/.profile: + * + * + * export G_FILENAME_ENCODING=ISO-8859-1 + * + * + * Glib provides the functions g_filename_to_utf8() and + * g_filename_from_utf8() to perform the necessary conversions. These + * functions convert file names from the encoding specified in + * G_FILENAME_ENCODING to UTF-8 and vice-versa. + * illustrates how + * these functions are used to convert between UTF-8 and the + * encoding for file names in the file system. + * + *
+ * Conversion between File Name Encodings + * + *
+ * + * Checklist for Application Writers + * + * This section is a practical summary of the detailed + * description above. You can use this as a checklist of + * things to do to make sure your applications process file + * name encodings correctly. + * + * + * + * If you get a file name from the file system from a function + * such as readdir(3) or gtk_file_chooser_get_filename(), + * you do not need to do any conversion to pass that + * file name to functions like open(2), rename(2), or + * fopen(3) — those are "raw" file names which the file + * system understands. + * + * + * If you need to display a file name, convert it to UTF-8 first by + * using g_filename_to_utf8(). If conversion fails, display a string like + * "Unknown file name". Do not + * convert this string back into the encoding used for file names if you + * wish to pass it to the file system; use the original file name instead. + * For example, the document window of a word processor could display + * "Unknown file name" in its title bar but still let the user save the + * file, as it would keep the raw file name internally. This can happen + * if the user has not set the G_FILENAME_ENCODING + * environment variable even though he has files whose names are not + * encoded in UTF-8. + * + * + * If your user interface lets the user type a file name for saving or + * renaming, convert it to the encoding used for file names in the file + * system by using g_filename_from_utf8(). Pass the converted file name + * to functions like fopen(3). If conversion fails, ask the user to enter + * a different file name. This can happen if the user types Japanese + * characters when G_FILENAME_ENCODING is set to + * ISO-8859-1, for example. + * + * + * + * + */ + +/* We try to terminate strings in unknown charsets with this many zero bytes + * to ensure that multibyte strings really are nul-terminated when we return + * them from g_convert() and friends. + */ +#define NUL_TERMINATOR_LENGTH 4 + +G_DEFINE_QUARK (g_convert_error, g_convert_error) static gboolean try_conversion (const char *to_codeset, @@ -91,7 +229,8 @@ try_to_aliases (const char **to_aliases, return FALSE; } -extern const char **_g_charset_get_aliases (const char *canonical_name) G_GNUC_INTERNAL; +G_GNUC_INTERNAL extern const char ** +_g_charset_get_aliases (const char *canonical_name); /** * g_iconv_open: @@ -159,7 +298,7 @@ g_iconv_open (const gchar *to_codeset, * * Return value: count of non-reversible conversions, or -1 on error **/ -size_t +gsize g_iconv (GIConv converter, gchar **inbuf, gsize *inbytes_left, @@ -195,6 +334,8 @@ g_iconv_close (GIConv converter) } +#ifdef NEED_ICONV_CACHE + #define ICONV_CACHE_SIZE (16) struct _iconv_cache_bucket { @@ -227,7 +368,7 @@ iconv_cache_init (void) } -/** +/* * iconv_cache_bucket_new: * @key: cache key * @cd: iconv descriptor @@ -235,15 +376,17 @@ iconv_cache_init (void) * Creates a new cache bucket, inserts it into the cache and * increments the cache size. * + * This assumes ownership of @key. + * * Returns a pointer to the newly allocated cache bucket. **/ static struct _iconv_cache_bucket * -iconv_cache_bucket_new (const gchar *key, GIConv cd) +iconv_cache_bucket_new (gchar *key, GIConv cd) { struct _iconv_cache_bucket *bucket; bucket = g_new (struct _iconv_cache_bucket, 1); - bucket->key = g_strdup (key); + bucket->key = key; bucket->refcount = 1; bucket->used = TRUE; bucket->cd = cd; @@ -260,7 +403,7 @@ iconv_cache_bucket_new (const gchar *key, GIConv cd) } -/** +/* * iconv_cache_bucket_expire: * @node: cache bucket's node * @bucket: cache bucket @@ -305,7 +448,7 @@ iconv_cache_bucket_expire (GList *node, struct _iconv_cache_bucket *bucket) } -/** +/* * iconv_cache_expire_unused: * * Expires as many unused cache buckets as it needs to in order to get @@ -336,13 +479,24 @@ open_converter (const gchar *to_codeset, GError **error) { struct _iconv_cache_bucket *bucket; - gchar *key; + gchar *key, *dyn_key, auto_key[80]; GIConv cd; + gsize len_from_codeset, len_to_codeset; /* create our key */ - key = g_alloca (strlen (from_codeset) + strlen (to_codeset) + 2); - _g_sprintf (key, "%s:%s", from_codeset, to_codeset); - + len_from_codeset = strlen (from_codeset); + len_to_codeset = strlen (to_codeset); + if (len_from_codeset + len_to_codeset + 2 < sizeof (auto_key)) + { + key = auto_key; + dyn_key = NULL; + } + else + key = dyn_key = g_malloc (len_from_codeset + len_to_codeset + 2); + memcpy (key, from_codeset, len_from_codeset); + key[len_from_codeset] = ':'; + strcpy (key + len_from_codeset + 1, to_codeset); + G_LOCK (iconv_cache_lock); /* make sure the cache has been initialized */ @@ -351,6 +505,8 @@ open_converter (const gchar *to_codeset, bucket = g_hash_table_lookup (iconv_cache, key); if (bucket) { + g_free (dyn_key); + if (bucket->used) { cd = g_iconv_open (to_codeset, from_codeset); @@ -379,12 +535,15 @@ open_converter (const gchar *to_codeset, else { cd = g_iconv_open (to_codeset, from_codeset); - if (cd == (GIConv) -1) - goto error; + if (cd == (GIConv) -1) + { + g_free (dyn_key); + goto error; + } iconv_cache_expire_unused (); - - bucket = iconv_cache_bucket_new (key, cd); + + bucket = iconv_cache_bucket_new (dyn_key ? dyn_key : g_strdup (key), cd); } g_hash_table_insert (iconv_open_hash, cd, bucket->key); @@ -398,14 +557,17 @@ open_converter (const gchar *to_codeset, G_UNLOCK (iconv_cache_lock); /* Something went wrong. */ - if (errno == EINVAL) - g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_NO_CONVERSION, - _("Conversion from character set '%s' to '%s' is not supported"), - from_codeset, to_codeset); - else - g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED, - _("Could not open converter from '%s' to '%s'"), - from_codeset, to_codeset); + if (error) + { + if (errno == EINVAL) + g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_NO_CONVERSION, + _("Conversion from character set '%s' to '%s' is not supported"), + from_codeset, to_codeset); + else + g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED, + _("Could not open converter from '%s' to '%s'"), + from_codeset, to_codeset); + } return cd; } @@ -459,74 +621,52 @@ close_converter (GIConv converter) return 0; } +#else /* !NEED_ICONV_CACHE */ -/** - * g_convert: - * @str: the string to convert - * @len: the length of the string - * @to_codeset: name of character set into which to convert @str - * @from_codeset: character set of @str. - * @bytes_read: location to store the number of bytes in the - * input string that were successfully converted, or %NULL. - * Even if the conversion was successful, this may be - * less than @len if there were partial characters - * at the end of the input. If the error - * #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value - * stored will the byte offset after the last valid - * input sequence. - * @bytes_written: the number of bytes stored in the output buffer (not - * including the terminating nul). - * @error: location to store the error occuring, or %NULL to ignore - * errors. Any of the errors in #GConvertError may occur. - * - * Converts a string from one character set to another. - * - * Return value: If the conversion was successful, a newly allocated - * nul-terminated string, which must be freed with - * g_free(). Otherwise %NULL and @error will be set. - **/ -gchar* -g_convert (const gchar *str, - gssize len, - const gchar *to_codeset, - const gchar *from_codeset, - gsize *bytes_read, - gsize *bytes_written, - GError **error) +static GIConv +open_converter (const gchar *to_codeset, + const gchar *from_codeset, + GError **error) { - gchar *res; GIConv cd; - - g_return_val_if_fail (str != NULL, NULL); - g_return_val_if_fail (to_codeset != NULL, NULL); - g_return_val_if_fail (from_codeset != NULL, NULL); - - cd = open_converter (to_codeset, from_codeset, error); + + cd = g_iconv_open (to_codeset, from_codeset); if (cd == (GIConv) -1) { - if (bytes_read) - *bytes_read = 0; - - if (bytes_written) - *bytes_written = 0; - - return NULL; + /* Something went wrong. */ + if (error) + { + if (errno == EINVAL) + g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_NO_CONVERSION, + _("Conversion from character set '%s' to '%s' is not supported"), + from_codeset, to_codeset); + else + g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED, + _("Could not open converter from '%s' to '%s'"), + from_codeset, to_codeset); + } } - - res = g_convert_with_iconv (str, len, cd, - bytes_read, bytes_written, - error); - close_converter (cd); + return cd; +} - return res; +static int +close_converter (GIConv cd) +{ + if (cd == (GIConv) -1) + return 0; + + return g_iconv_close (cd); } +#endif /* NEED_ICONV_CACHE */ + /** * g_convert_with_iconv: * @str: the string to convert - * @len: the length of the string + * @len: the length of the string, or -1 if the string is + * nul-terminated. * @converter: conversion descriptor from g_iconv_open() * @bytes_read: location to store the number of bytes in the * input string that were successfully converted, or %NULL. @@ -538,10 +678,25 @@ g_convert (const gchar *str, * input sequence. * @bytes_written: the number of bytes stored in the output buffer (not * including the terminating nul). - * @error: location to store the error occuring, or %NULL to ignore + * @error: location to store the error occurring, or %NULL to ignore * errors. Any of the errors in #GConvertError may occur. * - * Converts a string from one character set to another. + * Converts a string from one character set to another. + * + * Note that you should use g_iconv() for streaming + * conversions + * + * Despite the fact that @byes_read can return information about partial + * characters, the g_convert_... functions + * are not generally suitable for streaming. If the underlying converter + * being used maintains internal state, then this won't be preserved + * across successive calls to g_convert(), g_convert_with_iconv() or + * g_convert_with_fallback(). (An example of this is the GNU C converter + * for CP1255 which does not emit a base character until it knows that + * the next character is not a mark that could combine with the base + * character.) + * + * . * * Return value: If the conversion was successful, a newly allocated * nul-terminated string, which must be freed with @@ -563,8 +718,9 @@ g_convert_with_iconv (const gchar *str, gsize err; gsize outbuf_size; gboolean have_error = FALSE; + gboolean done = FALSE; + gboolean reset = FALSE; - g_return_val_if_fail (str != NULL, NULL); g_return_val_if_fail (converter != (GIConv) -1, NULL); if (len < 0) @@ -572,49 +728,68 @@ g_convert_with_iconv (const gchar *str, p = str; inbytes_remaining = len; - outbuf_size = len + 1; /* + 1 for nul in case len == 1 */ + outbuf_size = len + NUL_TERMINATOR_LENGTH; - outbytes_remaining = outbuf_size - 1; /* -1 for nul */ + outbytes_remaining = outbuf_size - NUL_TERMINATOR_LENGTH; outp = dest = g_malloc (outbuf_size); - again: - - err = g_iconv (converter, (char **)&p, &inbytes_remaining, &outp, &outbytes_remaining); - - if (err == (size_t) -1) + while (!done && !have_error) { - switch (errno) + if (reset) + err = g_iconv (converter, NULL, &inbytes_remaining, &outp, &outbytes_remaining); + else + err = g_iconv (converter, (char **)&p, &inbytes_remaining, &outp, &outbytes_remaining); + + if (err == (gsize) -1) { - case EINVAL: - /* Incomplete text, do not report an error */ - break; - case E2BIG: - { - size_t used = outp - dest; - - outbuf_size *= 2; - dest = g_realloc (dest, outbuf_size); + switch (errno) + { + case EINVAL: + /* Incomplete text, do not report an error */ + done = TRUE; + break; + case E2BIG: + { + gsize used = outp - dest; + + outbuf_size *= 2; + dest = g_realloc (dest, outbuf_size); - outp = dest + used; - outbytes_remaining = outbuf_size - used - 1; /* -1 for nul */ - - goto again; - } - case EILSEQ: - g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE, - _("Invalid byte sequence in conversion input")); - have_error = TRUE; - break; - default: - g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED, - _("Error during conversion: %s"), - g_strerror (errno)); - have_error = TRUE; - break; + outp = dest + used; + outbytes_remaining = outbuf_size - used - NUL_TERMINATOR_LENGTH; + } + break; + case EILSEQ: + g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE, + _("Invalid byte sequence in conversion input")); + have_error = TRUE; + break; + default: + { + int errsv = errno; + + g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED, + _("Error during conversion: %s"), + g_strerror (errsv)); + } + have_error = TRUE; + break; + } + } + else + { + if (!reset) + { + /* call g_iconv with NULL inbuf to cleanup shift state */ + reset = TRUE; + inbytes_remaining = 0; + } + else + done = TRUE; } } - *outp = '\0'; + memset (outp, 0, NUL_TERMINATOR_LENGTH); if (bytes_read) *bytes_read = p - str; @@ -624,8 +799,8 @@ g_convert_with_iconv (const gchar *str, { if (!have_error) { - g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT, - _("Partial character sequence at end of input")); + g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT, + _("Partial character sequence at end of input")); have_error = TRUE; } } @@ -644,16 +819,90 @@ g_convert_with_iconv (const gchar *str, } /** + * g_convert: + * @str: the string to convert + * @len: the length of the string, or -1 if the string is + * nul-terminated + + Note that some encodings may allow nul bytes to + occur inside strings. In that case, using -1 for + the @len parameter is unsafe. + + . + * @to_codeset: name of character set into which to convert @str + * @from_codeset: character set of @str. + * @bytes_read: (out): location to store the number of bytes in the + * input string that were successfully converted, or %NULL. + * Even if the conversion was successful, this may be + * less than @len if there were partial characters + * at the end of the input. If the error + * #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value + * stored will the byte offset after the last valid + * input sequence. + * @bytes_written: (out): the number of bytes stored in the output buffer (not + * including the terminating nul). + * @error: location to store the error occurring, or %NULL to ignore + * errors. Any of the errors in #GConvertError may occur. + * + * Converts a string from one character set to another. + * + * Note that you should use g_iconv() for streaming + * conversions. + * + * Return value: If the conversion was successful, a newly allocated + * nul-terminated string, which must be freed with + * g_free(). Otherwise %NULL and @error will be set. + **/ +gchar* +g_convert (const gchar *str, + gssize len, + const gchar *to_codeset, + const gchar *from_codeset, + gsize *bytes_read, + gsize *bytes_written, + GError **error) +{ + gchar *res; + GIConv cd; + + g_return_val_if_fail (str != NULL, NULL); + g_return_val_if_fail (to_codeset != NULL, NULL); + g_return_val_if_fail (from_codeset != NULL, NULL); + + cd = open_converter (to_codeset, from_codeset, error); + + if (cd == (GIConv) -1) + { + if (bytes_read) + *bytes_read = 0; + + if (bytes_written) + *bytes_written = 0; + + return NULL; + } + + res = g_convert_with_iconv (str, len, cd, + bytes_read, bytes_written, + error); + + close_converter (cd); + + return res; +} + +/** * g_convert_with_fallback: * @str: the string to convert - * @len: the length of the string + * @len: the length of the string, or -1 if the string is + * nul-terminated. * @to_codeset: name of character set into which to convert @str * @from_codeset: character set of @str. * @fallback: UTF-8 string to use in place of character not - * present in the target encoding. (This must be - * in the target encoding), if %NULL, characters - * not in the target encoding will be represented - * as Unicode escapes \uxxxx or \Uxxxxyyyy. + * present in the target encoding. (The string must be + * representable in the target encoding). + If %NULL, characters not in the target encoding will + be represented as Unicode escapes \uxxxx or \Uxxxxyyyy. * @bytes_read: location to store the number of bytes in the * input string that were successfully converted, or %NULL. * Even if the conversion was successful, this may be @@ -661,17 +910,20 @@ g_convert_with_iconv (const gchar *str, * at the end of the input. * @bytes_written: the number of bytes stored in the output buffer (not * including the terminating nul). - * @error: location to store the error occuring, or %NULL to ignore + * @error: location to store the error occurring, or %NULL to ignore * errors. Any of the errors in #GConvertError may occur. * * Converts a string from one character set to another, possibly * including fallback sequences for characters not representable * in the output. Note that it is not guaranteed that the specification * for the fallback sequences in @fallback will be honored. Some - * systems may do a approximate conversion from @from_codeset + * systems may do an approximate conversion from @from_codeset * to @to_codeset in their iconv() functions, * in which case GLib will simply return that approximate conversion. * + * Note that you should use g_iconv() for streaming + * conversions. + * * Return value: If the conversion was successful, a newly allocated * nul-terminated string, which must be freed with * g_free(). Otherwise %NULL and @error will be set. @@ -681,7 +933,7 @@ g_convert_with_fallback (const gchar *str, gssize len, const gchar *to_codeset, const gchar *from_codeset, - gchar *fallback, + const gchar *fallback, gsize *bytes_read, gsize *bytes_written, GError **error) @@ -763,17 +1015,17 @@ g_convert_with_fallback (const gchar *str, */ p = utf8; - outbuf_size = len + 1; /* + 1 for nul in case len == 1 */ - outbytes_remaining = outbuf_size - 1; /* -1 for nul */ + outbuf_size = len + NUL_TERMINATOR_LENGTH; + outbytes_remaining = outbuf_size - NUL_TERMINATOR_LENGTH; outp = dest = g_malloc (outbuf_size); while (!done && !have_error) { - size_t inbytes_tmp = inbytes_remaining; + gsize inbytes_tmp = inbytes_remaining; err = g_iconv (cd, (char **)&p, &inbytes_tmp, &outp, &outbytes_remaining); inbytes_remaining = inbytes_tmp; - if (err == (size_t) -1) + if (err == (gsize) -1) { switch (errno) { @@ -782,13 +1034,13 @@ g_convert_with_fallback (const gchar *str, break; case E2BIG: { - size_t used = outp - dest; + gsize used = outp - dest; outbuf_size *= 2; dest = g_realloc (dest, outbuf_size); outp = dest + used; - outbytes_remaining = outbuf_size - used - 1; /* -1 for nul */ + outbytes_remaining = outbuf_size - used - NUL_TERMINATOR_LENGTH; break; } @@ -803,7 +1055,7 @@ g_convert_with_fallback (const gchar *str, have_error = TRUE; break; } - else + else if (p) { if (!fallback) { @@ -818,12 +1070,18 @@ g_convert_with_fallback (const gchar *str, save_inbytes = inbytes_remaining - (save_p - p); p = insert_str; inbytes_remaining = strlen (p); + break; } - break; + /* fall thru if p is NULL */ default: - g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED, - _("Error during conversion: %s"), - g_strerror (errno)); + { + int errsv = errno; + + g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED, + _("Error during conversion: %s"), + g_strerror (errsv)); + } + have_error = TRUE; break; } @@ -838,6 +1096,12 @@ g_convert_with_fallback (const gchar *str, inbytes_remaining = save_inbytes; save_p = NULL; } + else if (p) + { + /* call g_iconv with NULL inbuf to cleanup shift state */ + p = NULL; + inbytes_remaining = 0; + } else done = TRUE; } @@ -845,7 +1109,7 @@ g_convert_with_fallback (const gchar *str, /* Cleanup */ - *outp = '\0'; + memset (outp, 0, NUL_TERMINATOR_LENGTH); close_converter (cd); @@ -888,8 +1152,8 @@ strdup_len (const gchar *string, if (bytes_written) *bytes_written = 0; - g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE, - _("Invalid byte sequence in conversion input")); + g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE, + _("Invalid byte sequence in conversion input")); return NULL; } @@ -913,9 +1177,10 @@ strdup_len (const gchar *string, /** * g_locale_to_utf8: - * @opsysstring: a string in the encoding of the current locale + * @opsysstring: a string in the encoding of the current locale. On Windows + * this means the system codepage. * @len: the length of the string, or -1 if the string is - * nul-terminated. + * nul-terminated. * @bytes_read: location to store the number of bytes in the * input string that were successfully converted, or %NULL. * Even if the conversion was successful, this may be @@ -926,12 +1191,13 @@ strdup_len (const gchar *string, * input sequence. * @bytes_written: the number of bytes stored in the output buffer (not * including the terminating nul). - * @error: location to store the error occuring, or %NULL to ignore + * @error: location to store the error occurring, or %NULL to ignore * errors. Any of the errors in #GConvertError may occur. * * Converts a string which is in the encoding used for strings by * the C runtime (usually the same as that used by the operating - * system) in the current locale into a UTF-8 string. + * system) in the current locale into a + * UTF-8 string. * * Return value: The converted string, or %NULL on an error. **/ @@ -955,7 +1221,7 @@ g_locale_to_utf8 (const gchar *opsysstring, * g_locale_from_utf8: * @utf8string: a UTF-8 encoded string * @len: the length of the string, or -1 if the string is - * nul-terminated. + * nul-terminated. * @bytes_read: location to store the number of bytes in the * input string that were successfully converted, or %NULL. * Even if the conversion was successful, this may be @@ -966,12 +1232,13 @@ g_locale_to_utf8 (const gchar *opsysstring, * input sequence. * @bytes_written: the number of bytes stored in the output buffer (not * including the terminating nul). - * @error: location to store the error occuring, or %NULL to ignore + * @error: location to store the error occurring, or %NULL to ignore * errors. Any of the errors in #GConvertError may occur. * * Converts a string from UTF-8 to the encoding used for strings by * the C runtime (usually the same as that used by the operating - * system) in the current locale. + * system) in the current locale. On + * Windows this means the system codepage. * * Return value: The converted string, or %NULL on an error. **/ @@ -998,7 +1265,7 @@ typedef struct _GFilenameCharsetCache GFilenameCharsetCache; struct _GFilenameCharsetCache { gboolean is_utf8; gchar *charset; - gchar *filename_charset; + gchar **filename_charsets; }; static void @@ -1006,45 +1273,55 @@ filename_charset_cache_free (gpointer data) { GFilenameCharsetCache *cache = data; g_free (cache->charset); - g_free (cache->filename_charset); + g_strfreev (cache->filename_charsets); g_free (cache); } -/* - * get_filename_charset: - * @charset: return location for the name of the filename encoding +/** + * g_get_filename_charsets: + * @charsets: return location for the %NULL-terminated list of encoding names * - * Determines the preferred character set used for filenames by - * consulting the environment variables G_FILENAME_ENCODING and - * G_BROKEN_FILENAMES. + * Determines the preferred character sets used for filenames. + * The first character set from the @charsets is the filename encoding, the + * subsequent character sets are used when trying to generate a displayable + * representation of a filename, see g_filename_display_name(). * - * G_FILENAME_ENCODING may be set to a comma-separated list of character - * set names. The special token "@locale" is taken to mean the character set - * for the current locale. The first character set from the list is taken - * as the filename encoding. - * If G_FILENAME_ENCODING is not set, but G_BROKEN_FILENAMES is, the - * character set of the current locale is taken as the filename encoding. + * On Unix, the character sets are determined by consulting the + * environment variables G_FILENAME_ENCODING and + * G_BROKEN_FILENAMES. On Windows, the character set + * used in the GLib API is always UTF-8 and said environment variables + * have no effect. * - * The returned @charset belongs to GLib and must not be freed. + * G_FILENAME_ENCODING may be set to a comma-separated list + * of character set names. The special token "@locale" is taken to + * mean the character set for the current + * locale. If G_FILENAME_ENCODING is not set, but + * G_BROKEN_FILENAMES is, the character set of the current + * locale is taken as the filename encoding. If neither environment variable + * is set, UTF-8 is taken as the filename encoding, but the character + * set of the current locale is also put in the list of encodings. + * + * The returned @charsets belong to GLib and must not be freed. * * Note that on Unix, regardless of the locale character set or - * G_FILENAME_ENCODING value, the actual file names present on a - * system might be in any random encoding or just gibberish. + * G_FILENAME_ENCODING value, the actual file names present + * on a system might be in any random encoding or just gibberish. * - * Return value: %TRUE - * if the charset used for filename is UTF-8. + * Return value: %TRUE if the filename encoding is UTF-8. + * + * Since: 2.6 */ -static gboolean -get_filename_charset (const gchar **filename_charset) +gboolean +g_get_filename_charsets (const gchar ***filename_charsets) { - static GStaticPrivate cache_private = G_STATIC_PRIVATE_INIT; - GFilenameCharsetCache *cache = g_static_private_get (&cache_private); + static GPrivate cache_private = G_PRIVATE_INIT (filename_charset_cache_free); + GFilenameCharsetCache *cache = g_private_get (&cache_private); const gchar *charset; - + if (!cache) { cache = g_new0 (GFilenameCharsetCache, 1); - g_static_private_set (&cache_private, cache, filename_charset_cache_free); + g_private_set (&cache_private, cache); } g_get_charset (&charset); @@ -1052,93 +1329,101 @@ get_filename_charset (const gchar **filename_charset) if (!(cache->charset && strcmp (cache->charset, charset) == 0)) { const gchar *new_charset; - gchar *p, *q; + gchar *p; + gint i; g_free (cache->charset); - g_free (cache->filename_charset); + g_strfreev (cache->filename_charsets); cache->charset = g_strdup (charset); p = getenv ("G_FILENAME_ENCODING"); - if (p != NULL) + if (p != NULL && p[0] != '\0') { - q = strchr (p, ','); - if (!q) - q = p + strlen (p); + cache->filename_charsets = g_strsplit (p, ",", 0); + cache->is_utf8 = (strcmp (cache->filename_charsets[0], "UTF-8") == 0); - if (strncmp ("@locale", p, q - p) == 0) - { - cache->is_utf8 = g_get_charset (&new_charset); - cache->filename_charset = g_strdup (new_charset); - } - else + for (i = 0; cache->filename_charsets[i]; i++) { - cache->filename_charset = g_strndup (p, q - p); - cache->is_utf8 = (strcmp (cache->filename_charset, "UTF-8") == 0); + if (strcmp ("@locale", cache->filename_charsets[i]) == 0) + { + g_get_charset (&new_charset); + g_free (cache->filename_charsets[i]); + cache->filename_charsets[i] = g_strdup (new_charset); + } } } else if (getenv ("G_BROKEN_FILENAMES") != NULL) { + cache->filename_charsets = g_new0 (gchar *, 2); cache->is_utf8 = g_get_charset (&new_charset); - cache->filename_charset = g_strdup (new_charset); + cache->filename_charsets[0] = g_strdup (new_charset); } else { - cache->filename_charset = g_strdup ("UTF-8"); + cache->filename_charsets = g_new0 (gchar *, 3); cache->is_utf8 = TRUE; + cache->filename_charsets[0] = g_strdup ("UTF-8"); + if (!g_get_charset (&new_charset)) + cache->filename_charsets[1] = g_strdup (new_charset); } } - if (filename_charset) - *filename_charset = cache->filename_charset; + if (filename_charsets) + *filename_charsets = (const gchar **)cache->filename_charsets; return cache->is_utf8; } #else /* G_PLATFORM_WIN32 */ -static gboolean -get_filename_charset (const gchar **filename_charset) +gboolean +g_get_filename_charsets (const gchar ***filename_charsets) { + static const gchar *charsets[] = { + "UTF-8", + NULL + }; + #ifdef G_OS_WIN32 /* On Windows GLib pretends that the filename charset is UTF-8 */ - if (filename_charset) - *filename_charset = "UTF-8"; + if (filename_charsets) + *filename_charsets = charsets; + return TRUE; #else - /* Cygwin works like before */ - g_get_charset (filename_charset); - return FALSE; -#endif -} + gboolean result; -#ifdef G_OS_WIN32 + /* Cygwin works like before */ + result = g_get_charset (&(charsets[0])); -static gboolean -old_get_filename_charset (const gchar **filename_charset) -{ - g_get_charset (filename_charset); - return FALSE; -} + if (filename_charsets) + *filename_charsets = charsets; + return result; #endif +} #endif /* G_PLATFORM_WIN32 */ -/* This is called from g_thread_init(). It's used to - * initialize some static data in a threadsafe way. - */ -void -_g_convert_thread_init (void) +static gboolean +get_filename_charset (const gchar **filename_charset) { - const gchar *dummy; - (void) get_filename_charset (&dummy); + const gchar **charsets; + gboolean is_utf8; + + is_utf8 = g_get_filename_charsets (&charsets); + + if (filename_charset) + *filename_charset = charsets[0]; + + return is_utf8; } /** * g_filename_to_utf8: * @opsysstring: a string in the encoding for filenames * @len: the length of the string, or -1 if the string is - * nul-terminated. + * nul-terminated. * @bytes_read: location to store the number of bytes in the * input string that were successfully converted, or %NULL. * Even if the conversion was successful, this may be @@ -1149,11 +1434,13 @@ _g_convert_thread_init (void) * input sequence. * @bytes_written: the number of bytes stored in the output buffer (not * including the terminating nul). - * @error: location to store the error occuring, or %NULL to ignore + * @error: location to store the error occurring, or %NULL to ignore * errors. Any of the errors in #GConvertError may occur. * - * Converts a string which is in the encoding used for filenames - * into a UTF-8 string. + * Converts a string which is in the encoding used by GLib for + * filenames into a UTF-8 string. Note that on Windows GLib uses UTF-8 + * for filenames; on other platforms, this function indirectly depends on + * the current locale. * * Return value: The converted string, or %NULL on an error. **/ @@ -1166,6 +1453,8 @@ g_filename_to_utf8 (const gchar *opsysstring, { const gchar *charset; + g_return_val_if_fail (opsysstring != NULL, NULL); + if (get_filename_charset (&charset)) return strdup_len (opsysstring, len, bytes_read, bytes_written, error); else @@ -1173,11 +1462,14 @@ g_filename_to_utf8 (const gchar *opsysstring, "UTF-8", charset, bytes_read, bytes_written, error); } -#ifdef G_OS_WIN32 +#if defined (G_OS_WIN32) && !defined (_WIN64) #undef g_filename_to_utf8 -/* Binary compatibility version. Not for newly compiled code. */ +/* Binary compatibility version. Not for newly compiled code. Also not needed for + * 64-bit versions as there should be no old deployed binaries that would use + * the old versions. + */ gchar* g_filename_to_utf8 (const gchar *opsysstring, @@ -1188,7 +1480,9 @@ g_filename_to_utf8 (const gchar *opsysstring, { const gchar *charset; - if (old_get_filename_charset (&charset)) + g_return_val_if_fail (opsysstring != NULL, NULL); + + if (g_get_charset (&charset)) return strdup_len (opsysstring, len, bytes_read, bytes_written, error); else return g_convert (opsysstring, len, @@ -1212,10 +1506,13 @@ g_filename_to_utf8 (const gchar *opsysstring, * input sequence. * @bytes_written: the number of bytes stored in the output buffer (not * including the terminating nul). - * @error: location to store the error occuring, or %NULL to ignore + * @error: location to store the error occurring, or %NULL to ignore * errors. Any of the errors in #GConvertError may occur. * - * Converts a string from UTF-8 to the encoding used for filenames. + * Converts a string from UTF-8 to the encoding GLib uses for + * filenames. Note that on Windows GLib uses UTF-8 for filenames; + * on other platforms, this function indirectly depends on the + * current locale. * * Return value: The converted string, or %NULL on an error. **/ @@ -1235,7 +1532,7 @@ g_filename_from_utf8 (const gchar *utf8string, charset, "UTF-8", bytes_read, bytes_written, error); } -#ifdef G_OS_WIN32 +#if defined (G_OS_WIN32) && !defined (_WIN64) #undef g_filename_from_utf8 @@ -1250,7 +1547,7 @@ g_filename_from_utf8 (const gchar *utf8string, { const gchar *charset; - if (old_get_filename_charset (&charset)) + if (g_get_charset (&charset)) return strdup_len (utf8string, len, bytes_read, bytes_written, error); else return g_convert (utf8string, len, @@ -1537,17 +1834,17 @@ hostname_validate (const char *hostname) /** * g_filename_from_uri: * @uri: a uri describing a filename (escaped, encoded in ASCII). - * @hostname: Location to store hostname for the URI, or %NULL. + * @hostname: (out) (allow-none): Location to store hostname for the URI, or %NULL. * If there is no hostname in the URI, %NULL will be * stored in this location. - * @error: location to store the error occuring, or %NULL to ignore + * @error: location to store the error occurring, or %NULL to ignore * errors. Any of the errors in #GConvertError may occur. * * Converts an escaped ASCII-encoded URI to a local filename in the * encoding used for filenames. * - * Return value: a newly-allocated string holding the resulting - * filename, or %NULL on an error. + * Return value: (type filename): a newly-allocated string holding + * the resulting filename, or %NULL on an error. **/ gchar * g_filename_from_uri (const gchar *uri, @@ -1670,23 +1967,48 @@ g_filename_from_uri (const gchar *uri, return result; } +#if defined (G_OS_WIN32) && !defined (_WIN64) + +#undef g_filename_from_uri + +gchar * +g_filename_from_uri (const gchar *uri, + gchar **hostname, + GError **error) +{ + gchar *utf8_filename; + gchar *retval = NULL; + + utf8_filename = g_filename_from_uri_utf8 (uri, hostname, error); + if (utf8_filename) + { + retval = g_locale_from_utf8 (utf8_filename, -1, NULL, NULL, error); + g_free (utf8_filename); + } + return retval; +} + +#endif + /** * g_filename_to_uri: - * @filename: an absolute filename specified in the encoding - * used for filenames by the operating system. - * @hostname: A UTF-8 encoded hostname, or %NULL for none. - * @error: location to store the error occuring, or %NULL to ignore + * @filename: an absolute filename specified in the GLib file name encoding, + * which is the on-disk file name bytes on Unix, and UTF-8 on + * Windows + * @hostname: (allow-none): A UTF-8 encoded hostname, or %NULL for none. + * @error: location to store the error occurring, or %NULL to ignore * errors. Any of the errors in #GConvertError may occur. * - * Converts an absolute filename to an escaped ASCII-encoded URI. + * Converts an absolute filename to an escaped ASCII-encoded URI, with the path + * component following Section 3.3. of RFC 2396. * * Return value: a newly-allocated string holding the resulting * URI, or %NULL on an error. **/ gchar * -g_filename_to_uri (const gchar *filename, - const gchar *hostname, - GError **error) +g_filename_to_uri (const gchar *filename, + const gchar *hostname, + GError **error) { char *escaped_uri; @@ -1704,8 +2026,8 @@ g_filename_to_uri (const gchar *filename, !(g_utf8_validate (hostname, -1, NULL) && hostname_validate (hostname))) { - g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE, - _("Invalid hostname")); + g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE, + _("Invalid hostname")); return NULL; } @@ -1720,6 +2042,31 @@ g_filename_to_uri (const gchar *filename, return escaped_uri; } +#if defined (G_OS_WIN32) && !defined (_WIN64) + +#undef g_filename_to_uri + +gchar * +g_filename_to_uri (const gchar *filename, + const gchar *hostname, + GError **error) +{ + gchar *utf8_filename; + gchar *retval = NULL; + + utf8_filename = g_locale_to_utf8 (filename, -1, NULL, NULL, error); + + if (utf8_filename) + { + retval = g_filename_to_uri_utf8 (utf8_filename, hostname, error); + g_free (utf8_filename); + } + + return retval; +} + +#endif + /** * g_uri_list_extract_uris: * @uri_list: an URI list @@ -1728,9 +2075,9 @@ g_filename_to_uri (const gchar *filename, * mime type defined in RFC 2483 into individual URIs, * discarding any comments. The URIs are not validated. * - * Returns: a newly allocated %NULL-terminated list of - * strings holding the individual URIs. The array should - * be freed with g_strfreev(). + * Returns: (transfer full): a newly allocated %NULL-terminated list + * of strings holding the individual URIs. The array should be freed + * with g_strfreev(). * * Since: 2.6 */ @@ -1792,3 +2139,108 @@ g_uri_list_extract_uris (const gchar *uri_list) return result; } + +/** + * g_filename_display_basename: + * @filename: an absolute pathname in the GLib file name encoding + * + * Returns the display basename for the particular filename, guaranteed + * to be valid UTF-8. The display name might not be identical to the filename, + * for instance there might be problems converting it to UTF-8, and some files + * can be translated in the display. + * + * If GLib cannot make sense of the encoding of @filename, as a last resort it + * replaces unknown characters with U+FFFD, the Unicode replacement character. + * You can search the result for the UTF-8 encoding of this character (which is + * "\357\277\275" in octal notation) to find out if @filename was in an invalid + * encoding. + * + * You must pass the whole absolute pathname to this functions so that + * translation of well known locations can be done. + * + * This function is preferred over g_filename_display_name() if you know the + * whole path, as it allows translation. + * + * Return value: a newly allocated string containing + * a rendition of the basename of the filename in valid UTF-8 + * + * Since: 2.6 + **/ +gchar * +g_filename_display_basename (const gchar *filename) +{ + char *basename; + char *display_name; + + g_return_val_if_fail (filename != NULL, NULL); + + basename = g_path_get_basename (filename); + display_name = g_filename_display_name (basename); + g_free (basename); + return display_name; +} + +/** + * g_filename_display_name: + * @filename: a pathname hopefully in the GLib file name encoding + * + * Converts a filename into a valid UTF-8 string. The conversion is + * not necessarily reversible, so you should keep the original around + * and use the return value of this function only for display purposes. + * Unlike g_filename_to_utf8(), the result is guaranteed to be non-%NULL + * even if the filename actually isn't in the GLib file name encoding. + * + * If GLib cannot make sense of the encoding of @filename, as a last resort it + * replaces unknown characters with U+FFFD, the Unicode replacement character. + * You can search the result for the UTF-8 encoding of this character (which is + * "\357\277\275" in octal notation) to find out if @filename was in an invalid + * encoding. + * + * If you know the whole pathname of the file you should use + * g_filename_display_basename(), since that allows location-based + * translation of filenames. + * + * Return value: a newly allocated string containing + * a rendition of the filename in valid UTF-8 + * + * Since: 2.6 + **/ +gchar * +g_filename_display_name (const gchar *filename) +{ + gint i; + const gchar **charsets; + gchar *display_name = NULL; + gboolean is_utf8; + + is_utf8 = g_get_filename_charsets (&charsets); + + if (is_utf8) + { + if (g_utf8_validate (filename, -1, NULL)) + display_name = g_strdup (filename); + } + + if (!display_name) + { + /* Try to convert from the filename charsets to UTF-8. + * Skip the first charset if it is UTF-8. + */ + for (i = is_utf8 ? 1 : 0; charsets[i]; i++) + { + display_name = g_convert (filename, -1, "UTF-8", charsets[i], + NULL, NULL, NULL); + + if (display_name) + break; + } + } + + /* if all conversions failed, we replace invalid UTF-8 + * by a question mark + */ + if (!display_name) + display_name = _g_utf8_make_valid (filename); + + return display_name; +}