1 /* gcharset.c - Charset information
3 * Copyright (C) 2011 Red Hat, Inc.
5 * SPDX-License-Identifier: LGPL-2.1-or-later
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
24 #include "gcharsetprivate.h"
29 #include "gmessages.h"
30 #include "gstrfuncs.h"
32 #include "gthreadprivate.h"
37 #include "libcharset/libcharset.h"
42 #if (HAVE_LANGINFO_TIME_CODESET || HAVE_LANGINFO_CODESET)
48 #define WIN32_LEAN_AND_MEAN
52 G_LOCK_DEFINE_STATIC (aliases);
57 static GHashTable *alias_hash = NULL;
64 alias_hash = g_hash_table_new (g_str_hash, g_str_equal);
66 aliases = _g_locale_get_charset_aliases ();
67 while (*aliases != '\0')
69 const char *canonical;
71 const char **alias_array;
75 aliases += strlen (aliases) + 1;
77 aliases += strlen (aliases) + 1;
79 alias_array = g_hash_table_lookup (alias_hash, canonical);
82 while (alias_array[count])
86 alias_array = g_renew (const char *, alias_array, count + 2);
87 alias_array[count] = alias;
88 alias_array[count + 1] = NULL;
90 g_hash_table_insert (alias_hash, (char *)canonical, alias_array);
99 /* As an abuse of the alias table, the following routines gets
100 * the charsets that are aliases for the canonical name.
103 _g_charset_get_aliases (const char *canonical_name)
105 GHashTable *alias_hash = get_alias_hash ();
107 return g_hash_table_lookup (alias_hash, canonical_name);
111 g_utf8_get_charset_internal (const char *raw_data,
114 /* Allow CHARSET to override the charset of any locale category. Users should
115 * probably never be setting this — instead, just add the charset after a `.`
116 * in `LANGUAGE`/`LC_ALL`/`LC_*`/`LANG`. I can’t find any reference (in
117 * `git log`, code comments, or man pages) to this environment variable being
118 * standardised or documented or even used anywhere outside GLib. Perhaps it
119 * should eventually be removed. */
120 const char *charset = g_getenv ("CHARSET");
122 if (charset && *charset)
126 if (charset && strstr (charset, "UTF-8"))
132 /* The libcharset code tries to be thread-safe without
133 * a lock, but has a memory leak and a missing memory
134 * barrier, so we lock for it
137 charset = _g_locale_charset_unalias (raw_data);
140 if (charset && *charset)
144 if (charset && strstr (charset, "UTF-8"))
150 /* Assume this for compatibility at present. */
156 typedef struct _GCharsetCache GCharsetCache;
158 struct _GCharsetCache {
165 charset_cache_free (gpointer data)
167 GCharsetCache *cache = data;
169 g_free (cache->charset);
175 * @charset: (out) (optional) (transfer none): return location for character set
178 * Obtains the character set for the [current locale][setlocale]; you
179 * might use this character set as an argument to g_convert(), to convert
180 * from the current locale's encoding to some other encoding. (Frequently
181 * g_locale_to_utf8() and g_locale_from_utf8() are nice shortcuts, though.)
183 * On Windows the character set returned by this function is the
184 * so-called system default ANSI code-page. That is the character set
185 * used by the "narrow" versions of C library and Win32 functions that
186 * handle file names. It might be different from the character set
187 * used by the C library's current locale.
189 * On Linux, the character set is found by consulting nl_langinfo() if
190 * available. If not, the environment variables `LC_ALL`, `LC_CTYPE`, `LANG`
191 * and `CHARSET` are queried in order. nl_langinfo() returns the C locale if
192 * no locale has been loaded by setlocale().
194 * The return value is %TRUE if the locale's encoding is UTF-8, in that
195 * case you can perhaps avoid calling g_convert().
197 * The string returned in @charset is not allocated, and should not be
200 * Returns: %TRUE if the returned charset is UTF-8
203 g_get_charset (const char **charset)
205 static GPrivate cache_private = G_PRIVATE_INIT (charset_cache_free);
206 GCharsetCache *cache = g_private_get (&cache_private);
210 cache = g_private_set_alloc0 (&cache_private, sizeof (GCharsetCache));
213 raw = _g_locale_charset_raw ();
216 if (cache->raw == NULL || strcmp (cache->raw, raw) != 0)
218 const gchar *new_charset;
221 g_free (cache->charset);
222 cache->raw = g_strdup (raw);
223 cache->is_utf8 = g_utf8_get_charset_internal (raw, &new_charset);
224 cache->charset = g_strdup (new_charset);
228 *charset = cache->charset;
230 return cache->is_utf8;
234 * Do the same as g_get_charset() but it temporarily set locale (LC_ALL to
235 * LC_TIME) to correctly check for charset about time conversion relatives.
237 * Returns: %TRUE if the returned charset is UTF-8
240 _g_get_time_charset (const char **charset)
242 static GPrivate cache_private = G_PRIVATE_INIT (charset_cache_free);
243 GCharsetCache *cache = g_private_get (&cache_private);
247 cache = g_private_set_alloc0 (&cache_private, sizeof (GCharsetCache));
249 #ifdef HAVE_LANGINFO_TIME_CODESET
250 raw = nl_langinfo (_NL_TIME_CODESET);
253 raw = _g_locale_charset_raw ();
257 if (cache->raw == NULL || strcmp (cache->raw, raw) != 0)
259 const gchar *new_charset;
262 g_free (cache->charset);
263 cache->raw = g_strdup (raw);
264 cache->is_utf8 = g_utf8_get_charset_internal (raw, &new_charset);
265 cache->charset = g_strdup (new_charset);
269 *charset = cache->charset;
271 return cache->is_utf8;
274 * Do the same as g_get_charset() but it temporarily set locale (LC_ALL to
275 * LC_CTYPE) to correctly check for charset about CTYPE conversion relatives.
277 * Returns: %TRUE if the returned charset is UTF-8
280 _g_get_ctype_charset (const char **charset)
282 static GPrivate cache_private = G_PRIVATE_INIT (charset_cache_free);
283 GCharsetCache *cache = g_private_get (&cache_private);
287 cache = g_private_set_alloc0 (&cache_private, sizeof (GCharsetCache));
289 #ifdef HAVE_LANGINFO_CODESET
290 raw = nl_langinfo (CODESET);
293 raw = _g_locale_charset_raw ();
297 if (cache->raw == NULL || strcmp (cache->raw, raw) != 0)
299 const gchar *new_charset;
302 g_free (cache->charset);
303 cache->raw = g_strdup (raw);
304 cache->is_utf8 = g_utf8_get_charset_internal (raw, &new_charset);
305 cache->charset = g_strdup (new_charset);
309 *charset = cache->charset;
311 return cache->is_utf8;
317 * Gets the character set for the current locale.
319 * Returns: a newly allocated string containing the name
320 * of the character set. This string must be freed with g_free().
325 const gchar *charset;
327 g_get_charset (&charset);
329 return g_strdup (charset);
333 * g_get_console_charset:
334 * @charset: (out) (optional) (transfer none): return location for character set
337 * Obtains the character set used by the console attached to the process,
338 * which is suitable for printing output to the terminal.
340 * Usually this matches the result returned by g_get_charset(), but in
341 * environments where the locale's character set does not match the encoding
342 * of the console this function tries to guess a more suitable value instead.
344 * On Windows the character set returned by this function is the
345 * output code page used by the console associated with the calling process.
346 * If the codepage can't be determined (for example because there is no
347 * console attached) UTF-8 is assumed.
349 * The return value is %TRUE if the locale's encoding is UTF-8, in that
350 * case you can perhaps avoid calling g_convert().
352 * The string returned in @charset is not allocated, and should not be
355 * Returns: %TRUE if the returned charset is UTF-8
360 g_get_console_charset (const char **charset)
363 static GPrivate cache_private = G_PRIVATE_INIT (charset_cache_free);
364 GCharsetCache *cache = g_private_get (&cache_private);
367 char buf[2 + 20 + 1]; /* "CP" + G_MAXUINT64 (to be safe) in decimal form (20 bytes) + "\0" */
368 const gchar *raw = NULL;
371 cache = g_private_set_alloc0 (&cache_private, sizeof (GCharsetCache));
373 /* first try to query $LANG (works for Cygwin/MSYS/MSYS2 and others using mintty) */
374 locale = g_getenv ("LANG");
375 if (locale != NULL && locale[0] != '\0')
377 /* If the locale name contains an encoding after the dot, return it. */
378 const char *dot = strchr (locale, '.');
382 const char *modifier;
385 /* Look for the possible @... trailer and remove it, if any. */
386 modifier = strchr (dot, '@');
387 if (modifier == NULL)
389 else if ((gsize) (modifier - dot) < sizeof (buf))
391 memcpy (buf, dot, modifier - dot);
392 buf[modifier - dot] = '\0';
397 /* next try querying console codepage using native win32 API */
400 cp = GetConsoleOutputCP ();
403 sprintf (buf, "CP%u", cp);
406 else if (GetLastError () != ERROR_INVALID_HANDLE)
408 gchar *emsg = g_win32_error_message (GetLastError ());
409 g_warning ("Failed to determine console output code page: %s. "
410 "Falling back to UTF-8", emsg);
414 /* fall-back to UTF-8 if the rest failed (it's a universal default) */
418 if (cache->raw == NULL || strcmp (cache->raw, raw) != 0)
420 const gchar *new_charset;
423 g_free (cache->charset);
424 cache->raw = g_strdup (raw);
425 cache->is_utf8 = g_utf8_get_charset_internal (raw, &new_charset);
426 cache->charset = g_strdup (new_charset);
430 *charset = cache->charset;
432 return cache->is_utf8;
434 /* assume the locale settings match the console encoding on non-Windows OSs */
435 return g_get_charset (charset);
441 /* read an alias file for the locales */
443 read_aliases (const gchar *file,
444 GHashTable *alias_table)
449 fp = fopen (file, "re");
452 while (fgets (buf, 256, fp))
458 /* Line is a comment */
459 if ((buf[0] == '#') || (buf[0] == '\0'))
462 /* Reads first column */
463 for (p = buf, q = NULL; *p; p++) {
464 if ((*p == '\t') || (*p == ' ') || (*p == ':')) {
467 while ((*q == '\t') || (*q == ' ')) {
473 /* The line only had one column */
474 if (!q || *q == '\0')
477 /* Read second column */
478 for (p = q; *p; p++) {
479 if ((*p == '\t') || (*p == ' ')) {
485 /* Add to alias table if necessary */
486 if (!g_hash_table_lookup (alias_table, buf)) {
487 g_hash_table_insert (alias_table, g_strdup (buf), g_strdup (q));
496 unalias_lang (char *lang)
499 static GHashTable *alias_table = NULL;
503 if (g_once_init_enter (&alias_table))
505 GHashTable *table = g_hash_table_new (g_str_hash, g_str_equal);
506 read_aliases ("/usr/share/locale/locale.alias", table);
507 g_once_init_leave (&alias_table, table);
511 while ((p = g_hash_table_lookup (alias_table, lang)) && (strcmp (p, lang) != 0))
516 static gboolean said_before = FALSE;
518 g_warning ("Too many alias levels for a locale, "
519 "may indicate a loop");
528 /* Mask for components of locale spec. The ordering here is from
529 * least significant to most significant
533 COMPONENT_CODESET = 1 << 0,
534 COMPONENT_TERRITORY = 1 << 1,
535 COMPONENT_MODIFIER = 1 << 2
538 /* Break an X/Open style locale specification into components
541 explode_locale (const gchar *locale,
547 const gchar *uscore_pos;
549 const gchar *dot_pos;
553 uscore_pos = strchr (locale, '_');
554 dot_pos = strchr (uscore_pos ? uscore_pos : locale, '.');
555 at_pos = strchr (dot_pos ? dot_pos : (uscore_pos ? uscore_pos : locale), '@');
559 mask |= COMPONENT_MODIFIER;
560 *modifier = g_strdup (at_pos);
563 at_pos = locale + strlen (locale);
567 mask |= COMPONENT_CODESET;
568 *codeset = g_strndup (dot_pos, at_pos - dot_pos);
575 mask |= COMPONENT_TERRITORY;
576 *territory = g_strndup (uscore_pos, dot_pos - uscore_pos);
579 uscore_pos = dot_pos;
581 *language = g_strndup (locale, uscore_pos - locale);
587 * Compute all interesting variants for a given locale name -
588 * by stripping off different components of the value.
590 * For simplicity, we assume that the locale is in
591 * X/Open format: language[_territory][.codeset][@modifier]
593 * TODO: Extend this to handle the CEN format (see the GNUlibc docs)
594 * as well. We could just copy the code from glibc wholesale
595 * but it is big, ugly, and complicated, so I'm reluctant
596 * to do so when this should handle 99% of the time...
599 append_locale_variants (GPtrArray *array,
602 gchar *language = NULL;
603 gchar *territory = NULL;
604 gchar *codeset = NULL;
605 gchar *modifier = NULL;
610 g_return_if_fail (locale != NULL);
612 mask = explode_locale (locale, &language, &territory, &codeset, &modifier);
614 /* Iterate through all possible combinations, from least attractive
615 * to most attractive.
617 for (j = 0; j <= mask; ++j)
621 if ((i & ~mask) == 0)
623 gchar *val = g_strconcat (language,
624 (i & COMPONENT_TERRITORY) ? territory : "",
625 (i & COMPONENT_CODESET) ? codeset : "",
626 (i & COMPONENT_MODIFIER) ? modifier : "",
628 g_ptr_array_add (array, val);
633 if (mask & COMPONENT_CODESET)
635 if (mask & COMPONENT_TERRITORY)
637 if (mask & COMPONENT_MODIFIER)
642 * g_get_locale_variants:
643 * @locale: a locale identifier
645 * Returns a list of derived variants of @locale, which can be used to
646 * e.g. construct locale-dependent filenames or search paths. The returned
647 * list is sorted from most desirable to least desirable.
648 * This function handles territory, charset and extra locale modifiers. See
649 * [`setlocale(3)`](man:setlocale) for information about locales and their format.
651 * @locale itself is guaranteed to be returned in the output.
653 * For example, if @locale is `fr_BE`, then the returned list
654 * is `fr_BE`, `fr`. If @locale is `en_GB.UTF-8@euro`, then the returned list
655 * is `en_GB.UTF-8@euro`, `en_GB.UTF-8`, `en_GB@euro`, `en_GB`, `en.UTF-8@euro`,
656 * `en.UTF-8`, `en@euro`, `en`.
658 * If you need the list of variants for the current locale,
659 * use g_get_language_names().
661 * Returns: (transfer full) (array zero-terminated=1) (element-type utf8): a newly
662 * allocated array of newly allocated strings with the locale variants. Free with
668 g_get_locale_variants (const gchar *locale)
672 g_return_val_if_fail (locale != NULL, NULL);
674 array = g_ptr_array_sized_new (8);
675 append_locale_variants (array, locale);
676 g_ptr_array_add (array, NULL);
678 return (gchar **) g_ptr_array_free (array, FALSE);
681 /* The following is (partly) taken from the gettext package.
682 Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc. */
685 guess_category_value (const gchar *category_name)
689 /* The highest priority value is the 'LANGUAGE' environment
690 variable. This is a GNU extension. */
691 retval = g_getenv ("LANGUAGE");
692 if ((retval != NULL) && (retval[0] != '\0'))
695 /* 'LANGUAGE' is not set. So we have to proceed with the POSIX
696 methods of looking to 'LC_ALL', 'LC_xxx', and 'LANG'. On some
697 systems this can be done by the 'setlocale' function itself. */
699 /* Setting of LC_ALL overwrites all other. */
700 retval = g_getenv ("LC_ALL");
701 if ((retval != NULL) && (retval[0] != '\0'))
704 /* Next comes the name of the desired category. */
705 retval = g_getenv (category_name);
706 if ((retval != NULL) && (retval[0] != '\0'))
709 /* Last possibility is the LANG environment variable. */
710 retval = g_getenv ("LANG");
711 if ((retval != NULL) && (retval[0] != '\0'))
714 #ifdef G_PLATFORM_WIN32
715 /* g_win32_getlocale() first checks for LC_ALL, LC_MESSAGES and
716 * LANG, which we already did above. Oh well. The main point of
717 * calling g_win32_getlocale() is to get the thread's locale as used
718 * by Windows and the Microsoft C runtime (in the "English_United
719 * States" format) translated into the Unixish format.
722 char *locale = g_win32_getlocale ();
723 retval = g_intern_string (locale);
732 typedef struct _GLanguageNamesCache GLanguageNamesCache;
734 struct _GLanguageNamesCache {
736 gchar **language_names;
740 language_names_cache_free (gpointer data)
742 GLanguageNamesCache *cache = data;
743 g_free (cache->languages);
744 g_strfreev (cache->language_names);
749 * g_get_language_names:
751 * Computes a list of applicable locale names, which can be used to
752 * e.g. construct locale-dependent filenames or search paths. The returned
753 * list is sorted from most desirable to least desirable and always contains
754 * the default locale "C".
756 * For example, if LANGUAGE=de:en_US, then the returned list is
757 * "de", "en_US", "en", "C".
759 * This function consults the environment variables `LANGUAGE`, `LC_ALL`,
760 * `LC_MESSAGES` and `LANG` to find the list of locales specified by the
763 * Returns: (array zero-terminated=1) (transfer none): a %NULL-terminated array of strings owned by GLib
764 * that must not be modified or freed.
768 const gchar * const *
769 g_get_language_names (void)
771 return g_get_language_names_with_category ("LC_MESSAGES");
775 * g_get_language_names_with_category:
776 * @category_name: a locale category name
778 * Computes a list of applicable locale names with a locale category name,
779 * which can be used to construct the fallback locale-dependent filenames
780 * or search paths. The returned list is sorted from most desirable to
781 * least desirable and always contains the default locale "C".
783 * This function consults the environment variables `LANGUAGE`, `LC_ALL`,
784 * @category_name, and `LANG` to find the list of locales specified by the
787 * g_get_language_names() returns g_get_language_names_with_category("LC_MESSAGES").
789 * Returns: (array zero-terminated=1) (transfer none): a %NULL-terminated array of strings owned by
790 * the thread g_get_language_names_with_category was called from.
791 * It must not be modified or freed. It must be copied if planned to be used in another thread.
795 const gchar * const *
796 g_get_language_names_with_category (const gchar *category_name)
798 static GPrivate cache_private = G_PRIVATE_INIT ((void (*)(gpointer)) g_hash_table_unref);
799 GHashTable *cache = g_private_get (&cache_private);
800 const gchar *languages;
801 GLanguageNamesCache *name_cache;
803 g_return_val_if_fail (category_name != NULL, NULL);
807 cache = g_hash_table_new_full (g_str_hash, g_str_equal,
808 g_free, language_names_cache_free);
809 g_private_set (&cache_private, cache);
812 languages = guess_category_value (category_name);
816 name_cache = (GLanguageNamesCache *) g_hash_table_lookup (cache, category_name);
817 if (!(name_cache && name_cache->languages &&
818 strcmp (name_cache->languages, languages) == 0))
823 g_hash_table_remove (cache, category_name);
825 array = g_ptr_array_sized_new (8);
827 alist = g_strsplit (languages, ":", 0);
828 for (a = alist; *a; a++)
829 append_locale_variants (array, unalias_lang (*a));
831 g_ptr_array_add (array, g_strdup ("C"));
832 g_ptr_array_add (array, NULL);
834 name_cache = g_new0 (GLanguageNamesCache, 1);
835 name_cache->languages = g_strdup (languages);
836 name_cache->language_names = (gchar **) g_ptr_array_free (array, FALSE);
837 g_hash_table_insert (cache, g_strdup (category_name), name_cache);
840 return (const gchar * const *) name_cache->language_names;