1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8; -*- */
4 * Michael Zucchi <notzed@ximian.com>
5 * Jeffrey Stedfast <fejj@ximian.com>
6 * Dan Winship <danw@ximian.com>
8 * Copyright (C) 1999-2008 Novell, Inc. (www.novell.com)
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of version 2 of the GNU Lesser General Public
12 * License as published by the Free Software Foundation.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
39 #include "camel-iconv.h"
42 * if you want to build the charset map, compile this with something like:
43 * gcc -DBUILD_MAP camel-charset-map.c `pkg-config --cflags --libs glib-2.0`
44 * (plus any -I/-L/-l flags you need for iconv), then run it as
45 * ./a.out > camel-charset-map-private.h
47 * Note that the big-endian variant isn't tested...
49 * The tables genereated work like this:
51 * An indirect array for each page of unicode character
52 * Each array element has an indirect pointer to one of the bytes of
53 * the generated bitmask.
59 gchar *name; /* charset name */
60 gint multibyte; /* charset type */
61 guint bit; /* assigned bit */
63 /* These are the 8bit character sets (other than iso-8859-1,
64 * which is special-cased) which are supported by both other
65 * mailers and the GNOME environment. Note that the order
66 * they're listed in is the order they'll be tried in, so put
67 * the more-popular ones first.
69 { "iso-8859-2", 0, 0 }, /* Central/Eastern European */
70 { "iso-8859-4", 0, 0 }, /* Baltic */
71 { "koi8-r", 0, 0 }, /* Russian */
72 { "koi8-u", 0, 0 }, /* Ukranian */
73 { "iso-8859-5", 0, 0 }, /* Least-popular Russian encoding */
74 { "iso-8859-6", 0, 0 }, /* Arabic */
75 { "iso-8859-7", 0, 0 }, /* Greek */
76 { "iso-8859-8", 0, 0 }, /* Hebrew; Visual */
77 { "iso-8859-9", 0, 0 }, /* Turkish */
78 { "iso-8859-13", 0, 0 }, /* Baltic again */
79 { "iso-8859-15", 0, 0 }, /* New-and-improved iso-8859-1, but most
80 * programs that support this support UTF8
82 { "windows-1251", 0, 0 }, /* Russian */
84 /* These are the multibyte character sets which are commonly
85 * supported by other mail clients. Note: order for multibyte
86 * charsets does not affect priority unlike the 8bit charsets
89 { "iso-2022-jp", 1, 0 }, /* Japanese designed for use over the Net */
90 { "Shift-JIS", 1, 0 }, /* Japanese as used by Windows and MacOS systems */
91 { "euc-jp", 1, 0 }, /* Japanese traditionally used on Unix systems */
92 { "euc-kr", 1, 0 }, /* Korean */
93 { "iso-2022-kr", 1, 0 }, /* Korean (less popular than euc-kr) */
94 { "gb2312", 1, 0 }, /* Simplified Chinese */
95 { "Big5", 1, 0 }, /* Traditional Chinese */
100 guint encoding_map[256 * 256];
102 #if G_BYTE_ORDER == G_BIG_ENDIAN
103 #define UCS "UCS-4BE"
105 #define UCS "UCS-4LE"
109 block_hash (gconstpointer v)
115 for (i = 0; i < 256; i++)
116 h = (h << 5) - h + *p++;
122 block_equal (gconstpointer v1,
125 return !memcmp (v1, v2, 256);
128 gint main (gint argc, gchar **argv)
130 guchar *block = NULL;
132 GHashTable *table_hash;
133 gsize inleft, outleft;
134 gchar *inbuf, *outbuf;
141 /* dont count the terminator */
142 bytes = (G_N_ELEMENTS (tables) + 7 - 1) / 8;
143 g_assert (bytes <= 4);
145 for (i = 0; i < 128; i++)
148 for (j = 0; tables[j].name && !tables[j].multibyte; j++) {
149 cd = iconv_open (UCS, tables[j].name);
151 inleft = sizeof (in);
152 outbuf = (gchar *) out;
153 outleft = sizeof (out);
154 while (iconv (cd, &inbuf, &inleft, &outbuf, &outleft) == -1) {
155 if (errno == EILSEQ) {
160 "iconv (%s->UCS4, ..., %d, ..., %d): %s",
161 tables[j].name, inleft, outleft,
168 for (i = 0; i < 128 - outleft / 4; i++) {
169 encoding_map[i] |= bit;
170 encoding_map[out[i]] |= bit;
177 /* Mutibyte tables */
178 for (; tables[j].name && tables[j].multibyte; j++) {
179 cd = iconv_open (tables[j].name, UCS);
180 if (cd == (iconv_t) -1)
183 for (c = 128, i = 0; c < 65535 && i < 65535; c++) {
184 inbuf = (gchar *) &c;
187 outleft = sizeof (in);
189 if (iconv (cd, &inbuf, &inleft, &outbuf, &outleft) != (gsize) -1) {
190 /* this is a legal character in charset table[j].name */
191 iconv (cd, NULL, NULL, &outbuf, &outleft);
192 encoding_map[i++] |= bit;
193 encoding_map[c] |= bit;
195 /* reset the iconv descriptor */
196 iconv (cd, NULL, NULL, NULL, NULL);
206 printf ("/* This file is automatically generated: DO NOT EDIT */\n\n");
208 table_hash = g_hash_table_new_full (block_hash, block_equal, g_free, g_free);
210 for (i = 0; i < 256; i++) {
211 for (k = 0; k < bytes; k++) {
212 gchar name[32], *alias;
213 gint has_bits = FALSE;
216 /* we reuse malloc'd blocks that are not added to the
217 * hash table to avoid unnecessary malloc/free's */
218 block = g_malloc (256);
221 for (j = 0; j < 256; j++) {
222 if ((block[j] = (encoding_map[i * 256 + j] >> (k * 8)) & 0xff))
229 sprintf (name, "m%02x%x", i, k);
231 if ((alias = g_hash_table_lookup (table_hash, block))) {
232 /* this block is identical to an earlier block, just alias it */
233 printf ("#define %s %s\n\n", name, alias);
235 /* unique block, dump it */
236 g_hash_table_insert (table_hash, block, g_strdup (name));
238 printf ("static guchar %s[256] = {\n\t", name);
239 for (j = 0; j < 256; j++) {
240 printf ("0x%02x, ", block[j]);
241 if (((j + 1) & 7) == 0 && j < 255)
246 /* force the next loop to malloc a new block */
252 g_hash_table_destroy (table_hash);
255 printf ("static const struct {\n");
256 for (k = 0; k < bytes; k++)
257 printf ("\tconst guchar *bits%d;\n", k);
259 printf ("} camel_charmap[256] = {\n\t");
260 for (i = 0; i < 256; i++) {
262 for (k = 0; k < bytes; k++) {
263 for (j = 0; j < 256; j++) {
264 if ((encoding_map[i * 256 + j] & (0xff << (k * 8))) != 0)
269 printf ("m%02x%x, ", i, k);
275 if (((i + 1) & 3) == 0 && i < 255)
281 "static const struct {\n"
282 "\tconst gchar *name;\n"
284 "} camel_charinfo[] = {\n");
285 for (j = 0; tables[j].name; j++)
287 "\t{ \"%s\", 0x%08x },\n",
288 tables[j].name, tables[j].bit);
291 printf ("#define charset_mask(x) \\\n");
292 for (k = 0; k < bytes; k++) {
299 "(camel_charmap[(x) >> 8].bits%d ? "
300 "camel_charmap[(x) >> 8].bits%d[(x) & 0xff] << %d : 0)",
313 #include "camel-charset-map.h"
314 #include "camel-charset-map-private.h"
315 #include "camel-utf8.h"
318 camel_charset_init (CamelCharset *c)
320 c->mask = (guint) ~0;
325 camel_charset_step (CamelCharset *cc,
329 const guchar *inptr = (const guchar *) in;
330 const guchar *inend = inptr + len;
338 /* check what charset a given string will fit in */
339 while ((c = camel_utf8_getc_limit (&inptr, inend)) != 0xffff) {
341 mask &= charset_mask (c);
343 if (c >= 128 && c < 256)
344 level = MAX (level, 1);
358 /* gets the best charset from the mask of chars in it */
360 camel_charset_best_mask (guint mask)
362 const gchar *locale_lang, *lang;
365 locale_lang = camel_iconv_locale_language ();
366 for (i = 0; i < G_N_ELEMENTS (camel_charinfo); i++) {
367 if (camel_charinfo[i].bit & mask) {
368 lang = camel_iconv_charset_language (camel_charinfo[i].name);
370 if (!locale_lang || (lang && !strncmp (locale_lang, lang, 2)))
371 return camel_charinfo[i].name;
379 camel_charset_best_name (CamelCharset *charset)
381 if (charset->level == 1)
383 else if (charset->level == 2)
384 return camel_charset_best_mask (charset->mask);
389 /* finds the minimum charset for this string NULL means US-ASCII */
391 camel_charset_best (const gchar *in,
394 CamelCharset charset;
396 camel_charset_init (&charset);
397 camel_charset_step (&charset, in, len);
398 return camel_charset_best_name (&charset);
402 * camel_charset_iso_to_windows:
403 * @isocharset: a canonicalised ISO charset
405 * Returns: the equivalent Windows charset.
408 camel_charset_iso_to_windows (const gchar *isocharset)
410 /* According to http://czyborra.com/charsets/codepages.html,
411 * the charset mapping is as follows:
413 * us-ascii maps to windows-cp1252
414 * iso-8859-1 maps to windows-cp1252
415 * iso-8859-2 maps to windows-cp1250
416 * iso-8859-3 maps to windows-cp????
417 * iso-8859-4 maps to windows-cp????
418 * iso-8859-5 maps to windows-cp1251
419 * iso-8859-6 maps to windows-cp1256
420 * iso-8859-7 maps to windows-cp1253
421 * iso-8859-8 maps to windows-cp1255
422 * iso-8859-9 maps to windows-cp1254
423 * iso-8859-10 maps to windows-cp????
424 * iso-8859-11 maps to windows-cp????
425 * iso-8859-12 maps to windows-cp????
426 * iso-8859-13 maps to windows-cp1257
429 * - I'm going to assume that since iso-8859-4 and
430 * iso-8859-13 are Baltic that it also maps to
434 if (!g_ascii_strcasecmp (isocharset, "iso-8859-1") || !g_ascii_strcasecmp (isocharset, "us-ascii"))
435 return "windows-cp1252";
436 else if (!g_ascii_strcasecmp (isocharset, "iso-8859-2"))
437 return "windows-cp1250";
438 else if (!g_ascii_strcasecmp (isocharset, "iso-8859-4"))
439 return "windows-cp1257";
440 else if (!g_ascii_strcasecmp (isocharset, "iso-8859-5"))
441 return "windows-cp1251";
442 else if (!g_ascii_strcasecmp (isocharset, "iso-8859-6"))
443 return "windows-cp1256";
444 else if (!g_ascii_strcasecmp (isocharset, "iso-8859-7"))
445 return "windows-cp1253";
446 else if (!g_ascii_strcasecmp (isocharset, "iso-8859-8"))
447 return "windows-cp1255";
448 else if (!g_ascii_strcasecmp (isocharset, "iso-8859-9"))
449 return "windows-cp1254";
450 else if (!g_ascii_strcasecmp (isocharset, "iso-8859-13"))
451 return "windows-cp1257";
456 #endif /* BUILD_MAP */