1 /* Convert using charmaps and possibly iconv().
2 Copyright (C) 2001, 2005, 2006 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@redhat.com>, 2001.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License version 2 as
8 published by the Free Software Foundation.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software Foundation,
17 Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
31 #include "iconv_prog.h"
34 /* Prototypes for a few program-wide used functions. */
35 extern void *xmalloc (size_t __n);
36 extern void *xcalloc (size_t __n, size_t __s);
44 struct convtable *sub;
50 static inline struct convtable *
53 return (struct convtable *) xcalloc (1, sizeof (struct convtable));
58 is_term (struct convtable *tbl, unsigned int idx)
60 return tbl->term[idx / 8] & (1 << (idx % 8));
65 clear_term (struct convtable *tbl, unsigned int idx)
67 tbl->term[idx / 8] &= ~(1 << (idx % 8));
72 set_term (struct convtable *tbl, unsigned int idx)
74 tbl->term[idx / 8] |= 1 << (idx % 8);
78 /* Generate the conversion table. */
79 static struct convtable *use_from_charmap (struct charmap_t *from_charmap,
81 static struct convtable *use_to_charmap (const char *from_code,
82 struct charmap_t *to_charmap);
83 static struct convtable *use_both_charmaps (struct charmap_t *from_charmap,
84 struct charmap_t *to_charmap);
86 /* Prototypes for the functions doing the actual work. */
87 static int process_block (struct convtable *tbl, char *addr, size_t len,
89 static int process_fd (struct convtable *tbl, int fd, FILE *output);
90 static int process_file (struct convtable *tbl, FILE *input, FILE *output);
94 charmap_conversion (const char *from_code, struct charmap_t *from_charmap,
95 const char *to_code, struct charmap_t *to_charmap,
96 int argc, int remaining, char *argv[], FILE *output)
98 struct convtable *cvtbl;
99 int status = EXIT_SUCCESS;
101 /* We have three different cases to handle:
103 - both, from_charmap and to_charmap, are available. This means we
104 can assume that the symbolic names match and use them to create
107 - only from_charmap is available. In this case we can only hope that
108 the symbolic names used are of the <Uxxxx> form in which case we
109 can use a UCS4->"to_code" iconv() conversion for the second step.
111 - only to_charmap is available. This is similar, only that we would
112 use iconv() for the "to_code"->UCS4 conversion.
114 We first create a table which maps input bytes into output bytes.
115 Once this is done we can handle all three of the cases above
117 if (from_charmap != NULL)
119 if (to_charmap == NULL)
120 cvtbl = use_from_charmap (from_charmap, to_code);
122 cvtbl = use_both_charmaps (from_charmap, to_charmap);
126 assert (to_charmap != NULL);
127 cvtbl = use_to_charmap (from_code, to_charmap);
130 /* If we couldn't generate a table stop now. */
134 /* We can now start the conversion. */
135 if (remaining == argc)
137 if (process_file (cvtbl, stdin, output) != 0)
138 status = EXIT_FAILURE;
148 printf ("%s:\n", argv[remaining]);
149 if (strcmp (argv[remaining], "-") == 0)
153 fd = open (argv[remaining], O_RDONLY);
157 error (0, errno, _("cannot open input file `%s'"),
159 status = EXIT_FAILURE;
164 #ifdef _POSIX_MAPPED_FILES
165 /* We have possibilities for reading the input file. First try
166 to mmap() it since this will provide the fastest solution. */
167 if (fstat (fd, &st) == 0
168 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
169 fd, 0)) != MAP_FAILED))
171 /* Yes, we can use mmap(). The descriptor is not needed
174 error (EXIT_FAILURE, errno,
175 _("error while closing input `%s'"), argv[remaining]);
177 if (process_block (cvtbl, addr, st.st_size, output) < 0)
179 /* Something went wrong. */
180 status = EXIT_FAILURE;
182 /* We don't need the input data anymore. */
183 munmap ((void *) addr, st.st_size);
185 /* We cannot go on with producing output since it might
186 lead to problem because the last output might leave
187 the output stream in an undefined state. */
191 /* We don't need the input data anymore. */
192 munmap ((void *) addr, st.st_size);
195 #endif /* _POSIX_MAPPED_FILES */
197 /* Read the file in pieces. */
198 if (process_fd (cvtbl, fd, output) != 0)
200 /* Something went wrong. */
201 status = EXIT_FAILURE;
203 /* We don't need the input file anymore. */
206 /* We cannot go on with producing output since it might
207 lead to problem because the last output might leave
208 the output stream in an undefined state. */
212 /* Now close the file. */
216 while (++remaining < argc);
224 add_bytes (struct convtable *tbl, struct charseq *in, struct charseq *out)
229 assert (in->nbytes > 0);
231 byte = ((unsigned char *) in->bytes)[n];
232 while (n + 1 < in->nbytes)
234 if (is_term (tbl, byte) || tbl->val[byte].sub == NULL)
236 /* Note that we simply ignore a definition for a byte sequence
237 which is also the prefix for a longer one. */
238 clear_term (tbl, byte);
240 (struct convtable *) xcalloc (1, sizeof (struct convtable));
243 tbl = tbl->val[byte].sub;
245 byte = ((unsigned char *) in->bytes)[++n];
248 /* Only add the new sequence if there is none yet and the byte sequence
249 is not part of an even longer one. */
250 if (! is_term (tbl, byte) && tbl->val[byte].sub == NULL)
252 set_term (tbl, byte);
253 tbl->val[byte].out = out;
258 static struct convtable *
259 use_from_charmap (struct charmap_t *from_charmap, const char *to_code)
261 /* We iterate over all entries in the from_charmap and for those which
262 have a known UCS4 representation we use an iconv() call to determine
263 the mapping to the to_code charset. */
264 struct convtable *rettbl;
271 cd = iconv_open (to_code, "WCHAR_T");
272 if (cd == (iconv_t) -1)
273 /* We cannot do anything. */
276 rettbl = allocate_table ();
278 while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
281 struct charseq *in = (struct charseq *) data;
283 if (in->ucs4 != UNINITIALIZED_CHAR_VALUE)
285 /* There is a chance. Try the iconv module. */
286 wchar_t inbuf[1] = { in->ucs4 };
287 unsigned char outbuf[64];
288 char *inptr = (char *) inbuf;
289 size_t inlen = sizeof (inbuf);
290 char *outptr = (char *) outbuf;
291 size_t outlen = sizeof (outbuf);
293 (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
295 if (outptr != (char *) outbuf)
297 /* We got some output. Good, use it. */
298 struct charseq *newp;
300 outlen = sizeof (outbuf) - outlen;
301 assert ((char *) outbuf + outlen == outptr);
303 newp = (struct charseq *) xmalloc (sizeof (struct charseq)
305 newp->name = in->name;
306 newp->ucs4 = in->ucs4;
307 newp->nbytes = outlen;
308 memcpy (newp->bytes, outbuf, outlen);
310 add_bytes (rettbl, in, newp);
313 /* Clear any possible state left behind. */
314 (void) iconv (cd, NULL, NULL, NULL, NULL);
324 static struct convtable *
325 use_to_charmap (const char *from_code, struct charmap_t *to_charmap)
327 /* We iterate over all entries in the to_charmap and for those which
328 have a known UCS4 representation we use an iconv() call to determine
329 the mapping to the from_code charset. */
330 struct convtable *rettbl;
337 /* Note that the conversion we use here is the reverse direction. Without
338 exhaustive search we cannot figure out which input yields the UCS4
339 character we are looking for. Therefore we determine it the other
341 cd = iconv_open (from_code, "WCHAR_T");
342 if (cd == (iconv_t) -1)
343 /* We cannot do anything. */
346 rettbl = allocate_table ();
348 while (iterate_table (&to_charmap->char_table, &ptr, &key, &keylen, &data)
351 struct charseq *out = (struct charseq *) data;
353 if (out->ucs4 != UNINITIALIZED_CHAR_VALUE)
355 /* There is a chance. Try the iconv module. */
356 wchar_t inbuf[1] = { out->ucs4 };
357 unsigned char outbuf[64];
358 char *inptr = (char *) inbuf;
359 size_t inlen = sizeof (inbuf);
360 char *outptr = (char *) outbuf;
361 size_t outlen = sizeof (outbuf);
363 (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
365 if (outptr != (char *) outbuf)
367 /* We got some output. Good, use it. */
376 unsigned char bytes[outlen];
380 outlen = sizeof (outbuf) - outlen;
381 assert ((char *) outbuf + outlen == outptr);
383 new.mem.name = out->name;
384 new.mem.ucs4 = out->ucs4;
385 new.mem.nbytes = outlen;
386 memcpy (new.mem.bytes, outbuf, outlen);
388 add_bytes (rettbl, &new.seq, out);
391 /* Clear any possible state left behind. */
392 (void) iconv (cd, NULL, NULL, NULL, NULL);
402 static struct convtable *
403 use_both_charmaps (struct charmap_t *from_charmap,
404 struct charmap_t *to_charmap)
406 /* In this case we iterate over all the entries in the from_charmap,
407 determine the internal name, and find an appropriate entry in the
408 to_charmap (if it exists). */
409 struct convtable *rettbl = allocate_table ();
415 while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
418 struct charseq *in = (struct charseq *) data;
419 struct charseq *out = charmap_find_value (to_charmap, key, keylen);
422 add_bytes (rettbl, in, out);
430 process_block (struct convtable *tbl, char *addr, size_t len, FILE *output)
436 struct convtable *cur = tbl;
437 unsigned char *curp = (unsigned char *) addr;
438 unsigned int byte = *curp;
442 while (! is_term (cur, byte))
443 if (cur->val[byte].sub == NULL)
445 /* This is a invalid sequence. Skip the first byte if we are
446 ignoring errors. Otherwise punt. */
449 error (0, 0, _("illegal input sequence at position %Zd"), n);
453 n -= curp - (unsigned char *) addr;
455 byte = *(curp = (unsigned char *) ++addr);
464 cur = cur->val[byte].sub;
469 incomplete character or shift sequence at end of buffer"));
476 /* We found a final byte. Write the output bytes. */
477 out = cur->val[byte].out;
478 for (cnt = 0; cnt < out->nbytes; ++cnt)
479 fputc_unlocked (out->bytes[cnt], output);
481 addr = (char *) curp + 1;
490 process_fd (struct convtable *tbl, int fd, FILE *output)
492 /* We have a problem with reading from a descriptor since we must not
493 provide the iconv() function an incomplete character or shift
494 sequence at the end of the buffer. Since we have to deal with
495 arbitrary encodings we must read the whole text in a buffer and
496 process it in one step. */
497 static char *inbuf = NULL;
498 static size_t maxlen = 0;
502 while (actlen < maxlen)
504 ssize_t n = read (fd, inptr, maxlen - actlen);
507 /* No more text to read. */
512 /* Error while reading. */
513 error (0, errno, _("error while reading the input"));
521 if (actlen == maxlen)
527 /* Increase the buffer. */
528 new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
529 if (new_inbuf == NULL)
531 error (0, errno, _("unable to allocate buffer for input"));
536 inptr = inbuf + actlen;
540 n = read (fd, inptr, maxlen - actlen);
543 /* No more text to read. */
548 /* Error while reading. */
549 error (0, errno, _("error while reading the input"));
556 while (actlen < maxlen);
559 /* Break again so we leave both loops. */
563 /* Now we have all the input in the buffer. Process it in one run. */
564 return process_block (tbl, inbuf, actlen, output);
569 process_file (struct convtable *tbl, FILE *input, FILE *output)
571 /* This should be safe since we use this function only for `stdin' and
572 we haven't read anything so far. */
573 return process_fd (tbl, fileno (input), output);