1 /* Convert text in given files from the specified from-set to the to-set.
2 Copyright (C) 1998-2011, 2012 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published
8 by the Free Software Foundation; version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, see <http://www.gnu.org/licenses/>. */
35 #ifdef _POSIX_MAPPED_FILES
36 # include <sys/mman.h>
39 #include <gconv_int.h>
40 #include "iconv_prog.h"
41 #include "iconvconfig.h"
43 /* Get libc version number. */
44 #include "../version.h"
46 #define PACKAGE _libc_intl_domainname
49 /* Name and version of program. */
50 static void print_version (FILE *stream, struct argp_state *state);
51 void (*argp_program_version_hook) (FILE *, struct argp_state *) = print_version;
53 #define OPT_VERBOSE 1000
56 /* Definitions of arguments for argp functions. */
57 static const struct argp_option options[] =
59 { NULL, 0, NULL, 0, N_("Input/Output format specification:") },
60 { "from-code", 'f', "NAME", 0, N_("encoding of original text") },
61 { "to-code", 't', "NAME", 0, N_("encoding for output") },
62 { NULL, 0, NULL, 0, N_("Information:") },
63 { "list", 'l', NULL, 0, N_("list all known coded character sets") },
64 { NULL, 0, NULL, 0, N_("Output control:") },
65 { NULL, 'c', NULL, 0, N_("omit invalid characters from output") },
66 { "output", 'o', "FILE", 0, N_("output file") },
67 { "silent", 's', NULL, 0, N_("suppress warnings") },
68 { "verbose", OPT_VERBOSE, NULL, 0, N_("print progress information") },
69 { NULL, 0, NULL, 0, NULL }
72 /* Short description of program. */
73 static const char doc[] = N_("\
74 Convert encoding of given files from one encoding to another.");
76 /* Strings for arguments in help texts. */
77 static const char args_doc[] = N_("[FILE...]");
79 /* Prototype for option handler. */
80 static error_t parse_opt (int key, char *arg, struct argp_state *state);
82 /* Function to print some extra text in the help message. */
83 static char *more_help (int key, const char *text, void *input);
85 /* Data structure to communicate with argp functions. */
86 static struct argp argp =
88 options, parse_opt, args_doc, doc, NULL, more_help
91 /* Code sets to convert from and to respectively. An empty string as the
92 default causes the 'iconv_open' function to look up the charset of the
93 currently selected locale and use it. */
94 static const char *from_code = "";
95 static const char *to_code = "";
97 /* File to write output to. If NULL write to stdout. */
98 static const char *output_file;
100 /* Nonzero if verbose ouput is wanted. */
103 /* Nonzero if list of all coded character sets is wanted. */
106 /* If nonzero omit invalid character from output. */
109 /* Prototypes for the functions doing the actual work. */
110 static int process_block (iconv_t cd, char *addr, size_t len, FILE **output,
111 const char *output_file);
112 static int process_fd (iconv_t cd, int fd, FILE **output,
113 const char *output_file);
114 static int process_file (iconv_t cd, FILE *input, FILE **output,
115 const char *output_file);
116 static void print_known_names (void) internal_function;
120 main (int argc, char *argv[])
122 int status = EXIT_SUCCESS;
125 const char *orig_to_code;
126 struct charmap_t *from_charmap = NULL;
127 struct charmap_t *to_charmap = NULL;
129 /* Set locale via LC_ALL. */
130 setlocale (LC_ALL, "");
132 /* Set the text message domain. */
133 textdomain (_libc_intl_domainname);
135 /* Parse and process arguments. */
136 argp_parse (&argp, argc, argv, 0, &remaining, NULL);
138 /* List all coded character sets if wanted. */
141 print_known_names ();
145 /* If we have to ignore errors make sure we use the appropriate name for
146 the to-character-set. */
147 orig_to_code = to_code;
150 const char *errhand = strchrnul (to_code, '/');
158 errhand = strchrnul (errhand, '/');
163 errhand = strchr (errhand, '\0');
167 newp = (char *) alloca (errhand - to_code + nslash + 7 + 1);
168 cp = mempcpy (newp, to_code, errhand - to_code);
173 memcpy (cp, "IGNORE", sizeof ("IGNORE"));
178 /* POSIX 1003.2b introduces a silly thing: the arguments to -t anf -f
179 can be file names of charmaps. In this case iconv will have to read
180 those charmaps and use them to do the conversion. But there are
181 holes in the specification. There is nothing said that if -f is a
182 charmap filename that -t must be, too. And vice versa. There is
183 also no word about the symbolic names used. What if they don't
185 if (strchr (from_code, '/') != NULL)
186 /* The from-name might be a charmap file name. Try reading the
188 from_charmap = charmap_read (from_code, /*0, 1*/1, 0, 0, 0);
190 if (strchr (orig_to_code, '/') != NULL)
191 /* The to-name might be a charmap file name. Try reading the
193 to_charmap = charmap_read (orig_to_code, /*0, 1,*/1, 0, 0, 0);
196 /* At this point we have to handle two cases. The first one is
197 where a charmap is used for the from- or to-charset, or both. We
198 handle this special since it is very different from the sane way of
199 doing things. The other case allows converting using the iconv()
201 if (from_charmap != NULL || to_charmap != NULL)
202 /* Construct the conversion table and do the conversion. */
203 status = charmap_conversion (from_code, from_charmap, to_code, to_charmap,
204 argc, remaining, argv, output_file);
207 /* Let's see whether we have these coded character sets. */
208 cd = iconv_open (to_code, from_code);
209 if (cd == (iconv_t) -1)
213 /* Try to be nice with the user and tell her which of the
214 two encoding names is wrong. This is possible because
215 all supported encodings can be converted from/to Unicode,
216 in other words, because the graph of encodings is
219 (iconv_open ("UTF-8", from_code) == (iconv_t) -1
222 (iconv_open (to_code, "UTF-8") == (iconv_t) -1
224 const char *from_pretty =
225 (from_code[0] ? from_code : nl_langinfo (CODESET));
226 const char *to_pretty =
227 (orig_to_code[0] ? orig_to_code : nl_langinfo (CODESET));
234 conversions from `%s' and to `%s' are not supported"),
235 from_pretty, to_pretty);
238 _("conversion from `%s' is not supported"),
245 _("conversion to `%s' is not supported"),
249 _("conversion from `%s' to `%s' is not supported"),
250 from_pretty, to_pretty);
253 argp_help (&argp, stderr, ARGP_HELP_SEE,
254 program_invocation_short_name);
258 error (EXIT_FAILURE, errno,
259 _("failed to start conversion processing"));
262 /* The output file. Will be opened when we are ready to produce
266 /* Now process the remaining files. Write them to stdout or the file
267 specified with the `-o' parameter. If we have no file given as
268 the parameter process all from stdin. */
269 if (remaining == argc)
271 if (process_file (cd, stdin, &output, output_file) != 0)
272 status = EXIT_FAILURE;
277 #ifdef _POSIX_MAPPED_FILES
284 fprintf (stderr, "%s:\n", argv[remaining]);
285 if (strcmp (argv[remaining], "-") == 0)
289 fd = open (argv[remaining], O_RDONLY);
293 error (0, errno, _("cannot open input file `%s'"),
295 status = EXIT_FAILURE;
300 #ifdef _POSIX_MAPPED_FILES
301 /* We have possibilities for reading the input file. First try
302 to mmap() it since this will provide the fastest solution. */
303 if (fstat (fd, &st) == 0
304 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
305 fd, 0)) != MAP_FAILED))
307 /* Yes, we can use mmap(). The descriptor is not needed
310 error (EXIT_FAILURE, errno,
311 _("error while closing input `%s'"),
314 ret = process_block (cd, addr, st.st_size, &output,
317 /* We don't need the input data anymore. */
318 munmap ((void *) addr, st.st_size);
322 status = EXIT_FAILURE;
325 /* We cannot go on with producing output since it might
326 lead to problem because the last output might leave
327 the output stream in an undefined state. */
332 #endif /* _POSIX_MAPPED_FILES */
334 /* Read the file in pieces. */
335 ret = process_fd (cd, fd, &output, output_file);
337 /* Now close the file. */
342 /* Something went wrong. */
343 status = EXIT_FAILURE;
346 /* We cannot go on with producing output since it might
347 lead to problem because the last output might leave
348 the output stream in an undefined state. */
353 while (++remaining < argc);
355 /* Close the output file now. */
356 if (output != NULL && fclose (output))
357 error (EXIT_FAILURE, errno, _("error while closing output file"));
364 /* Handle program arguments. */
366 parse_opt (int key, char *arg, struct argp_state *state)
380 /* Nothing, for now at least. We are not giving out any information
381 about missing character or so. */
384 /* Omit invalid characters from output. */
394 return ARGP_ERR_UNKNOWN;
401 more_help (int key, const char *text, void *input)
406 case ARGP_KEY_HELP_EXTRA:
407 /* We print some extra information. */
408 if (asprintf (&tp, gettext ("\
409 For bug reporting instructions, please see:\n\
410 %s.\n"), REPORT_BUGS_TO) < 0)
416 return (char *) text;
420 /* Print the version information. */
422 print_version (FILE *stream, struct argp_state *state)
424 fprintf (stream, "iconv %s%s\n", PKGVERSION, VERSION);
425 fprintf (stream, gettext ("\
426 Copyright (C) %s Free Software Foundation, Inc.\n\
427 This is free software; see the source for copying conditions. There is NO\n\
428 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
430 fprintf (stream, gettext ("Written by %s.\n"), "Ulrich Drepper");
435 write_output (const char *outbuf, const char *outptr, FILE **output,
436 const char *output_file)
438 /* We have something to write out. */
439 int errno_save = errno;
443 /* Determine output file. */
444 if (output_file != NULL && strcmp (output_file, "-") != 0)
446 *output = fopen (output_file, "w");
448 error (EXIT_FAILURE, errno, _("cannot open output file"));
454 if (fwrite (outbuf, 1, outptr - outbuf, *output) < (size_t) (outptr - outbuf)
457 /* Error occurred while printing the result. */
459 conversion stopped due to problem in writing the output"));
470 process_block (iconv_t cd, char *addr, size_t len, FILE **output,
471 const char *output_file)
473 #define OUTBUF_SIZE 32768
474 const char *start = addr;
475 char outbuf[OUTBUF_SIZE];
484 outlen = OUTBUF_SIZE;
485 n = iconv (cd, &addr, &len, &outptr, &outlen);
487 if (n == (size_t) -1 && omit_invalid && errno == EILSEQ)
496 if (outptr != outbuf)
498 ret = write_output (outbuf, outptr, output, output_file);
503 if (n != (size_t) -1)
505 /* All the input test is processed. For state-dependent
506 character sets we have to flush the state now. */
508 outlen = OUTBUF_SIZE;
509 n = iconv (cd, NULL, NULL, &outptr, &outlen);
511 if (outptr != outbuf)
513 ret = write_output (outbuf, outptr, output, output_file);
518 if (n != (size_t) -1)
521 if (omit_invalid && errno == EILSEQ)
530 /* iconv() ran into a problem. */
535 error (0, 0, _("illegal input sequence at position %ld"),
536 (long int) (addr - start));
540 incomplete character or shift sequence at end of buffer"));
543 error (0, 0, _("internal error (illegal descriptor)"));
546 error (0, 0, _("unknown iconv() error %d"), errno);
559 process_fd (iconv_t cd, int fd, FILE **output, const char *output_file)
561 /* we have a problem with reading from a desriptor since we must not
562 provide the iconv() function an incomplete character or shift
563 sequence at the end of the buffer. Since we have to deal with
564 arbitrary encodings we must read the whole text in a buffer and
565 process it in one step. */
566 static char *inbuf = NULL;
567 static size_t maxlen = 0;
571 while (actlen < maxlen)
573 ssize_t n = read (fd, inptr, maxlen - actlen);
576 /* No more text to read. */
581 /* Error while reading. */
582 error (0, errno, _("error while reading the input"));
590 if (actlen == maxlen)
596 /* Increase the buffer. */
597 new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
598 if (new_inbuf == NULL)
600 error (0, errno, _("unable to allocate buffer for input"));
605 inptr = inbuf + actlen;
609 n = read (fd, inptr, maxlen - actlen);
612 /* No more text to read. */
617 /* Error while reading. */
618 error (0, errno, _("error while reading the input"));
625 while (actlen < maxlen);
628 /* Break again so we leave both loops. */
632 /* Now we have all the input in the buffer. Process it in one run. */
633 return process_block (cd, inbuf, actlen, output, output_file);
638 process_file (iconv_t cd, FILE *input, FILE **output, const char *output_file)
640 /* This should be safe since we use this function only for `stdin' and
641 we haven't read anything so far. */
642 return process_fd (cd, fileno (input), output, output_file);
646 /* Print all known character sets/encodings. */
647 static void *printlist;
648 static size_t column;
649 static int not_first;
652 insert_print_list (const void *nodep, VISIT value, int level)
654 if (value == leaf || value == postorder)
656 const struct gconv_alias *s = *(const struct gconv_alias **) nodep;
657 tsearch (s->fromname, &printlist, (__compar_fn_t) strverscmp);
662 do_print_human (const void *nodep, VISIT value, int level)
664 if (value == leaf || value == postorder)
666 const char *s = *(const char **) nodep;
667 size_t len = strlen (s);
670 while (len > 0 && s[len - 1] == '/')
673 for (cnt = 0; cnt < len; ++cnt)
674 if (isalnum (s[cnt]))
684 if (column > 2 && column + len > 77)
686 fputs ("\n ", stdout);
698 fwrite (s, len, 1, stdout);
704 do_print (const void *nodep, VISIT value, int level)
706 if (value == leaf || value == postorder)
708 const char *s = *(const char **) nodep;
716 add_known_names (struct gconv_module *node)
718 if (node->left != NULL)
719 add_known_names (node->left);
720 if (node->right != NULL)
721 add_known_names (node->right);
724 if (strcmp (node->from_string, "INTERNAL") != 0)
725 tsearch (node->from_string, &printlist, (__compar_fn_t) strverscmp);
726 if (strcmp (node->to_string, "INTERNAL") != 0)
727 tsearch (node->to_string, &printlist, (__compar_fn_t) strverscmp);
731 while (node != NULL);
738 const struct gconvcache_header *header;
740 const struct hash_entry *hashtab;
743 header = (const struct gconvcache_header *) __gconv_get_cache ();
744 strtab = (char *) header + header->string_offset;
745 hashtab = (struct hash_entry *) ((char *) header + header->hash_offset);
747 for (cnt = 0; cnt < header->hash_size; ++cnt)
748 if (hashtab[cnt].string_offset != 0)
750 const char *str = strtab + hashtab[cnt].string_offset;
752 if (strcmp (str, "INTERNAL") != 0)
753 tsearch (str, &printlist, (__compar_fn_t) strverscmp);
760 print_known_names (void)
765 /* We must initialize the internal databases first. */
766 h = iconv_open ("L1", "L1");
769 /* See whether we have a cache. */
770 cache = __gconv_get_cache ();
772 /* Yep, use only this information. */
776 struct gconv_module *modules;
778 /* No, then use the information read from the gconv-modules file.
779 First add the aliases. */
780 twalk (__gconv_get_alias_db (), insert_print_list);
782 /* Add the from- and to-names from the known modules. */
783 modules = __gconv_get_modules_db ();
785 add_known_names (modules);
788 bool human_readable = isatty (fileno (stdout));
792 The following list contain all the coded character sets known. This does\n\
793 not necessarily mean that all combinations of these names can be used for\n\
794 the FROM and TO command line parameters. One coded character set can be\n\
795 listed with several different names (aliases).\n\n "), stdout);
797 /* Now print the collected names. */
799 twalk (printlist, human_readable ? do_print_human : do_print);
801 if (human_readable && column != 0)