1 /* xgettext librep backend.
2 Copyright (C) 2001-2003, 2005-2009 Free Software Foundation, Inc.
4 This file was written by Bruno Haible <haible@clisp.cons.org>, 2001.
6 This program is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program. If not, see <http://www.gnu.org/licenses/>. */
40 #define _(s) gettext(s)
43 /* Summary of librep syntax:
44 - ';' starts a comment until end of line.
45 - Block comments start with '#|' and end with '|#'.
46 - Numbers are constituted of an optional prefix (#b, #B for binary,
47 #o, #O for octal, #d, #D for decimal, #x, #X for hexadecimal,
48 #e, #E for exact, #i, #I for inexact), an optional sign (+ or -), and
50 - Characters are written as '?' followed by the character, possibly
51 with an escape sequence, for examples '?a', '?\n', '?\177'.
52 - Strings are delimited by double quotes. Backslash introduces an escape
53 sequence. The following are understood: '\n', '\r', '\f', '\t', '\a',
54 '\\', '\^C', '\012' (octal), '\x12' (hexadecimal).
55 - Symbols: can contain meta-characters - whitespace or any from ()[]'";|\' -
56 if preceded by backslash or enclosed in |...|.
57 - Keywords: written as #:SYMBOL.
60 The reader is implemented in librep-0.14/src/lisp.c. */
63 /* ====================== Keyword set customization. ====================== */
65 /* If true extract all strings. */
66 static bool extract_all = false;
68 static hash_table keywords;
69 static bool default_keywords = true;
73 x_librep_extract_all ()
80 x_librep_keyword (const char *name)
83 default_keywords = false;
87 struct callshape shape;
90 if (keywords.table == NULL)
91 hash_init (&keywords, 100);
93 split_keywordspec (name, &end, &shape);
95 /* The characters between name and end should form a valid Lisp
97 colon = strchr (name, ':');
98 if (colon == NULL || colon >= end)
99 insert_keyword_callshape (&keywords, name, end - name, &shape);
103 /* Finish initializing the keywords hash table.
104 Called after argument processing, before each file is processed. */
108 if (default_keywords)
110 /* When adding new keywords here, also update the documentation in
112 x_librep_keyword ("_");
113 default_keywords = false;
118 init_flag_table_librep ()
120 xgettext_record_flag ("_:1:pass-librep-format");
121 xgettext_record_flag ("format:2:librep-format");
125 /* ======================== Reading of characters. ======================== */
127 /* Real filename, used in error messages about the input file. */
128 static const char *real_file_name;
130 /* Logical filename and line number, used to label the extracted messages. */
131 static char *logical_file_name;
132 static int line_number;
134 /* The input file stream. */
138 /* Fetch the next character from the input file. */
147 error (EXIT_FAILURE, errno, _("\
148 error while reading \"%s\""), real_file_name);
156 /* Put back the last fetched character, not EOF. */
166 /* ========================== Reading of tokens. ========================== */
169 /* A token consists of a sequence of characters. */
172 int allocated; /* number of allocated 'token_char's */
173 int charcount; /* number of used 'token_char's */
174 char *chars; /* the token's constituents */
177 /* Initialize a 'struct token'. */
179 init_token (struct token *tp)
182 tp->chars = XNMALLOC (tp->allocated, char);
186 /* Free the memory pointed to by a 'struct token'. */
188 free_token (struct token *tp)
193 /* Ensure there is enough room in the token for one more character. */
195 grow_token (struct token *tp)
197 if (tp->charcount == tp->allocated)
200 tp->chars = (char *) xrealloc (tp->chars, tp->allocated * sizeof (char));
204 /* Read the next token. If 'first' is given, it points to the first
205 character, which has already been read. Returns true for a symbol,
206 false for a number. */
208 read_token (struct token *tp, const int *first)
211 /* Variables for speculative number parsing: */
215 bool rational = false;
216 bool exponent = false;
217 bool had_sign = false;
218 bool expecting_prefix = false;
227 for (;; c = do_getc ())
234 case ' ': case '\t': case '\n': case '\f': case '\r':
235 case '(': case ')': case '[': case ']':
236 case '\'': case '"': case ';': case ',': case '`':
243 /* Invalid, but be tolerant. */
246 tp->chars[tp->charcount++] = c;
254 if (c == EOF || c == '|')
257 tp->chars[tp->charcount++] = c;
264 if (expecting_prefix)
287 expecting_prefix = false;
288 nfirst = tp->charcount + 1;
290 else if (tp->charcount == nfirst
291 && (c == '+' || c == '-' || c == '#'))
298 expecting_prefix = true;
302 nfirst = tp->charcount + 1;
314 else if (!(c >= '0' && c <= '9'))
327 nfirst = tp->charcount + 1;
329 case '0': case '1': case '2': case '3': case '4':
330 case '5': case '6': case '7':
332 nfirst = tp->charcount;
334 case '.': case 'E': case 'e':
352 if (exact && radix == 10 && !rational)
358 if (exact && !rational)
366 if (!rational && !exponent)
377 if (exponent && (c == '+' || c == '-'))
380 && !(c >= '0' && c <= '0' + radix - 1))
381 || (radix == 16 && !c_isxdigit (c)))
395 tp->chars[tp->charcount++] = c;
401 if (radix > 0 && nfirst < tp->charcount)
402 return false; /* number */
404 return true; /* symbol */
408 /* ========================= Accumulating comments ========================= */
412 static size_t bufmax;
413 static size_t buflen;
424 if (buflen >= bufmax)
426 bufmax = 2 * bufmax + 10;
427 buffer = xrealloc (buffer, bufmax);
429 buffer[buflen++] = c;
433 comment_line_end (size_t chars_to_remove)
435 buflen -= chars_to_remove;
437 && (buffer[buflen - 1] == ' ' || buffer[buflen - 1] == '\t'))
439 if (chars_to_remove == 0 && buflen >= bufmax)
441 bufmax = 2 * bufmax + 10;
442 buffer = xrealloc (buffer, bufmax);
444 buffer[buflen] = '\0';
445 savable_comment_add (buffer);
449 /* These are for tracking whether comments count as immediately before
451 static int last_comment_line;
452 static int last_non_comment_line;
455 /* ========================= Accumulating messages ========================= */
458 static message_list_ty *mlp;
461 /* ============== Reading of objects. See CLHS 2 "Syntax". ============== */
464 /* We are only interested in symbols (e.g. GETTEXT or NGETTEXT) and strings.
465 Other objects need not to be represented precisely. */
468 t_symbol, /* symbol */
469 t_string, /* string */
470 t_other, /* other kind of real object */
471 t_dot, /* '.' pseudo object */
472 t_close, /* ')' or ']' pseudo object */
473 t_eof /* EOF marker */
478 enum object_type type;
479 struct token *token; /* for t_symbol and t_string */
480 int line_number_at_start; /* for t_string */
483 /* Free the memory pointed to by a 'struct object'. */
485 free_object (struct object *op)
487 if (op->type == t_symbol || op->type == t_string)
489 free_token (op->token);
494 /* Convert a t_symbol/t_string token to a char*. */
496 string_of_object (const struct object *op)
501 if (!(op->type == t_symbol || op->type == t_string))
503 n = op->token->charcount;
504 str = XNMALLOC (n + 1, char);
505 memcpy (str, op->token->chars, n);
510 /* Context lookup table. */
511 static flag_context_list_table_ty *flag_context_list_table;
513 /* Returns the character represented by an escape sequence. */
515 do_getc_escaped (int c)
536 case '0': case '1': case '2': case '3': case '4':
537 case '5': case '6': case '7':
544 if (c >= '0' && c <= '7')
546 n = (n << 3) + (c - '0');
550 if (c >= '0' && c <= '7')
551 n = (n << 3) + (c - '0');
559 return (unsigned char) n;
570 else if (c >= '0' && c <= '9')
571 n = (n << 4) + (c - '0');
572 else if (c >= 'A' && c <= 'F')
573 n = (n << 4) + (c - 'A' + 10);
574 else if (c >= 'a' && c <= 'f')
575 n = (n << 4) + (c - 'a' + 10);
582 return (unsigned char) n;
589 /* Read the next object. */
591 read_object (struct object *op, flag_context_ty outer_context)
606 /* Comments assumed to be grouped with a message must immediately
607 precede it, with no non-whitespace token on a line between
609 if (last_non_comment_line > last_comment_line)
610 savable_comment_reset ();
613 case ' ': case '\t': case '\f': case '\r':
618 int arg = 0; /* Current argument number. */
619 flag_context_list_iterator_ty context_iter;
620 const struct callshapes *shapes = NULL;
621 struct arglist_parser *argparser = NULL;
626 flag_context_ty inner_context;
629 inner_context = null_context;
632 inherited_context (outer_context,
633 flag_context_list_iterator_advance (
636 read_object (&inner, inner_context);
638 /* Recognize end of list. */
639 if (inner.type == t_close)
642 /* Don't bother converting "()" to "NIL". */
643 last_non_comment_line = line_number;
644 if (argparser != NULL)
645 arglist_parser_done (argparser, arg);
649 /* Dots are not allowed in every position.
652 /* EOF inside list is illegal. But be tolerant. */
653 if (inner.type == t_eof)
658 /* This is the function position. */
659 if (inner.type == t_symbol)
661 char *symbol_name = string_of_object (&inner);
664 if (hash_find_entry (&keywords,
665 symbol_name, strlen (symbol_name),
668 shapes = (const struct callshapes *) keyword_value;
670 argparser = arglist_parser_alloc (mlp, shapes);
673 flag_context_list_iterator (
674 flag_context_list_table_lookup (
675 flag_context_list_table,
676 symbol_name, strlen (symbol_name)));
681 context_iter = null_context_list_iterator;
685 /* These are the argument positions. */
686 if (argparser != NULL && inner.type == t_string)
687 arglist_parser_remember (argparser, arg,
688 string_of_object (&inner),
691 inner.line_number_at_start,
695 free_object (&inner);
698 if (argparser != NULL)
699 arglist_parser_done (argparser, arg);
702 last_non_comment_line = line_number;
711 read_object (&inner, null_context);
713 /* Recognize end of vector. */
714 if (inner.type == t_close)
717 last_non_comment_line = line_number;
721 /* Dots are not allowed. But be tolerant. */
723 /* EOF inside vector is illegal. But be tolerant. */
724 if (inner.type == t_eof)
727 free_object (&inner);
731 last_non_comment_line = line_number;
735 /* Tell the caller about the end of list or vector.
736 Unmatched closing parenthesis is illegal. But be tolerant. */
738 last_non_comment_line = line_number;
744 /* The ,@ handling inside lists is wrong anyway, because
745 ,@form expands to an unknown number of elements. */
746 if (c != EOF && c != '@')
755 read_object (&inner, null_context);
757 /* Dots and EOF are not allowed here. But be tolerant. */
759 free_object (&inner);
762 last_non_comment_line = line_number;
768 bool all_semicolons = true;
770 last_comment_line = line_number;
775 if (c == EOF || c == '\n' || c == '\f' || c == '\r')
778 all_semicolons = false;
781 /* We skip all leading white space, but not EOLs. */
782 if (!(buflen == 0 && (c == ' ' || c == '\t')))
786 comment_line_end (0);
792 op->token = XMALLOC (struct token);
793 init_token (op->token);
794 op->line_number_at_start = line_number;
799 /* Invalid input. Be tolerant, no error message. */
807 /* Invalid input. Be tolerant, no error message. */
810 /* Ignore escaped newline. */
814 c = do_getc_escaped (c);
816 /* Invalid input. Be tolerant, no error message. */
818 grow_token (op->token);
819 op->token->chars[op->token->charcount++] = c;
824 grow_token (op->token);
825 op->token->chars[op->token->charcount++] = c;
834 pos.file_name = logical_file_name;
835 pos.line_number = op->line_number_at_start;
836 remember_a_message (mlp, NULL, string_of_object (op),
837 null_context, &pos, NULL, savable_comment);
839 last_non_comment_line = line_number;
846 /* Invalid input. Be tolerant, no error message. */
852 /* Invalid input. Be tolerant, no error message. */
856 c = do_getc_escaped (c);
858 /* Invalid input. Be tolerant, no error message. */
863 last_non_comment_line = line_number;
867 /* Dispatch macro handling. */
870 /* Invalid input. Be tolerant, no error message. */
880 /* Skip comment until !# */
890 if (c == EOF || c == '#')
898 /* EOF not allowed here. But be tolerant. */
909 read_object (&inner, null_context);
910 /* Dots and EOF are not allowed here.
912 free_object (&inner);
914 last_non_comment_line = line_number;
923 read_object (&inner, null_context);
924 /* Dots and EOF are not allowed here.
926 free_object (&inner);
928 last_non_comment_line = line_number;
951 comment_line_end (0);
977 /* We skip all leading white space. */
978 if (!(buflen == 0 && (c == ' ' || c == '\t')))
982 comment_line_end (1);
990 /* EOF not allowed here. But be tolerant. */
994 last_comment_line = line_number;
1002 read_token (&token, &first);
1003 free_token (&token);
1005 last_non_comment_line = line_number;
1012 last_non_comment_line = line_number;
1025 read_token (&token, &c);
1026 free_token (&token);
1028 last_non_comment_line = line_number;
1033 /* Invalid input. Be tolerant, no error message. */
1035 last_non_comment_line = line_number;
1047 op->token = XMALLOC (struct token);
1048 symbol = read_token (op->token, &c);
1049 if (op->token->charcount == 1 && op->token->chars[0] == '.')
1051 free_token (op->token);
1054 last_non_comment_line = line_number;
1059 free_token (op->token);
1062 last_non_comment_line = line_number;
1065 /* Distinguish between "foo" and "foo#bar". */
1069 struct token second_token;
1071 free_token (op->token);
1073 read_token (&second_token, NULL);
1074 free_token (&second_token);
1076 last_non_comment_line = line_number;
1083 op->type = t_symbol;
1084 last_non_comment_line = line_number;
1094 extract_librep (FILE *f,
1095 const char *real_filename, const char *logical_filename,
1096 flag_context_list_table_ty *flag_table,
1097 msgdomain_list_ty *mdlp)
1099 mlp = mdlp->item[0]->messages;
1102 real_file_name = real_filename;
1103 logical_file_name = xstrdup (logical_filename);
1106 last_comment_line = -1;
1107 last_non_comment_line = -1;
1109 flag_context_list_table = flag_table;
1113 /* Eat tokens until eof is seen. When read_object returns
1114 due to an unbalanced closing parenthesis, just restart it. */
1117 struct object toplevel_object;
1119 read_object (&toplevel_object, null_context);
1121 if (toplevel_object.type == t_eof)
1124 free_object (&toplevel_object);
1128 /* Close scanner. */
1130 real_file_name = NULL;
1131 logical_file_name = NULL;