1 /* tr -- a filter to translate characters
2 Copyright (C) 91, 1995-2008 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
17 /* Written by Jim Meyering */
23 #include <sys/types.h>
29 #include "safe-read.h"
32 /* The official name of this program (e.g., no `g' prefix). */
33 #define PROGRAM_NAME "tr"
35 #define AUTHORS proper_name ("Jim Meyering")
37 enum { N_CHARS = UCHAR_MAX + 1 };
39 /* An unsigned integer type big enough to hold a repeat count or an
40 unsigned character. POSIX requires support for repeat counts as
41 high as 2**31 - 1. Since repeat counts might need to expand to
42 match the length of an argument string, we need at least size_t to
43 avoid arbitrary internal limits. It doesn't cost much to use
45 typedef uintmax_t count;
47 /* The value for Spec_list->state that indicates to
48 get_next that it should initialize the tail pointer.
49 Its value should be as large as possible to avoid conflict
50 a valid value for the state field -- and that may be as
51 large as any valid repeat_count. */
52 #define BEGIN_STATE (UINTMAX_MAX - 1)
54 /* The value for Spec_list->state that indicates to
55 get_next that the element pointed to by Spec_list->tail is
56 being considered for the first time on this pass through the
57 list -- it indicates that get_next should make any necessary
59 #define NEW_ELEMENT (BEGIN_STATE + 1)
61 /* The maximum possible repeat count. Due to how the states are
62 implemented, it can be as much as BEGIN_STATE. */
63 #define REPEAT_COUNT_MAXIMUM BEGIN_STATE
65 /* The following (but not CC_NO_CLASS) are indices into the array of
66 valid character class strings. */
69 CC_ALNUM = 0, CC_ALPHA = 1, CC_BLANK = 2, CC_CNTRL = 3,
70 CC_DIGIT = 4, CC_GRAPH = 5, CC_LOWER = 6, CC_PRINT = 7,
71 CC_PUNCT = 8, CC_SPACE = 9, CC_UPPER = 10, CC_XDIGIT = 11,
75 /* Character class to which a character (returned by get_next) belonged;
76 but it is set only if the construct from which the character was obtained
77 was one of the character classes [:upper:] or [:lower:]. The value
78 is used only when translating and then, only to make sure that upper
79 and lower class constructs have the same relative positions in string1
81 enum Upper_Lower_class
88 /* The type of a List_element. See build_spec_list for more details. */
89 enum Range_element_type
98 /* One construct in one of tr's argument strings.
99 For example, consider the POSIX version of the classic tr command:
100 tr -cs 'a-zA-Z_' '[\n*]'
101 String1 has 3 constructs, two of which are ranges (a-z and A-Z),
102 and a single normal character, `_'. String2 has one construct. */
105 enum Range_element_type type;
106 struct List_element *next;
109 unsigned char normal_char;
112 unsigned char first_char;
113 unsigned char last_char;
116 enum Char_class char_class;
117 unsigned char equiv_code;
120 unsigned char the_repeated_char;
128 /* Each of tr's argument strings is parsed into a form that is easier
129 to work with: a linked list of constructs (struct List_element).
130 Each Spec_list structure also encapsulates various attributes of
131 the corresponding argument string. The attributes are used mainly
132 to verify that the strings are valid in the context of any options
133 specified (like -s, -d, or -c). The main exception is the member
134 `tail', which is first used to construct the list. After construction,
135 it is used by get_next to save its state when traversing the list.
136 The member `state' serves a similar function. */
139 /* Points to the head of the list of range elements.
140 The first struct is a dummy; its members are never used. */
141 struct List_element *head;
143 /* When appending, points to the last element. When traversing via
144 get_next(), points to the element to process next. Setting
145 Spec_list.state to the value BEGIN_STATE before calling get_next
146 signals get_next to initialize tail to point to head->next. */
147 struct List_element *tail;
149 /* Used to save state between calls to get_next. */
152 /* Length, in the sense that length ('a-z[:digit:]123abc')
153 is 42 ( = 26 + 10 + 6). */
156 /* The number of [c*] and [c*0] constructs that appear in this spec. */
157 size_t n_indefinite_repeats;
159 /* If n_indefinite_repeats is nonzero, this points to the List_element
160 corresponding to the last [c*] or [c*0] construct encountered in
161 this spec. Otherwise it is undefined. */
162 struct List_element *indefinite_repeat_element;
164 /* True if this spec contains at least one equivalence
165 class construct e.g. [=c=]. */
166 bool has_equiv_class;
168 /* True if this spec contains at least one character class
169 construct. E.g. [:digit:]. */
172 /* True if this spec contains at least one of the character class
173 constructs (all but upper and lower) that aren't allowed in s2. */
174 bool has_restricted_char_class;
177 /* A representation for escaped string1 or string2. As a string is parsed,
178 any backslash-escaped characters (other than octal or \a, \b, \f, \n,
179 etc.) are marked as such in this structure by setting the corresponding
180 entry in the ESCAPED vector. */
188 /* Return nonzero if the Ith character of escaped string ES matches C
189 and is not escaped itself. */
191 es_match (struct E_string const *es, size_t i, char c)
193 return es->s[i] == c && !es->escaped[i];
196 /* When true, each sequence in the input of a repeated character
197 (call it c) is replaced (in the output) by a single occurrence of c
198 for every c in the squeeze set. */
199 static bool squeeze_repeats = false;
201 /* When true, removes characters in the delete set from input. */
202 static bool delete = false;
204 /* Use the complement of set1 in place of set1. */
205 static bool complement = false;
207 /* When tr is performing translation and string1 is longer than string2,
208 POSIX says that the result is unspecified. That gives the implementor
209 of a POSIX conforming version of tr two reasonable choices for the
210 semantics of this case.
212 * The BSD tr pads string2 to the length of string1 by
213 repeating the last character in string2.
215 * System V tr ignores characters in string1 that have no
216 corresponding character in string2. That is, string1 is effectively
217 truncated to the length of string2.
219 When nonzero, this flag causes GNU tr to imitate the behavior
220 of System V tr when translating with string1 longer than string2.
221 The default is to emulate BSD tr. This flag is ignored in modes where
222 no translation is performed. Emulating the System V tr
223 in this exceptional case causes the relatively common BSD idiom:
225 tr -cs A-Za-z0-9 '\012'
227 to break (it would convert only zero bytes, rather than all
228 non-alphanumerics, to newlines).
230 WARNING: This switch does not provide general BSD or System V
231 compatibility. For example, it doesn't disable the interpretation
232 of the POSIX constructs [:alpha:], [=c=], and [c*10], so if by
233 some unfortunate coincidence you use such constructs in scripts
234 expecting to use some other version of tr, the scripts will break. */
235 static bool truncate_set1 = false;
237 /* An alias for (!delete && non_option_args == 2).
238 It is set in main and used there and in validate(). */
239 static bool translating;
241 static char io_buf[BUFSIZ];
243 static char const *const char_class_name[] =
245 "alnum", "alpha", "blank", "cntrl", "digit", "graph",
246 "lower", "print", "punct", "space", "upper", "xdigit"
248 enum { N_CHAR_CLASSES = sizeof char_class_name / sizeof char_class_name[0] };
250 /* Array of boolean values. A character `c' is a member of the
251 squeeze set if and only if in_squeeze_set[c] is true. The squeeze
252 set is defined by the last (possibly, the only) string argument
253 on the command line when the squeeze option is given. */
254 static bool in_squeeze_set[N_CHARS];
256 /* Array of boolean values. A character `c' is a member of the
257 delete set if and only if in_delete_set[c] is true. The delete
258 set is defined by the first (or only) string argument on the
259 command line when the delete option is given. */
260 static bool in_delete_set[N_CHARS];
262 /* Array of character values defining the translation (if any) that
263 tr is to perform. Translation is performed only when there are
264 two specification strings and the delete switch is not given. */
265 static char xlate[N_CHARS];
267 static struct option const long_options[] =
269 {"complement", no_argument, NULL, 'c'},
270 {"delete", no_argument, NULL, 'd'},
271 {"squeeze-repeats", no_argument, NULL, 's'},
272 {"truncate-set1", no_argument, NULL, 't'},
273 {GETOPT_HELP_OPTION_DECL},
274 {GETOPT_VERSION_OPTION_DECL},
281 if (status != EXIT_SUCCESS)
282 fprintf (stderr, _("Try `%s --help' for more information.\n"),
287 Usage: %s [OPTION]... SET1 [SET2]\n\
291 Translate, squeeze, and/or delete characters from standard input,\n\
292 writing to standard output.\n\
294 -c, -C, --complement first complement SET1\n\
295 -d, --delete delete characters in SET1, do not translate\n\
296 -s, --squeeze-repeats replace each input sequence of a repeated character\n\
297 that is listed in SET1 with a single occurrence\n\
299 -t, --truncate-set1 first truncate SET1 to length of SET2\n\
301 fputs (HELP_OPTION_DESCRIPTION, stdout);
302 fputs (VERSION_OPTION_DESCRIPTION, stdout);
305 SETs are specified as strings of characters. Most represent themselves.\n\
306 Interpreted sequences are:\n\
308 \\NNN character with octal value NNN (1 to 3 octal digits)\n\
315 \\t horizontal tab\n\
319 CHAR1-CHAR2 all characters from CHAR1 to CHAR2 in ascending order\n\
320 [CHAR*] in SET2, copies of CHAR until length of SET1\n\
321 [CHAR*REPEAT] REPEAT copies of CHAR, REPEAT octal if starting with 0\n\
322 [:alnum:] all letters and digits\n\
323 [:alpha:] all letters\n\
324 [:blank:] all horizontal whitespace\n\
325 [:cntrl:] all control characters\n\
326 [:digit:] all digits\n\
329 [:graph:] all printable characters, not including space\n\
330 [:lower:] all lower case letters\n\
331 [:print:] all printable characters, including space\n\
332 [:punct:] all punctuation characters\n\
333 [:space:] all horizontal or vertical whitespace\n\
334 [:upper:] all upper case letters\n\
335 [:xdigit:] all hexadecimal digits\n\
336 [=CHAR=] all characters which are equivalent to CHAR\n\
340 Translation occurs if -d is not given and both SET1 and SET2 appear.\n\
341 -t may be used only when translating. SET2 is extended to length of\n\
342 SET1 by repeating its last character as necessary. \
346 of SET2 are ignored. Only [:lower:] and [:upper:] are guaranteed to\n\
347 expand in ascending order; used in SET2 while translating, they may\n\
348 only be used in pairs to specify case conversion. \
351 -s uses SET1 if not\n\
352 translating nor deleting; else squeezing uses SET2 and occurs after\n\
353 translation or deletion.\n\
355 emit_bug_reporting_address ();
360 /* Return nonzero if the character C is a member of the
361 equivalence class containing the character EQUIV_CLASS. */
364 is_equiv_class_member (unsigned char equiv_class, unsigned char c)
366 return (equiv_class == c);
369 /* Return true if the character C is a member of the
370 character class CHAR_CLASS. */
373 is_char_class_member (enum Char_class char_class, unsigned char c)
380 result = isalnum (c);
383 result = isalpha (c);
386 result = isblank (c);
389 result = iscntrl (c);
392 result = isdigit (c);
395 result = isgraph (c);
398 result = islower (c);
401 result = isprint (c);
404 result = ispunct (c);
407 result = isspace (c);
410 result = isupper (c);
413 result = isxdigit (c);
424 es_free (struct E_string *es)
430 /* Perform the first pass over each range-spec argument S, converting all
431 \c and \ddd escapes to their one-byte representations. If an invalid
432 quote sequence is found print an error message and return false;
433 Otherwise set *ES to the resulting string and return true.
434 The resulting array of characters may contain zero-bytes;
435 however, on input, S is assumed to be null-terminated, and hence
436 cannot contain actual (non-escaped) zero bytes. */
439 unquote (char const *s, struct E_string *es)
442 size_t len = strlen (s);
444 es->s = xmalloc (len);
445 es->escaped = xcalloc (len, sizeof es->escaped[0]);
448 for (i = 0; s[i]; i++)
456 es->escaped[j] = true;
492 oct_digit = s[i + 2] - '0';
493 if (0 <= oct_digit && oct_digit <= 7)
495 c = 8 * c + oct_digit;
497 oct_digit = s[i + 2] - '0';
498 if (0 <= oct_digit && oct_digit <= 7)
500 if (8 * c + oct_digit < N_CHARS)
502 c = 8 * c + oct_digit;
507 /* A 3-digit octal number larger than \377 won't
508 fit in 8 bits. So we stop when adding the
509 next digit would put us over the limit and
510 give a warning about the ambiguity. POSIX
511 isn't clear on this, and we interpret this
512 lack of clarity as meaning the resulting behavior
513 is undefined, which means we're allowed to issue
515 error (0, 0, _("warning: the ambiguous octal escape \
516 \\%c%c%c is being\n\tinterpreted as the 2-byte sequence \\0%c%c, %c"),
517 s[i], s[i + 1], s[i + 2],
518 s[i], s[i + 1], s[i + 2]);
524 error (0, 0, _("warning: an unescaped backslash "
525 "at end of string is not portable"));
526 /* POSIX is not clear about this. */
527 es->escaped[j] = false;
547 /* If CLASS_STR is a valid character class string, return its index
548 in the global char_class_name array. Otherwise, return CC_NO_CLASS. */
550 static enum Char_class
551 look_up_char_class (char const *class_str, size_t len)
555 for (i = 0; i < N_CHAR_CLASSES; i++)
556 if (strncmp (class_str, char_class_name[i], len) == 0
557 && strlen (char_class_name[i]) == len)
562 /* Return a newly allocated string with a printable version of C.
563 This function is used solely for formatting error messages. */
566 make_printable_char (unsigned char c)
568 char *buf = xmalloc (5);
577 sprintf (buf, "\\%03o", c);
582 /* Return a newly allocated copy of S which is suitable for printing.
583 LEN is the number of characters in S. Most non-printing
584 (isprint) characters are represented by a backslash followed by
585 3 octal digits. However, the characters represented by \c escapes
586 where c is one of [abfnrtv] are represented by their 2-character \c
587 sequences. This function is used solely for printing error messages. */
590 make_printable_str (char const *s, size_t len)
592 /* Worst case is that every character expands to a backslash
593 followed by a 3-character octal escape sequence. */
594 char *printable_buf = xnmalloc (len + 1, 4);
595 char *p = printable_buf;
598 for (i = 0; i < len; i++)
601 char const *tmp = NULL;
602 unsigned char c = s[i];
637 sprintf (buf, "\\%03o", c);
643 return printable_buf;
646 /* Append a newly allocated structure representing a
647 character C to the specification list LIST. */
650 append_normal_char (struct Spec_list *list, unsigned char c)
652 struct List_element *new;
654 new = xmalloc (sizeof *new);
656 new->type = RE_NORMAL_CHAR;
657 new->u.normal_char = c;
659 list->tail->next = new;
663 /* Append a newly allocated structure representing the range
664 of characters from FIRST to LAST to the specification list LIST.
665 Return false if LAST precedes FIRST in the collating sequence,
666 true otherwise. This means that '[c-c]' is acceptable. */
669 append_range (struct Spec_list *list, unsigned char first, unsigned char last)
671 struct List_element *new;
675 char *tmp1 = make_printable_char (first);
676 char *tmp2 = make_printable_char (last);
679 _("range-endpoints of `%s-%s' are in reverse collating sequence order"),
685 new = xmalloc (sizeof *new);
687 new->type = RE_RANGE;
688 new->u.range.first_char = first;
689 new->u.range.last_char = last;
691 list->tail->next = new;
696 /* If CHAR_CLASS_STR is a valid character class string, append a
697 newly allocated structure representing that character class to the end
698 of the specification list LIST and return true. If CHAR_CLASS_STR is not
699 a valid string return false. */
702 append_char_class (struct Spec_list *list,
703 char const *char_class_str, size_t len)
705 enum Char_class char_class;
706 struct List_element *new;
708 char_class = look_up_char_class (char_class_str, len);
709 if (char_class == CC_NO_CLASS)
711 new = xmalloc (sizeof *new);
713 new->type = RE_CHAR_CLASS;
714 new->u.char_class = char_class;
716 list->tail->next = new;
721 /* Append a newly allocated structure representing a [c*n]
722 repeated character construct to the specification list LIST.
723 THE_CHAR is the single character to be repeated, and REPEAT_COUNT
724 is a non-negative repeat count. */
727 append_repeated_char (struct Spec_list *list, unsigned char the_char,
730 struct List_element *new;
732 new = xmalloc (sizeof *new);
734 new->type = RE_REPEATED_CHAR;
735 new->u.repeated_char.the_repeated_char = the_char;
736 new->u.repeated_char.repeat_count = repeat_count;
738 list->tail->next = new;
742 /* Given a string, EQUIV_CLASS_STR, from a [=str=] context and
743 the length of that string, LEN, if LEN is exactly one, append
744 a newly allocated structure representing the specified
745 equivalence class to the specification list, LIST and return true.
746 If LEN is not 1, return false. */
749 append_equiv_class (struct Spec_list *list,
750 char const *equiv_class_str, size_t len)
752 struct List_element *new;
756 new = xmalloc (sizeof *new);
758 new->type = RE_EQUIV_CLASS;
759 new->u.equiv_code = *equiv_class_str;
761 list->tail->next = new;
766 /* Search forward starting at START_IDX for the 2-char sequence
767 (PRE_BRACKET_CHAR,']') in the string P of length P_LEN. If such
768 a sequence is found, set *RESULT_IDX to the index of the first
769 character and return true. Otherwise return false. P may contain
773 find_closing_delim (const struct E_string *es, size_t start_idx,
774 char pre_bracket_char, size_t *result_idx)
778 for (i = start_idx; i < es->len - 1; i++)
779 if (es->s[i] == pre_bracket_char && es->s[i + 1] == ']'
780 && !es->escaped[i] && !es->escaped[i + 1])
788 /* Parse the bracketed repeat-char syntax. If the P_LEN characters
789 beginning with P[ START_IDX ] comprise a valid [c*n] construct,
790 then set *CHAR_TO_REPEAT, *REPEAT_COUNT, and *CLOSING_BRACKET_IDX
791 and return zero. If the second character following
792 the opening bracket is not `*' or if no closing bracket can be
793 found, return -1. If a closing bracket is found and the
794 second char is `*', but the string between the `*' and `]' isn't
795 empty, an octal number, or a decimal number, print an error message
799 find_bracketed_repeat (const struct E_string *es, size_t start_idx,
800 unsigned char *char_to_repeat, count *repeat_count,
801 size_t *closing_bracket_idx)
805 assert (start_idx + 1 < es->len);
806 if (!es_match (es, start_idx + 1, '*'))
809 for (i = start_idx + 2; i < es->len && !es->escaped[i]; i++)
813 size_t digit_str_len = i - start_idx - 2;
815 *char_to_repeat = es->s[start_idx];
816 if (digit_str_len == 0)
818 /* We've matched [c*] -- no explicit repeat count. */
823 /* Here, we have found [c*s] where s should be a string
824 of octal (if it starts with `0') or decimal digits. */
825 char const *digit_str = &es->s[start_idx + 2];
827 if ((xstrtoumax (digit_str, &d_end, *digit_str == '0' ? 8 : 10,
830 || REPEAT_COUNT_MAXIMUM < *repeat_count
831 || digit_str + digit_str_len != d_end)
833 char *tmp = make_printable_str (digit_str, digit_str_len);
835 _("invalid repeat count %s in [c*n] construct"),
841 *closing_bracket_idx = i;
845 return -1; /* No bracket found. */
848 /* Return true if the string at ES->s[IDX] matches the regular
849 expression `\*[0-9]*\]', false otherwise. The string does not
850 match if any of its characters are escaped. */
853 star_digits_closebracket (const struct E_string *es, size_t idx)
857 if (!es_match (es, idx, '*'))
860 for (i = idx + 1; i < es->len; i++)
861 if (!ISDIGIT (to_uchar (es->s[i])) || es->escaped[i])
862 return es_match (es, i, ']');
866 /* Convert string UNESCAPED_STRING (which has been preprocessed to
867 convert backslash-escape sequences) of length LEN characters into
868 a linked list of the following 5 types of constructs:
869 - [:str:] Character class where `str' is one of the 12 valid strings.
870 - [=c=] Equivalence class where `c' is any single character.
871 - [c*n] Repeat the single character `c' `n' times. n may be omitted.
872 However, if `n' is present, it must be a non-negative octal or
874 - r-s Range of characters from `r' to `s'. The second endpoint must
875 not precede the first in the current collating sequence.
876 - c Any other character is interpreted as itself. */
879 build_spec_list (const struct E_string *es, struct Spec_list *result)
886 /* The main for-loop below recognizes the 4 multi-character constructs.
887 A character that matches (in its context) none of the multi-character
888 constructs is classified as `normal'. Since all multi-character
889 constructs have at least 3 characters, any strings of length 2 or
890 less are composed solely of normal characters. Hence, the index of
891 the outer for-loop runs only as far as LEN-2. */
893 for (i = 0; i + 2 < es->len; /* empty */)
895 if (es_match (es, i, '['))
897 bool matched_multi_char_construct;
898 size_t closing_bracket_idx;
899 unsigned char char_to_repeat;
903 matched_multi_char_construct = true;
904 if (es_match (es, i + 1, ':') || es_match (es, i + 1, '='))
906 size_t closing_delim_idx;
908 if (find_closing_delim (es, i + 2, p[i + 1], &closing_delim_idx))
910 size_t opnd_str_len = closing_delim_idx - 1 - (i + 2) + 1;
911 char const *opnd_str = p + i + 2;
913 if (opnd_str_len == 0)
916 error (0, 0, _("missing character class name `[::]'"));
919 _("missing equivalence class character `[==]'"));
925 /* FIXME: big comment. */
926 if (!append_char_class (result, opnd_str, opnd_str_len))
928 if (star_digits_closebracket (es, i + 2))
929 goto try_bracketed_repeat;
932 char *tmp = make_printable_str (opnd_str,
934 error (0, 0, _("invalid character class %s"),
943 /* FIXME: big comment. */
944 if (!append_equiv_class (result, opnd_str, opnd_str_len))
946 if (star_digits_closebracket (es, i + 2))
947 goto try_bracketed_repeat;
950 char *tmp = make_printable_str (opnd_str,
953 _("%s: equivalence class operand must be a single character"),
961 i = closing_delim_idx + 2;
964 /* Else fall through. This could be [:*] or [=*]. */
967 try_bracketed_repeat:
969 /* Determine whether this is a bracketed repeat range
970 matching the RE \[.\*(dec_or_oct_number)?\]. */
971 err = find_bracketed_repeat (es, i + 1, &char_to_repeat,
973 &closing_bracket_idx);
976 append_repeated_char (result, char_to_repeat, repeat_count);
977 i = closing_bracket_idx + 1;
981 matched_multi_char_construct = false;
985 /* Found a string that looked like [c*n] but the
986 numeric part was invalid. */
990 if (matched_multi_char_construct)
993 /* We reach this point if P does not match [:str:], [=c=],
994 [c*n], or [c*]. Now, see if P looks like a range `[-c'
995 (from `[' to `c'). */
998 /* Look ahead one char for ranges like a-z. */
999 if (es_match (es, i + 1, '-'))
1001 if (!append_range (result, p[i], p[i + 2]))
1007 append_normal_char (result, p[i]);
1012 /* Now handle the (2 or fewer) remaining characters p[i]..p[es->len - 1]. */
1013 for (; i < es->len; i++)
1014 append_normal_char (result, p[i]);
1019 /* Advance past the current construct.
1020 S->tail must be non-NULL. */
1022 skip_construct (struct Spec_list *s)
1024 s->tail = s->tail->next;
1025 s->state = NEW_ELEMENT;
1028 /* Given a Spec_list S (with its saved state implicit in the values
1029 of its members `tail' and `state'), return the next single character
1030 in the expansion of S's constructs. If the last character of S was
1031 returned on the previous call or if S was empty, this function
1032 returns -1. For example, successive calls to get_next where S
1033 represents the spec-string 'a-d[y*3]' will return the sequence
1034 of values a, b, c, d, y, y, y, -1. Finally, if the construct from
1035 which the returned character comes is [:upper:] or [:lower:], the
1036 parameter CLASS is given a value to indicate which it was. Otherwise
1037 CLASS is set to UL_NONE. This value is used only when constructing
1038 the translation table to verify that any occurrences of upper and
1039 lower class constructs in the spec-strings appear in the same relative
1043 get_next (struct Spec_list *s, enum Upper_Lower_class *class)
1045 struct List_element *p;
1052 if (s->state == BEGIN_STATE)
1054 s->tail = s->head->next;
1055 s->state = NEW_ELEMENT;
1064 case RE_NORMAL_CHAR:
1065 return_val = p->u.normal_char;
1066 s->state = NEW_ELEMENT;
1071 if (s->state == NEW_ELEMENT)
1072 s->state = p->u.range.first_char;
1075 return_val = s->state;
1076 if (s->state == p->u.range.last_char)
1079 s->state = NEW_ELEMENT;
1086 switch (p->u.char_class)
1099 if (s->state == NEW_ELEMENT)
1101 for (i = 0; i < N_CHARS; i++)
1102 if (is_char_class_member (p->u.char_class, i))
1104 assert (i < N_CHARS);
1107 assert (is_char_class_member (p->u.char_class, s->state));
1108 return_val = s->state;
1109 for (i = s->state + 1; i < N_CHARS; i++)
1110 if (is_char_class_member (p->u.char_class, i))
1117 s->state = NEW_ELEMENT;
1121 case RE_EQUIV_CLASS:
1122 /* FIXME: this assumes that each character is alone in its own
1123 equivalence class (which appears to be correct for my
1124 LC_COLLATE. But I don't know of any function that allows
1125 one to determine a character's equivalence class. */
1127 return_val = p->u.equiv_code;
1128 s->state = NEW_ELEMENT;
1132 case RE_REPEATED_CHAR:
1133 /* Here, a repeat count of n == 0 means don't repeat at all. */
1134 if (p->u.repeated_char.repeat_count == 0)
1137 s->state = NEW_ELEMENT;
1138 return_val = get_next (s, class);
1142 if (s->state == NEW_ELEMENT)
1147 return_val = p->u.repeated_char.the_repeated_char;
1148 if (s->state == p->u.repeated_char.repeat_count)
1151 s->state = NEW_ELEMENT;
1164 /* This is a minor kludge. This function is called from
1165 get_spec_stats to determine the cardinality of a set derived
1166 from a complemented string. It's a kludge in that some of the
1167 same operations are (duplicated) performed in set_initialize. */
1170 card_of_complement (struct Spec_list *s)
1173 int cardinality = N_CHARS;
1174 bool in_set[N_CHARS] = { 0, };
1176 s->state = BEGIN_STATE;
1177 while ((c = get_next (s, NULL)) != -1)
1179 cardinality -= (!in_set[c]);
1185 /* Gather statistics about the spec-list S in preparation for the tests
1186 in validate that determine the consistency of the specs. This function
1187 is called at most twice; once for string1, and again for any string2.
1188 LEN_S1 < 0 indicates that this is the first call and that S represents
1189 string1. When LEN_S1 >= 0, it is the length of the expansion of the
1190 constructs in string1, and we can use its value to resolve any
1191 indefinite repeat construct in S (which represents string2). Hence,
1192 this function has the side-effect that it converts a valid [c*]
1193 construct in string2 to [c*n] where n is large enough (or 0) to give
1194 string2 the same length as string1. For example, with the command
1195 tr a-z 'A[\n*]Z' on the second call to get_spec_stats, LEN_S1 would
1196 be 26 and S (representing string2) would be converted to 'A[\n*24]Z'. */
1199 get_spec_stats (struct Spec_list *s)
1201 struct List_element *p;
1204 s->n_indefinite_repeats = 0;
1205 s->has_equiv_class = false;
1206 s->has_restricted_char_class = false;
1207 s->has_char_class = false;
1208 for (p = s->head->next; p; p = p->next)
1216 case RE_NORMAL_CHAR:
1221 assert (p->u.range.last_char >= p->u.range.first_char);
1222 len = p->u.range.last_char - p->u.range.first_char + 1;
1226 s->has_char_class = true;
1227 for (i = 0; i < N_CHARS; i++)
1228 if (is_char_class_member (p->u.char_class, i))
1230 switch (p->u.char_class)
1236 s->has_restricted_char_class = true;
1241 case RE_EQUIV_CLASS:
1242 for (i = 0; i < N_CHARS; i++)
1243 if (is_equiv_class_member (p->u.equiv_code, i))
1245 s->has_equiv_class = true;
1248 case RE_REPEATED_CHAR:
1249 if (p->u.repeated_char.repeat_count > 0)
1250 len = p->u.repeated_char.repeat_count;
1253 s->indefinite_repeat_element = p;
1254 ++(s->n_indefinite_repeats);
1263 /* Check for arithmetic overflow in computing length. Also, reject
1264 any length greater than the maximum repeat count, in case the
1265 length is later used to compute the repeat count for an
1266 indefinite element. */
1267 new_length = length + len;
1268 if (! (length <= new_length && new_length <= REPEAT_COUNT_MAXIMUM))
1269 error (EXIT_FAILURE, 0, _("too many characters in set"));
1270 length = new_length;
1277 get_s1_spec_stats (struct Spec_list *s1)
1279 get_spec_stats (s1);
1281 s1->length = card_of_complement (s1);
1285 get_s2_spec_stats (struct Spec_list *s2, count len_s1)
1287 get_spec_stats (s2);
1288 if (len_s1 >= s2->length && s2->n_indefinite_repeats == 1)
1290 s2->indefinite_repeat_element->u.repeated_char.repeat_count =
1291 len_s1 - s2->length;
1292 s2->length = len_s1;
1297 spec_init (struct Spec_list *spec_list)
1299 struct List_element *new = xmalloc (sizeof *new);
1300 spec_list->head = spec_list->tail = new;
1301 spec_list->head->next = NULL;
1304 /* This function makes two passes over the argument string S. The first
1305 one converts all \c and \ddd escapes to their one-byte representations.
1306 The second constructs a linked specification list, SPEC_LIST, of the
1307 characters and constructs that comprise the argument string. If either
1308 of these passes detects an error, this function returns false. */
1311 parse_str (char const *s, struct Spec_list *spec_list)
1314 bool ok = unquote (s, &es) && build_spec_list (&es, spec_list);
1319 /* Given two specification lists, S1 and S2, and assuming that
1320 S1->length > S2->length, append a single [c*n] element to S2 where c
1321 is the last character in the expansion of S2 and n is the difference
1322 between the two lengths.
1323 Upon successful completion, S2->length is set to S1->length. The only
1324 way this function can fail to make S2 as long as S1 is when S2 has
1325 zero-length, since in that case, there is no last character to repeat.
1326 So S2->length is required to be at least 1.
1328 Providing this functionality allows the user to do some pretty
1329 non-BSD (and non-portable) things: For example, the command
1330 tr -cs '[:upper:]0-9' '[:lower:]'
1331 is almost guaranteed to give results that depend on your collating
1335 string2_extend (const struct Spec_list *s1, struct Spec_list *s2)
1337 struct List_element *p;
1338 unsigned char char_to_repeat;
1341 assert (translating);
1342 assert (s1->length > s2->length);
1343 assert (s2->length > 0);
1348 case RE_NORMAL_CHAR:
1349 char_to_repeat = p->u.normal_char;
1352 char_to_repeat = p->u.range.last_char;
1355 for (i = N_CHARS - 1; i >= 0; i--)
1356 if (is_char_class_member (p->u.char_class, i))
1362 case RE_REPEATED_CHAR:
1363 char_to_repeat = p->u.repeated_char.the_repeated_char;
1366 case RE_EQUIV_CLASS:
1367 /* This shouldn't happen, because validate exits with an error
1368 if it finds an equiv class in string2 when translating. */
1377 append_repeated_char (s2, char_to_repeat, s1->length - s2->length);
1378 s2->length = s1->length;
1381 /* Return true if S is a non-empty list in which exactly one
1382 character (but potentially, many instances of it) appears.
1383 E.g., [X*] or xxxxxxxx. */
1386 homogeneous_spec_list (struct Spec_list *s)
1390 s->state = BEGIN_STATE;
1392 if ((b = get_next (s, NULL)) == -1)
1395 while ((c = get_next (s, NULL)) != -1)
1402 /* Die with an error message if S1 and S2 describe strings that
1403 are not valid with the given command line switches.
1404 A side effect of this function is that if a valid [c*] or
1405 [c*0] construct appears in string2, it is converted to [c*n]
1406 with a value for n that makes s2->length == s1->length. By
1407 the same token, if the --truncate-set1 option is not
1408 given, S2 may be extended. */
1411 validate (struct Spec_list *s1, struct Spec_list *s2)
1413 get_s1_spec_stats (s1);
1414 if (s1->n_indefinite_repeats > 0)
1416 error (EXIT_FAILURE, 0,
1417 _("the [c*] repeat construct may not appear in string1"));
1422 get_s2_spec_stats (s2, s1->length);
1424 if (s2->n_indefinite_repeats > 1)
1426 error (EXIT_FAILURE, 0,
1427 _("only one [c*] repeat construct may appear in string2"));
1432 if (s2->has_equiv_class)
1434 error (EXIT_FAILURE, 0,
1435 _("[=c=] expressions may not appear in string2 \
1436 when translating"));
1439 if (s1->length > s2->length)
1443 /* string2 must be non-empty unless --truncate-set1 is
1444 given or string1 is empty. */
1446 if (s2->length == 0)
1447 error (EXIT_FAILURE, 0,
1448 _("when not truncating set1, string2 must be non-empty"));
1449 string2_extend (s1, s2);
1453 if (complement && s1->has_char_class
1454 && ! (s2->length == s1->length && homogeneous_spec_list (s2)))
1456 error (EXIT_FAILURE, 0,
1457 _("when translating with complemented character classes,\
1458 \nstring2 must map all characters in the domain to one"));
1461 if (s2->has_restricted_char_class)
1463 error (EXIT_FAILURE, 0,
1464 _("when translating, the only character classes that may \
1465 appear in\nstring2 are `upper' and `lower'"));
1469 /* Not translating. */
1471 if (s2->n_indefinite_repeats > 0)
1472 error (EXIT_FAILURE, 0,
1473 _("the [c*] construct may appear in string2 only \
1474 when translating"));
1479 /* Read buffers of SIZE bytes via the function READER (if READER is
1480 NULL, read from stdin) until EOF. When non-NULL, READER is either
1481 read_and_delete or read_and_xlate. After each buffer is read, it is
1482 processed and written to stdout. The buffers are processed so that
1483 multiple consecutive occurrences of the same character in the input
1484 stream are replaced by a single occurrence of that character if the
1485 character is in the squeeze set. */
1488 squeeze_filter (char *buf, size_t size, size_t (*reader) (char *, size_t))
1490 /* A value distinct from any character that may have been stored in a
1491 buffer as the result of a block-read in the function squeeze_filter. */
1492 enum { NOT_A_CHAR = CHAR_MAX + 1 };
1494 int char_to_squeeze = NOT_A_CHAR;
1504 nr = reader (buf, size);
1512 if (char_to_squeeze == NOT_A_CHAR)
1515 /* Here, by being a little tricky, we can get a significant
1516 performance increase in most cases when the input is
1517 reasonably large. Since tr will modify the input only
1518 if two consecutive (and identical) input characters are
1519 in the squeeze set, we can step by two through the data
1520 when searching for a character in the squeeze set. This
1521 means there may be a little more work in a few cases and
1522 perhaps twice as much work in the worst cases where most
1523 of the input is removed by squeezing repeats. But most
1524 uses of this functionality seem to remove less than 20-30%
1526 for (; i < nr && !in_squeeze_set[to_uchar (buf[i])]; i += 2)
1529 /* There is a special case when i == nr and we've just
1530 skipped a character (the last one in buf) that is in
1532 if (i == nr && in_squeeze_set[to_uchar (buf[i - 1])])
1536 out_len = nr - begin;
1539 char_to_squeeze = buf[i];
1540 /* We're about to output buf[begin..i]. */
1541 out_len = i - begin + 1;
1543 /* But since we stepped by 2 in the loop above,
1544 out_len may be one too large. */
1545 if (i > 0 && buf[i - 1] == char_to_squeeze)
1548 /* Advance i to the index of first character to be
1549 considered when looking for a char different from
1554 && fwrite (&buf[begin], 1, out_len, stdout) != out_len)
1555 error (EXIT_FAILURE, errno, _("write error"));
1558 if (char_to_squeeze != NOT_A_CHAR)
1560 /* Advance i to index of first char != char_to_squeeze
1561 (or to nr if all the rest of the characters in this
1562 buffer are the same as char_to_squeeze). */
1563 for (; i < nr && buf[i] == char_to_squeeze; i++)
1566 char_to_squeeze = NOT_A_CHAR;
1567 /* If (i >= nr) we've squeezed the last character in this buffer.
1568 So now we have to read a new buffer and continue comparing
1569 characters against char_to_squeeze. */
1575 plain_read (char *buf, size_t size)
1577 size_t nr = safe_read (STDIN_FILENO, buf, size);
1578 if (nr == SAFE_READ_ERROR)
1579 error (EXIT_FAILURE, errno, _("read error"));
1583 /* Read buffers of SIZE bytes from stdin until one is found that
1584 contains at least one character not in the delete set. Store
1585 in the array BUF, all characters from that buffer that are not
1586 in the delete set, and return the number of characters saved
1590 read_and_delete (char *buf, size_t size)
1594 /* This enclosing do-while loop is to make sure that
1595 we don't return zero (indicating EOF) when we've
1596 just deleted all the characters in a buffer. */
1600 size_t nr = plain_read (buf, size);
1605 /* This first loop may be a waste of code, but gives much
1606 better performance when no characters are deleted in
1607 the beginning of a buffer. It just avoids the copying
1608 of buf[i] into buf[n_saved] when it would be a NOP. */
1610 for (i = 0; i < nr && !in_delete_set[to_uchar (buf[i])]; i++)
1614 for (++i; i < nr; i++)
1615 if (!in_delete_set[to_uchar (buf[i])])
1616 buf[n_saved++] = buf[i];
1618 while (n_saved == 0);
1623 /* Read at most SIZE bytes from stdin into the array BUF. Then
1624 perform the in-place and one-to-one mapping specified by the global
1625 array `xlate'. Return the number of characters read, or 0 upon EOF. */
1628 read_and_xlate (char *buf, size_t size)
1630 size_t bytes_read = plain_read (buf, size);
1633 for (i = 0; i < bytes_read; i++)
1634 buf[i] = xlate[to_uchar (buf[i])];
1639 /* Initialize a boolean membership set, IN_SET, with the character
1640 values obtained by traversing the linked list of constructs S
1641 using the function `get_next'. IN_SET is expected to have been
1642 initialized to all zeros by the caller. If COMPLEMENT_THIS_SET
1643 is true the resulting set is complemented. */
1646 set_initialize (struct Spec_list *s, bool complement_this_set, bool *in_set)
1651 s->state = BEGIN_STATE;
1652 while ((c = get_next (s, NULL)) != -1)
1654 if (complement_this_set)
1655 for (i = 0; i < N_CHARS; i++)
1656 in_set[i] = (!in_set[i]);
1660 main (int argc, char **argv)
1663 int non_option_args;
1666 struct Spec_list buf1, buf2;
1667 struct Spec_list *s1 = &buf1;
1668 struct Spec_list *s2 = &buf2;
1670 initialize_main (&argc, &argv);
1671 set_program_name (argv[0]);
1672 setlocale (LC_ALL, "");
1673 bindtextdomain (PACKAGE, LOCALEDIR);
1674 textdomain (PACKAGE);
1676 atexit (close_stdout);
1678 while ((c = getopt_long (argc, argv, "+cCdst", long_options, NULL)) != -1)
1692 squeeze_repeats = true;
1696 truncate_set1 = true;
1699 case_GETOPT_HELP_CHAR;
1701 case_GETOPT_VERSION_CHAR (PROGRAM_NAME, AUTHORS);
1704 usage (EXIT_FAILURE);
1709 non_option_args = argc - optind;
1710 translating = (non_option_args == 2 && !delete);
1711 min_operands = 1 + (delete == squeeze_repeats);
1712 max_operands = 1 + (delete <= squeeze_repeats);
1714 if (non_option_args < min_operands)
1716 if (non_option_args == 0)
1717 error (0, 0, _("missing operand"));
1720 error (0, 0, _("missing operand after %s"), quote (argv[argc - 1]));
1721 fprintf (stderr, "%s\n",
1723 ? ("Two strings must be given when "
1724 "both deleting and squeezing repeats.")
1725 : "Two strings must be given when translating."));
1727 usage (EXIT_FAILURE);
1730 if (max_operands < non_option_args)
1732 error (0, 0, _("extra operand %s"), quote (argv[optind + max_operands]));
1733 if (non_option_args == 2)
1734 fprintf (stderr, "%s\n",
1735 _("Only one string may be given when "
1736 "deleting without squeezing repeats."));
1737 usage (EXIT_FAILURE);
1741 if (!parse_str (argv[optind], s1))
1742 exit (EXIT_FAILURE);
1744 if (non_option_args == 2)
1747 if (!parse_str (argv[optind + 1], s2))
1748 exit (EXIT_FAILURE);
1755 /* Use binary I/O, since `tr' is sometimes used to transliterate
1756 non-printable characters, or characters which are stripped away
1757 by text-mode reads (like CR and ^Z). */
1758 if (O_BINARY && ! isatty (STDIN_FILENO))
1759 freopen (NULL, "rb", stdin);
1760 if (O_BINARY && ! isatty (STDOUT_FILENO))
1761 freopen (NULL, "wb", stdout);
1763 if (squeeze_repeats && non_option_args == 1)
1765 set_initialize (s1, complement, in_squeeze_set);
1766 squeeze_filter (io_buf, sizeof io_buf, plain_read);
1768 else if (delete && non_option_args == 1)
1770 set_initialize (s1, complement, in_delete_set);
1774 size_t nr = read_and_delete (io_buf, sizeof io_buf);
1777 if (fwrite (io_buf, 1, nr, stdout) != nr)
1778 error (EXIT_FAILURE, errno, _("write error"));
1781 else if (squeeze_repeats && delete && non_option_args == 2)
1783 set_initialize (s1, complement, in_delete_set);
1784 set_initialize (s2, false, in_squeeze_set);
1785 squeeze_filter (io_buf, sizeof io_buf, read_and_delete);
1787 else if (translating)
1792 bool *in_s1 = in_delete_set;
1794 set_initialize (s1, false, in_s1);
1795 s2->state = BEGIN_STATE;
1796 for (i = 0; i < N_CHARS; i++)
1798 for (i = 0; i < N_CHARS; i++)
1802 int ch = get_next (s2, NULL);
1803 assert (ch != -1 || truncate_set1);
1806 /* This will happen when tr is invoked like e.g.
1807 tr -cs A-Za-z0-9 '\012'. */
1818 bool case_convert = false;
1819 enum Upper_Lower_class class_s1;
1820 enum Upper_Lower_class class_s2;
1822 for (i = 0; i < N_CHARS; i++)
1824 s1->state = BEGIN_STATE;
1825 s2->state = BEGIN_STATE;
1828 /* When the previous pair identified case-converting classes,
1829 advance S1 and S2 so that each points to the following
1833 skip_construct (s1);
1834 skip_construct (s2);
1835 case_convert = false;
1838 c1 = get_next (s1, &class_s1);
1839 c2 = get_next (s2, &class_s2);
1841 /* When translating and there is an [:upper:] or [:lower:]
1842 class in SET2, then there must be a corresponding [:lower:]
1843 or [:upper:] class in SET1. */
1844 if (class_s1 == UL_NONE
1845 && (class_s2 == UL_LOWER || class_s2 == UL_UPPER))
1846 error (EXIT_FAILURE, 0,
1847 _("misaligned [:upper:] and/or [:lower:] construct"));
1849 if (class_s1 == UL_LOWER && class_s2 == UL_UPPER)
1851 case_convert = true;
1852 for (i = 0; i < N_CHARS; i++)
1854 xlate[i] = toupper (i);
1856 else if (class_s1 == UL_UPPER && class_s2 == UL_LOWER)
1858 case_convert = true;
1859 for (i = 0; i < N_CHARS; i++)
1861 xlate[i] = tolower (i);
1863 else if ((class_s1 == UL_LOWER && class_s2 == UL_LOWER)
1864 || (class_s1 == UL_UPPER && class_s2 == UL_UPPER))
1866 /* POSIX says the behavior of `tr "[:upper:]" "[:upper:]"'
1867 is undefined. Treat it as a no-op. */
1871 /* The following should have been checked by validate... */
1872 if (c1 == -1 || c2 == -1)
1877 assert (c1 == -1 || truncate_set1);
1879 if (squeeze_repeats)
1881 set_initialize (s2, false, in_squeeze_set);
1882 squeeze_filter (io_buf, sizeof io_buf, read_and_xlate);
1888 size_t bytes_read = read_and_xlate (io_buf, sizeof io_buf);
1889 if (bytes_read == 0)
1891 if (fwrite (io_buf, 1, bytes_read, stdout) != bytes_read)
1892 error (EXIT_FAILURE, errno, _("write error"));
1897 if (close (STDIN_FILENO) != 0)
1898 error (EXIT_FAILURE, errno, _("standard input"));
1900 exit (EXIT_SUCCESS);