1 /* uniq -- remove duplicate lines from a sorted file
2 Copyright (C) 86, 91, 1995-2003, Free Software Foundation, Inc.
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2, or (at your option)
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software Foundation,
16 Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
18 /* Written by Richard Stallman and David MacKenzie. */
24 #include <sys/types.h>
28 #include "linebuffer.h"
30 #include "hard-locale.h"
34 #include "memcasecmp.h"
36 /* The official name of this program (e.g., no `g' prefix). */
37 #define PROGRAM_NAME "uniq"
39 #define WRITTEN_BY _("Written by Richard Stallman and David MacKenzie.")
41 #define SWAP_LINES(A, B) \
44 struct linebuffer *_tmp; \
51 /* The name this program was run with. */
54 /* Nonzero if the LC_COLLATE locale is hard. */
55 static int hard_LC_COLLATE;
57 /* Number of fields to skip on each line when doing comparisons. */
58 static size_t skip_fields;
60 /* Number of chars to skip after skipping any fields. */
61 static size_t skip_chars;
63 /* Number of chars to compare. */
64 static size_t check_chars;
68 count_occurrences, /* -c Print count before output lines. */
69 count_none /* Default. Do not print counts. */
72 /* Whether and how to precede the output lines with a count of the number of
73 times they occurred in the input. */
74 static enum countmode countmode;
76 /* Which lines to output: unique lines, the first of a group of
77 repeated lines, and the second and subsequented of a group of
79 static bool output_unique;
80 static bool output_first_repeated;
81 static bool output_later_repeated;
83 /* If nonzero, ignore case when comparing. */
84 static int ignore_case;
88 /* No delimiters output. --all-repeated[=none] */
91 /* Delimiter precedes all groups. --all-repeated=prepend */
94 /* Delimit all groups. --all-repeated=separate */
98 static char const *const delimit_method_string[] =
100 "none", "prepend", "separate", 0
103 static enum delimit_method const delimit_method_map[] =
105 DM_NONE, DM_PREPEND, DM_SEPARATE
108 /* Select whether/how to delimit groups of duplicate lines. */
109 static enum delimit_method delimit_groups;
111 static struct option const longopts[] =
113 {"count", no_argument, NULL, 'c'},
114 {"repeated", no_argument, NULL, 'd'},
115 {"all-repeated", optional_argument, NULL, 'D'},
116 {"ignore-case", no_argument, NULL, 'i'},
117 {"unique", no_argument, NULL, 'u'},
118 {"skip-fields", required_argument, NULL, 'f'},
119 {"skip-chars", required_argument, NULL, 's'},
120 {"check-chars", required_argument, NULL, 'w'},
121 {GETOPT_HELP_OPTION_DECL},
122 {GETOPT_VERSION_OPTION_DECL},
130 fprintf (stderr, _("Try `%s --help' for more information.\n"),
135 Usage: %s [OPTION]... [INPUT [OUTPUT]]\n\
139 Discard all but one of successive identical lines from INPUT (or\n\
140 standard input), writing to OUTPUT (or standard output).\n\
144 Mandatory arguments to long options are mandatory for short options too.\n\
147 -c, --count prefix lines by the number of occurrences\n\
148 -d, --repeated only print duplicate lines\n\
151 -D, --all-repeated[=delimit-method] print all duplicate lines\n\
152 delimit-method={none(default),prepend,separate}\n\
153 Delimiting is done with blank lines.\n\
154 -f, --skip-fields=N avoid comparing the first N fields\n\
155 -i, --ignore-case ignore differences in case when comparing\n\
156 -s, --skip-chars=N avoid comparing the first N characters\n\
157 -u, --unique only print unique lines\n\
160 -w, --check-chars=N compare no more than N characters in lines\n\
162 fputs (HELP_OPTION_DESCRIPTION, stdout);
163 fputs (VERSION_OPTION_DESCRIPTION, stdout);
166 A field is a run of whitespace, then non-whitespace characters.\n\
167 Fields are skipped before chars.\n\
169 printf (_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
171 exit (status == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
174 /* Convert OPT to size_t, reporting an error using MSGID if it does
178 size_opt (char const *opt, char const *msgid)
180 unsigned long int size;
181 if (xstrtoul (opt, NULL, 10, &size, "") != LONGINT_OK
183 error (EXIT_FAILURE, 0, "%s: %s", opt, _(msgid));
187 /* Given a linebuffer LINE,
188 return a pointer to the beginning of the line's field to be compared. */
191 find_field (const struct linebuffer *line)
193 register size_t count;
194 register char *lp = line->buffer;
195 register size_t size = line->length - 1;
196 register size_t i = 0;
198 for (count = 0; count < skip_fields && i < size; count++)
200 while (i < size && ISBLANK (lp[i]))
202 while (i < size && !ISBLANK (lp[i]))
206 for (count = 0; count < skip_chars && i < size; count++)
212 /* Return zero if two strings OLD and NEW match, nonzero if not.
213 OLD and NEW point not to the beginnings of the lines
214 but rather to the beginnings of the fields to compare.
215 OLDLEN and NEWLEN are their lengths. */
218 different (char *old, char *new, size_t oldlen, size_t newlen)
220 if (check_chars < oldlen)
221 oldlen = check_chars;
222 if (check_chars < newlen)
223 newlen = check_chars;
227 /* FIXME: This should invoke strcoll somehow. */
228 return oldlen != newlen || memcasecmp (old, new, oldlen);
230 else if (HAVE_SETLOCALE && hard_LC_COLLATE)
231 return xmemcoll (old, oldlen, new, newlen);
233 return oldlen != newlen || memcmp (old, new, oldlen);
236 /* Output the line in linebuffer LINE to stream STREAM
237 provided that the switches say it should be output.
238 MATCH is true if the line matches the previous line.
239 If requested, print the number of times it occurred, as well;
240 LINECOUNT + 1 is the number of times that the line occurred. */
243 writeline (struct linebuffer const *line, FILE *stream,
244 bool match, int linecount)
246 if (! (linecount == 0 ? output_unique
247 : !match ? output_first_repeated
248 : output_later_repeated))
251 if (countmode == count_occurrences)
252 fprintf (stream, "%7d ", linecount + 1);
254 fwrite (line->buffer, sizeof (char), line->length, stream);
257 /* Process input file INFILE with output to OUTFILE.
258 If either is "-", use the standard I/O stream for it instead. */
261 check_file (const char *infile, const char *outfile)
265 struct linebuffer lb1, lb2;
266 struct linebuffer *thisline, *prevline;
268 if (STREQ (infile, "-"))
271 istream = fopen (infile, "r");
273 error (EXIT_FAILURE, errno, "%s", infile);
275 if (STREQ (outfile, "-"))
278 ostream = fopen (outfile, "w");
280 error (EXIT_FAILURE, errno, "%s", outfile);
285 initbuffer (thisline);
286 initbuffer (prevline);
288 /* The duplication in the following `if' and `else' blocks is an
289 optimization to distinguish the common case (in which none of
290 the following options has been specified: --count, -repeated,
291 --all-repeated, --unique) from the others. In the common case,
292 this optimization lets uniq output each different line right away,
293 without waiting to see if the next one is different. */
295 if (output_unique && output_first_repeated && countmode == count_none)
297 char *prevfield IF_LINT (= NULL);
298 size_t prevlen IF_LINT (= 0);
300 while (!feof (istream))
304 if (readlinebuffer (thisline, istream) == 0)
306 thisfield = find_field (thisline);
307 thislen = thisline->length - 1 - (thisfield - thisline->buffer);
308 if (prevline->length == 0
309 || different (thisfield, prevfield, thislen, prevlen))
311 fwrite (thisline->buffer, sizeof (char),
312 thisline->length, ostream);
314 SWAP_LINES (prevline, thisline);
315 prevfield = thisfield;
325 int first_delimiter = 1;
327 if (readlinebuffer (prevline, istream) == 0)
329 prevfield = find_field (prevline);
330 prevlen = prevline->length - 1 - (prevfield - prevline->buffer);
332 while (!feof (istream))
337 if (readlinebuffer (thisline, istream) == 0)
339 thisfield = find_field (thisline);
340 thislen = thisline->length - 1 - (thisfield - thisline->buffer);
341 match = !different (thisfield, prevfield, thislen, prevlen);
346 if (delimit_groups != DM_NONE)
350 if (match_count) /* a previous match */
351 first_delimiter = 0; /* Only used when DM_SEPARATE */
353 else if (match_count == 1)
355 if ((delimit_groups == DM_PREPEND)
356 || (delimit_groups == DM_SEPARATE
357 && !first_delimiter))
358 putc ('\n', ostream);
362 if (!match || output_later_repeated)
364 writeline (prevline, ostream, match, match_count);
365 SWAP_LINES (prevline, thisline);
366 prevfield = thisfield;
373 writeline (prevline, ostream, false, match_count);
377 if (ferror (istream) || fclose (istream) == EOF)
378 error (EXIT_FAILURE, errno, _("error reading %s"), infile);
380 /* Close ostream only if it's not stdout -- the latter is closed
381 via the atexit-invoked close_stdout. */
382 if (ostream != stdout && (ferror (ostream) || fclose (ostream) == EOF))
383 error (EXIT_FAILURE, errno, _("error writing %s"), outfile);
390 main (int argc, char **argv)
393 bool posixly_correct = (getenv ("POSIXLY_CORRECT") != NULL);
394 bool obsolete_skip_fields = false;
398 file[0] = file[1] = "-";
399 initialize_main (&argc, &argv);
400 program_name = argv[0];
401 setlocale (LC_ALL, "");
402 bindtextdomain (PACKAGE, LOCALEDIR);
403 textdomain (PACKAGE);
404 hard_LC_COLLATE = hard_locale (LC_COLLATE);
406 atexit (close_stdout);
410 check_chars = SIZE_MAX;
411 output_unique = output_first_repeated = true;
412 output_later_repeated = false;
413 countmode = count_none;
414 delimit_groups = DM_NONE;
418 /* Parse an operand with leading "+" as a file after "--" was
419 seen; or if pedantic and a file was seen; or if not
423 || (posixly_correct && nfiles != 0)
424 || ((optc = getopt_long (argc, argv,
425 "-0123456789Dcdf:is:uw:", longopts, NULL))
432 error (0, 0, _("extra operand `%s'"), argv[optind]);
433 usage (EXIT_FAILURE);
435 file[nfiles++] = argv[optind++];
441 unsigned long int size;
443 && posix2_version () < 200112
444 && xstrtoul (optarg, NULL, 10, &size, "") == LONGINT_OK
447 else if (nfiles == 2)
449 error (0, 0, _("extra operand `%s'"), optarg);
450 usage (EXIT_FAILURE);
453 file[nfiles++] = optarg;
468 size_t s = skip_fields;
469 skip_fields = s * 10 + optc - '0';
470 if (SIZE_MAX / 10 < s || skip_fields < s)
471 error (EXIT_FAILURE, 0, "%s",
472 _("invalid number of fields to skip"));
473 obsolete_skip_fields = true;
478 countmode = count_occurrences;
482 output_unique = false;
486 output_unique = false;
487 output_later_repeated = true;
489 delimit_groups = DM_NONE;
491 delimit_groups = XARGMATCH ("--all-repeated", optarg,
492 delimit_method_string,
496 case 'f': /* Like '-#'. */
497 skip_fields = size_opt (optarg,
498 N_("invalid number of fields to skip"));
505 case 's': /* Like '+#'. */
506 skip_chars = size_opt (optarg,
507 N_("invalid number of bytes to skip"));
511 output_first_repeated = false;
515 check_chars = size_opt (optarg,
516 N_("invalid number of bytes to compare"));
519 case_GETOPT_HELP_CHAR;
521 case_GETOPT_VERSION_CHAR (PROGRAM_NAME, WRITTEN_BY);
524 usage (EXIT_FAILURE);
528 if (obsolete_skip_fields && 200112 <= posix2_version ())
530 error (0, 0, _("`-%lu' option is obsolete; use `-f %lu'"),
531 (unsigned long) skip_fields, (unsigned long) skip_fields);
532 usage (EXIT_FAILURE);
535 if (countmode == count_occurrences && output_later_repeated)
538 _("printing all duplicated lines and repeat counts is meaningless"));
539 usage (EXIT_FAILURE);
542 check_file (file[0], file[1]);