1 /* uniq -- remove duplicate lines from a sorted file
2 Copyright (C) 86, 91, 1995-2003, Free Software Foundation, Inc.
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2, or (at your option)
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software Foundation,
16 Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
18 /* Written by Richard Stallman and David MacKenzie. */
24 #include <sys/types.h>
28 #include "linebuffer.h"
30 #include "hard-locale.h"
34 #include "memcasecmp.h"
36 /* The official name of this program (e.g., no `g' prefix). */
37 #define PROGRAM_NAME "uniq"
39 #define AUTHORS "Richard Stallman", "David MacKenzie"
41 #define SWAP_LINES(A, B) \
44 struct linebuffer *_tmp; \
51 /* The name this program was run with. */
54 /* Nonzero if the LC_COLLATE locale is hard. */
55 static int hard_LC_COLLATE;
57 /* Number of fields to skip on each line when doing comparisons. */
58 static size_t skip_fields;
60 /* Number of chars to skip after skipping any fields. */
61 static size_t skip_chars;
63 /* Number of chars to compare. */
64 static size_t check_chars;
68 count_occurrences, /* -c Print count before output lines. */
69 count_none /* Default. Do not print counts. */
72 /* Whether and how to precede the output lines with a count of the number of
73 times they occurred in the input. */
74 static enum countmode countmode;
76 /* Which lines to output: unique lines, the first of a group of
77 repeated lines, and the second and subsequented of a group of
79 static bool output_unique;
80 static bool output_first_repeated;
81 static bool output_later_repeated;
83 /* If nonzero, ignore case when comparing. */
84 static int ignore_case;
88 /* No delimiters output. --all-repeated[=none] */
91 /* Delimiter precedes all groups. --all-repeated=prepend */
94 /* Delimit all groups. --all-repeated=separate */
98 static char const *const delimit_method_string[] =
100 "none", "prepend", "separate", 0
103 static enum delimit_method const delimit_method_map[] =
105 DM_NONE, DM_PREPEND, DM_SEPARATE
108 /* Select whether/how to delimit groups of duplicate lines. */
109 static enum delimit_method delimit_groups;
111 static struct option const longopts[] =
113 {"count", no_argument, NULL, 'c'},
114 {"repeated", no_argument, NULL, 'd'},
115 {"all-repeated", optional_argument, NULL, 'D'},
116 {"ignore-case", no_argument, NULL, 'i'},
117 {"unique", no_argument, NULL, 'u'},
118 {"skip-fields", required_argument, NULL, 'f'},
119 {"skip-chars", required_argument, NULL, 's'},
120 {"check-chars", required_argument, NULL, 'w'},
121 {GETOPT_HELP_OPTION_DECL},
122 {GETOPT_VERSION_OPTION_DECL},
130 fprintf (stderr, _("Try `%s --help' for more information.\n"),
135 Usage: %s [OPTION]... [INPUT [OUTPUT]]\n\
139 Discard all but one of successive identical lines from INPUT (or\n\
140 standard input), writing to OUTPUT (or standard output).\n\
144 Mandatory arguments to long options are mandatory for short options too.\n\
147 -c, --count prefix lines by the number of occurrences\n\
148 -d, --repeated only print duplicate lines\n\
151 -D, --all-repeated[=delimit-method] print all duplicate lines\n\
152 delimit-method={none(default),prepend,separate}\n\
153 Delimiting is done with blank lines.\n\
154 -f, --skip-fields=N avoid comparing the first N fields\n\
155 -i, --ignore-case ignore differences in case when comparing\n\
156 -s, --skip-chars=N avoid comparing the first N characters\n\
157 -u, --unique only print unique lines\n\
160 -w, --check-chars=N compare no more than N characters in lines\n\
162 fputs (HELP_OPTION_DESCRIPTION, stdout);
163 fputs (VERSION_OPTION_DESCRIPTION, stdout);
166 A field is a run of whitespace, then non-whitespace characters.\n\
167 Fields are skipped before chars.\n\
169 printf (_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
171 exit (status == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
174 /* Convert OPT to size_t, reporting an error using MSGID if it does
178 size_opt (char const *opt, char const *msgid)
180 unsigned long int size;
181 if (xstrtoul (opt, NULL, 10, &size, "") != LONGINT_OK
183 error (EXIT_FAILURE, 0, "%s: %s", opt, _(msgid));
187 /* Given a linebuffer LINE,
188 return a pointer to the beginning of the line's field to be compared. */
191 find_field (const struct linebuffer *line)
193 register size_t count;
194 register char *lp = line->buffer;
195 register size_t size = line->length - 1;
196 register size_t i = 0;
198 for (count = 0; count < skip_fields && i < size; count++)
200 while (i < size && ISBLANK (lp[i]))
202 while (i < size && !ISBLANK (lp[i]))
206 for (count = 0; count < skip_chars && i < size; count++)
212 /* Return zero if two strings OLD and NEW match, nonzero if not.
213 OLD and NEW point not to the beginnings of the lines
214 but rather to the beginnings of the fields to compare.
215 OLDLEN and NEWLEN are their lengths. */
218 different (char *old, char *new, size_t oldlen, size_t newlen)
220 if (check_chars < oldlen)
221 oldlen = check_chars;
222 if (check_chars < newlen)
223 newlen = check_chars;
227 /* FIXME: This should invoke strcoll somehow. */
228 return oldlen != newlen || memcasecmp (old, new, oldlen);
230 else if (HAVE_SETLOCALE && hard_LC_COLLATE)
231 return xmemcoll (old, oldlen, new, newlen);
233 return oldlen != newlen || memcmp (old, new, oldlen);
236 /* Output the line in linebuffer LINE to stream STREAM
237 provided that the switches say it should be output.
238 MATCH is true if the line matches the previous line.
239 If requested, print the number of times it occurred, as well;
240 LINECOUNT + 1 is the number of times that the line occurred. */
243 writeline (struct linebuffer const *line, FILE *stream,
244 bool match, int linecount)
246 if (! (linecount == 0 ? output_unique
247 : !match ? output_first_repeated
248 : output_later_repeated))
251 if (countmode == count_occurrences)
252 fprintf (stream, "%7d ", linecount + 1);
254 fwrite (line->buffer, sizeof (char), line->length, stream);
257 /* Process input file INFILE with output to OUTFILE.
258 If either is "-", use the standard I/O stream for it instead. */
261 check_file (const char *infile, const char *outfile)
265 struct linebuffer lb1, lb2;
266 struct linebuffer *thisline, *prevline;
268 if (STREQ (infile, "-"))
271 istream = fopen (infile, "r");
273 error (EXIT_FAILURE, errno, "%s", infile);
275 if (STREQ (outfile, "-"))
278 ostream = fopen (outfile, "w");
280 error (EXIT_FAILURE, errno, "%s", outfile);
285 initbuffer (thisline);
286 initbuffer (prevline);
288 /* The duplication in the following `if' and `else' blocks is an
289 optimization to distinguish the common case (in which none of
290 the following options has been specified: --count, -repeated,
291 --all-repeated, --unique) from the others. In the common case,
292 this optimization lets uniq output each different line right away,
293 without waiting to see if the next one is different. */
295 if (output_unique && output_first_repeated && countmode == count_none)
297 char *prevfield IF_LINT (= NULL);
298 size_t prevlen IF_LINT (= 0);
300 while (!feof (istream))
304 if (readlinebuffer (thisline, istream) == 0)
306 thisfield = find_field (thisline);
307 thislen = thisline->length - 1 - (thisfield - thisline->buffer);
308 if (prevline->length == 0
309 || different (thisfield, prevfield, thislen, prevlen))
311 fwrite (thisline->buffer, sizeof (char),
312 thisline->length, ostream);
314 SWAP_LINES (prevline, thisline);
315 prevfield = thisfield;
325 int first_delimiter = 1;
327 if (readlinebuffer (prevline, istream) == 0)
329 prevfield = find_field (prevline);
330 prevlen = prevline->length - 1 - (prevfield - prevline->buffer);
332 while (!feof (istream))
337 if (readlinebuffer (thisline, istream) == 0)
339 if (ferror (istream))
343 thisfield = find_field (thisline);
344 thislen = thisline->length - 1 - (thisfield - thisline->buffer);
345 match = !different (thisfield, prevfield, thislen, prevlen);
350 if (delimit_groups != DM_NONE)
354 if (match_count) /* a previous match */
355 first_delimiter = 0; /* Only used when DM_SEPARATE */
357 else if (match_count == 1)
359 if ((delimit_groups == DM_PREPEND)
360 || (delimit_groups == DM_SEPARATE
361 && !first_delimiter))
362 putc ('\n', ostream);
366 if (!match || output_later_repeated)
368 writeline (prevline, ostream, match, match_count);
369 SWAP_LINES (prevline, thisline);
370 prevfield = thisfield;
377 writeline (prevline, ostream, false, match_count);
381 if (ferror (istream) || fclose (istream) == EOF)
382 error (EXIT_FAILURE, errno, _("error reading %s"), infile);
384 if (ferror (ostream))
385 error (EXIT_FAILURE, 0, _("error writing %s"), outfile);
386 /* Close ostream only if it's not stdout -- the latter is closed
387 via the atexit-invoked close_stdout. */
388 if (ostream != stdout && fclose (ostream) != 0)
389 error (EXIT_FAILURE, errno, _("error writing %s"), outfile);
396 main (int argc, char **argv)
399 bool posixly_correct = (getenv ("POSIXLY_CORRECT") != NULL);
400 bool obsolete_skip_fields = false;
404 file[0] = file[1] = "-";
405 initialize_main (&argc, &argv);
406 program_name = argv[0];
407 setlocale (LC_ALL, "");
408 bindtextdomain (PACKAGE, LOCALEDIR);
409 textdomain (PACKAGE);
410 hard_LC_COLLATE = hard_locale (LC_COLLATE);
412 atexit (close_stdout);
416 check_chars = SIZE_MAX;
417 output_unique = output_first_repeated = true;
418 output_later_repeated = false;
419 countmode = count_none;
420 delimit_groups = DM_NONE;
424 /* Parse an operand with leading "+" as a file after "--" was
425 seen; or if pedantic and a file was seen; or if not
429 || (posixly_correct && nfiles != 0)
430 || ((optc = getopt_long (argc, argv,
431 "-0123456789Dcdf:is:uw:", longopts, NULL))
438 error (0, 0, _("extra operand `%s'"), argv[optind]);
439 usage (EXIT_FAILURE);
441 file[nfiles++] = argv[optind++];
447 unsigned long int size;
449 && posix2_version () < 200112
450 && xstrtoul (optarg, NULL, 10, &size, "") == LONGINT_OK
453 else if (nfiles == 2)
455 error (0, 0, _("extra operand `%s'"), optarg);
456 usage (EXIT_FAILURE);
459 file[nfiles++] = optarg;
474 size_t s = skip_fields;
475 skip_fields = s * 10 + optc - '0';
476 if (SIZE_MAX / 10 < s || skip_fields < s)
477 error (EXIT_FAILURE, 0, "%s",
478 _("invalid number of fields to skip"));
479 obsolete_skip_fields = true;
484 countmode = count_occurrences;
488 output_unique = false;
492 output_unique = false;
493 output_later_repeated = true;
495 delimit_groups = DM_NONE;
497 delimit_groups = XARGMATCH ("--all-repeated", optarg,
498 delimit_method_string,
502 case 'f': /* Like '-#'. */
503 skip_fields = size_opt (optarg,
504 N_("invalid number of fields to skip"));
511 case 's': /* Like '+#'. */
512 skip_chars = size_opt (optarg,
513 N_("invalid number of bytes to skip"));
517 output_first_repeated = false;
521 check_chars = size_opt (optarg,
522 N_("invalid number of bytes to compare"));
525 case_GETOPT_HELP_CHAR;
527 case_GETOPT_VERSION_CHAR (PROGRAM_NAME, AUTHORS);
530 usage (EXIT_FAILURE);
534 if (obsolete_skip_fields && 200112 <= posix2_version ())
536 error (0, 0, _("`-%lu' option is obsolete; use `-f %lu'"),
537 (unsigned long) skip_fields, (unsigned long) skip_fields);
538 usage (EXIT_FAILURE);
541 if (countmode == count_occurrences && output_later_repeated)
544 _("printing all duplicated lines and repeat counts is meaningless"));
545 usage (EXIT_FAILURE);
548 check_file (file[0], file[1]);