1 /* join - join lines of two files on a common field
2 Copyright (C) 91, 1995-2003 Free Software Foundation, Inc.
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2, or (at your option)
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software Foundation,
16 Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 Written by Mike Haertel, mike@gnu.ai.mit.edu. */
24 #include <sys/types.h>
29 #include "hard-locale.h"
30 #include "linebuffer.h"
31 #include "memcasecmp.h"
35 /* The official name of this program (e.g., no `g' prefix). */
36 #define PROGRAM_NAME "join"
38 #define WRITTEN_BY _("Written by Mike Haertel.")
40 #define join system_join
42 /* Undefine, to avoid warning about redefinition on some systems. */
45 #define min(A, B) ((A) < (B) ? (A) : (B))
46 #define max(A, B) ((A) > (B) ? (A) : (B))
48 /* An element of the list identifying which fields to print for each
52 /* File number: 0, 1, or 2. 0 means use the join field.
53 1 means use the first file argument, 2 the second. */
56 /* Field index (zero-based), specified only when FILE is 1 or 2. */
62 /* A field of a line. */
65 const unsigned char *beg; /* First character in field. */
66 size_t len; /* The length of the field. */
69 /* A line read from an input file. */
72 struct linebuffer buf; /* The line itself. */
73 int nfields; /* Number of elements in `fields'. */
74 int nfields_allocated; /* Number of elements in `fields'. */
78 /* One or more consecutive lines read from a file that all have the
79 same join field value. */
82 int count; /* Elements used in `lines'. */
83 int alloc; /* Elements allocated in `lines'. */
87 /* The name this program was run with. */
90 /* Nonzero if the LC_COLLATE locale is hard. */
91 static int hard_LC_COLLATE;
93 /* If nonzero, print unpairable lines in file 1 or 2. */
94 static int print_unpairables_1, print_unpairables_2;
96 /* If nonzero, print pairable lines. */
97 static int print_pairables;
99 /* Empty output field filler. */
100 static char *empty_filler;
102 /* Field to join on. */
103 static int join_field_1, join_field_2;
105 /* List of fields to print. */
106 static struct outlist outlist_head;
108 /* Last element in `outlist', where a new element can be added. */
109 static struct outlist *outlist_end = &outlist_head;
111 /* Tab character separating fields; if this is NUL fields are separated
112 by any nonempty string of white space, otherwise by exactly one
114 static unsigned char tab;
116 /* When using getopt_long_only, no long option can start with
117 a character that is a short option. */
118 static struct option const longopts[] =
120 {"ignore-case", no_argument, NULL, 'i'},
121 {"j", required_argument, NULL, 'j'},
122 {"j1", required_argument, NULL, '1'},
123 {"j2", required_argument, NULL, '2'},
124 {GETOPT_HELP_OPTION_DECL},
125 {GETOPT_VERSION_OPTION_DECL},
129 /* Used to print non-joining lines */
130 static struct line uni_blank;
132 /* If nonzero, ignore case when comparing join fields. */
133 static int ignore_case;
139 fprintf (stderr, _("Try `%s --help' for more information.\n"),
144 Usage: %s [OPTION]... FILE1 FILE2\n\
148 For each pair of input lines with identical join fields, write a line to\n\
149 standard output. The default join field is the first, delimited\n\
150 by whitespace. When FILE1 or FILE2 (not both) is -, read standard input.\n\
152 -a FILENUM print unpairable lines coming from file FILENUM, where\n\
153 FILENUM is 1 or 2, corresponding to FILE1 or FILE2\n\
154 -e EMPTY replace missing input fields with EMPTY\n\
157 -i, --ignore-case ignore differences in case when comparing fields\n\
158 -j FIELD (obsolescent) equivalent to `-1 FIELD -2 FIELD'\n\
159 -j1 FIELD (obsolescent) equivalent to `-1 FIELD'\n\
160 -j2 FIELD (obsolescent) equivalent to `-2 FIELD'\n\
161 -o FORMAT obey FORMAT while constructing output line\n\
162 -t CHAR use CHAR as input and output field separator\n\
165 -v FILENUM like -a FILENUM, but suppress joined output lines\n\
166 -1 FIELD join on this FIELD of file 1\n\
167 -2 FIELD join on this FIELD of file 2\n\
169 fputs (HELP_OPTION_DESCRIPTION, stdout);
170 fputs (VERSION_OPTION_DESCRIPTION, stdout);
173 Unless -t CHAR is given, leading blanks separate fields and are ignored,\n\
174 else fields are separated by CHAR. Any FIELD is a field number counted\n\
175 from 1. FORMAT is one or more comma or blank separated specifications,\n\
176 each being `FILENUM.FIELD' or `0'. Default FORMAT outputs the join field,\n\
177 the remaining fields from FILE1, the remaining fields from FILE2, all\n\
178 separated by CHAR.\n\
180 printf (_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
182 exit (status == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
186 ADD_FIELD (struct line *line, const unsigned char *field, size_t len)
188 if (line->nfields >= line->nfields_allocated)
190 line->nfields_allocated = (3 * line->nfields_allocated) / 2 + 1;
191 line->fields = xrealloc (line->fields,
192 (line->nfields_allocated
193 * sizeof (struct field)));
195 line->fields[line->nfields].beg = field;
196 line->fields[line->nfields].len = len;
200 /* Fill in the `fields' structure in LINE. */
203 xfields (struct line *line)
206 unsigned char *ptr0 = (unsigned char *) line->buf.buffer;
211 lim = ptr0 + line->buf.length - 1;
215 /* Skip leading blanks before the first field. */
216 while (ptr < lim && ISBLANK (*ptr))
220 for (i = 0; ptr < lim; ++i)
227 while (ptr < lim && *ptr != tab)
229 ADD_FIELD (line, beg, ptr - beg);
238 while (ptr < lim && !ISBLANK (*ptr))
240 ADD_FIELD (line, beg, ptr - beg);
241 while (ptr < lim && ISBLANK (*ptr))
246 if (ptr != ptr0 && ((!tab && ISBLANK (ptr[-1])) || ptr[-1] == tab))
248 /* Add one more (empty) field because the last character of the
249 line was a delimiter. */
250 ADD_FIELD (line, NULL, 0);
254 /* Read a line from FP into LINE and split it into fields.
255 Return 0 if EOF, 1 otherwise. */
258 get_line (FILE *fp, struct line *line)
260 initbuffer (&line->buf);
262 if (! readlinebuffer (&line->buf, fp))
265 error (EXIT_FAILURE, errno, _("read error"));
266 free (line->buf.buffer);
267 line->buf.buffer = NULL;
271 line->nfields_allocated = 0;
279 freeline (struct line *line)
282 free (line->buf.buffer);
283 line->buf.buffer = NULL;
287 initseq (struct seq *seq)
291 seq->lines = xmalloc (seq->alloc * sizeof (struct line));
294 /* Read a line from FP and add it to SEQ. Return 0 if EOF, 1 otherwise. */
297 getseq (FILE *fp, struct seq *seq)
299 if (seq->count == seq->alloc)
302 seq->lines = (struct line *)
303 xrealloc (seq->lines, seq->alloc * sizeof (struct line));
306 if (get_line (fp, &seq->lines[seq->count]))
315 delseq (struct seq *seq)
318 for (i = 0; i < seq->count; i++)
319 if (seq->lines[i].buf.buffer)
320 freeline (&seq->lines[i]);
324 /* Return <0 if the join field in LINE1 compares less than the one in LINE2;
325 >0 if it compares greater; 0 if it compares equal.
326 Report an error and exit if the comparison fails. */
329 keycmp (struct line *line1, struct line *line2)
331 /* Start of field to compare in each file. */
332 const unsigned char *beg1, *beg2;
334 size_t len1, len2; /* Length of fields to compare. */
337 if (join_field_1 < line1->nfields)
339 beg1 = line1->fields[join_field_1].beg;
340 len1 = line1->fields[join_field_1].len;
348 if (join_field_2 < line2->nfields)
350 beg2 = line2->fields[join_field_2].beg;
351 len2 = line2->fields[join_field_2].len;
360 return len2 == 0 ? 0 : -1;
364 /* Use an if-statement here rather than a function variable to
365 avoid portability hassles of getting a non-conflicting declaration
369 /* FIXME: ignore_case does not work with NLS (in particular,
370 with multibyte chars). */
371 diff = memcasecmp (beg1, beg2, min (len1, len2));
375 if (HAVE_SETLOCALE && hard_LC_COLLATE)
376 return xmemcoll ((char *) beg1, len1, (char *) beg2, len2);
377 diff = memcmp (beg1, beg2, min (len1, len2));
382 return len1 < len2 ? -1 : len1 != len2;
385 /* Print field N of LINE if it exists and is nonempty, otherwise
386 `empty_filler' if it is nonempty. */
389 prfield (int n, struct line *line)
393 if (n < line->nfields)
395 len = line->fields[n].len;
397 fwrite (line->fields[n].beg, 1, len, stdout);
398 else if (empty_filler)
399 fputs (empty_filler, stdout);
401 else if (empty_filler)
402 fputs (empty_filler, stdout);
405 /* Print the join of LINE1 and LINE2. */
408 prjoin (struct line *line1, struct line *line2)
410 const struct outlist *outlist;
412 outlist = outlist_head.next;
415 const struct outlist *o;
425 if (line1 == &uni_blank)
428 field = join_field_2;
433 field = join_field_1;
438 line = (o->file == 1 ? line1 : line2);
439 assert (o->field >= 0);
442 prfield (field, line);
446 putchar (tab ? tab : ' ');
454 if (line1 == &uni_blank)
461 prfield (join_field_1, line1);
462 for (i = 0; i < join_field_1 && i < line1->nfields; ++i)
464 putchar (tab ? tab : ' ');
467 for (i = join_field_1 + 1; i < line1->nfields; ++i)
469 putchar (tab ? tab : ' ');
473 for (i = 0; i < join_field_2 && i < line2->nfields; ++i)
475 putchar (tab ? tab : ' ');
478 for (i = join_field_2 + 1; i < line2->nfields; ++i)
480 putchar (tab ? tab : ' ');
487 /* Print the join of the files in FP1 and FP2. */
490 join (FILE *fp1, FILE *fp2)
492 struct seq seq1, seq2;
494 int diff, i, j, eof1, eof2;
496 /* Read the first line of each file. */
502 while (seq1.count && seq2.count)
504 diff = keycmp (&seq1.lines[0], &seq2.lines[0]);
507 if (print_unpairables_1)
508 prjoin (&seq1.lines[0], &uni_blank);
509 freeline (&seq1.lines[0]);
516 if (print_unpairables_2)
517 prjoin (&uni_blank, &seq2.lines[0]);
518 freeline (&seq2.lines[0]);
524 /* Keep reading lines from file1 as long as they continue to
525 match the current line from file2. */
528 if (!getseq (fp1, &seq1))
534 while (!keycmp (&seq1.lines[seq1.count - 1], &seq2.lines[0]));
536 /* Keep reading lines from file2 as long as they continue to
537 match the current line from file1. */
540 if (!getseq (fp2, &seq2))
546 while (!keycmp (&seq1.lines[0], &seq2.lines[seq2.count - 1]));
550 for (i = 0; i < seq1.count - 1; ++i)
551 for (j = 0; j < seq2.count - 1; ++j)
552 prjoin (&seq1.lines[i], &seq2.lines[j]);
555 for (i = 0; i < seq1.count - 1; ++i)
556 freeline (&seq1.lines[i]);
559 seq1.lines[0] = seq1.lines[seq1.count - 1];
565 for (i = 0; i < seq2.count - 1; ++i)
566 freeline (&seq2.lines[i]);
569 seq2.lines[0] = seq2.lines[seq2.count - 1];
576 if (print_unpairables_1 && seq1.count)
578 prjoin (&seq1.lines[0], &uni_blank);
579 freeline (&seq1.lines[0]);
580 while (get_line (fp1, &line))
582 prjoin (&line, &uni_blank);
587 if (print_unpairables_2 && seq2.count)
589 prjoin (&uni_blank, &seq2.lines[0]);
590 freeline (&seq2.lines[0]);
591 while (get_line (fp2, &line))
593 prjoin (&uni_blank, &line);
602 /* Add a field spec for field FIELD of file FILE to `outlist'. */
605 add_field (int file, int field)
609 assert (file == 0 || file == 1 || file == 2);
610 assert (file == 0 ? field < 0 : field >= 0);
612 o = xmalloc (sizeof (struct outlist));
617 /* Add to the end of the list so the fields are in the right order. */
618 outlist_end->next = o;
622 /* Convert a single field specifier string, S, to a *FILE_INDEX, *FIELD_INDEX
623 pair. In S, the field index string is 1-based; *FIELD_INDEX is zero-based.
624 If S is valid, return zero. Otherwise, give a diagnostic, don't update
625 *FILE_INDEX or *FIELD_INDEX, and return nonzero. */
628 decode_field_spec (const char *s, int *file_index, int *field_index)
632 /* The first character must be 0, 1, or 2. */
639 /* Give *field_index an invalid value. */
645 /* `0' must be all alone -- no `.FIELD'. */
646 error (0, 0, _("invalid field specifier: `%s'"), s);
652 if (s[1] == '.' && s[2] != '\0')
657 s_err = xstrtol (s + 2, NULL, 10, &tmp_long, "");
658 if (s_err != LONGINT_OK || tmp_long <= 0 || tmp_long > INT_MAX)
660 error (0, 0, _("invalid field number: `%s'"), s + 2);
664 *file_index = s[0] - '0';
665 /* Convert to a zero-based index. */
666 *field_index = (int) tmp_long - 1;
673 error (0, 0, _("invalid file number in field spec: `%s'"), s);
679 /* Add the comma or blank separated field spec(s) in STR to `outlist'.
680 Return nonzero to indicate failure. */
683 add_field_list (const char *c_str)
687 /* Make a writable copy of c_str. */
688 str = (char *) alloca (strlen (c_str) + 1);
695 int file_index, field_index;
698 p = strpbrk (p, ", \t");
701 invalid = decode_field_spec (spec_item, &file_index, &field_index);
704 add_field (file_index, field_index);
705 uni_blank.nfields = max (uni_blank.nfields, field_index);
711 /* Create a blank line with COUNT fields separated by tabs. */
714 make_blank (struct line *blank, int count)
717 unsigned char *buffer;
718 struct field *fields;
719 blank->nfields = count;
720 blank->buf.size = blank->buf.length = count + 1;
721 blank->buf.buffer = xmalloc (blank->buf.size);
722 buffer = (unsigned char *) blank->buf.buffer;
723 blank->fields = fields =
724 xmalloc (sizeof (struct field) * count);
725 for (i = 0; i < count; i++)
728 fields[i].beg = &buffer[i];
735 main (int argc, char **argv)
739 int optc, prev_optc = 0, nfiles;
741 initialize_main (&argc, &argv);
742 program_name = argv[0];
743 setlocale (LC_ALL, "");
744 bindtextdomain (PACKAGE, LOCALEDIR);
745 textdomain (PACKAGE);
746 hard_LC_COLLATE = hard_locale (LC_COLLATE);
748 atexit (close_stdout);
750 /* Initialize this before parsing options. In parsing options,
751 it may be increased. */
752 uni_blank.nfields = 1;
757 while ((optc = getopt_long_only (argc, argv, "-a:e:i1:2:o:t:v:", longopts,
772 if (xstrtol (optarg, NULL, 10, &val, "") != LONGINT_OK
773 || (val != 1 && val != 2))
774 error (EXIT_FAILURE, 0, _("invalid field number: `%s'"), optarg);
776 print_unpairables_1 = 1;
778 print_unpairables_2 = 1;
782 empty_filler = optarg;
790 if (xstrtol (optarg, NULL, 10, &val, "") != LONGINT_OK
791 || val <= 0 || val > INT_MAX)
793 error (EXIT_FAILURE, 0,
794 _("invalid field number for file 1: `%s'"), optarg);
796 join_field_1 = (int) val - 1;
800 if (xstrtol (optarg, NULL, 10, &val, "") != LONGINT_OK
801 || val <= 0 || val > INT_MAX)
802 error (EXIT_FAILURE, 0,
803 _("invalid field number for file 2: `%s'"), optarg);
804 join_field_2 = (int) val - 1;
808 if (xstrtol (optarg, NULL, 10, &val, "") != LONGINT_OK
809 || val <= 0 || val > INT_MAX)
810 error (EXIT_FAILURE, 0, _("invalid field number: `%s'"), optarg);
811 join_field_1 = join_field_2 = (int) val - 1;
815 if (add_field_list (optarg))
823 case 1: /* Non-option argument. */
824 if (prev_optc == 'o' && optind <= argc - 2)
826 if (add_field_list (optarg))
829 /* Might be continuation of args to -o. */
830 continue; /* Don't change `prev_optc'. */
835 error (0, 0, _("too many non-option arguments"));
836 usage (EXIT_FAILURE);
838 names[nfiles++] = optarg;
841 case_GETOPT_HELP_CHAR;
843 case_GETOPT_VERSION_CHAR (PROGRAM_NAME, WRITTEN_BY);
846 usage (EXIT_FAILURE);
851 /* Now that we've seen the options, we can construct the blank line
853 make_blank (&uni_blank, uni_blank.nfields);
857 error (0, 0, _("too few non-option arguments"));
858 usage (EXIT_FAILURE);
861 fp1 = STREQ (names[0], "-") ? stdin : fopen (names[0], "r");
863 error (EXIT_FAILURE, errno, "%s", names[0]);
864 fp2 = STREQ (names[1], "-") ? stdin : fopen (names[1], "r");
866 error (EXIT_FAILURE, errno, "%s", names[1]);
868 error (EXIT_FAILURE, errno, _("both files cannot be standard input"));
871 if (fp1 != stdin && fclose (fp1) == EOF)
872 error (EXIT_FAILURE, errno, "%s", names[0]);
873 if (fp2 != stdin && fclose (fp2) == EOF)
874 error (EXIT_FAILURE, errno, "%s", names[1]);
875 if ((fp1 == stdin || fp2 == stdin) && fclose (stdin) == EOF)
876 error (EXIT_FAILURE, errno, "-");