1 /* xdelta3 - delta compression tools and library
2 * Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
3 * 2009, 2010, 2011, 2012, 2013 Joshua P. MacDonald
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 /* This is all the extra stuff you need for convenience to users in a
21 * command line application. It contains these major components:
23 * 1. VCDIFF tools 2. external compression support (this is
24 * POSIX-specific). 3. a general read/write loop that handles all of
25 * the Xdelta decode/encode/VCDIFF-print functions 4. command-line
26 * interpreter 5. an Xdelta application header which stores default
27 * filename, external compression settings 6. output/error printing
28 * 7. basic file support and OS interface
31 /* TODO list: 1. do exact gzip-like filename, stdout handling. make a
32 * .vcdiff extension, refuse to encode to stdout without -cf, etc.
33 * 2. Allow the user to add a comment string to the app header without
34 * disturbing the default behavior.
37 /* On error handling and printing:
39 * The xdelta library sets stream->msg to indicate what condition
40 * caused an internal failure, but many failures originate here and
41 * are printed here. The return convention is 0 for success, as
42 * throughout Xdelta code, but special attention is required here for
43 * the operating system calls with different error handling. See the
44 * main_file_* routines. All errors in this file have a message
45 * printed at the time of occurance. Since some of these calls occur
46 * within calls to the library, the error may end up being printed
47 * again with a more general error message.
50 /*********************************************************************/
65 /* Combines xd3_strerror() and strerror() */
66 const char* xd3_mainerror(int err_num);
68 #include "xdelta3-internal.h"
71 xsnprintf_func (char *str, int n, const char *fmt, ...)
76 ret = vsnprintf_func (str, n, fmt, a);
85 /* If none are set, default to posix. */
86 #if (XD3_POSIX + XD3_STDIO + XD3_WIN32) == 0
91 /* Handle externally-compressed inputs. */
92 #ifndef EXTERNAL_COMPRESSION
93 #define EXTERNAL_COMPRESSION 1
96 #define PRINTHDR_SPECIAL -4378291
98 /* The number of soft-config variables. */
99 #define XD3_SOFTCFG_VARCNT 7
101 /* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an
102 * error message from the library. */
103 #define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", \
104 xd3_errstring (stream), xd3_mainerror (ret)
107 #include <unistd.h> /* close, read, write... */
108 #include <sys/types.h>
113 #include <unistd.h> /* lots */
114 #include <sys/time.h> /* gettimeofday() */
115 #include <sys/stat.h> /* stat() and fstat() */
117 #if defined(_MSC_VER)
118 #define strtoll _strtoi64
120 #include <sys/types.h>
121 #include <sys/stat.h>
123 # define WIFEXITED(stat) (((*((int *) &(stat))) & 0xff) == 0)
126 # define WEXITSTATUS(stat) (((*((int *) &(stat))) >> 8) & 0xff)
130 //# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
132 # define S_ISREG(m) 1
134 #endif /* !S_ISREG */
136 // For standard input/output handles
137 static STARTUPINFO winStartupInfo;
140 /**********************************************************************
142 *********************************************************************/
144 /* These flags (mainly pertaining to main_read() operations) are set
145 * in the main_file->flags variable. All are related to with external
146 * decompression support.
148 * RD_FIRST causes the external decompression check when the input is
151 * RD_NONEXTERNAL disables external decompression for reading a
152 * compressed input, in the case of Xdelta inputs. Note: Xdelta is
153 * supported as an external compression type, which makes is the
154 * reason for this flag. An example to justify this is: to create a
155 * delta between two files that are VCDIFF-compressed. Two external
156 * Xdelta decoders are run to supply decompressed source and target
157 * inputs to the Xdelta encoder. */
161 RD_NONEXTERNAL = (1 << 1),
162 RD_DECOMPSET = (1 << 2),
163 RD_MAININPUT = (1 << 3),
166 /* Main commands. For example, CMD_PRINTHDR is the "xdelta printhdr"
186 #define CMD_DEFAULT CMD_ENCODE
187 #define IS_ENCODE(cmd) (cmd == CMD_ENCODE)
189 #define CMD_DEFAULT CMD_DECODE
190 #define IS_ENCODE(cmd) (0)
193 typedef struct _main_merge main_merge;
194 typedef struct _main_merge_list main_merge_list;
196 /* Various strings and magic values used to detect and call external
197 * compression. See below for examples. */
200 const char *recomp_cmdname;
201 const char *recomp_options;
203 const char *decomp_cmdname;
204 const char *decomp_options;
214 struct _main_merge_list
216 main_merge_list *next;
217 main_merge_list *prev;
222 const char *filename;
224 main_merge_list link;
227 XD3_MAKELIST(main_merge_list,main_merge,link);
229 /* TODO: really need to put options in a struct so that internal
230 * callers can easily reset state. */
232 #define DEFAULT_VERBOSE 0
234 /* Program options: various command line flags and options. */
235 static int option_stdout = 0;
236 static int option_force = 0;
237 static int option_verbose = DEFAULT_VERBOSE;
238 static int option_quiet = 0;
239 static int option_use_appheader = 1;
240 static uint8_t* option_appheader = NULL;
241 static int option_use_secondary = 0;
242 static const char* option_secondary = NULL;
243 static int option_use_checksum = 1;
244 static const char* option_smatch_config = NULL;
245 static int option_no_compress = 0;
246 static int option_no_output = 0; /* do not write output */
247 static const char *option_source_filename = NULL;
249 static int option_level = XD3_DEFAULT_LEVEL;
250 static usize_t option_iopt_size = XD3_DEFAULT_IOPT_SIZE;
251 static usize_t option_winsize = XD3_DEFAULT_WINSIZE;
252 /* Note: option_srcwinsz is restricted from [16Kb, 4Gb], because
253 * addresses in the large hash checksum are 32 bits. The flag is read
254 * as xoff_t, so that 4Gb != 0. */
255 static xoff_t option_srcwinsz = XD3_DEFAULT_SRCWINSZ;
256 static usize_t option_sprevsz = XD3_DEFAULT_SPREVSZ;
258 /* These variables are supressed to avoid their use w/o support. main() warns
259 * appropriately when external compression is not enabled. */
260 #if EXTERNAL_COMPRESSION
261 static int num_subprocs = 0;
262 static int option_force2 = 0;
263 static int option_decompress_inputs = 1;
264 static int option_recompress_outputs = 1;
267 /* This is for comparing "printdelta" output without attention to
268 * copy-instruction modes. */
270 static int option_print_cpymode = 1; /* Note: see reset_defaults(). */
273 /* Static variables */
274 IF_DEBUG(static int main_mallocs = 0;)
276 static char* program_name = NULL;
277 static uint8_t* appheader_used = NULL;
278 static uint8_t* main_bdata = NULL;
279 static usize_t main_bsize = 0;
281 /* Hacks for VCDIFF tools, recode command. */
282 static int allow_fake_source = 0;
284 /* recode_stream is used by both recode/merge for reading vcdiff inputs */
285 static xd3_stream *recode_stream = NULL;
287 /* merge_stream is used by merge commands for storing the source encoding */
288 static xd3_stream *merge_stream = NULL;
290 /* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is
291 * false just so the program knows the mapping of IDENT->NAME. */
292 static main_extcomp extcomp_types[] =
294 { "bzip2", "-c", "bzip2", "-dc", "B", "BZh", 3, 0 },
295 { "gzip", "-c", "gzip", "-dc", "G", "\037\213", 2, 0 },
296 { "compress", "-c", "uncompress", "-c", "Z", "\037\235", 2, 0 },
298 /* Xz is lzma with a magic number http://tukaani.org/xz/format.html */
299 { "xz", "-c", "xz", "-dc", "Y", "\xfd\x37\x7a\x58\x5a\x00", 2, 0 },
302 static int main_input (xd3_cmd cmd, main_file *ifile,
303 main_file *ofile, main_file *sfile);
304 static void main_get_appheader (xd3_stream *stream, main_file *ifile,
305 main_file *output, main_file *sfile);
307 static int main_getblk_func (xd3_stream *stream,
310 static void main_free (void *ptr);
311 static void* main_malloc (size_t size);
313 static int main_file_stat (main_file *xfile, xoff_t *size);
314 static int main_file_seek (main_file *xfile, xoff_t pos);
315 static int main_read_primary_input (main_file *file,
320 static const char* main_format_bcnt (xoff_t r, shortbuf *buf);
321 static int main_help (void);
324 static int xd3_merge_input_output (xd3_stream *stream,
325 xd3_whole_state *source);
328 /* The code in xdelta3-blk.h is essentially part of this unit, see
330 #include "xdelta3-blkcache.h"
332 void (*xprintf_message_func)(const char*msg) = NULL;
335 xprintf (const char *fmt, ...)
341 size = vsnprintf_func (buf, 1000, fmt, a);
345 size = sizeof(buf) - 1;
348 if (xprintf_message_func != NULL) {
349 xprintf_message_func(buf);
351 size_t ignore = fwrite(buf, 1, size, stderr);
359 /* $Format: " XPR(NTR \"Xdelta version $Xdelta3Version$, Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, Joshua MacDonald\\n\");" $ */
360 XPR(NTR "Xdelta version 3.0.8, Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Joshua MacDonald\n");
361 XPR(NTR "Xdelta comes with ABSOLUTELY NO WARRANTY.\n");
362 XPR(NTR "This is free software, and you are welcome to redistribute it\n");
363 XPR(NTR "under certain conditions; see \"COPYING\" for details.\n");
372 XPR(NTR "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION);
373 XPR(NTR "REGRESSION_TEST=%d\n", REGRESSION_TEST);
374 XPR(NTR "SECONDARY_DJW=%d\n", SECONDARY_DJW);
375 XPR(NTR "SECONDARY_FGK=%d\n", SECONDARY_FGK);
376 XPR(NTR "SECONDARY_LZMA=%d\n", SECONDARY_LZMA);
377 XPR(NTR "UNALIGNED_OK=%d\n", UNALIGNED_OK);
378 XPR(NTR "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS);
379 XPR(NTR "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE);
380 XPR(NTR "XD3_DEBUG=%d\n", XD3_DEBUG);
381 XPR(NTR "XD3_ENCODER=%d\n", XD3_ENCODER);
382 XPR(NTR "XD3_POSIX=%d\n", XD3_POSIX);
383 XPR(NTR "XD3_STDIO=%d\n", XD3_STDIO);
384 XPR(NTR "XD3_WIN32=%d\n", XD3_WIN32);
385 XPR(NTR "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64);
386 XPR(NTR "XD3_DEFAULT_LEVEL=%d\n", XD3_DEFAULT_LEVEL);
387 XPR(NTR "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE);
388 XPR(NTR "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ);
389 XPR(NTR "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ);
390 XPR(NTR "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE);
391 XPR(NTR "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE);
392 XPR(NTR "sizeof(void*)=%d\n", (int)sizeof(void*));
393 XPR(NTR "sizeof(int)=%d\n", (int)sizeof(int));
394 XPR(NTR "sizeof(size_t)=%d\n", (int)sizeof(size_t));
395 XPR(NTR "sizeof(uint32_t)=%d\n", (int)sizeof(uint32_t));
396 XPR(NTR "sizeof(uint64_t)=%d\n", (int)sizeof(uint64_t));
397 XPR(NTR "sizeof(usize_t)=%d\n", (int)sizeof(usize_t));
398 XPR(NTR "sizeof(xoff_t)=%d\n", (int)sizeof(xoff_t));
408 option_verbose = DEFAULT_VERBOSE;
410 option_appheader = NULL;
411 option_use_secondary = 0;
412 option_secondary = NULL;
413 option_smatch_config = NULL;
414 option_no_compress = 0;
415 option_no_output = 0;
416 option_source_filename = NULL;
418 appheader_used = NULL;
421 allow_fake_source = 0;
422 option_smatch_config = NULL;
426 option_use_appheader = 1;
427 option_use_checksum = 1;
428 #if EXTERNAL_COMPRESSION
430 option_decompress_inputs = 1;
431 option_recompress_outputs = 1;
435 option_print_cpymode = 1;
437 option_level = XD3_DEFAULT_LEVEL;
438 option_iopt_size = XD3_DEFAULT_IOPT_SIZE;
439 option_winsize = XD3_DEFAULT_WINSIZE;
440 option_srcwinsz = XD3_DEFAULT_SRCWINSZ;
441 option_sprevsz = XD3_DEFAULT_SPREVSZ;
445 main_malloc1 (size_t size)
447 void* r = malloc (size);
448 if (r == NULL) { XPR(NT "malloc: %s\n", xd3_mainerror (ENOMEM)); }
452 void* main_bufalloc (size_t size) {
454 return VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
456 return main_malloc1(size);
461 main_malloc (size_t size)
463 void *r = main_malloc1 (size);
464 if (r) { IF_DEBUG (main_mallocs += 1); }
469 main_alloc (void *opaque,
473 return main_malloc1 (items * size);
477 main_free1 (void *opaque, void *ptr)
483 main_free (void *ptr)
487 IF_DEBUG (main_mallocs -= 1);
488 main_free1 (NULL, ptr);
489 IF_DEBUG (XD3_ASSERT(main_mallocs >= 0));
493 void main_buffree (void *ptr) {
495 VirtualFree(ptr, 0, MEM_RELEASE);
497 main_free1(NULL, ptr);
501 /* This ensures that (ret = errno) always indicates failure, in case errno was
502 * accidentally not set. If this prints there's a bug somewhere. */
509 XPR(NT "you found a bug: expected errno != 0\n");
510 errno = XD3_INTERNAL;
514 DWORD err_num = GetLastError();
515 if (err_num == NO_ERROR)
517 err_num = XD3_INTERNAL;
524 xd3_mainerror(int err_num) {
526 const char* x = xd3_strerror (err_num);
531 return strerror(err_num);
533 static char err_buf[256];
534 const char* x = xd3_strerror (err_num);
539 memset (err_buf, 0, 256);
540 FormatMessage (FORMAT_MESSAGE_FROM_SYSTEM |
541 FORMAT_MESSAGE_IGNORE_INSERTS,
543 MAKELANGID (LANG_NEUTRAL, SUBLANG_DEFAULT),
545 if (err_buf[0] != 0 && err_buf[strlen(err_buf) - 1] == '\n')
547 err_buf[strlen(err_buf) - 1] = 0;
554 get_millisecs_now (void)
559 gettimeofday (& tv, NULL);
561 return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000;
565 __int64 *pi = (__int64*)&ft;
567 SystemTimeToFileTime(&st, &ft);
568 return (long)((*pi) / 10000);
572 /* Always >= 1 millisec, right? */
574 get_millisecs_since (void)
576 static long last = 0;
577 long now = get_millisecs_now();
578 long diff = now - last;
584 main_format_bcnt (xoff_t r, shortbuf *buf)
586 static const char* fmts[] = { "B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB" };
589 for (i = 0; i < SIZEOF_ARRAY(fmts) - 1; i += 1)
595 short_sprintf (*buf, "0 %s", fmts[i]);
599 if (r >= 1 && r < 10)
601 short_sprintf (*buf, "%.2f %s", (double) r, fmts[i]);
605 if (r >= 10 && r < 100)
607 short_sprintf (*buf, "%.1f %s", (double) r, fmts[i]);
611 if (r >= 100 && r < 1000)
613 short_sprintf (*buf, "%"Q"u %s", r, fmts[i]);
621 short_sprintf (*buf, "%.2f %s", (double) r / 1024.0, fmts[i + 1]);
627 short_sprintf (*buf, "%.1f %s", (double) r / 1024.0, fmts[i + 1]);
638 main_format_rate (xoff_t bytes, long millis, shortbuf *buf)
640 xoff_t r = (xoff_t)(1.0 * bytes / (1.0 * millis / 1000.0));
641 static shortbuf lbuf;
643 main_format_bcnt (r, &lbuf);
644 short_sprintf (*buf, "%s/s", lbuf.buf);
649 main_format_millis (long millis, shortbuf *buf)
653 short_sprintf (*buf, "%lu ms", millis);
655 else if (millis < 10000)
657 short_sprintf (*buf, "%.1f sec", millis / 1000.0);
661 short_sprintf (*buf, "%lu sec", millis / 1000L);
666 /* A safe version of strtol for xoff_t. */
668 main_strtoxoff (const char* s, xoff_t *xo, char which)
673 XD3_ASSERT(s && *s != 0);
676 /* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */
677 #if SIZEOF_XOFF_T == 4
678 long xx = strtol (s, &e, 0);
680 long long xx = strtoll (s, &e, 0);
685 XPR(NT "-%c: negative integer: %s\n", which, s);
694 XPR(NT "-%c: invalid integer: %s\n", which, s);
703 main_atoux (const char* arg, xoff_t *xo, xoff_t low,
704 xoff_t high, char which)
709 if ((ret = main_strtoxoff (arg, & x, which))) { return ret; }
713 XPR(NT "-%c: minimum value: %"Q"u\n", which, low);
716 if (high != 0 && x > high)
718 XPR(NT "-%c: maximum value: %"Q"u\n", which, high);
726 main_atou (const char* arg, usize_t *uo, usize_t low,
727 usize_t high, char which)
731 if ((ret = main_atoux (arg, &xo, low, high, which)))
739 /******************************************************************
741 ******************************************************************/
743 /* With all the variation in file system-call semantics, arguments,
744 * return values and error-handling for the POSIX and STDIO file APIs,
745 * the insides of these functions make me sick, which is why these
748 #define XOPEN_OPNAME (xfile->mode == XO_READ ? "read" : "write")
749 #define XOPEN_STDIO (xfile->mode == XO_READ ? "rb" : "wb")
750 #define XOPEN_POSIX (xfile->mode == XO_READ ? \
751 O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC)
752 #define XOPEN_MODE (xfile->mode == XO_READ ? 0 : 0666)
754 #define XF_ERROR(op, name, ret) \
755 do { if (!option_quiet) { XPR(NT "file %s failed: %s: %s: %s\n", (op), \
756 XOPEN_OPNAME, (name), xd3_mainerror (ret)); } } while (0)
759 #define XFNO(f) fileno(f->file)
760 #define XSTDOUT_XF(f) { (f)->file = stdout; (f)->filename = "/dev/stdout"; }
761 #define XSTDIN_XF(f) { (f)->file = stdin; (f)->filename = "/dev/stdin"; }
764 #define XFNO(f) f->file
765 #define XSTDOUT_XF(f) \
766 { (f)->file = STDOUT_FILENO; (f)->filename = "/dev/stdout"; }
767 #define XSTDIN_XF(f) \
768 { (f)->file = STDIN_FILENO; (f)->filename = "/dev/stdin"; }
772 #define XSTDOUT_XF(f) { \
773 (f)->file = GetStdHandle(STD_OUTPUT_HANDLE); \
774 (f)->filename = "(stdout)"; \
776 #define XSTDIN_XF(f) { \
777 (f)->file = GetStdHandle(STD_INPUT_HANDLE); \
778 (f)->filename = "(stdin)"; \
783 main_file_init (main_file *xfile)
785 memset (xfile, 0, sizeof (*xfile));
791 xfile->file = INVALID_HANDLE_VALUE;
796 main_file_isopen (main_file *xfile)
799 return xfile->file != NULL;
802 return xfile->file != -1;
805 return xfile->file != INVALID_HANDLE_VALUE;
810 main_file_close (main_file *xfile)
814 if (! main_file_isopen (xfile))
820 ret = fclose (xfile->file);
824 ret = close (xfile->file);
828 if (!CloseHandle(xfile->file)) {
831 xfile->file = INVALID_HANDLE_VALUE;
834 if (ret != 0) { XF_ERROR ("close", xfile->filename, ret = get_errno ()); }
839 main_file_cleanup (main_file *xfile)
841 XD3_ASSERT (xfile != NULL);
843 if (main_file_isopen (xfile))
845 main_file_close (xfile);
848 if (xfile->snprintf_buf != NULL)
850 main_free(xfile->snprintf_buf);
851 xfile->snprintf_buf = NULL;
854 if (xfile->filename_copy != NULL)
856 main_free(xfile->filename_copy);
857 xfile->filename_copy = NULL;
862 main_file_open (main_file *xfile, const char* name, int mode)
868 XD3_ASSERT (name != NULL);
869 XD3_ASSERT (! main_file_isopen (xfile));
872 XPR(NT "invalid file name: empty string\n");
877 xfile->file = fopen (name, XOPEN_STDIO);
879 ret = (xfile->file == NULL) ? get_errno () : 0;
882 /* TODO: Should retry this call if interrupted, similar to read/write */
883 if ((ret = open (name, XOPEN_POSIX, XOPEN_MODE)) < 0)
894 xfile->file = CreateFile(name,
895 (mode == XO_READ) ? GENERIC_READ : GENERIC_WRITE,
900 (option_force ? CREATE_ALWAYS : CREATE_NEW),
901 FILE_ATTRIBUTE_NORMAL,
903 if (xfile->file == INVALID_HANDLE_VALUE)
908 if (ret) { XF_ERROR ("open", name, ret); }
909 else { xfile->realname = name; xfile->nread = 0; }
914 main_file_stat (main_file *xfile, xoff_t *size)
918 if (GetFileType(xfile->file) != FILE_TYPE_DISK)
922 # if (_WIN32_WINNT >= 0x0500)
925 if (GetFileSizeEx(xfile->file, &li) == 0)
933 DWORD filesize = GetFileSize(xfile->file, NULL);
934 if (filesize == INVALID_FILE_SIZE)
943 if (fstat (XFNO (xfile), & sbuf) < 0)
949 if (! S_ISREG (sbuf.st_mode))
953 (*size) = sbuf.st_size;
959 main_file_exists (main_file *xfile)
962 return stat (xfile->filename, & sbuf) == 0 && S_ISREG (sbuf.st_mode);
965 #if (XD3_POSIX || EXTERNAL_COMPRESSION)
966 /* POSIX-generic code takes a function pointer to read() or write().
967 * This calls the function repeatedly until the buffer is full or EOF.
968 * The NREAD parameter is not set for write, NULL is passed. Return
969 * is signed, < 0 indicate errors, otherwise byte count. */
970 typedef int (xd3_posix_func) (int fd, uint8_t *buf, usize_t size);
973 xd3_posix_io (int fd, uint8_t *buf, size_t size,
974 xd3_posix_func *func, size_t *nread)
981 size_t tryread = min(size - nproc, 1U << 30);
982 ssize_t result = (*func) (fd, buf + nproc, tryread);
987 if (ret != EAGAIN && ret != EINTR)
994 if (nread != NULL && result == 0) { break; }
998 if (nread != NULL) { (*nread) = nproc; }
1005 xd3_win32_io (HANDLE file, uint8_t *buf, size_t size,
1006 int is_read, size_t *nread)
1011 while (nproc < size)
1013 DWORD nproc2 = 0; /* hmm */
1014 DWORD nremain = size - nproc;
1016 ReadFile (file, buf + nproc, nremain, &nproc2, NULL) :
1017 WriteFile (file, buf + nproc, nremain, &nproc2, NULL)) == 0)
1020 if (ret != ERROR_HANDLE_EOF && ret != ERROR_BROKEN_PIPE)
1024 /* By falling through here, we'll break this loop in the
1025 * read case in case of eof or broken pipe. */
1030 if (nread != NULL && nproc2 == 0) { break; }
1032 if (nread != NULL) { (*nread) = nproc; }
1037 /* POSIX is unbuffered, while STDIO is buffered. main_file_read()
1038 * should always be called on blocks. */
1040 main_file_read (main_file *ifile,
1051 result = fread (buf, 1, size, ifile->file);
1053 if (result < size && ferror (ifile->file))
1063 ret = xd3_posix_io (ifile->file, buf, size, (xd3_posix_func*) &read, nread);
1065 ret = xd3_win32_io (ifile->file, buf, size, 1 /* is_read */, nread);
1070 XPR(NT "%s: %s: %s\n", msg, ifile->filename, xd3_mainerror (ret));
1074 if (option_verbose > 4) { XPR(NT "read %s: %zu bytes\n",
1075 ifile->filename, (*nread)); }
1076 ifile->nread += (*nread);
1083 main_file_write (main_file *ofile, uint8_t *buf, usize_t size, const char *msg)
1090 result = fwrite (buf, 1, size, ofile->file);
1092 if (result != size) { ret = get_errno (); }
1095 ret = xd3_posix_io (ofile->file, buf, size, (xd3_posix_func*) &write, NULL);
1098 ret = xd3_win32_io (ofile->file, buf, size, 0, NULL);
1104 XPR(NT "%s: %s: %s\n", msg, ofile->filename, xd3_mainerror (ret));
1108 if (option_verbose > 5) { XPR(NT "write %s: %u bytes\n",
1109 ofile->filename, size); }
1110 ofile->nwrite += size;
1117 main_file_seek (main_file *xfile, xoff_t pos)
1122 if (fseek (xfile->file, pos, SEEK_SET) != 0) { ret = get_errno (); }
1125 if ((xoff_t) lseek (xfile->file, pos, SEEK_SET) != pos)
1126 { ret = get_errno (); }
1129 # if (_WIN32_WINNT >= 0x0500)
1130 LARGE_INTEGER move, out;
1131 move.QuadPart = pos;
1132 if (SetFilePointerEx(xfile->file, move, &out, FILE_BEGIN) == 0)
1137 if (SetFilePointer(xfile->file, (LONG)pos, NULL, FILE_BEGIN) ==
1138 INVALID_SET_FILE_POINTER)
1148 /* This function simply writes the stream output buffer, if there is
1149 * any, for encode, decode and recode commands. (The VCDIFF tools use
1150 * main_print_func()). */
1152 main_write_output (xd3_stream* stream, main_file *ofile)
1156 if (option_no_output)
1161 if (stream->avail_out > 0 &&
1162 (ret = main_file_write (ofile, stream->next_out,
1163 stream->avail_out, "write failed")))
1172 main_set_secondary_flags (xd3_config *config)
1175 if (option_use_secondary)
1177 /* The default secondary compressor is DJW, if it's compiled. */
1178 if (option_secondary == NULL)
1182 config->flags |= XD3_SEC_DJW;
1187 if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK)
1189 config->flags |= XD3_SEC_FGK;
1191 else if (strcmp (option_secondary, "lzma") == 0 && SECONDARY_LZMA)
1193 config->flags |= XD3_SEC_LZMA;
1195 else if (strncmp (option_secondary, "djw", 3) == 0 && SECONDARY_DJW)
1197 usize_t level = XD3_DEFAULT_SECONDARY_LEVEL;
1199 config->flags |= XD3_SEC_DJW;
1201 if (strlen (option_secondary) > 3 &&
1202 (ret = main_atou (option_secondary + 3,
1210 /* XD3_SEC_NOXXXX flags disable secondary compression on
1211 * a per-section basis. For djw, ngroups=1 indicates
1212 * minimum work, ngroups=0 uses default settings, which
1213 * is > 1 groups by default. */
1214 if (level < 1) { config->flags |= XD3_SEC_NODATA; }
1215 if (level < 7) { config->sec_data.ngroups = 1; }
1216 else { config->sec_data.ngroups = 0; }
1218 if (level < 3) { config->flags |= XD3_SEC_NOINST; }
1219 if (level < 8) { config->sec_inst.ngroups = 1; }
1220 else { config->sec_inst.ngroups = 0; }
1222 if (level < 5) { config->flags |= XD3_SEC_NOADDR; }
1223 if (level < 9) { config->sec_addr.ngroups = 1; }
1224 else { config->sec_addr.ngroups = 0; }
1226 else if (strcmp (option_secondary, "none") == 0 && SECONDARY_DJW)
1234 XPR(NT "unrecognized secondary compressor type: %s\n",
1245 /******************************************************************
1247 *****************************************************************/
1250 #include "xdelta3-merge.h"
1252 /* The following macros let VCDIFF print using main_file_write(),
1255 * VC(UT "trying to be portable: %d\n", x)VE;
1257 #define SNPRINTF_BUFSIZE 1024
1258 #define VC do { if (((ret = xsnprintf_func
1259 #define UT (char*)xfile->snprintf_buf, SNPRINTF_BUFSIZE,
1260 #define VE ) >= SNPRINTF_BUFSIZE \
1261 && (ret = main_print_overflow(ret)) != 0) \
1262 || (ret = main_file_write(xfile, xfile->snprintf_buf, \
1263 (usize_t)ret, "print")) != 0) \
1264 { return ret; } } while (0)
1267 main_print_overflow (int x)
1269 XPR(NT "internal print buffer overflow: %d bytes\n", x);
1270 return XD3_INTERNAL;
1273 /* This function prints a single VCDIFF window. */
1275 main_print_window (xd3_stream* stream, main_file *xfile)
1280 VC(UT " Offset Code Type1 Size1 @Addr1 + Type2 Size2 @Addr2\n")VE;
1282 while (stream->inst_sect.buf < stream->inst_sect.buf_max)
1284 usize_t code = stream->inst_sect.buf[0];
1285 const uint8_t *addr_before = stream->addr_sect.buf;
1286 const uint8_t *inst_before = stream->inst_sect.buf;
1289 usize_t size_before = size;
1291 if ((ret = xd3_decode_instruction (stream)))
1293 XPR(NT "instruction decode error at %"Q"u: %s\n",
1294 stream->dec_winstart + size, stream->msg);
1298 addr_bytes = (usize_t)(stream->addr_sect.buf - addr_before);
1299 inst_bytes = (usize_t)(stream->inst_sect.buf - inst_before);
1301 VC(UT " %06"Q"u %03u %s %6u", stream->dec_winstart + size,
1302 option_print_cpymode ? code : 0,
1303 xd3_rtype_to_string ((xd3_rtype) stream->dec_current1.type,
1304 option_print_cpymode),
1305 stream->dec_current1.size)VE;
1307 if (stream->dec_current1.type != XD3_NOOP)
1309 if (stream->dec_current1.type >= XD3_CPY)
1311 if (stream->dec_current1.addr >= stream->dec_cpylen)
1314 stream->dec_current1.addr - stream->dec_cpylen)VE;
1319 stream->dec_cpyoff + stream->dec_current1.addr)VE;
1327 size += stream->dec_current1.size;
1330 if (stream->dec_current2.type != XD3_NOOP)
1333 xd3_rtype_to_string ((xd3_rtype) stream->dec_current2.type,
1334 option_print_cpymode),
1335 stream->dec_current2.size)VE;
1337 if (stream->dec_current2.type >= XD3_CPY)
1339 if (stream->dec_current2.addr >= stream->dec_cpylen)
1342 stream->dec_current2.addr - stream->dec_cpylen)VE;
1347 stream->dec_cpyoff + stream->dec_current2.addr)VE;
1351 size += stream->dec_current2.size;
1356 if (option_verbose &&
1357 addr_bytes + inst_bytes >= (size - size_before) &&
1358 (stream->dec_current1.type >= XD3_CPY ||
1359 stream->dec_current2.type >= XD3_CPY))
1361 VC(UT " %06"Q"u (inefficiency) %u encoded as %u bytes\n",
1362 stream->dec_winstart + size_before,
1364 addr_bytes + inst_bytes)VE;
1368 if (stream->dec_tgtlen != size && (stream->flags & XD3_SKIP_WINDOW) == 0)
1370 XPR(NT "target window size inconsistency");
1371 return XD3_INTERNAL;
1374 if (stream->dec_position != stream->dec_maxpos)
1376 XPR(NT "target window position inconsistency");
1377 return XD3_INTERNAL;
1380 if (stream->addr_sect.buf != stream->addr_sect.buf_max)
1382 XPR(NT "address section inconsistency");
1383 return XD3_INTERNAL;
1390 main_print_vcdiff_file (main_file *xfile, main_file *file, const char *type)
1392 int ret; /* Used by above macros */
1395 VC(UT "XDELTA filename (%s): %s\n", type,
1398 if (file->compressor)
1400 VC(UT "XDELTA ext comp (%s): %s\n", type,
1401 file->compressor->recomp_cmdname)VE;
1406 /* This function prints a VCDIFF input, mainly for debugging purposes. */
1408 main_print_func (xd3_stream* stream, main_file *xfile)
1412 if (option_no_output)
1417 if (xfile->snprintf_buf == NULL)
1419 if ((xfile->snprintf_buf =
1420 (uint8_t*)main_malloc(SNPRINTF_BUFSIZE)) == NULL)
1426 if (stream->dec_winstart == 0)
1428 VC(UT "VCDIFF version: 0\n")VE;
1429 VC(UT "VCDIFF header size: %d\n",
1430 stream->dec_hdrsize)VE;
1431 VC(UT "VCDIFF header indicator: ")VE;
1432 if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0)
1433 VC(UT "VCD_SECONDARY ")VE;
1434 if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0)
1435 VC(UT "VCD_CODETABLE ")VE;
1436 if ((stream->dec_hdr_ind & VCD_APPHEADER) != 0)
1437 VC(UT "VCD_APPHEADER ")VE;
1438 if (stream->dec_hdr_ind == 0)
1442 IF_SEC(VC(UT "VCDIFF secondary compressor: %s\n",
1443 stream->sec_type ? stream->sec_type->name : "none")VE);
1444 IF_NSEC(VC(UT "VCDIFF secondary compressor: unsupported\n")VE);
1446 if (stream->dec_hdr_ind & VCD_APPHEADER)
1450 ret = xd3_get_appheader (stream, & apphead, & appheadsz);
1452 if (ret == 0 && appheadsz > 0)
1454 int sq = option_quiet;
1456 XD3_ASSERT (apphead != NULL);
1457 VC(UT "VCDIFF application header: ")VE;
1458 if ((ret = main_file_write (xfile, apphead,
1459 appheadsz, "print")) != 0)
1463 main_file_init (& i);
1464 main_file_init (& o);
1465 main_file_init (& s);
1467 main_get_appheader (stream, &i, & o, & s);
1469 if ((ret = main_print_vcdiff_file (xfile, & o, "output")))
1471 if ((ret = main_print_vcdiff_file (xfile, & s, "source")))
1473 main_file_cleanup (& i);
1474 main_file_cleanup (& o);
1475 main_file_cleanup (& s);
1484 VC(UT "VCDIFF window number: %"Q"u\n", stream->current_window)VE;
1485 VC(UT "VCDIFF window indicator: ")VE;
1486 if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE ")VE;
1487 if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET ")VE;
1488 if ((stream->dec_win_ind & VCD_ADLER32) != 0) VC(UT "VCD_ADLER32 ")VE;
1489 if (stream->dec_win_ind == 0) VC(UT "none")VE;
1492 if ((stream->dec_win_ind & VCD_ADLER32) != 0)
1494 VC(UT "VCDIFF adler32 checksum: %08X\n",
1495 (usize_t)stream->dec_adler32)VE;
1498 if (stream->dec_del_ind != 0)
1500 VC(UT "VCDIFF delta indicator: ")VE;
1501 if ((stream->dec_del_ind & VCD_DATACOMP) != 0) VC(UT "VCD_DATACOMP ")VE;
1502 if ((stream->dec_del_ind & VCD_INSTCOMP) != 0) VC(UT "VCD_INSTCOMP ")VE;
1503 if ((stream->dec_del_ind & VCD_ADDRCOMP) != 0) VC(UT "VCD_ADDRCOMP ")VE;
1504 if (stream->dec_del_ind == 0) VC(UT "none")VE;
1508 if (stream->dec_winstart != 0)
1510 VC(UT "VCDIFF window at offset: %"Q"u\n", stream->dec_winstart)VE;
1513 if (SRCORTGT (stream->dec_win_ind))
1515 VC(UT "VCDIFF copy window length: %u\n",
1516 (usize_t)stream->dec_cpylen)VE;
1517 VC(UT "VCDIFF copy window offset: %"Q"u\n",
1518 stream->dec_cpyoff)VE;
1521 VC(UT "VCDIFF delta encoding length: %u\n",
1522 (usize_t)stream->dec_enclen)VE;
1523 VC(UT "VCDIFF target window length: %u\n",
1524 (usize_t)stream->dec_tgtlen)VE;
1526 VC(UT "VCDIFF data section length: %u\n",
1527 (usize_t)stream->data_sect.size)VE;
1528 VC(UT "VCDIFF inst section length: %u\n",
1529 (usize_t)stream->inst_sect.size)VE;
1530 VC(UT "VCDIFF addr section length: %u\n",
1531 (usize_t)stream->addr_sect.size)VE;
1534 if ((stream->flags & XD3_JUST_HDR) != 0)
1536 /* Print a header -- finished! */
1537 ret = PRINTHDR_SPECIAL;
1539 else if ((stream->flags & XD3_SKIP_WINDOW) == 0)
1541 ret = main_print_window (stream, xfile);
1548 main_recode_copy (xd3_stream* stream,
1554 XD3_ASSERT(output != NULL);
1555 XD3_ASSERT(output->next_page == NULL);
1557 if ((ret = xd3_decode_allocate (recode_stream,
1562 XPR(NT XD3_LIB_ERRMSG (stream, ret));
1566 memcpy (output->base,
1567 /* Note: decoder advances buf, so get base of buffer with
1569 input->buf_max - input->size,
1571 output->next = input->size;
1575 // Re-encode one window
1577 main_recode_func (xd3_stream* stream, main_file *ofile)
1580 xd3_source decode_source;
1582 XD3_ASSERT(stream->dec_state == DEC_FINISH);
1583 XD3_ASSERT(recode_stream->enc_state == ENC_INIT ||
1584 recode_stream->enc_state == ENC_INPUT);
1586 // Copy partial decoder output to partial encoder inputs
1587 if ((ret = main_recode_copy (recode_stream,
1588 DATA_HEAD(recode_stream),
1589 &stream->data_sect)) ||
1590 (ret = main_recode_copy (recode_stream,
1591 INST_HEAD(recode_stream),
1592 &stream->inst_sect)) ||
1593 (ret = main_recode_copy (recode_stream,
1594 ADDR_HEAD(recode_stream),
1595 &stream->addr_sect)))
1600 // This jumps to xd3_emit_hdr()
1601 recode_stream->enc_state = ENC_FLUSH;
1602 recode_stream->avail_in = stream->dec_tgtlen;
1604 if (SRCORTGT (stream->dec_win_ind))
1606 recode_stream->src = & decode_source;
1607 decode_source.srclen = stream->dec_cpylen;
1608 decode_source.srcbase = stream->dec_cpyoff;
1611 if (option_use_checksum &&
1612 (stream->dec_win_ind & VCD_ADLER32) != 0)
1614 recode_stream->flags |= XD3_ADLER32_RECODE;
1615 recode_stream->recode_adler32 = stream->dec_adler32;
1618 if (option_use_appheader != 0 &&
1619 option_appheader != NULL)
1621 xd3_set_appheader (recode_stream, option_appheader,
1622 (usize_t) strlen ((char*) option_appheader));
1624 else if (option_use_appheader != 0 &&
1625 option_appheader == NULL)
1627 if (stream->dec_appheader != NULL)
1629 xd3_set_appheader (recode_stream,
1630 stream->dec_appheader, stream->dec_appheadsz);
1637 switch((ret = xd3_encode_input (recode_stream)))
1640 /* finished recoding one window */
1641 stream->total_out = recode_stream->total_out;
1645 /* main_file_write below */
1650 case XD3_WINFINISH: {
1656 return XD3_INTERNAL;
1662 if ((ret = main_write_output (recode_stream, ofile)))
1667 xd3_consume_output (recode_stream);
1670 #endif /* VCDIFF_TOOLS */
1672 /*******************************************************************
1674 ******************************************************************/
1677 /* Modifies static state. */
1679 main_init_recode_stream (void)
1682 int stream_flags = XD3_ADLER32_NOVER | XD3_SKIP_EMIT;
1684 xd3_config recode_config;
1686 XD3_ASSERT (recode_stream == NULL);
1688 if ((recode_stream = (xd3_stream*) main_malloc(sizeof(xd3_stream))) == NULL)
1693 recode_flags = (stream_flags & XD3_SEC_TYPE);
1695 recode_config.alloc = main_alloc;
1696 recode_config.freef = main_free1;
1698 xd3_init_config(&recode_config, recode_flags);
1700 if ((ret = main_set_secondary_flags (&recode_config)) ||
1701 (ret = xd3_config_stream (recode_stream, &recode_config)) ||
1702 (ret = xd3_encode_init_partial (recode_stream)) ||
1703 (ret = xd3_whole_state_init (recode_stream)))
1705 XPR(NT XD3_LIB_ERRMSG (recode_stream, ret));
1706 xd3_free_stream (recode_stream);
1707 recode_stream = NULL;
1714 /* This processes the sequence of -m arguments. The final input
1715 * is processed as part of the ordinary main_input() loop. */
1717 main_merge_arguments (main_merge_list* merges)
1721 main_merge *merge = NULL;
1722 xd3_stream merge_input;
1724 if (main_merge_list_empty (merges))
1729 if ((ret = xd3_config_stream (& merge_input, NULL)) ||
1730 (ret = xd3_whole_state_init (& merge_input)))
1732 XPR(NT XD3_LIB_ERRMSG (& merge_input, ret));
1736 merge = main_merge_list_front (merges);
1737 while (!main_merge_list_end (merges, merge))
1740 main_file_init (& mfile);
1741 mfile.filename = merge->filename;
1742 mfile.flags = RD_NONEXTERNAL;
1744 if ((ret = main_file_open (& mfile, merge->filename, XO_READ)))
1749 ret = main_input (CMD_MERGE_ARG, & mfile, NULL, NULL);
1755 /* The first merge source is the next merge input. */
1756 xd3_swap_whole_state (& recode_stream->whole_target,
1757 & merge_input.whole_target);
1761 /* Merge the recode_stream with merge_input. */
1762 ret = xd3_merge_input_output (recode_stream,
1763 & merge_input.whole_target);
1765 /* Save the next merge source in merge_input. */
1766 xd3_swap_whole_state (& recode_stream->whole_target,
1767 & merge_input.whole_target);
1771 main_file_cleanup (& mfile);
1773 if (recode_stream != NULL)
1775 xd3_free_stream (recode_stream);
1776 main_free (recode_stream);
1777 recode_stream = NULL;
1780 if (main_bdata != NULL)
1782 main_buffree (main_bdata);
1792 merge = main_merge_list_next (merge);
1795 XD3_ASSERT (merge_stream == NULL);
1797 if ((merge_stream = (xd3_stream*) main_malloc (sizeof(xd3_stream))) == NULL)
1803 if ((ret = xd3_config_stream (merge_stream, NULL)) ||
1804 (ret = xd3_whole_state_init (merge_stream)))
1806 XPR(NT XD3_LIB_ERRMSG (& merge_input, ret));
1810 xd3_swap_whole_state (& merge_stream->whole_target,
1811 & merge_input.whole_target);
1814 xd3_free_stream (& merge_input);
1818 /* This processes each window of the final merge input. This routine
1819 * does not output, it buffers the entire delta into memory. */
1821 main_merge_func (xd3_stream* stream, main_file *no_write)
1825 if ((ret = xd3_whole_append_window (stream)))
1834 /* This is called after all windows have been read, as a final step in
1835 * main_input(). This is only called for the final merge step. */
1837 main_merge_output (xd3_stream *stream, main_file *ofile)
1840 usize_t inst_pos = 0;
1841 xoff_t output_pos = 0;
1842 xd3_source recode_source;
1843 usize_t window_num = 0;
1844 int at_least_once = 0;
1846 /* merge_stream is set if there were arguments. this stream's input
1847 * needs to be applied to the merge_stream source. */
1848 if ((merge_stream != NULL) &&
1849 (ret = xd3_merge_input_output (stream,
1850 & merge_stream->whole_target)))
1852 XPR(NT XD3_LIB_ERRMSG (stream, ret));
1856 if (option_use_appheader != 0 &&
1857 option_appheader != NULL)
1859 xd3_set_appheader (recode_stream, option_appheader,
1860 (usize_t) strlen ((char*) option_appheader));
1863 /* Enter the ENC_INPUT state and bypass the next_in == NULL test
1864 * and (leftover) input buffering logic. */
1865 XD3_ASSERT(recode_stream->enc_state == ENC_INIT);
1866 recode_stream->enc_state = ENC_INPUT;
1867 recode_stream->next_in = main_bdata;
1868 recode_stream->flags |= XD3_FLUSH;
1870 /* This encodes the entire target. */
1871 while (inst_pos < stream->whole_target.instlen || !at_least_once)
1873 xoff_t window_start = output_pos;
1874 int window_srcset = 0;
1875 xoff_t window_srcmin = 0;
1876 xoff_t window_srcmax = 0;
1877 usize_t window_pos = 0;
1878 usize_t window_size;
1880 /* at_least_once ensures that we encode at least one window,
1881 * which handles the 0-byte case. */
1884 XD3_ASSERT (recode_stream->enc_state == ENC_INPUT);
1886 if ((ret = xd3_encode_input (recode_stream)) != XD3_WINSTART)
1888 XPR(NT "invalid merge state: %s\n", xd3_mainerror (ret));
1892 /* Window sizes must match from the input to the output, so that
1893 * target copies are in-range (and so that checksums carry
1895 XD3_ASSERT (window_num < stream->whole_target.wininfolen);
1896 window_size = stream->whole_target.wininfo[window_num].length;
1898 /* Output position should also match. */
1899 if (output_pos != stream->whole_target.wininfo[window_num].offset)
1901 XPR(NT "internal merge error: offset mismatch\n");
1905 if (option_use_checksum &&
1906 (stream->dec_win_ind & VCD_ADLER32) != 0)
1908 recode_stream->flags |= XD3_ADLER32_RECODE;
1909 recode_stream->recode_adler32 =
1910 stream->whole_target.wininfo[window_num].adler32;
1915 if (main_bsize < window_size)
1917 main_buffree (main_bdata);
1920 if ((main_bdata = (uint8_t*)
1921 main_bufalloc (window_size)) == NULL)
1925 main_bsize = window_size;
1928 /* This encodes a single target window. */
1929 while (window_pos < window_size &&
1930 inst_pos < stream->whole_target.instlen)
1932 xd3_winst *inst = &stream->whole_target.inst[inst_pos];
1933 usize_t take = min(inst->size, window_size - window_pos);
1939 if ((ret = xd3_emit_run (recode_stream, window_pos, take,
1940 &stream->whole_target.adds[inst->addr])))
1947 /* Adds are implicit, put them into the input buffer. */
1948 memcpy (main_bdata + window_pos,
1949 stream->whole_target.adds + inst->addr, take);
1952 default: /* XD3_COPY + copy mode */
1953 if (inst->mode != 0)
1955 if (window_srcset) {
1956 window_srcmin = min(window_srcmin, inst->addr);
1957 window_srcmax = max(window_srcmax, inst->addr + take);
1960 window_srcmin = inst->addr;
1961 window_srcmax = inst->addr + take;
1967 XD3_ASSERT (inst->addr >= window_start);
1968 addr = inst->addr - window_start;
1970 IF_DEBUG2 (XPR(NTR "[merge copy] winpos %u take %u addr %"Q"u mode %u\n",
1971 window_pos, take, addr, inst->mode));
1972 if ((ret = xd3_found_match (recode_stream, window_pos, take,
1973 addr, inst->mode != 0)))
1983 if (take == inst->size)
1989 /* Modify the instruction for the next pass. */
1990 if (inst->type != XD3_RUN)
1998 xd3_avail_input (recode_stream, main_bdata, window_pos);
2000 recode_stream->enc_state = ENC_INSTR;
2002 if (window_srcset) {
2003 recode_stream->srcwin_decided = 1;
2004 recode_stream->src = &recode_source;
2005 recode_source.srclen = (usize_t)(window_srcmax - window_srcmin);
2006 recode_source.srcbase = window_srcmin;
2007 recode_stream->taroff = recode_source.srclen;
2009 XD3_ASSERT (recode_source.srclen != 0);
2011 recode_stream->srcwin_decided = 0;
2012 recode_stream->src = NULL;
2013 recode_stream->taroff = 0;
2018 switch ((ret = xd3_encode_input (recode_stream)))
2024 /* main_file_write below */
2029 case XD3_WINFINISH: {
2035 return XD3_INTERNAL;
2041 if ((ret = main_write_output(recode_stream, ofile)))
2046 xd3_consume_output (recode_stream);
2056 /*******************************************************************
2057 Input decompression, output recompression
2058 ******************************************************************/
2060 #if EXTERNAL_COMPRESSION
2061 /* This is tricky POSIX-specific code with lots of fork(), pipe(),
2062 * dup(), waitpid(), and exec() business. Most of this code
2063 * originated in PRCS1, which did automatic package-file
2064 * decompression. It works with both XD3_POSIX and XD3_STDIO file
2067 * To automatically detect compressed inputs requires a child process
2068 * to reconstruct the input stream, which was advanced in order to
2069 * detect compression, because it may not be seekable. In other
2070 * words, the main program reads part of the input stream, and if it
2071 * detects a compressed input it then forks a pipe copier process,
2072 * which copies the first-read block out of the main-program's memory,
2073 * then streams the remaining compressed input into the
2074 * input-decompression pipe.
2079 #include <sys/stat.h>
2080 #include <sys/wait.h>
2082 /* Remember which pipe FD is which. */
2083 #define PIPE_READ_FD 0
2084 #define PIPE_WRITE_FD 1
2085 #define MAX_SUBPROCS 4 /* max(source + copier + output,
2086 source + copier + input + copier). */
2087 static pid_t ext_subprocs[MAX_SUBPROCS];
2089 /* Like write(), applies to a fd instead of a main_file, for the pipe
2090 * copier subprocess. Does not print an error, to facilitate ignoring
2091 * trailing garbage, see main_pipe_copier(). */
2093 main_pipe_write (int outfd, uint8_t *exist_buf, usize_t remain)
2097 if ((ret = xd3_posix_io (outfd, exist_buf, remain,
2098 (xd3_posix_func*) &write, NULL)))
2106 /* A simple error-reporting waitpid interface. */
2108 main_waitpid_check(pid_t pid)
2113 if (waitpid (pid, & status, 0) < 0)
2116 XPR(NT "external compression [pid %d] wait: %s\n",
2117 pid, xd3_mainerror (ret));
2119 else if (! WIFEXITED (status))
2121 // SIGPIPE will be delivered to the child process whenever it
2122 // writes data after this process closes the pipe,
2123 // happens if xdelta does not require access to the entire
2124 // source file. Considered normal.
2125 if (! WIFSIGNALED (status) || WTERMSIG (status) != SIGPIPE)
2128 XPR(NT "external compression [pid %d] signal %d\n", pid,
2129 WIFSIGNALED (status) ? WTERMSIG (status) : WSTOPSIG (status));
2131 else if (option_verbose)
2133 XPR(NT "external compression sigpipe\n");
2136 else if (WEXITSTATUS (status) != 0)
2139 if (option_verbose > 1)
2141 /* Presumably, the error was printed by the subprocess. */
2142 XPR(NT "external compression [pid %d] exit %d\n",
2143 pid, WEXITSTATUS (status));
2150 /* Wait for any existing child processes to check for abnormal exit. */
2152 main_external_compression_finish (void)
2157 for (i = 0; i < num_subprocs; i += 1)
2159 if (! ext_subprocs[i]) { continue; }
2161 if ((ret = main_waitpid_check (ext_subprocs[i])))
2166 ext_subprocs[i] = 0;
2172 /* Kills any outstanding compression process. */
2174 main_external_compression_cleanup (void)
2178 for (i = 0; i < num_subprocs; i += 1)
2180 if (! ext_subprocs[i]) { continue; }
2182 kill (ext_subprocs[i], SIGTERM);
2184 ext_subprocs[i] = 0;
2188 /* This runs as a forked process of main_input_decompress_setup() to
2189 * copy input to the decompression process. First, the available
2190 * input is copied out of the existing buffer, then the buffer is
2191 * reused to continue reading from the compressed input file. */
2193 main_pipe_copier (uint8_t *pipe_buf,
2194 usize_t pipe_bufsize,
2202 /* Prevent SIGPIPE signals, allow EPIPE return values instead. This
2203 * is safe to comment-out, except that the -F flag will not work
2204 * properly (the parent would need to treat WTERMSIG(status) ==
2206 struct sigaction sa;
2207 sa.sa_handler = SIG_IGN;
2208 sigaction (SIGPIPE, &sa, NULL);
2212 /* force_drain will be set when option_force and EPIPE cause us
2213 * to skip data. This is reset each time through the loop, so
2214 * the break condition below works. */
2215 int force_drain = 0;
2216 if (nread > 0 && (ret = main_pipe_write (outfd, pipe_buf, nread)))
2220 /* This causes the loop to continue reading until nread
2227 XPR(NT "pipe write failed: %s\n", xd3_mainerror (ret));
2232 if (nread < pipe_bufsize && !force_drain)
2237 if ((ret = main_file_read (ifile, pipe_buf, pipe_bufsize,
2238 & nread, "pipe read failed")) < 0)
2244 if (option_verbose && skipped != 0)
2246 XPR(NT "skipping %"Q"u bytes in %s\n",
2247 skipped, ifile->filename);
2252 /* This function is called after we have read some amount of data from
2253 * the input file and detected a compressed input. Here we start a
2254 * decompression subprocess by forking twice. The first process runs
2255 * the decompression command, the second process copies data to the
2256 * input of the first. */
2258 main_input_decompress_setup (const main_extcomp *decomp,
2261 usize_t input_bufsize,
2263 usize_t pipe_bufsize,
2267 /* The two pipes: input and output file descriptors. */
2268 int outpipefd[2], inpipefd[2];
2269 int input_fd = -1; /* The resulting input_fd (output of decompression). */
2270 pid_t decomp_id, copier_id; /* The two subprocs. */
2273 outpipefd[0] = outpipefd[1] = -1;
2274 inpipefd[0] = inpipefd[1] = -1;
2276 if (pipe (outpipefd) || pipe (inpipefd))
2278 XPR(NT "pipe failed: %s\n", xd3_mainerror (ret = get_errno ()));
2282 if ((decomp_id = fork ()) < 0)
2284 XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ()));
2288 /* The first child runs the decompression process: */
2291 if (option_verbose > 2)
2293 XPR(NT "external decompression pid %d\n", getpid ());
2296 /* Setup pipes: write to the outpipe, read from the inpipe. */
2297 if (dup2 (outpipefd[PIPE_WRITE_FD], STDOUT_FILENO) < 0 ||
2298 dup2 (inpipefd[PIPE_READ_FD], STDIN_FILENO) < 0 ||
2299 close (outpipefd[PIPE_READ_FD]) ||
2300 close (outpipefd[PIPE_WRITE_FD]) ||
2301 close (inpipefd[PIPE_READ_FD]) ||
2302 close (inpipefd[PIPE_WRITE_FD]) ||
2303 execlp (decomp->decomp_cmdname, decomp->decomp_cmdname,
2304 decomp->decomp_options,
2305 option_force2 ? "-f" : NULL,
2308 XPR(NT "child process %s failed to execute: %s\n",
2309 decomp->decomp_cmdname, xd3_mainerror (get_errno ()));
2315 XD3_ASSERT(num_subprocs < MAX_SUBPROCS);
2316 ext_subprocs[num_subprocs++] = decomp_id;
2318 if ((copier_id = fork ()) < 0)
2320 XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ()));
2324 /* The second child runs the copier process: */
2329 if (option_verbose > 2)
2331 XPR(NT "child pipe-copier pid %d\n", getpid ());
2334 if (close (inpipefd[PIPE_READ_FD]) ||
2335 close (outpipefd[PIPE_READ_FD]) ||
2336 close (outpipefd[PIPE_WRITE_FD]) ||
2337 main_pipe_copier (pipe_buf, pipe_bufsize, pipe_avail,
2338 ifile, inpipefd[PIPE_WRITE_FD]) ||
2339 close (inpipefd[PIPE_WRITE_FD]))
2341 XPR(NT "child copier process failed: %s\n",
2342 xd3_mainerror (get_errno ()));
2349 XD3_ASSERT(num_subprocs < MAX_SUBPROCS);
2350 ext_subprocs[num_subprocs++] = copier_id;
2352 /* The parent closes both pipes after duplicating the output of
2354 input_fd = dup (outpipefd[PIPE_READ_FD]);
2357 main_file_close (ifile) ||
2358 close (outpipefd[PIPE_READ_FD]) ||
2359 close (outpipefd[PIPE_WRITE_FD]) ||
2360 close (inpipefd[PIPE_READ_FD]) ||
2361 close (inpipefd[PIPE_WRITE_FD]))
2363 XPR(NT "dup/close failed: %s\n", xd3_mainerror (ret = get_errno ()));
2368 /* Note: fdopen() acquires the fd, closes it when finished. */
2369 if ((ifile->file = fdopen (input_fd, "r")) == NULL)
2371 XPR(NT "fdopen failed: %s\n", xd3_mainerror (ret = get_errno ()));
2376 ifile->file = input_fd;
2379 ifile->compressor = decomp;
2381 /* Now the input file is decompressed. */
2382 return main_file_read (ifile, input_buf, input_bufsize,
2383 nread, "input decompression failed");
2387 close (outpipefd[PIPE_READ_FD]);
2388 close (outpipefd[PIPE_WRITE_FD]);
2389 close (inpipefd[PIPE_READ_FD]);
2390 close (inpipefd[PIPE_WRITE_FD]);
2395 /* This routine is called when the first buffer of input data is read
2396 * by the main program (unless input decompression is disabled by
2397 * command-line option). If it recognizes the magic number of a known
2398 * input type it invokes decompression.
2400 * Skips decompression if the decompression type or the file type is
2403 * Behaves exactly like main_file_read, otherwise.
2405 * This function uses a separate buffer to read the first small block
2406 * of input. If a compressed input is detected, the separate buffer
2407 * is passed to the pipe copier. This avoids using the same size
2408 * buffer in both cases. */
2410 main_secondary_decompress_check (main_file *file,
2417 usize_t try_read = min (input_size, XD3_ALLOCSIZE);
2418 size_t check_nread = 0;
2419 uint8_t check_buf[XD3_ALLOCSIZE]; /* TODO: stack limit */
2420 const main_extcomp *decompressor = NULL;
2422 if ((ret = main_file_read (file, check_buf,
2424 & check_nread, "input read failed")))
2429 if (file->flags & RD_DECOMPSET)
2431 /* This allows the application header to override the magic
2432 * number, for whatever reason. */
2433 decompressor = file->compressor;
2437 for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
2439 const main_extcomp *decomp = & extcomp_types[i];
2441 if (check_nread > decomp->magic_size)
2443 /* The following expr checks if we are trying to read a
2444 * VCDIFF input, in which case do not treat it as
2445 * "secondary" decompression. */
2446 int skip_this_type = (decomp->flags & RD_NONEXTERNAL) &&
2447 (file->flags & RD_NONEXTERNAL);
2454 if (memcmp (check_buf, decomp->magic, decomp->magic_size) == 0)
2456 decompressor = decomp;
2463 if (decompressor != NULL)
2467 XPR(NT "externally compressed input: %s %s%s < %s\n",
2468 decompressor->decomp_cmdname,
2469 decompressor->decomp_options,
2470 (option_force2 ? " -f" : ""),
2472 if (file->flags & RD_MAININPUT)
2475 "WARNING: the encoder is automatically decompressing the input file;\n");
2477 "WARNING: the decoder will automatically recompress the output file;\n");
2479 "WARNING: this may result in different compressed data and checksums\n");
2481 "WARNING: despite being identical data; if this is an issue, use -D\n");
2483 "WARNING: to avoid decompression and/or use -R to avoid recompression\n");
2485 "WARNING: and/or manually decompress the input file; if you know the\n");
2487 "WARNING: compression settings that will produce identical output\n");
2489 "WARNING: you may set those flags using the environment (e.g., GZIP=-9)\n");
2493 file->size_known = 0;
2494 return main_input_decompress_setup (decompressor, file,
2495 input_buf, input_size,
2496 check_buf, XD3_ALLOCSIZE,
2497 check_nread, nread);
2500 /* Now read the rest of the input block. */
2503 if (check_nread == try_read)
2505 ret = main_file_read (file,
2506 input_buf + try_read,
2507 input_size - try_read,
2509 "input read failed");
2512 memcpy (input_buf, check_buf, check_nread);
2514 (*nread) += check_nread;
2519 /* Initiate re-compression of the output stream. This is easier than
2520 * input decompression because we know beforehand that the stream will
2521 * be compressed, whereas the input has already been read when we
2522 * decide it should be decompressed. Thus, it only requires one
2523 * subprocess and one pipe. */
2525 main_recompress_output (main_file *ofile)
2527 pid_t recomp_id; /* One subproc. */
2528 int pipefd[2]; /* One pipe. */
2531 const main_extcomp *recomp = ofile->compressor;
2533 pipefd[0] = pipefd[1] = -1;
2537 XPR(NT "pipe failed: %s\n", xd3_mainerror (ret = get_errno ()));
2541 if ((recomp_id = fork ()) < 0)
2543 XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ()));
2547 /* The child runs the recompression process: */
2550 if (option_verbose > 2)
2552 XPR(NT "external recompression pid %d\n", getpid ());
2555 /* Setup pipes: write to the output file, read from the pipe. */
2556 if (dup2 (XFNO (ofile), STDOUT_FILENO) < 0 ||
2557 dup2 (pipefd[PIPE_READ_FD], STDIN_FILENO) < 0 ||
2558 close (pipefd[PIPE_READ_FD]) ||
2559 close (pipefd[PIPE_WRITE_FD]) ||
2560 execlp (recomp->recomp_cmdname, recomp->recomp_cmdname,
2561 recomp->recomp_options,
2562 option_force2 ? "-f" : NULL,
2565 XPR(NT "child process %s failed to execute: %s\n",
2566 recomp->recomp_cmdname, xd3_mainerror (get_errno ()));
2572 XD3_ASSERT(num_subprocs < MAX_SUBPROCS);
2573 ext_subprocs[num_subprocs++] = recomp_id;
2575 /* The parent closes both pipes after duplicating the output-fd for
2576 * writing to the compression pipe. */
2577 output_fd = dup (pipefd[PIPE_WRITE_FD]);
2579 if (output_fd < 0 ||
2580 main_file_close (ofile) ||
2581 close (pipefd[PIPE_READ_FD]) ||
2582 close (pipefd[PIPE_WRITE_FD]))
2584 XPR(NT "close failed: %s\n", xd3_mainerror (ret = get_errno ()));
2589 /* Note: fdopen() acquires the fd, closes it when finished. */
2590 if ((ofile->file = fdopen (output_fd, "w")) == NULL)
2592 XPR(NT "fdopen failed: %s\n", xd3_mainerror (ret = get_errno ()));
2597 ofile->file = output_fd;
2600 /* Now the output file will be compressed. */
2605 close (pipefd[PIPE_READ_FD]);
2606 close (pipefd[PIPE_WRITE_FD]);
2609 #endif /* EXTERNAL_COMPRESSION */
2611 /* Identify the compressor that was used based on its ident string,
2612 * which is passed in the application header. */
2613 static const main_extcomp*
2614 main_ident_compressor (const char *ident)
2618 for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1)
2620 if (strcmp (extcomp_types[i].ident, ident) == 0)
2622 return & extcomp_types[i];
2629 /* Return the main_extcomp record to use for this identifier, if possible. */
2630 static const main_extcomp*
2631 main_get_compressor (const char *ident)
2633 const main_extcomp *ext = main_ident_compressor (ident);
2639 XPR(NT "warning: cannot recompress output: "
2640 "unrecognized external compression ID: %s\n", ident);
2644 else if (! EXTERNAL_COMPRESSION)
2648 XPR(NT "warning: external support not compiled: "
2649 "original input was compressed: %s\n", ext->recomp_cmdname);
2659 /*********************************************************************
2661 *******************************************************************/
2665 main_apphead_string (const char* x)
2669 if (x == NULL) { return ""; }
2671 if (strcmp (x, "/dev/stdin") == 0 ||
2672 strcmp (x, "/dev/stdout") == 0 ||
2673 strcmp (x, "/dev/stderr") == 0) { return "-"; }
2675 // TODO: this is not portable
2676 return (y = strrchr (x, '/')) == NULL ? x : y + 1;
2680 main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile)
2682 /* The user may disable the application header. Once the appheader
2683 * is set, this disables setting it again. */
2684 if (appheader_used || ! option_use_appheader) { return 0; }
2686 /* The user may specify the application header, otherwise format the
2688 if (option_appheader)
2690 appheader_used = option_appheader;
2700 iname = main_apphead_string (input->filename);
2701 icomp = (input->compressor == NULL) ? "" : input->compressor->ident;
2702 len = (usize_t) strlen (iname) + (usize_t) strlen (icomp) + 2;
2704 if (sfile->filename != NULL)
2706 sname = main_apphead_string (sfile->filename);
2707 scomp = (sfile->compressor == NULL) ? "" : sfile->compressor->ident;
2708 len += (usize_t) strlen (sname) + (usize_t) strlen (scomp) + 2;
2715 if ((appheader_used = (uint8_t*) main_malloc (len)) == NULL)
2720 if (sfile->filename == NULL)
2722 snprintf_func ((char*)appheader_used, len, "%s/%s", iname, icomp);
2726 snprintf_func ((char*)appheader_used, len, "%s/%s/%s/%s",
2727 iname, icomp, sname, scomp);
2731 xd3_set_appheader (stream, appheader_used,
2732 (usize_t) strlen ((char*)appheader_used));
2739 main_get_appheader_params (main_file *file, char **parsed,
2740 int output, const char *type,
2743 /* Set the filename if it was not specified. If output, option_stdout (-c)
2745 if (file->filename == NULL &&
2746 ! (output && option_stdout) &&
2747 strcmp (parsed[0], "-") != 0)
2749 file->filename = parsed[0];
2751 if (other->filename != NULL) {
2752 /* Take directory from the other file, if it has one. */
2753 /* TODO: This results in nonsense names like /dev/foo.tar.gz
2754 * and probably the filename-default logic interferes with
2755 * multi-file operation and the standard file extension?
2756 * Possibly the name header is bad, should be off by default.
2757 * Possibly we just want to remember external/compression
2759 const char *last_slash = strrchr(other->filename, '/');
2761 if (last_slash != NULL) {
2762 usize_t dlen = (usize_t) (last_slash - other->filename);
2764 XD3_ASSERT(file->filename_copy == NULL);
2765 file->filename_copy =
2766 (char*) main_malloc(dlen + 2 + (usize_t) strlen(file->filename));
2768 strncpy(file->filename_copy, other->filename, dlen);
2769 file->filename_copy[dlen] = '/';
2770 strcpy(file->filename_copy + dlen + 1, parsed[0]);
2772 file->filename = file->filename_copy;
2778 XPR(NT "using default %s filename: %s\n", type, file->filename);
2782 /* Set the compressor, initiate de/recompression later. */
2783 if (file->compressor == NULL && *parsed[1] != 0)
2785 file->flags |= RD_DECOMPSET;
2786 file->compressor = main_get_compressor (parsed[1]);
2791 main_get_appheader (xd3_stream *stream, main_file *ifile,
2792 main_file *output, main_file *sfile)
2798 /* The user may disable the application header. Once the appheader
2799 * is set, this disables setting it again. */
2800 if (! option_use_appheader) { return; }
2802 ret = xd3_get_appheader (stream, & apphead, & appheadsz);
2804 /* Ignore failure, it only means we haven't received a header yet. */
2805 if (ret != 0) { return; }
2809 const int kMaxArgs = 4;
2810 char *start = (char*)apphead;
2813 char *parsed[kMaxArgs];
2815 memset (parsed, 0, sizeof (parsed));
2817 while ((slash = strchr (start, '/')) != NULL && place < (kMaxArgs-1))
2820 parsed[place++] = start;
2824 parsed[place++] = start;
2826 /* First take the output parameters. */
2827 if (place == 2 || place == 4)
2829 main_get_appheader_params (output, parsed, 1, "output", ifile);
2832 /* Then take the source parameters. */
2835 main_get_appheader_params (sfile, parsed+2, 0, "source", ifile);
2839 option_use_appheader = 0;
2843 /*********************************************************************
2845 **********************************************************************/
2847 /* This function acts like the above except it may also try to
2848 * recognize a compressed input (source or target) when the first
2849 * buffer of data is read. The EXTERNAL_COMPRESSION code is called to
2850 * search for magic numbers. */
2852 main_read_primary_input (main_file *file,
2857 #if EXTERNAL_COMPRESSION
2858 if (option_decompress_inputs && file->flags & RD_FIRST)
2860 file->flags &= ~RD_FIRST;
2861 return main_secondary_decompress_check (file, buf, size, nread);
2865 return main_file_read (file, buf, size, nread, "input read failed");
2868 /* Open the main output file, sets a default file name, initiate
2869 * recompression. This function is expected to fprint any error
2872 main_open_output (xd3_stream *stream, main_file *ofile)
2876 if (option_no_output)
2881 if (ofile->filename == NULL)
2885 if (option_verbose > 1)
2887 XPR(NT "using standard output: %s\n", ofile->filename);
2892 /* Stat the file to check for overwrite. */
2893 if (option_force == 0 && main_file_exists (ofile))
2897 XPR(NT "to overwrite output file specify -f: %s\n",
2903 if ((ret = main_file_open (ofile, ofile->filename, XO_WRITE)))
2908 if (option_verbose > 1) { XPR(NT "output %s\n", ofile->filename); }
2911 #if EXTERNAL_COMPRESSION
2912 /* Do output recompression. */
2913 if (ofile->compressor != NULL && option_recompress_outputs == 1)
2917 XPR(NT "externally compressed output: %s %s%s > %s\n",
2918 ofile->compressor->recomp_cmdname,
2919 ofile->compressor->recomp_options,
2920 (option_force2 ? " -f" : ""),
2924 if ((ret = main_recompress_output (ofile)))
2935 main_get_winsize (main_file *ifile) {
2936 xoff_t file_size = 0;
2937 usize_t size = option_winsize;
2938 static shortbuf iszbuf;
2940 if (main_file_stat (ifile, &file_size) == 0)
2942 size = (usize_t) min(file_size, (xoff_t) size);
2945 size = max(size, XD3_ALLOCSIZE);
2947 if (option_verbose > 1)
2949 XPR(NT "input %s window size %s\n",
2951 main_format_bcnt (size, &iszbuf));
2957 /*********************************************************************
2959 ********************************************************************/
2961 /* This is a generic input function. It calls the xd3_encode_input or
2962 * xd3_decode_input functions and makes calls to the various input
2963 * handling routines above, which coordinate external decompression.
2966 main_input (xd3_cmd cmd,
2975 int stream_flags = 0;
2978 xoff_t last_total_in = 0;
2979 xoff_t last_total_out = 0;
2981 int stdout_only = 0;
2982 int (*input_func) (xd3_stream*);
2983 int (*output_func) (xd3_stream*, main_file *);
2985 memset (& stream, 0, sizeof (stream));
2986 memset (& source, 0, sizeof (source));
2987 memset (& config, 0, sizeof (config));
2989 config.alloc = main_alloc;
2990 config.freef = main_free1;
2992 config.iopt_size = option_iopt_size;
2993 config.sprevsz = option_sprevsz;
2997 start_time = get_millisecs_now ();
2999 if (option_use_checksum) { stream_flags |= XD3_ADLER32; }
3001 /* main_input setup. */
3005 if (1) { case CMD_PRINTHDR: stream_flags |= XD3_JUST_HDR; }
3006 else if (1) { case CMD_PRINTHDRS: stream_flags |= XD3_SKIP_WINDOW; }
3007 else { case CMD_PRINTDELTA: stream_flags |= XD3_SKIP_EMIT; }
3008 ifile->flags |= RD_NONEXTERNAL;
3009 input_func = xd3_decode_input;
3010 output_func = main_print_func;
3011 stream_flags |= XD3_ADLER32_NOVER;
3018 /* No source will be read */
3019 stream_flags |= XD3_ADLER32_NOVER | XD3_SKIP_EMIT;
3020 ifile->flags |= RD_NONEXTERNAL;
3021 input_func = xd3_decode_input;
3023 if ((ret = main_init_recode_stream ()))
3025 return EXIT_FAILURE;
3028 if (cmd == CMD_RECODE) { output_func = main_recode_func; }
3029 else { output_func = main_merge_func; }
3031 #endif /* VCDIFF_TOOLS */
3036 input_func = xd3_encode_input;
3037 output_func = main_write_output;
3039 if (option_no_compress) { stream_flags |= XD3_NOCOMPRESS; }
3040 if (option_smatch_config)
3042 const char *s = option_smatch_config;
3044 int values[XD3_SOFTCFG_VARCNT];
3047 config.smatch_cfg = XD3_SMATCH_SOFT;
3049 for (got = 0; got < XD3_SOFTCFG_VARCNT; got += 1, s = e + 1)
3051 values[got] = strtol (s, &e, 10);
3053 if ((values[got] < 0) ||
3055 (got < XD3_SOFTCFG_VARCNT-1 && *e == 0) ||
3056 (got == XD3_SOFTCFG_VARCNT-1 && *e != 0))
3058 XPR(NT "invalid string match specifier (-C) %d: %s\n",
3060 return EXIT_FAILURE;
3064 config.smatcher_soft.large_look = values[0];
3065 config.smatcher_soft.large_step = values[1];
3066 config.smatcher_soft.small_look = values[2];
3067 config.smatcher_soft.small_chain = values[3];
3068 config.smatcher_soft.small_lchain = values[4];
3069 config.smatcher_soft.max_lazy = values[5];
3070 config.smatcher_soft.long_enough = values[6];
3074 if (option_verbose > 2)
3076 XPR(NT "compression level: %d\n", option_level);
3078 if (option_level == 0)
3080 stream_flags |= XD3_NOCOMPRESS;
3081 config.smatch_cfg = XD3_SMATCH_FASTEST;
3083 else if (option_level == 1)
3084 { config.smatch_cfg = XD3_SMATCH_FASTEST; }
3085 else if (option_level == 2)
3086 { config.smatch_cfg = XD3_SMATCH_FASTER; }
3087 else if (option_level <= 5)
3088 { config.smatch_cfg = XD3_SMATCH_FAST; }
3089 else if (option_level == 6)
3090 { config.smatch_cfg = XD3_SMATCH_DEFAULT; }
3092 { config.smatch_cfg = XD3_SMATCH_SLOW; }
3097 if (option_use_checksum == 0) { stream_flags |= XD3_ADLER32_NOVER; }
3098 ifile->flags |= RD_NONEXTERNAL;
3099 input_func = xd3_decode_input;
3100 output_func = main_write_output;
3103 XPR(NT "internal error\n");
3104 return EXIT_FAILURE;
3107 main_bsize = winsize = main_get_winsize (ifile);
3109 if ((main_bdata = (uint8_t*) main_bufalloc (winsize)) == NULL)
3111 return EXIT_FAILURE;
3114 config.winsize = winsize;
3115 config.getblk = main_getblk_func;
3116 config.flags = stream_flags;
3118 if ((ret = main_set_secondary_flags (&config)) ||
3119 (ret = xd3_config_stream (& stream, & config)))
3121 XPR(NT XD3_LIB_ERRMSG (& stream, ret));
3122 return EXIT_FAILURE;
3126 if ((cmd == CMD_MERGE || cmd == CMD_MERGE_ARG) &&
3127 (ret = xd3_whole_state_init (& stream)))
3129 XPR(NT XD3_LIB_ERRMSG (& stream, ret));
3130 return EXIT_FAILURE;
3134 if (cmd != CMD_DECODE)
3136 /* When not decoding, set source now. The decoder delays this
3137 * step until XD3_GOTHEADER. */
3138 if (sfile && sfile->filename != NULL)
3140 if ((ret = main_set_source (& stream, cmd, sfile, & source)))
3142 return EXIT_FAILURE;
3145 XD3_ASSERT(stream.src != NULL);
3149 if (cmd == CMD_PRINTHDR ||
3150 cmd == CMD_PRINTHDRS ||
3151 cmd == CMD_PRINTDELTA ||
3154 if (sfile->filename == NULL)
3156 allow_fake_source = 1;
3157 sfile->filename = "<placeholder>";
3158 main_set_source (& stream, cmd, sfile, & source);
3162 /* This times each window. */
3163 get_millisecs_since ();
3165 /* Main input loop. */
3168 xoff_t input_offset;
3169 xoff_t input_remain;
3172 input_offset = ifile->nread;
3174 input_remain = XOFF_T_MAX - input_offset;
3176 try_read = (usize_t) min ((xoff_t) config.winsize, input_remain);
3178 if ((ret = main_read_primary_input (ifile, main_bdata,
3179 try_read, & nread)))
3181 return EXIT_FAILURE;
3184 /* If we've reached EOF tell the stream to flush. */
3185 if (nread < try_read)
3187 stream.flags |= XD3_FLUSH;
3191 /* After the first main_read_primary_input completes, we know
3192 * all the information needed to encode the application
3194 if (cmd == CMD_ENCODE &&
3195 (ret = main_set_appheader (& stream, ifile, sfile)))
3197 return EXIT_FAILURE;
3200 xd3_avail_input (& stream, main_bdata, nread);
3202 /* If we read zero bytes after encoding at least one window... */
3203 if (nread == 0 && stream.current_window > 0) {
3208 ret = input_func (& stream);
3217 XD3_ASSERT (stream.current_window == 0);
3219 /* Need to process the appheader as soon as possible. It may
3220 * contain a suggested default filename/decompression routine for
3221 * the ofile, and it may contain default/decompression routine for
3223 if (cmd == CMD_DECODE)
3225 /* May need to set the sfile->filename if none was given. */
3226 main_get_appheader (& stream, ifile, ofile, sfile);
3228 /* Now open the source file. */
3229 if ((sfile->filename != NULL) &&
3230 (ret = main_set_source (& stream, cmd, sfile, & source)))
3232 return EXIT_FAILURE;
3239 /* e.g., set or unset XD3_SKIP_WINDOW. */
3245 /* Defer opening the output file until the stream produces its
3246 * first output for both encoder and decoder, this way we
3247 * delay long enough for the decoder to receive the
3248 * application header. (Or longer if there are skipped
3249 * windows, but I can't think of any reason not to delay
3251 if (ofile != NULL &&
3252 ! main_file_isopen (ofile) &&
3253 (ret = main_open_output (& stream, ofile)) != 0)
3255 return EXIT_FAILURE;
3258 if ((ret = output_func (& stream, ofile)) &&
3259 (ret != PRINTHDR_SPECIAL))
3261 return EXIT_FAILURE;
3264 if (ret == PRINTHDR_SPECIAL)
3266 xd3_abort_stream (& stream);
3273 xd3_consume_output (& stream);
3279 if (IS_ENCODE (cmd) || cmd == CMD_DECODE || cmd == CMD_RECODE)
3281 if (! option_quiet && IS_ENCODE (cmd) &&
3282 main_file_isopen (sfile))
3284 /* Warn when no source copies are found */
3285 if (option_verbose && ! xd3_encoder_used_source (& stream))
3287 XPR(NT "warning: input window %"Q"u..%"Q"u has "
3288 "no source copies\n",
3289 stream.current_window * winsize,
3290 (stream.current_window+1) * winsize);
3291 XD3_ASSERT (stream.src != NULL);
3294 /* Limited i-buffer size affects source copies
3295 * when the sourcewin is decided early. */
3296 if (option_verbose > 1 &&
3297 stream.srcwin_decided_early &&
3298 stream.i_slots_used > stream.iopt_size)
3300 XPR(NT "warning: input position %"Q"u overflowed "
3301 "instruction buffer, needed %u (vs. %u), "
3302 "consider changing -I\n",
3303 stream.current_window * winsize,
3304 stream.i_slots_used, stream.iopt_size);
3310 shortbuf rrateavg, wrateavg, tm;
3312 shortbuf trdb, twdb;
3314 long millis = get_millisecs_since ();
3315 usize_t this_read = (usize_t)(stream.total_in -
3317 usize_t this_write = (usize_t)(stream.total_out -
3319 last_total_in = stream.total_in;
3320 last_total_out = stream.total_out;
3322 if (option_verbose > 1)
3324 XPR(NT "%"Q"u: in %s (%s): out %s (%s): "
3325 "total in %s: out %s: %s: srcpos %s\n",
3326 stream.current_window,
3327 main_format_bcnt (this_read, &rdb),
3328 main_format_rate (this_read, millis, &rrateavg),
3329 main_format_bcnt (this_write, &wdb),
3330 main_format_rate (this_write, millis, &wrateavg),
3331 main_format_bcnt (stream.total_in, &trdb),
3332 main_format_bcnt (stream.total_out, &twdb),
3333 main_format_millis (millis, &tm),
3334 main_format_bcnt (sfile->source_position, &srcpos));
3338 XPR(NT "%"Q"u: in %s: out %s: total in %s: "
3340 stream.current_window,
3341 main_format_bcnt (this_read, &rdb),
3342 main_format_bcnt (this_write, &wdb),
3343 main_format_bcnt (stream.total_in, &trdb),
3344 main_format_bcnt (stream.total_out, &twdb),
3345 main_format_millis (millis, &tm));
3353 /* input_func() error */
3354 XPR(NT XD3_LIB_ERRMSG (& stream, ret));
3355 if (! option_quiet && ret == XD3_INVALID_INPUT)
3357 XPR(NT "normally this indicates that the source file is incorrect\n");
3358 XPR(NT "please verify the source file with sha1sum or equivalent\n");
3360 return EXIT_FAILURE;
3363 while (nread == config.winsize);
3365 /* Close the inputs. (ifile must be open, sfile may be open) */
3366 main_file_close (ifile);
3369 main_file_close (sfile);
3373 if (cmd == CMD_MERGE &&
3374 (ret = main_merge_output (& stream, ofile)))
3376 return EXIT_FAILURE;
3379 if (cmd == CMD_MERGE_ARG)
3381 xd3_swap_whole_state (& stream.whole_target,
3382 & recode_stream->whole_target);
3384 #endif /* VCDIFF_TOOLS */
3386 /* If output file is not open yet because of delayed-open, it means
3387 * we never encountered a window in the delta, but it could have had
3388 * a VCDIFF header? TODO: solve this elsewhere. For now, it prints
3389 * "nothing to output" below, but the check doesn't happen in case
3390 * of option_no_output. */
3391 if (! option_no_output && ofile != NULL)
3393 if (!stdout_only && ! main_file_isopen (ofile))
3395 XPR(NT "nothing to output: %s\n", ifile->filename);
3396 return EXIT_FAILURE;
3399 /* Have to close the output before calling
3400 * main_external_compression_finish, or else it hangs. */
3401 if (main_file_close (ofile) != 0)
3403 return EXIT_FAILURE;
3407 #if EXTERNAL_COMPRESSION
3408 if ((ret = main_external_compression_finish ()))
3410 XPR(NT "external compression commands failed\n");
3411 return EXIT_FAILURE;
3415 if ((ret = xd3_close_stream (& stream)))
3417 XPR(NT XD3_LIB_ERRMSG (& stream, ret));
3418 return EXIT_FAILURE;
3422 if (option_verbose > 1 && cmd == CMD_ENCODE)
3424 XPR(NT "scanner configuration: %s\n", stream.smatcher.name);
3425 XPR(NT "target hash table size: %u\n", stream.small_hash.size);
3426 if (sfile != NULL && sfile->filename != NULL)
3428 XPR(NT "source hash table size: %u\n", stream.large_hash.size);
3432 if (option_verbose > 2 && cmd == CMD_ENCODE)
3434 XPR(NT "source copies: %"Q"u (%"Q"u bytes)\n",
3435 stream.n_scpy, stream.l_scpy);
3436 XPR(NT "target copies: %"Q"u (%"Q"u bytes)\n",
3437 stream.n_tcpy, stream.l_tcpy);
3438 XPR(NT "adds: %"Q"u (%"Q"u bytes)\n", stream.n_add, stream.l_add);
3439 XPR(NT "runs: %"Q"u (%"Q"u bytes)\n", stream.n_run, stream.l_run);
3443 xd3_free_stream (& stream);
3448 long end_time = get_millisecs_now ();
3449 xoff_t nwrite = ofile != NULL ? ofile->nwrite : 0;
3451 XPR(NT "finished in %s; input %"Q"u output %"Q"u bytes (%0.2f%%)\n",
3452 main_format_millis (end_time - start_time, &tm),
3453 ifile->nread, nwrite, 100.0 * nwrite / ifile->nread);
3456 return EXIT_SUCCESS;
3459 /* free memory before exit, reset single-use variables. */
3463 if (appheader_used != NULL &&
3464 appheader_used != option_appheader)
3466 main_free (appheader_used);
3467 appheader_used = NULL;
3470 main_buffree (main_bdata);
3476 if (recode_stream != NULL)
3478 xd3_free_stream (recode_stream);
3479 main_free (recode_stream);
3480 recode_stream = NULL;
3483 if (merge_stream != NULL)
3485 xd3_free_stream (merge_stream);
3486 main_free (merge_stream);
3487 merge_stream = NULL;
3490 XD3_ASSERT (main_mallocs == 0);
3494 setup_environment (int argc,
3502 char *p, *v = getenv("XDELTA");
3506 (*argv_free) = NULL;
3511 (*env_free) = (char*) main_malloc((usize_t) strlen(v) + 1);
3512 strcpy(*env_free, v);
3514 /* Space needed for extra args, at least # of spaces */
3516 for (p = *env_free; *p != 0; ) {
3522 (*argv_free) = (char**) main_malloc(sizeof(char*) * (n + 1));
3523 (*argv_out) = (*argv_free);
3524 (*argv_out)[0] = argv[0];
3525 (*argv_out)[n] = NULL;
3528 for (p = *env_free; *p != 0; ) {
3529 (*argv_out)[i++] = p;
3530 while (*p != ' ' && *p != 0) {
3538 for (i0 = 1; i0 < argc; i0++) {
3539 (*argv_out)[i++] = argv[i0];
3542 /* Counting spaces is an upper bound, argv stays NULL terminated. */
3545 (*argv_out)[i++] = NULL;
3549 #if PYTHON_MODULE || SWIG_MODULE || NOT_MAIN
3550 int xd3_main_cmdline (int argc, char **argv)
3552 int main (int argc, char **argv)
3555 static const char *flags =
3556 "0123456789cdefhnqvDFJNORVs:m:B:C:E:I:L:O:M:P:W:A::S::";
3561 main_merge_list merge_order;
3564 const char *my_optarg;
3565 const char *my_optstr;
3566 const char *sfilename;
3569 char **free_argv; /* malloc() in setup_environment() */
3570 char *free_value; /* malloc() in setup_environment() */
3574 GetStartupInfo(&winStartupInfo);
3575 setvbuf(stderr, NULL, _IONBF, 0); /* Do not buffer stderr */
3578 main_file_init (& ifile);
3579 main_file_init (& ofile);
3580 main_file_init (& sfile);
3581 main_merge_list_init (& merge_order);
3587 setup_environment(argc, argv, &env_argc, &env_argv,
3588 &free_argv, &free_value);
3594 program_name = env_argv[0];
3598 my_optstr = argv[my_optind];
3600 /* This doesn't use getopt() because it makes trouble for -P & python which
3601 * reenter main() and thus care about freeing all memory. I never had much
3602 * trust for getopt anyway, it's too opaque. This implements a fairly
3603 * standard non-long-option getopt with support for named operations (e.g.,
3604 * "xdelta3 [encode|decode|printhdr...] < in > out"). */
3607 if (*my_optstr == '-') { my_optstr += 1; }
3608 else if (cmd == CMD_NONE) { goto nonflag; }
3609 else { my_optstr = NULL; }
3615 if ((ret = *my_optstr++) == 0) { my_optind += 1; goto takearg; }
3617 /* Option handling: first check for one ':' following the option in
3618 * flags, then check for two. The syntax allows:
3620 * 1. -Afoo defines optarg="foo"
3621 * 2. -A foo defines optarg="foo"
3622 * 3. -A "" defines optarg="" (allows empty-string)
3623 * 4. -A [EOA or -moreargs] error (mandatory case)
3624 * 5. -A [EOA -moreargs] defines optarg=NULL (optional case)
3625 * 6. -A=foo defines optarg="foo"
3626 * 7. -A= defines optarg="" (mandatory case)
3627 * 8. -A= defines optarg=NULL (optional case)
3629 * See tests in test_command_line_arguments().
3631 s = strchr (flags, ret);
3632 if (s && s[1] && s[1] == ':')
3634 int option = s[2] && s[2] == ':';
3636 /* Case 1, set optarg to the remaining characters. */
3637 my_optarg = my_optstr;
3641 if (*my_optarg == 0)
3644 int have_arg = (my_optind < (argc - 1) &&
3645 *argv[my_optind+1] != '-');
3652 XPR(NT "-%c: requires an argument\n", ret);
3662 my_optarg = argv[++my_optind];
3666 else if (*my_optarg == '=')
3668 /* Remove the = in all cases. */
3671 if (option && *my_optarg == 0)
3681 /* case: if no '-' was found, maybe check for a command name. */
3683 if (strcmp (my_optstr, "decode") == 0) { cmd = CMD_DECODE; }
3684 else if (strcmp (my_optstr, "encode") == 0)
3689 XPR(NT "encoder support not compiled\n");
3690 return EXIT_FAILURE;
3693 else if (strcmp (my_optstr, "config") == 0) { cmd = CMD_CONFIG; }
3695 else if (strcmp (my_optstr, "test") == 0) { cmd = CMD_TEST; }
3698 else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; }
3699 else if (strcmp (my_optstr, "printhdrs") == 0)
3700 { cmd = CMD_PRINTHDRS; }
3701 else if (strcmp (my_optstr, "printdelta") == 0)
3702 { cmd = CMD_PRINTDELTA; }
3703 else if (strcmp (my_optstr, "recode") == 0) { cmd = CMD_RECODE; }
3704 else if (strcmp (my_optstr, "merge") == 0) { cmd = CMD_MERGE; }
3707 /* If no option was found and still no command, let the default
3708 * command be encode. The remaining args are treated as
3710 if (cmd == CMD_NONE)
3718 /* But if we find a command name, continue the getopt loop. */
3723 /* gzip-like options */
3724 case '0': case '1': case '2': case '3': case '4':
3725 case '5': case '6': case '7': case '8': case '9':
3726 option_level = ret - '0';
3728 case 'f': option_force = 1; break;
3730 #if EXTERNAL_COMPRESSION
3733 XPR(NT "warning: -F option ignored, "
3734 "external compression support was not compiled\n");
3737 case 'v': option_verbose += 1; option_quiet = 0; break;
3738 case 'q': option_quiet = 1; option_verbose = 0; break;
3739 case 'c': option_stdout = 1; break;
3741 if (cmd == CMD_NONE) { cmd = CMD_DECODE; }
3742 else { ret = main_help (); goto exit; }
3746 if (cmd == CMD_NONE) { cmd = CMD_ENCODE; }
3747 else { ret = main_help (); goto exit; }
3750 XPR(NT "encoder support not compiled\n");
3751 return EXIT_FAILURE;
3754 case 'n': option_use_checksum = 0; break;
3755 case 'N': option_no_compress = 1; break;
3756 case 'C': option_smatch_config = my_optarg; break;
3757 case 'J': option_no_output = 1; break;
3758 case 'S': if (my_optarg == NULL)
3760 option_use_secondary = 1;
3761 option_secondary = "none";
3765 option_use_secondary = 1;
3766 option_secondary = my_optarg;
3769 case 'A': if (my_optarg == NULL) { option_use_appheader = 0; }
3770 else { option_appheader = (uint8_t*) my_optarg; } break;
3773 if ((ret = main_atoux (my_optarg, & bsize,
3774 XD3_MINSRCWINSZ, XD3_MAXSRCWINSZ, 'B')))
3778 option_srcwinsz = bsize;
3782 if ((ret = main_atou (my_optarg, & option_iopt_size, 0,
3789 if ((ret = main_atou (my_optarg, & option_sprevsz, 0,
3796 if ((ret = main_atou (my_optarg, & option_winsize, XD3_ALLOCSIZE,
3797 XD3_HARDMAXWINSIZE, 'W')))
3803 #if EXTERNAL_COMPRESSION == 0
3804 if (option_verbose > 0)
3806 XPR(NT "warning: -D option ignored, "
3807 "external compression support was not compiled\n");
3810 option_decompress_inputs = 0;
3814 #if EXTERNAL_COMPRESSION == 0
3815 if (option_verbose > 0)
3817 XPR(NT "warning: -R option ignored, "
3818 "external compression support was not compiled\n");
3821 option_recompress_outputs = 0;
3825 if (sfilename != NULL)
3827 XPR(NT "specify only one source file\n");
3831 sfilename = my_optarg;
3834 if ((merge = (main_merge*)
3835 main_malloc (sizeof (main_merge))) == NULL)
3839 main_merge_list_push_back (& merge_order, merge);
3840 merge->filename = my_optarg;
3843 ret = main_version (); goto exit;
3845 ret = main_help (); goto exit;
3849 option_source_filename = sfilename;
3851 /* In case there were no arguments, set the default command. */
3852 if (cmd == CMD_NONE) { cmd = CMD_DEFAULT; }
3857 /* There may be up to two more arguments. */
3860 XPR(NT "too many filenames: %s ...\n", argv[2]);
3864 ifile.flags = RD_FIRST | RD_MAININPUT;
3865 sfile.flags = RD_FIRST;
3866 sfile.filename = option_source_filename;
3868 /* The infile takes the next argument, if there is one. But if not, infile
3869 * is set to stdin. */
3872 ifile.filename = argv[0];
3874 if ((ret = main_file_open (& ifile, ifile.filename, XO_READ)))
3881 XSTDIN_XF (& ifile);
3884 /* The ofile takes the following argument, if there is one. But if not, it
3885 * is left NULL until the application header is processed. It will be set
3886 * in main_open_output. */
3889 /* Check for conflicting arguments. */
3890 if (option_stdout && ! option_quiet)
3892 XPR(NT "warning: -c option overrides output filename: %s\n",
3896 if (! option_stdout) { ofile.filename = argv[1]; }
3900 if (cmd == CMD_MERGE &&
3901 (ret = main_merge_arguments (&merge_order)))
3905 #endif /* VCDIFF_TOOLS */
3911 case CMD_PRINTDELTA:
3918 ret = main_input (cmd, & ifile, & ofile, & sfile);
3924 ret = xd3_selftest ();
3929 ret = main_config ();
3945 #if EXTERNAL_COMPRESSION
3946 main_external_compression_cleanup ();
3949 main_file_cleanup (& ifile);
3950 main_file_cleanup (& ofile);
3951 main_file_cleanup (& sfile);
3953 while (! main_merge_list_empty (& merge_order))
3955 merge = main_merge_list_pop_front (& merge_order);
3959 main_free (free_argv);
3960 main_free (free_value);
3974 /* Note: update wiki when command-line features change */
3975 XPR(NTR "usage: xdelta3 [command/options] [input [output]]\n");
3976 XPR(NTR "make patch:\n");
3978 XPR(NTR " xdelta3.exe -e -s old_file new_file delta_file\n");
3980 XPR(NTR "apply patch:\n");
3982 XPR(NTR " xdelta3.exe -d -s old_file delta_file decoded_new_file\n");
3984 XPR(NTR "special command names:\n");
3985 XPR(NTR " config prints xdelta3 configuration\n");
3986 XPR(NTR " decode decompress the input\n");
3987 XPR(NTR " encode compress the input%s\n",
3988 XD3_ENCODER ? "" : " [Not compiled]");
3990 XPR(NTR " test run the builtin tests\n");
3993 XPR(NTR "special commands for VCDIFF inputs:\n");
3994 XPR(NTR " printdelta print information about the entire delta\n");
3995 XPR(NTR " printhdr print information about the first window\n");
3996 XPR(NTR " printhdrs print information about all windows\n");
3997 XPR(NTR " recode encode with new application/secondary settings\n");
3998 XPR(NTR " merge merge VCDIFF inputs (see below)\n");
4000 XPR(NTR "merge patches:\n");
4002 XPR(NTR " xdelta3 merge -m 1.vcdiff -m 2.vcdiff 3.vcdiff merged.vcdiff\n");
4004 XPR(NTR "standard options:\n");
4005 XPR(NTR " -0 .. -9 compression level\n");
4006 XPR(NTR " -c use stdout\n");
4007 XPR(NTR " -d decompress\n");
4008 XPR(NTR " -e compress%s\n",
4009 XD3_ENCODER ? "" : " [Not compiled]");
4010 XPR(NTR " -f force (overwrite, ignore trailing garbage)\n");
4011 #if EXTERNAL_COMPRESSION
4012 XPR(NTR " -F force the external-compression subprocess\n");
4014 XPR(NTR " -h show help\n");
4015 XPR(NTR " -q be quiet\n");
4016 XPR(NTR " -v be verbose (max 2)\n");
4017 XPR(NTR " -V show version\n");
4019 XPR(NTR "memory options:\n");
4020 XPR(NTR " -B bytes source window size\n");
4021 XPR(NTR " -W bytes input window size\n");
4022 XPR(NTR " -P size compression duplicates window\n");
4023 XPR(NTR " -I size instruction buffer size (0 = unlimited)\n");
4025 XPR(NTR "compression options:\n");
4026 XPR(NTR " -s source source file to copy from (if any)\n");
4027 XPR(NTR " -S [lzma|djw|fgk] enable/disable secondary compression\n");
4028 XPR(NTR " -N disable small string-matching compression\n");
4029 XPR(NTR " -D disable external decompression (encode/decode)\n");
4030 XPR(NTR " -R disable external recompression (decode)\n");
4031 XPR(NTR " -n disable checksum (encode/decode)\n");
4032 XPR(NTR " -C soft config (encode, undocumented)\n");
4033 XPR(NTR " -A [apphead] disable/provide application header (encode)\n");
4034 XPR(NTR " -J disable output (check/compute only)\n");
4035 XPR(NTR " -m arguments for \"merge\"\n");
4037 XPR(NTR "the XDELTA environment variable may contain extra args:\n");
4038 XPR(NTR " XDELTA=\"-s source-x.y.tar.gz\" \\\n");
4039 XPR(NTR " tar --use-compress-program=xdelta3 \\\n");
4040 XPR(NTR " -cf target-x.z.tar.gz.vcdiff target-x.y\n");
4041 return EXIT_FAILURE;