From d160e5d7ea6568f4595e08a98a791dae335117d7 Mon Sep 17 00:00:00 2001 From: DongHun Kwak Date: Fri, 18 Nov 2016 16:45:32 +0900 Subject: [PATCH] Imported Upstream version 1.1.7 Change-Id: I98f0deba129cff608fcb791b3bbbf7ee968f38c0 Signed-off-by: DongHun Kwak --- AUTHORS | 3 + COPYING | 2 +- ChangeLog | 4 ++ README | 4 +- pbzip2.1 | 2 +- pbzip2.cpp | 210 ++++++++++++++++++++++++++++++++++++++---------------------- pbzip2.spec | 5 +- 7 files changed, 150 insertions(+), 80 deletions(-) diff --git a/AUTHORS b/AUTHORS index 0f9c221..553dff7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -209,6 +209,9 @@ Yavor Nikolov - code to support throttling comp - Prevent deletion of input files on error (bug #874543) - Add more detailed kernel error messages - inspired by Gordon's patch (bug #874605) - Error-handling improvements - mainly for multi-archive scenarios (bug #883782) + - Fixed occasional failure on decompress with --ignore-trailing-garbage=1 + with multiple bad blocks in the archive (bug #886625) + - Fixed refusal to write to stdout on -dc from stdin (bug #886628) David James - provided patch to fix deadlock due to unsynchronized broadcast (bug #876686) Gordon - provided patch for improving I/O error messages (bug #874605) diff --git a/COPYING b/COPYING index e02b619..b841b73 100644 --- a/COPYING +++ b/COPYING @@ -37,4 +37,4 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Jeff Gilchrist, Ottawa, Canada. pbzip2@compression.ca -pbzip2 version 1.1.6 of Oct 30, 2011 +pbzip2 version 1.1.7 of Dec 11, 2011 diff --git a/ChangeLog b/ChangeLog index 58ea3d1..9a44dfa 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +Changes in 1.1.7 (Dec 11, 2011) +- Fixed refusal to write to stdout on -dc from stdin (bug #886628) +- Fixed occasional failure on decompress with --ignore-trailing-garbage=1 + with multiple bad blocks in the archive (bug #886625) Changes in 1.1.6 (Oct 30, 2011) - Fixed bug - deadlock due to unsynchronized broadcasts (bug #876686) - Prevent deletion of input files on error (bug #874543) diff --git a/README b/README index 9d2c3da..2917ed2 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ -Oct 30, 2011 +Dec 11, 2011 -Parallel BZIP2 v1.1.6 - by: Jeff Gilchrist +Parallel BZIP2 v1.1.7 - by: Jeff Gilchrist Available at: http://compression.ca/ This is the README for pbzip2, a parallel implementation of the diff --git a/pbzip2.1 b/pbzip2.1 index df746f7..7a610d9 100644 --- a/pbzip2.1 +++ b/pbzip2.1 @@ -1,6 +1,6 @@ .TH pbzip2 1 .SH NAME -pbzip2 \- parallel bzip2 file compressor, v1.1.6 +pbzip2 \- parallel bzip2 file compressor, v1.1.7 .SH SYNOPSIS .B pbzip2 .RB [ " \-123456789 " ] diff --git a/pbzip2.cpp b/pbzip2.cpp index 52edd27..7671fdf 100644 --- a/pbzip2.cpp +++ b/pbzip2.cpp @@ -201,6 +201,9 @@ * Gordon's patch (bug #874605) * - Error-handling improvements - mainly for multi-archive * scenarios (bug #883782) + * - Fixed occasional failure on decompress with --ignore-trailing-garbage=1 + * with multiple bad blocks in the archive (bug #886625) + * - Fixed refusal to write to stdout on -dc from stdin (bug #886628) * David James - provided patch to fix deadlock due to unsynchronized broadcast (bug #876686) * Gordon - provided patch for improving I/O error messages (bug #874605) * @@ -260,7 +263,7 @@ * * Jeff Gilchrist, Ottawa, Canada. * pbzip2@compression.ca - * pbzip2 version 1.1.6 of Oct 30, 2011 + * pbzip2 version 1.1.7 of Dec 11, 2011 * */ #include "pbzip2.h" @@ -316,6 +319,7 @@ static size_t NumBufferedTailBlocks = 0; static size_t NumBufferedBlocksMax = 0; static int NextBlockToWrite; static int LastGoodBlock; // set only to terminate write prematurely (ignoring garbage) +static int MinErrorBlock; // lowest so far block number which has errors (on decompress; could be trailing garbage) static size_t OutBufferPosToWrite; // = 0; // position in output buffer static int Verbosity = 0; static int QuietMode = 1; @@ -348,7 +352,7 @@ inline void syncSetProducerDone(int newValue); inline int syncGetTerminateFlag(); inline void syncSetTerminateFlag(int newValue); inline void syncSetFinishedFlag(int newValue); -inline void syncSetLastGoodBlock(int newValue); +inline void syncSetLastGoodBlock(int newValue, int errBlock); inline int syncGetLastGoodBlock(); void cleanupUnfinishedWork(); void cleanupAndQuit(int exitCode); @@ -384,7 +388,7 @@ ssize_t bufread(int hf, char *buf, size_t bsize); int detectCPUs(void); inline bool isIgnoredTrailingGarbage(); -int waitForPreviousBlock(int blockNum); +int waitForPreviousBlock(int blockNumToWait, int errBlockNumber); inline int getLastGoodBlockBeforeErr(int errBlockNumber, int outSequenceNumber); inline int issueDecompressError(int bzret, const outBuff * fileData, int outSequenceNumber, const bz_stream & strm, const char * errmsg, @@ -394,6 +398,8 @@ int decompressErrCheckSingle(int bzret, const outBuff * fileData, bool isTrailingGarbageErr); int decompressErrCheck(int bzret, const outBuff * fileData, int outSequenceNumber, const bz_stream & strm); +inline bool hasTrailingGarbage(int bzret, const outBuff * fileData, + const bz_stream & strm); int producerDecompressCheckInterrupt(int hInfile, outBuff *& fileData, int lastBlock); using pbzip2::ErrorContext; @@ -791,18 +797,41 @@ inline void syncSetFinishedFlag(int newValue) safe_mutex_unlock(&TerminateFlagMutex); } -inline void syncSetLastGoodBlock(int newValue) +/** + * Set last block which is maybe good (not guaranteed) and MinErrorBlock + * (lowest block with errors encountered so far). + * Only moving downwards has effect (attempts to raise the block numbers are ignored). + * -1 means infinity. + * + * + * @param newValue last block which is maybe good. -1 means +infinity. + * @param errBlock block number which has errors + */ +inline void syncSetLastGoodBlock(int newValue, int errBlock) { + bool changed = false; + safe_mutex_lock(OutMutex); #ifdef PBZIP_DEBUG unsigned long long thid = (unsigned long long) pthread_self(); - fprintf(stderr, "(%"PRIu64") syncSetLastGoodBlock: %d -> %d\n", thid, LastGoodBlock, newValue ); + fprintf(stderr, "(%"PRIu64") syncSetLastGoodBlock: %d -> %d; MinErrorBlock: %d -> %d\n", + thid, LastGoodBlock, newValue, MinErrorBlock, errBlock); #endif if ( (LastGoodBlock == -1) || (newValue < LastGoodBlock) ) { LastGoodBlock = newValue; + changed = true; + } + + if ( (MinErrorBlock == -1) || (errBlock < MinErrorBlock) ) + { + MinErrorBlock = errBlock; + changed = true; + } + if ( changed ) + { safe_cond_signal(&ErrStateChangeCond); safe_cond_signal(&OutBufferHeadNotEmpty); @@ -834,6 +863,16 @@ inline int syncGetLastGoodBlock() return ret; } +inline int syncGetMinErrorBlock() +{ + int ret; + safe_mutex_lock(OutMutex); + ret = MinErrorBlock; + safe_mutex_unlock(OutMutex); + + return ret; +} + inline bool isIgnoredTrailingGarbage() { return (IgnoreTrailingGarbageFlag != 0); @@ -874,16 +913,20 @@ int handle_error(ExitFlag exitFlag, int exitCode, const char *fmt, ...) /** * * @return -1 - terminate flag set (error) - * 0 - prev block is OK + * 0 - prev block is OK (i.e. we're on the first error here) * 2 - lower block number already in error state */ -int waitForPreviousBlock(int blockNum) +int waitForPreviousBlock(int blockNumToWait, int errBlockNumber) { #ifdef PBZIP_DEBUG unsigned long long thid = (unsigned long long) pthread_self(); - fprintf(stderr, "(%"PRIu64") waitForPreviousBlock before check: LastGoodBlock=%d; blockNum=%d; NextBlockToWrite=%d\n", + safe_mutex_lock(OutMutex); + fprintf( stderr, "(%"PRIu64") waitForPreviousBlock enter: LastGoodBlock=%d" + "; blockNumToWait=%d; NextBlockToWrite=%d; MinErrorBlock=%d; errBlockNumber=%d\n", thid, - LastGoodBlock, blockNum, NextBlockToWrite ); + LastGoodBlock, blockNumToWait, NextBlockToWrite, + MinErrorBlock, errBlockNumber ); + safe_mutex_unlock(OutMutex); #endif for (;;) @@ -891,8 +934,8 @@ int waitForPreviousBlock(int blockNum) if (syncGetTerminateFlag() != 0) { #ifdef PBZIP_DEBUG - fprintf(stderr, "(%"PRIu64") waitForPreviousBlock terminated [%d]: blockNum=%d\n", - thid, -1, blockNum ); + fprintf(stderr, "(%"PRIu64") waitForPreviousBlock terminated [%d]: blockNumToWait=%d\n", + thid, -1, blockNumToWait ); #endif return -1; } @@ -900,33 +943,34 @@ int waitForPreviousBlock(int blockNum) safe_mutex_lock(OutMutex); #ifdef PBZIP_DEBUG - fprintf(stderr, "(%"PRIu64") waitForPreviousBlock before check: LastGoodBlock=%d; blockNum=%d; NextBlockToWrite=%d\n", - thid, LastGoodBlock, blockNum, NextBlockToWrite ); + fprintf( stderr, "(%"PRIu64") waitForPreviousBlock before check: LastGoodBlock=%d; blockNumToWait=%d; NextBlockToWrite=%d; MinErrorBlock=%d\n", + thid, LastGoodBlock, blockNumToWait, NextBlockToWrite, MinErrorBlock ); #endif - if (blockNum <= NextBlockToWrite) + // This check should (min error block) be before next one (next block to write) + if ( (MinErrorBlock != -1) && (MinErrorBlock < errBlockNumber) ) { #ifdef PBZIP_DEBUG - fprintf(stderr, "(%"PRIu64") waitForPreviousBlock exit [%d]: LastGoodBlock=%d; blockNum=%d; NextBlockToWrite=%d\n", - thid, 0, LastGoodBlock, blockNum, NextBlockToWrite ); + fprintf( stderr, "(%"PRIu64") waitForPreviousBlock exit [%d]: LastGoodBlock=%d; blockNumToWait=%d; NextBlockToWrite=%d; MinErrorBlock=%d\n", + thid, 2, LastGoodBlock, blockNumToWait, NextBlockToWrite, MinErrorBlock ); #endif safe_mutex_unlock(OutMutex); - return 0; + return 2; } - - if ( (LastGoodBlock != -1) && (LastGoodBlock < blockNum) ) + + if (errBlockNumber <= NextBlockToWrite) { #ifdef PBZIP_DEBUG - fprintf(stderr, "(%"PRIu64") waitForPreviousBlock exit [%d]: LastGoodBlock=%d; blockNum=%d; NextBlockToWrite=%d\n", - thid, 2, LastGoodBlock, blockNum, NextBlockToWrite ); + fprintf( stderr, "(%"PRIu64") waitForPreviousBlock exit [%d]: LastGoodBlock=%d; blockNumToWait=%d; NextBlockToWrite=%d; MinErrorBlock=%d\n", + thid, 0, LastGoodBlock, blockNumToWait, NextBlockToWrite, MinErrorBlock ); #endif safe_mutex_unlock(OutMutex); - return 2; + return 0; } #ifdef PBZIP_DEBUG - fprintf(stderr, "(%"PRIu64") waitForPreviousBlock to sleep: LastGoodBlock=%d; blockNum=%d; NextBlockToWrite=%d\n", - thid, LastGoodBlock, blockNum, NextBlockToWrite ); + fprintf( stderr, "(%"PRIu64") waitForPreviousBlock to sleep: LastGoodBlock=%d; blockNumToWait=%d; NextBlockToWrite=%d; MinErrorBlock=%d\n", + thid, LastGoodBlock, blockNumToWait, NextBlockToWrite, MinErrorBlock ); #endif safe_cond_timed_wait(&ErrStateChangeCond, OutMutex, 1, "waitForPreviousBlock"); @@ -937,9 +981,10 @@ int waitForPreviousBlock(int blockNum) /** * - * @param errBlockNumber - * @param outSequenceNumber - * @return Last input block not after the given which resulted in good out blocks. + * @param errBlockNumber block from input file + * @param outSequenceNumber sequence in the output tail for the given block + * @return Last input block not after the given which possibly (not guaranteed) + * resulted in good out blocks. * -1 if such don't exist. */ inline int getLastGoodBlockBeforeErr(int errBlockNumber, int outSequenceNumber) @@ -971,6 +1016,14 @@ inline int issueDecompressError(int bzret, const outBuff * fileData, int outSequenceNumber, const bz_stream & strm, const char * errmsg, int exitCode) { + #ifdef PBZIP_DEBUG + unsigned long long thid = (unsigned long long) pthread_self(); + fprintf(stderr, "(%"PRIu64") enter issueDecompressError: msg=%s; ret=%d; block=%d; seq=%d; isLastInSeq=%d; avail_in=%d\n", + thid, + errmsg, bzret, fileData->blockNumber, + outSequenceNumber, (int)fileData->isLastInSequence, strm.avail_in); + #endif + handle_error(EF_EXIT, exitCode, "pbzip2: %s: ret=%d; block=%d; seq=%d; isLastInSeq=%d; avail_in=%d\n", errmsg, bzret, fileData->blockNumber, @@ -983,7 +1036,7 @@ inline int issueDecompressError(int bzret, const outBuff * fileData, * * * @param bzret - * @param fileData + * @param fileData block from input file * @param outSequenceNumber * @param strm * @param errmsg @@ -1000,9 +1053,11 @@ int decompressErrCheckSingle(int bzret, const outBuff * fileData, int lastGoodBlock = getLastGoodBlockBeforeErr(fileData->blockNumber, outSequenceNumber); #ifdef PBZIP_DEBUG - fprintf(stderr, "enter decompressErrCheckSingle: msg=%s; ret=%d; block=%d; seq=%d; isLastInSeq=%d; avail_in=%d; lastGoodBlock=%d\n", - errmsg, bzret, fileData->blockNumber, - outSequenceNumber, (int)fileData->isLastInSequence, strm.avail_in, lastGoodBlock); + unsigned long long thid = (unsigned long long) pthread_self(); + fprintf(stderr, "(%"PRIu64") enter decompressErrCheckSingle: msg=%s; ret=%d; block=%d; seq=%d; isLastInSeq=%d; avail_in=%d; lastGoodBlock=%d\n", + thid, + errmsg, bzret, fileData->blockNumber, + outSequenceNumber, (int)fileData->isLastInSequence, strm.avail_in, lastGoodBlock); #endif if ( (lastGoodBlock == -1) || !isIgnoredTrailingGarbage() ) @@ -1013,9 +1068,9 @@ int decompressErrCheckSingle(int bzret, const outBuff * fileData, else { // Cut off larger block numbers - syncSetLastGoodBlock(lastGoodBlock); + syncSetLastGoodBlock(lastGoodBlock, fileData->blockNumber); // wait until the state of previous block is known - int prevState = waitForPreviousBlock(lastGoodBlock); + int prevState = waitForPreviousBlock(lastGoodBlock, fileData->blockNumber); if (prevState == 0) { @@ -1048,6 +1103,21 @@ int decompressErrCheckSingle(int bzret, const outBuff * fileData, } /** + * Check if trailing garbage has been identified during the last decompression + * operation. + * + * @param bzret last bzip2 return code + * @param fileData should be initialized before calling this + * @param strm bzip2 library bz_stream + * @return true if trailing garbage has been detected. false otherwise + */ +inline bool hasTrailingGarbage(int bzret, const outBuff * fileData, const bz_stream & strm) +{ + return (bzret == BZ_STREAM_END) && + ( (strm.avail_in != 0) || !fileData->isLastInSequence ); +} + +/** * * @param bzret * @param fileData @@ -1061,8 +1131,7 @@ int decompressErrCheckSingle(int bzret, const outBuff * fileData, int decompressErrCheck(int bzret, const outBuff * fileData, int outSequenceNumber, const bz_stream & strm) { - if ( (bzret == BZ_STREAM_END) && - ((strm.avail_in != 0) || !fileData->isLastInSequence) ) + if ( hasTrailingGarbage( bzret, fileData, strm ) ) { // Potential trailing garbage return decompressErrCheckSingle(bzret, fileData, outSequenceNumber, strm, @@ -1499,12 +1568,14 @@ int consumerDecompressCheckInterrupt(const outBuff * lastElement) isInterrupted = true; #ifdef PBZIP_DEBUG - fprintf (stderr, "(%"PRIu64") producer_decompress: interrupt1 - TerminateFlag set.\n", thid); + fprintf (stderr, "(%"PRIu64") consumer_decompress: interrupt1 - TerminateFlag set.\n", thid); #endif } - if ( (syncGetLastGoodBlock() != -1) && - ( (lastElement == NULL) || (lastElement->blockNumber > syncGetLastGoodBlock()) - || lastElement->isLastInSequence ) ) + int minErrBlock = syncGetMinErrorBlock(); + if ( (minErrBlock != -1) && + ( (lastElement == NULL) + || (lastElement->blockNumber >= minErrBlock) + || lastElement->isLastInSequence ) ) { isInterrupted = true; @@ -1678,11 +1749,11 @@ void *consumer_decompress(void *q) outBuff * addret = NULL; unsigned int len = outSize - strm.avail_out; bool isLast = (bzret == BZ_STREAM_END); - - if ( isLast && ( (strm.avail_in != 0) || !fileData->isLastInSequence ) ) + + if ( hasTrailingGarbage( bzret, fileData, strm ) ) { - // trailng garbage detected - syncSetLastGoodBlock(fileData->blockNumber); + // trailing garbage detected + syncSetLastGoodBlock(fileData->blockNumber, fileData->blockNumber); } if (outSequenceNumber>0) @@ -3407,8 +3478,8 @@ int detectCPUs() */ void banner() { - fprintf(stderr, "Parallel BZIP2 v1.1.6 - by: Jeff Gilchrist [http://compression.ca]\n"); - fprintf(stderr, "[Oct. 30, 2011] (uses libbzip2 by Julian Seward)\n"); + fprintf(stderr, "Parallel BZIP2 v1.1.7 - by: Jeff Gilchrist [http://compression.ca]\n"); + fprintf(stderr, "[Dec. 11, 2011] (uses libbzip2 by Julian Seward)\n"); fprintf(stderr, "Major contributions: Yavor Nikolov \n"); return; @@ -3871,23 +3942,7 @@ int main(int argc, char* argv[]) FileList[FileListCount] = stdinFile; FileListCount++; } - else if (OutputStdOut == 1) - { - #ifndef WIN32 - if (isatty(fileno(stdout))) - #else - if (_isatty(_fileno(stdout))) - #endif - { - fprintf(stderr,"pbzip2: *ERROR: Won't write compressed data to terminal. Aborting!\n"); - fprintf(stderr,"pbzip2: For help type: %s -h\n", argv[0]); - return 1; - } - // expecting data from stdin - FileList[FileListCount] = stdinFile; - FileListCount++; - } - else if ((decompress == 1) && (argc == 2)) + else if (decompress == 1) { #ifndef WIN32 if (isatty(fileno(stdin))) @@ -3895,9 +3950,9 @@ int main(int argc, char* argv[]) if (_isatty(_fileno(stdin))) #endif { - fprintf(stderr,"pbzip2: *ERROR: Won't read compressed data from terminal. Aborting!\n"); - fprintf(stderr,"pbzip2: For help type: %s -h\n", argv[0]); - return 1; + fprintf(stderr,"pbzip2: *ERROR: Won't read compressed data from terminal. Aborting!\n"); + fprintf(stderr,"pbzip2: For help type: %s -h\n", argv[0]); + return 1; } // expecting data from stdin via TAR OutputStdOut = 1; @@ -3906,27 +3961,31 @@ int main(int argc, char* argv[]) FileListCount++; } else - { - // probably trying to input data from stdin - if (QuietMode != 1) - fprintf(stderr,"pbzip2: Assuming input data coming from stdin...\n\n"); + { + if (OutputStdOut == 0) + { + // probably trying to input data from stdin + if (QuietMode != 1) + fprintf(stderr,"pbzip2: Assuming input data coming from stdin...\n\n"); + + OutputStdOut = 1; + keep = 1; + } - OutputStdOut = 1; - keep = 1; #ifndef WIN32 if (isatty(fileno(stdout))) #else if (_isatty(_fileno(stdout))) #endif { - fprintf(stderr,"pbzip2: *ERROR: Won't write compressed data to terminal. Aborting!\n"); - fprintf(stderr,"pbzip2: For help type: %s -h\n", argv[0]); - return 1; + fprintf(stderr,"pbzip2: *ERROR: Won't write compressed data to terminal. Aborting!\n"); + fprintf(stderr,"pbzip2: For help type: %s -h\n", argv[0]); + return 1; } // expecting data from stdin FileList[FileListCount] = stdinFile; FileListCount++; - } + } } if (QuietMode != 1) @@ -4435,6 +4494,7 @@ int main(int argc, char* argv[]) } LastGoodBlock = -1; + MinErrorBlock = -1; // create output buffer outputBufferInit(NumBufferedBlocksMax); diff --git a/pbzip2.spec b/pbzip2.spec index 999b35d..f4dbb49 100644 --- a/pbzip2.spec +++ b/pbzip2.spec @@ -1,5 +1,5 @@ Name: pbzip2 -Version: 1.1.6 +Version: 1.1.7 Release: 1%{?dist} Summary: Parallel implementation of bzip2 URL: http://www.compression.ca/pbzip2/ @@ -52,6 +52,9 @@ rm -rf %{buildroot} %changelog +* Sun Dec 11 2011 Jeff Gilchrist - 1.1.7-1 +- Release 1.1.7 + * Sun Oct 30 2011 Jeff Gilchrist - 1.1.6-1 - Release 1.1.6 -- 2.7.4