From 6f6a97958e8723b4b4a3ccf352252b36cf7a0d85 Mon Sep 17 00:00:00 2001 From: Luis Reis Date: Tue, 9 Sep 2014 17:30:15 +0100 Subject: [PATCH] zlib: support concatenated gzip files Reviewed-By: Fedor Indutny PR-URL: https://github.com/joyent/node/pull/6442 --- lib/zlib.js | 9 ++- src/node_zlib.cc | 31 ++++++-- .../test-zlib-from-multiple-gzip-with-garbage.js | 83 +++++++++++++++++++ test/simple/test-zlib-from-multiple-gzip.js | 74 +++++++++++++++++ test/simple/test-zlib-from-multiple-huge-gzip.js | 93 ++++++++++++++++++++++ 5 files changed, 282 insertions(+), 8 deletions(-) create mode 100644 test/simple/test-zlib-from-multiple-gzip-with-garbage.js create mode 100644 test/simple/test-zlib-from-multiple-gzip.js create mode 100644 test/simple/test-zlib-from-multiple-huge-gzip.js diff --git a/lib/zlib.js b/lib/zlib.js index a44e69f..2acde58 100644 --- a/lib/zlib.js +++ b/lib/zlib.js @@ -580,7 +580,7 @@ Zlib.prototype._processChunk = function(chunk, flushFlag, cb) { self._buffer = new Buffer(self._chunkSize); } - if (availOutAfter === 0) { + if (availOutAfter === 0 || availInAfter > 0) { // Not actually done. Need to reprocess. // Also, update the availInBefore to the availInAfter value, // so that if we have to hit it a third (fourth, etc.) time, @@ -588,6 +588,13 @@ Zlib.prototype._processChunk = function(chunk, flushFlag, cb) { inOff += (availInBefore - availInAfter); availInBefore = availInAfter; + if (availOutAfter !== 0) { + // There is still some data available for reading. + // This is usually a concatenated stream, so, reset and restart. + self.reset(); + self._offset = 0; + } + if (!async) return true; diff --git a/src/node_zlib.cc b/src/node_zlib.cc index 4f0c938..95907dd 100644 --- a/src/node_zlib.cc +++ b/src/node_zlib.cc @@ -63,6 +63,11 @@ enum node_zlib_mode { UNZIP }; +enum node_zlib_error { + NO_ERROR, + FAILED, + WRITE_PENDING +}; void InitZlib(v8::Handle target); @@ -207,7 +212,7 @@ class ZCtx : public AsyncWrap { if (!async) { // sync version Process(work_req); - if (CheckError(ctx)) + if (CheckError(ctx) == NO_ERROR) AfterSync(ctx, args); return; } @@ -292,7 +297,7 @@ class ZCtx : public AsyncWrap { } - static bool CheckError(ZCtx* ctx) { + static node_zlib_error CheckError(ZCtx* ctx) { // Acceptable error states depend on the type of zlib stream. switch (ctx->err_) { case Z_OK: @@ -305,14 +310,18 @@ class ZCtx : public AsyncWrap { ZCtx::Error(ctx, "Missing dictionary"); else ZCtx::Error(ctx, "Bad dictionary"); - return false; + return FAILED; default: // something else. - ZCtx::Error(ctx, "Zlib error"); - return false; + if (ctx->strm_.total_out == 0) { + ZCtx::Error(ctx, "Zlib error"); + return FAILED; + } else { + return WRITE_PENDING; + } } - return true; + return NO_ERROR; } @@ -326,7 +335,8 @@ class ZCtx : public AsyncWrap { HandleScope handle_scope(env->isolate()); Context::Scope context_scope(env->context()); - if (!CheckError(ctx)) + node_zlib_error error = CheckError(ctx); + if (error == FAILED) return; Local avail_out = Integer::New(env->isolate(), @@ -340,6 +350,11 @@ class ZCtx : public AsyncWrap { Local args[2] = { avail_in, avail_out }; ctx->MakeCallback(env->callback_string(), ARRAY_SIZE(args), args); + if (error == WRITE_PENDING) { + ZCtx::Error(ctx, "Zlib error"); + return; + } + ctx->Unref(); if (ctx->pending_close_) ctx->Close(); @@ -557,10 +572,12 @@ class ZCtx : public AsyncWrap { switch (ctx->mode_) { case DEFLATE: case DEFLATERAW: + case GZIP: ctx->err_ = deflateReset(&ctx->strm_); break; case INFLATE: case INFLATERAW: + case GUNZIP: ctx->err_ = inflateReset(&ctx->strm_); break; default: diff --git a/test/simple/test-zlib-from-multiple-gzip-with-garbage.js b/test/simple/test-zlib-from-multiple-gzip-with-garbage.js new file mode 100644 index 0000000..f6a0185 --- /dev/null +++ b/test/simple/test-zlib-from-multiple-gzip-with-garbage.js @@ -0,0 +1,83 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +// test unzipping a file that was created by concatenating multiple gzip +// streams. + +var common = require('../common'); +var assert = require('assert'); +var zlib = require('zlib'); + +var util = require('util'); + +var gzipBuffer = new Buffer(128); +var gzipOffset = 0; + +var stream1 = '123\n'; +var stream2 = '456\n'; +var stream3 = '789\n'; + +function gzipAppend(data) { + data.copy(gzipBuffer, gzipOffset); + gzipOffset += data.length; +} + +function writeGzipStream(text, cb) { + var gzip = zlib.createGzip(); + gzip.on('data', gzipAppend); + gzip.write(text, function() { + gzip.flush(function() { + gzip.end(function() { + cb(); + }); + }); + }); +} + +function writeGarbageStream(text, cb) { + gzipAppend(new Buffer(text)); + cb(); +} + +writeGzipStream(stream1, function() { + writeGzipStream(stream2, function() { + writeGarbageStream(stream3, function() { + var gunzip = zlib.createGunzip(); + var gunzippedData = new Buffer(2 * 1024); + var gunzippedOffset = 0; + gunzip.on('data', function (data) { + data.copy(gunzippedData, gunzippedOffset); + gunzippedOffset += data.length; + }); + gunzip.on('error', function() { + assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset), + stream1 + stream2); + }); + gunzip.on('end', function() { + assert.fail('end event not expected'); + }); + + gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() { + gunzip.end(); + }); + }); + }); +}); diff --git a/test/simple/test-zlib-from-multiple-gzip.js b/test/simple/test-zlib-from-multiple-gzip.js new file mode 100644 index 0000000..6f4127a --- /dev/null +++ b/test/simple/test-zlib-from-multiple-gzip.js @@ -0,0 +1,74 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +// test unzipping a file that was created by concatenating multiple gzip +// streams. + +var common = require('../common'); +var assert = require('assert'); +var zlib = require('zlib'); + +var util = require('util'); + +var gzipBuffer = new Buffer(128); +var gzipOffset = 0; + +var stream1 = '123\n'; +var stream2 = '456\n'; +var stream3 = '789\n'; + +function gzipAppend(data) { + data.copy(gzipBuffer, gzipOffset); + gzipOffset += data.length; +} + +function writeGzipStream(text, cb) { + var gzip = zlib.createGzip(); + gzip.on('data', gzipAppend); + gzip.write(text, function() { + gzip.flush(function() { + gzip.end(function() { + cb(); + }); + }); + }); +} + +writeGzipStream(stream1, function() { + writeGzipStream(stream2, function() { + writeGzipStream(stream3, function() { + var gunzip = zlib.createGunzip(); + var gunzippedData = new Buffer(2 * 1024); + var gunzippedOffset = 0; + gunzip.on('data', function (data) { + data.copy(gunzippedData, gunzippedOffset); + gunzippedOffset += data.length; + }); + gunzip.on('end', function() { + assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset), stream1 + stream2 + stream3); + }); + + gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() { + gunzip.end(); + }); + }); + }); +}); diff --git a/test/simple/test-zlib-from-multiple-huge-gzip.js b/test/simple/test-zlib-from-multiple-huge-gzip.js new file mode 100644 index 0000000..5533aaf --- /dev/null +++ b/test/simple/test-zlib-from-multiple-huge-gzip.js @@ -0,0 +1,93 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +// test unzipping a file that was created by concatenating multiple gzip +// streams. + +var common = require('../common'); +var assert = require('assert'); +var zlib = require('zlib'); + +var util = require('util'); + +var HUGE = 64 * 1024; + +var originalBuffer = new Buffer(3 * HUGE); +var originalOffset = 0; + +var gzipBuffer = new Buffer(3 * HUGE); +var gzipOffset = 0; + +function getRandomLetter() { + return (Math.random() * (122 - 97)) + 97; +} + +function generateHugeStream() { + var buffer = new Buffer(HUGE); + for (var i = 0; i < HUGE; i++) + buffer.writeUInt8(getRandomLetter(), i); + + buffer.copy(originalBuffer, originalOffset); + originalOffset += HUGE; + + return buffer; +} + +function gzipAppend(data) { + data.copy(gzipBuffer, gzipOffset); + gzipOffset += data.length; +} + +function writeGzipStream(text, cb) { + var gzip = zlib.createGzip(); + gzip.on('data', gzipAppend); + gzip.write(text, function() { + gzip.flush(function() { + gzip.end(function() { + cb(); + }); + }); + }); +} + +writeGzipStream(generateHugeStream(), function() { + writeGzipStream(generateHugeStream(), function() { + writeGzipStream(generateHugeStream(), function() { + var gunzip = zlib.createGunzip(); + var gunzippedData = new Buffer(3 * HUGE); + var gunzippedOffset = 0; + gunzip.on('data', function (data) { + data.copy(gunzippedData, gunzippedOffset); + gunzippedOffset += data.length; + }); + gunzip.on('end', function() { + var gunzippedStr = gunzippedData.toString('utf8', 0, gunzippedOffset); + var originalStr = originalBuffer.toString('utf8', 0, 3 * HUGE); + + assert.equal(gunzippedStr, originalStr); + }); + + gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() { + gunzip.end(); + }); + }); + }); +}); -- 2.7.4