zlib: reduce memory consumption, release early

In zlibBuffer(), don't wait for the garbage collector to reclaim the zlib memory
but release it manually. Reduces memory consumption by a factor of 10 or more
with some workloads.

Test case:

  function f() {
    require('zlib').deflate('xxx', g);
  }
  function g() {
    setTimeout(f, 5);
  }
  f();

Observe RSS memory usage with and without this commit. After 10,000 iterations,
RSS stabilizes at ~35 MB with this commit. Without, RSS is over 300 MB and keeps
growing.

Cause: whenever the JS object heap hits the high-water mark, the V8 GC sweeps
it clean, then tries to grow it in order to avoid more sweeps in the near
future. Rule of thumb: the bigger the JS heap, the lazier the GC can be.

A side effect of a bigger heap is that objects now live longer. This is harmless
in general but it affects zlib context objects because those are tied to large
buffers that live outside the JS heap, on the order of 16K per context object.

Ergo, don't wait for the GC to reclaim the memory - it may take a long time.

Fixes #4172.

diff --git a/lib/zlib.js b/lib/zlib.js
index 7837f3f..b0826a8 100644 (file)
--- a/lib/zlib.js
+++ b/lib/zlib.js
@@ -150,7 +150,10 @@ function zlibBuffer(engine, buffer, callback) {
   }
 
   function onEnd() {
-    callback(null, Buffer.concat(buffers, nread));
+    var buf = Buffer.concat(buffers, nread);
+    buffers = [];
+    callback(null, buf);
+    engine._clear();
   }
 
   engine.on('error', onError);
@@ -353,6 +356,10 @@ Zlib.prototype.end = function end(chunk, cb) {
   return ret;
 };
 
+Zlib.prototype._clear = function() {
+  return this._binding.clear();
+};
+
 Zlib.prototype._process = function() {
   if (this._hadError) return;
 

diff --git a/src/node_zlib.cc b/src/node_zlib.cc
index deacb71..406c042 100644 (file)
--- a/src/node_zlib.cc
+++ b/src/node_zlib.cc
@@ -39,7 +39,8 @@ static Persistent<String> callback_sym;
 static Persistent<String> onerror_sym;
 
 enum node_zlib_mode {
-  DEFLATE = 1,
+  NONE,
+  DEFLATE,
   INFLATE,
   GZIP,
   GUNZIP,
@@ -60,17 +61,40 @@ class ZCtx : public ObjectWrap {
 
   ZCtx(node_zlib_mode mode) : ObjectWrap(), dictionary_(NULL), mode_(mode) {}
 
+
   ~ZCtx() {
+    Clear();
+  }
+
+
+  void Clear() {
+    assert(!write_in_progress_ && "write in progress");
+    assert(init_done_ && "clear before init");
+    assert(mode_ <= UNZIP);
+
     if (mode_ == DEFLATE || mode_ == GZIP || mode_ == DEFLATERAW) {
       (void)deflateEnd(&strm_);
     } else if (mode_ == INFLATE || mode_ == GUNZIP || mode_ == INFLATERAW ||
                mode_ == UNZIP) {
       (void)inflateEnd(&strm_);
     }
+    mode_ = NONE;
+
+    if (dictionary_ != NULL) {
+      delete[] dictionary_;
+      dictionary_ = NULL;
+    }
+  }
+
 
-    if (dictionary_ != NULL) delete[] dictionary_;
+  static Handle<Value> Clear(const Arguments& args) {
+    HandleScope scope;
+    ZCtx *ctx = ObjectWrap::Unwrap<ZCtx>(args.This());
+    ctx->Clear();
+    return scope.Close(Undefined());
   }
 
+
   // write(flush, in, in_off, in_len, out, out_off, out_len)
   static Handle<Value> Write(const Arguments& args) {
     HandleScope scope;
@@ -78,6 +102,7 @@ class ZCtx : public ObjectWrap {
 
     ZCtx *ctx = ObjectWrap::Unwrap<ZCtx>(args.This());
     assert(ctx->init_done_ && "write before init");
+    assert(ctx->mode_ != NONE && "already finalized");
 
     assert(!ctx->write_in_progress_ && "write already in progress");
     ctx->write_in_progress_ = true;
@@ -450,6 +475,7 @@ void InitZlib(Handle<Object> target) {
 
   NODE_SET_PROTOTYPE_METHOD(z, "write", ZCtx::Write);
   NODE_SET_PROTOTYPE_METHOD(z, "init", ZCtx::Init);
+  NODE_SET_PROTOTYPE_METHOD(z, "clear", ZCtx::Clear);
   NODE_SET_PROTOTYPE_METHOD(z, "reset", ZCtx::Reset);
 
   z->SetClassName(String::NewSymbol("Zlib"));