implement deflate store + high level deflate implementation + lint fixes

.jshintignore

@@ -1,4 +1,3 @@
 .git/
 node_modules/
 benchmark/implementations
-lib/zlib

.jshintrc

@@ -1,7 +1,7 @@
 {
   // Enforcing Options /////////////////////////////////////////////////////////
 
-  "bitwise"       : true,   // Prohibit bitwise operators (&, |, ^, etc.).
+  "bitwise"       : false,   // Prohibit bitwise operators (&, |, ^, etc.).
   "camelcase"     : false,  // true: Identifiers must be in camelCase
   "curly"         : true,   // Require {} for every new block or scope.
   "eqeqeq"        : true,   // Require triple equals i.e. `===`.

benchmark/implementations/deflate-pako/index.js

@@ -0,0 +1,9 @@
+'use strict'
+
+var pako = require('../../../index.js');
+
+exports.run = function(data) {
+  return pako.deflate(data, {
+    level: 0
+  });
+}

lib/deflate.js

@@ -1,8 +1,11 @@
 'use strict';
 
 
-//var zlib_deflate = require('./zlib/deflate.js');
+var zlib_deflate = require('./zlib/deflate.js');
 var utils = require('./zlib/utils');
+var c = require('./zlib/constants');
+var msg = require('./zlib/messages');
+var zstream = require('./zlib/zstream');
 
 
 /**
@@ -11,25 +14,76 @@ var utils = require('./zlib/utils');
  * @param {Object} [options] zlib options
  * @constructor
  */
-var Deflate = function(/*options*/) {
+var Deflate = function(options) {
+  options = utils.assign({
+    level: 6,
+    method: c.Z_DEFLATED,
+    chunkSize: 16384,
+    windowBits: 15,
+    memLevel: 8,
+    strategy: c.Z_DEFAULT_STRATEGY
+  },options || {});
+  this.options = options;
 
+  this.strm = new zstream();
+  this.strm.next_out = utils.arrayCreate(this.options.chunkSize);
+
+  var ret = zlib_deflate.deflateInit2(this.strm, options.level, options.method, options.windowBits, options.memLevel, options.strategy);
+  if (ret !== c.Z_OK) {
+    throw new Error(msg[ret]);
+  }
 };
 
-
 /**
  * Compresses the input data and fills output buffer with compressed data.
- * @return {Array|Uint8Array} compressed data
  */
-Deflate.prototype.push = function(/*data_in*/) {
+Deflate.prototype.push = function(data_in) {
+  var strm = this.strm;
 
+  strm.next_in = data_in;
+  strm.next_in_index = 0;
+  strm.avail_in = strm.next_in.length;
+
+  /* run deflate() on input until output buffer not full, finish
+   compression if all of source has been read in */
+  do {
+    strm.avail_out = this.options.chunkSize;
+    strm.next_out_index = 0;
+    zlib_deflate.deflate(strm, c.Z_NO_FLUSH);
+    //var ret = zlib_deflate.deflate(strm, c.Z_NO_FLUSH);    /* no bad return value */
+    //if (ret !== c.Z_STREAM_END && ret !== c.Z_OK) {
+    //  this.onEnd(ret);
+    //}
+    if(strm.next_out_index) {
+      var out = utils.arrayCreate(strm.next_out_index);
+      utils.arraySet(out, strm.next_out, 0, strm.next_out_index, 0);
+      this.onData(out);
+    }
+  } while (strm.avail_in > 0 || strm.avail_out === 0);
 };
 
 Deflate.prototype.flush = function() {
+  var strm = this.strm;
+  do {
+    strm.avail_out = this.options.chunkSize;
+    strm.next_out_index = 0;
+    var ret = zlib_deflate.deflate(strm, c.Z_FINISH);    /* no bad return value */
+    if (ret !== c.Z_STREAM_END && ret !== c.Z_OK) {
+      this.onEnd(ret);
+    }
 
+    if(strm.next_out_index) {
+      var out = utils.arrayCreate(strm.next_out_index);
+      utils.arraySet(out, strm.next_out, 0, strm.next_out_index, 0);
+      this.onData(out);
+    }
+  } while (strm.avail_out === 0);
 };
 
 Deflate.prototype.finish = function() {
-
+  this.flush();
+  zlib_deflate.deflateEnd(this.strm);
+  this.onEnd(c.Z_OK);
 };
 
 Deflate.prototype.onData = function(/*data_out*/) {
@@ -52,37 +106,38 @@ exports.Deflate = Deflate;
  */
 function deflate(input, options) {
   var result;
-  var chains = [];
+  var chunks = [];
   var deflator = new Deflate(options);
 
   deflator.onData = function(data_out) {
-    chains.push(data_out);
+    chunks.push(data_out);
   };
 
   deflator.onEnd = function(error) {
-    var i, l, len, pos, chain;
+    var i, l, len, pos, chunk;
 
     if (error) { throw error; }
 
     // calculate data length
     len = 0;
-    for (i=0, l=chains.length; i<l; i++) {
+    for (i=0, l=chunks.length; i<l; i++) {
-      len += chains[i].length;
+      len += chunks[i].length;
     }
 
-    // join chains
+    // join chunks
     result = utils.arrayCreate(len);
     pos = 0;
 
-    for (i=0, l=chains.length; i<l; i++) {
+    for (i=0, l=chunks.length; i<l; i++) {
-      chain = chains[i];
+      chunk = chunks[i];
-      len = chain.length;
+      len = chunk.length;
-      utils.arraySet(result, chain, 0, len, pos);
+      utils.arraySet(result, chunk, 0, len, pos);
       pos += len;
     }
   };
 
   deflator.push(input);
+  deflator.finish();
 
   return result;
 }

lib/zlib/adler32.js

@@ -1,23 +1,25 @@
 'use strict';
 
-var BASE = 65521;      /* largest prime smaller than 65536 */
+function adler32(adler, buf, len, start)
-var NMAX =  5552;
-
-function adler32(adler, buf, len)
 {
-  var i = 0;
+  var BASE = 65521;      /* largest prime smaller than 65536 */
+  var NMAX =  5552;
+
+  var i = start;
   /* split Adler-32 into component sums */
   var sum2 = (adler >> 16) & 0xffff;
   adler &= 0xffff;
 
   /* in case user likes doing a byte at a time, keep it fast */
-  if (len == 1) {
+  if (len === 1) {
     adler += buf[0];
-    if (adler >= BASE)
+    if (adler >= BASE) {
       adler -= BASE;
+    }
     sum2 += adler;
-    if (sum2 >= BASE)
+    if (sum2 >= BASE) {
       sum2 -= BASE;
+    }
     return (adler | (sum2 << 16)) >>> 0;
   }
 
@@ -27,30 +29,18 @@ function adler32(adler, buf, len)
       adler += buf[i];
       sum2 += adler;
     }
-    if (adler >= BASE)
+    if (adler >= BASE) {
       adler -= BASE;
+    }
     sum2 %= BASE;            /* only added so many BASE's */
     return (adler | (sum2 << 16)) >>> 0;
   }
 
-  var cursor = 0;
+  while (len>0) {
-  /* do length NMAX blocks -- requires just one modulo operation */
+    var cursor = len > NMAX ? NMAX : len;
-  while (len >= NMAX) {
+    len -= cursor;
-    len -= NMAX;
+    for(var j=0;j<cursor;j++) {
-    var next_cursor = cursor + NMAX;
+      adler += buf[i++];
-    for (i=cursor;i<next_cursor;i++) {
-      adler += buf[i];
-      sum2 += adler;
-    }
-
-    adler %= BASE;
-    sum2 %= BASE;
-  }
-
-  /* do remaining bytes (less than NMAX, still just one modulo) */
-  if (len) {                  /* avoid modulos if none remaining */
-    for (i=cursor;i<len;i++) {
-      adler += buf[i];
       sum2 += adler;
     }
 

lib/zlib/constants.js

@@ -39,5 +39,5 @@ module.exports = {
 
   Z_DEFLATED:               8,
   /* The deflate compression method */
-  Z_NULL:                   0
+  Z_NULL:                   -1
-}
+};

lib/zlib/inflate.js

@@ -1,16 +1,18 @@
-var utils = require('utils');
+'use strict';
+
+//var utils = require('utils');
 
 var ENOUGH_LENS = 852;
 var ENOUGH_DISTS = 592;
 var ENOUGH =  (ENOUGH_LENS+ENOUGH_DISTS);
 
-function code() {
+//function Code() {
-  this.op = 0;           /* operation, extra bits, table bits */
+//  this.op = 0;           /* operation, extra bits, table bits */
-  this.bits = 0;         /* bits in this part of the code */
+//  this.bits = 0;         /* bits in this part of the code */
-  this.val = 0;          /* offset in table or code value */
+//  this.val = 0;          /* offset in table or code value */
-};
+//}
 
-function inflate_state() {
+function InflateState() {
   this.mode = -1;             /* current inflate mode */
   this.last = 0;              /* true if processing last block */
   this.wrap = 0;              /* bit 0 true for zlib, bit 1 true for gzip */
@@ -55,29 +57,29 @@ function inflate_state() {
   this.sane = 0;                   /* if false, allow invalid distance too far */
   this.back = 0;                   /* bits back of last unprocessed length/lit */
   this.was = 0;                    /* initial length of match */
-};
+}
 
-function inflateResetKeep(strm) {
+function inflateResetKeep(/*strm*/) {
 
 }
 
-function inflateReset(strm) {
+function inflateReset(/*strm*/) {
 
 }
 
-function inflateReset2(strm, windowBits) {
+function inflateReset2(/*strm, windowBits*/) {
 
 }
 
-function inflateInit2(strm, windowBits, version, stream_size) {
+function inflateInit2(strm/*, windowBits, version, stream_size*/) {
+  strm.state = new InflateState();
+}
+
+function inflateInit(/*strm, version, stream_size*/) {
 
 }
 
-function inflateInit(strm, version, stream_size) {
+function inflatePrime(/*strm, bits, value*/) {
-
-}
-
-function inflatePrime(strm, bits, value) {
 
 }
 
@@ -91,9 +93,9 @@ function inflatePrime(strm, bits, value) {
  used for threaded applications, since the rewriting of the tables and virgin
  may not be thread-safe.
  */
-function fixedtables(state) {
+//function fixedtables(state) {
-
+//
-}
+//}
 
 /*
  Write out the inffixed.h that is #include'd above.  Defining MAKEFIXED also
@@ -113,9 +115,9 @@ function fixedtables(state) {
 
  a.out > inffixed.h
  */
-function makefixed() {
+//function makefixed() {
-
+//
-}
+//}
 
 /*
  Update the window with the last wsize (normally 32K) bytes written before
@@ -131,27 +133,27 @@ function makefixed() {
  output will fall in the output data, making match copies simpler and faster.
  The advantage may be dependent on the size of the processor's data caches.
  */
-function updatewindow(strm, end, copy) {
+//function updatewindow(strm, end, copy) {
+//
+//}
+
+function inflate(/*strm, flush*/) {
 
 }
 
-function inflate(strm, flush) {
+function inflateEnd(/*strm*/) {
 
 }
 
-function inflateEnd(strm) {
+function inflateGetDictionary(/*strm, dictionary, dictLength*/) {
 
 }
 
-function inflateGetDictionary(strm, dictionary, dictLength) {
+function inflateSetDictionary(/*strm, dictionary, dictLength*/) {
 
 }
 
-function inflateSetDictionary(strm, dictionary, dictLength) {
+function inflateGetHeader(/*strm, head*/) {
-
-}
-
-function inflateGetHeader(strm, head) {
 
 }
 
@@ -166,27 +168,27 @@ function inflateGetHeader(strm, head) {
  called again with more data and the *have state.  *have is initialized to
  zero for the first call.
  */
-function syncsearch(have, buf, len) {
+//function syncsearch(/*have, buf, len*/) {
+//
+//}
+
+function inflateSync(/*strm*/) {
 
 }
 
-function inflateSync(strm) {
+function inflateSyncPoint(/*strm*/) {
 
 }
 
-function inflateSyncPoint(strm) {
+function inflateCopy(/*dest, source*/) {
 
 }
 
-function inflateCopy(dest, source) {
+function inflateUndermine(/*strm, subvert*/) {
 
 }
 
-function inflateUndermine(strm, subvert) {
+function inflateMark(/*strm*/) {
-
-}
-
-function inflateMark(strm) {
 
 }
 
@@ -210,6 +212,8 @@ exports.inflateGetDictionary = inflateGetDictionary;
 
 exports.inflateGetHeader = inflateGetHeader;
 
+exports.inflateSetDictionary = inflateSetDictionary;
+
 exports.inflateSync = inflateSync;
 
 exports.inflateSyncPoint = inflateSyncPoint;

lib/zlib/messages.js

@@ -0,0 +1,13 @@
+'use strict';
+
+module.exports = {
+  '2':    'need dictionary',     /* Z_NEED_DICT       2  */
+  '1':    'stream end',          /* Z_STREAM_END      1  */
+  '0':    '',                    /* Z_OK              0  */
+  '-1':   'file error',          /* Z_ERRNO         (-1) */
+  '-2':   'stream error',        /* Z_STREAM_ERROR  (-2) */
+  '-3':   'data error',          /* Z_DATA_ERROR    (-3) */
+  '-4':   'insufficient memory', /* Z_MEM_ERROR     (-4) */
+  '-5':   'buffer error',        /* Z_BUF_ERROR     (-5) */
+  '-6':   'incompatible version' /* Z_VERSION_ERROR (-6) */
+};

lib/zlib/trees.js

@@ -0,0 +1,179 @@
+'use strict';
+
+var utils = require('./utils');
+
+var STORED_BLOCK = 0;
+var Buf_size = 16;
+
+/* ===========================================================================
+ * Output a short LSB first on the stream.
+ * IN assertion: there is enough room in pendingBuf.
+ */
+function put_short (s, w) {
+//    put_byte(s, (uch)((w) & 0xff));
+//    put_byte(s, (uch)((ush)(w) >> 8));
+  s.pending_buf[s.pending++] = (w) & 0xff;
+  s.pending_buf[s.pending++] = (w >>> 8) & 0xff;
+}
+
+/* ===========================================================================
+ * Send a value on a given number of bits.
+ * IN assertion: length <= 16 and value fits in length bits.
+ */
+function send_bits (s, value, length)
+{
+  var len = length, val = value;
+  if (s.bi_valid > Buf_size - len) {
+    s.bi_buf |= (val << s.bi_valid) & 0xffff;
+    put_short(s, s.bi_buf);
+    s.bi_buf = val >>> (Buf_size - s.bi_valid);
+    s.bi_valid += len - Buf_size;
+  } else {
+    s.bi_buf |=  ((value) << s.bi_valid) & 0xffff;
+    s.bi_valid += len;
+  }
+}
+
+/* ===========================================================================
+ * Flush the bit buffer and align the output on a byte boundary
+ */
+function bi_windup (s)
+{
+  if (s.bi_valid > 8) {
+    put_short(s, s.bi_buf);
+  } else if (s.bi_valid > 0) {
+    //put_byte(s, (Byte)s->bi_buf);
+    s.pending_buf[s.pending++] = s.bi_buf;
+  }
+  s.bi_buf = 0;
+  s.bi_valid = 0;
+}
+
+/* ===========================================================================
+ * Copy a stored block, storing first the length and its
+ * one's complement if requested.
+ */
+function copy_block (s, buf, len, header)
+//DeflateState *s;
+//charf    *buf;    /* the input data */
+//unsigned len;     /* its length */
+//int      header;  /* true if block header must be written */
+{
+  bi_windup(s);        /* align on byte boundary */
+
+  if (header) {
+    put_short(s, len);
+    put_short(s, ~len);
+  }
+//  while (len--) {
+//    put_byte(s, *buf++);
+//  }
+  utils.arraySet(s.pending_buf, s.window, buf, len, s.pending);
+  s.pending += len;
+}
+
+
+/* ===========================================================================
+ * Initialize the tree data structures for a new zlib stream.
+ */
+function _tr_init(s)
+{
+//  todo: tr_static_init();
+//
+//  s.l_desc.dyn_tree = s.dyn_ltree;
+//  s.l_desc.stat_desc = static_l_desc;
+//
+//  s.d_desc.dyn_tree = s.dyn_dtree;
+//  s.d_desc.stat_desc = static_d_desc;
+//
+//  s.bl_desc.dyn_tree = s.bl_tree;
+//  s.bl_desc.stat_desc = static_bl_desc;
+//
+  s.bi_buf = 0;
+  s.bi_valid = 0;
+//
+//  /* Initialize the first block of the first file: */
+// todo:init_block(s);
+}
+
+/* ===========================================================================
+ * Send a stored block
+ */
+function _tr_stored_block(s, buf, stored_len, last)
+//DeflateState *s;
+//charf *buf;       /* input block */
+//ulg stored_len;   /* length of input block */
+//int last;         /* one if this is the last block for a file */
+{
+  send_bits(s, (STORED_BLOCK<<1)+(last ? 1 : 0), 3);    /* send block type */
+  copy_block(s, buf, stored_len, 1); /* with header */
+}
+
+/* ===========================================================================
+ * Determine the best encoding for the current block: dynamic trees, static
+ * trees or store, and output the encoded block to the zip file.
+ */
+function _tr_flush_block(s, buf, stored_len, last)
+//DeflateState *s;
+//charf *buf;       /* input block, or NULL if too old */
+//ulg stored_len;   /* length of input block */
+//int last;         /* one if this is the last block for a file */
+{
+  var opt_lenb = 0, static_lenb = 0; /* opt_len and static_len in bytes */
+//  var max_blindex = 0;  /* index of last bit length code of non zero freq */
+
+  /* Build the Huffman trees unless a stored block is forced */
+  if (s.level > 0) {
+    /* Construct the literal and distance trees */
+    //todo: build_tree(s, s.l_desc);
+    //todo: build_tree(s, s->d_desc);
+
+    /* At this point, opt_len and static_len are the total bit lengths of
+     * the compressed block data, excluding the tree representations.
+     */
+
+    /* Build the bit length tree for the above two trees, and get the index
+     * in bl_order of the last bit length code to send.
+     */
+    //todo: max_blindex = build_bl_tree(s);
+
+    /* Determine the best encoding. Compute the block lengths in bytes. */
+    opt_lenb = (s.opt_len + 3 + 7) >>> 3;
+    static_lenb = (s.static_len + 3 + 7) >>> 3;
+
+    if (static_lenb <= opt_lenb) { opt_lenb = static_lenb; }
+
+  } else {
+    opt_lenb = static_lenb = stored_len + 5; /* force a stored block */
+  }
+
+  if ((stored_len+4 <= opt_lenb) && (buf !== -1)) {
+    /* 4: two words for the lengths */
+
+    /* The test buf != NULL is only necessary if LIT_BUFSIZE > WSIZE.
+     * Otherwise we can't have processed more than WSIZE input bytes since
+     * the last block flush, because compression would have been
+     * successful. If LIT_BUFSIZE <= WSIZE, it is never too late to
+     * transform a block into a stored block.
+     */
+    _tr_stored_block(s, buf, stored_len, last);
+  }
+//  else {
+//    //send_bits(s, (DYN_TREES<<1)+(last ? 1 : 0), 3);
+//    //todo: send_all_trees(s, s.l_desc.max_code+1, s.d_desc.max_code+1,max_blindex+1);
+//    //todo: compress_block(s, s->dyn_ltree,s->dyn_dtree);
+//  }
+
+  /* The above check is made mod 2^32, for files larger than 512 MB
+   * and uLong implemented on 32 bits.
+   */
+  //todo: init_block(s);
+
+  if (last) {
+    bi_windup(s);
+  }
+}
+
+exports._tr_init = _tr_init;
+exports._tr_stored_block = _tr_stored_block;
+exports._tr_flush_block = _tr_flush_block;

lib/zlib/utils.js

@@ -1,11 +1,10 @@
 'use strict';
 
-
 exports.assign = function(obj /*from1, from2, from3, ...*/) {
   var sources = Array.prototype.slice.call(arguments, 1);
   while (sources.length) {
     var source = sources.shift();
-    if (!source) { continue }
+    if (!source) { continue; }
 
     if (typeof(source) !== 'object') {
       throw new TypeError(source + 'must be non-object');
@@ -26,8 +25,9 @@ exports.arraySet = function(dest, src, src_offs, len, dest_offs) {
   for(var i=0; i<len; i++) {
     dest[dest_offs + i] = src[src_offs + i];
   }
-}
+};
+
 
 exports.arrayCreate = function(length) {
   return new Array(length);
-}
+};

lib/zlib/zstream.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var c = require('constants');
 
 function ZStream() {
@@ -8,13 +10,13 @@ function ZStream() {
   /* total number of input bytes read so far */
   this.total_in = 0;
   /* next output byte should be put there */
-  this.next_out = c.Z_NULL;
+  //this.next_out = c.Z_NULL;
   /* remaining free space at next_out */
   this.avail_out = 0;
   /* total number of bytes output so far */
   this.total_out = 0;
   /* last error message, NULL if no error */
-  this.msg = c.Z_NULL;
+  //this.msg = c.Z_NULL;
   /* not visible by applications */
   this.state = c.Z_NULL;
   /* best guess about the data type: binary or text */

test/deflate.js

@@ -31,33 +31,33 @@ describe.skip('Deflate defaults', function () {
 });
 
 
-describe.skip('Deflate levels', function () {
+describe('Deflate levels', function () {
 
-  it('level 9', function(done) {
+  it.skip('level 9', function(done) {
     testDeflate(zlib.createDeflate, pako.deflate, sample, { level: 9 }, done);
   });
-  it('level 8', function(done) {
+  it.skip('level 8', function(done) {
     testDeflate(zlib.createDeflate, pako.deflate, sample, { level: 8 }, done);
   });
-  it('level 7', function(done) {
+  it.skip('level 7', function(done) {
     testDeflate(zlib.createDeflate, pako.deflate, sample, { level: 7 }, done);
   });
-  it('level 6', function(done) {
+  it.skip('level 6', function(done) {
     testDeflate(zlib.createDeflate, pako.deflate, sample, { level: 6 }, done);
   });
-  it('level 5', function(done) {
+  it.skip('level 5', function(done) {
     testDeflate(zlib.createDeflate, pako.deflate, sample, { level: 5 }, done);
   });
-  it('level 4', function(done) {
+  it.skip('level 4', function(done) {
     testDeflate(zlib.createDeflate, pako.deflate, sample, { level: 4 }, done);
   });
-  it('level 3', function(done) {
+  it.skip('level 3', function(done) {
     testDeflate(zlib.createDeflate, pako.deflate, sample, { level: 3 }, done);
   });
-  it('level 2', function(done) {
+  it.skip('level 2', function(done) {
     testDeflate(zlib.createDeflate, pako.deflate, sample, { level: 2 }, done);
   });
-  it('level 1', function(done) {
+  it.skip('level 1', function(done) {
     testDeflate(zlib.createDeflate, pako.deflate, sample, { level: 1 }, done);
   });
   it('level 0', function(done) {
@@ -151,4 +151,4 @@ describe.skip('Deflate strategy', function () {
     testDeflate(zlib.createDeflate, pako.deflate, sample, { strategy: 4 }, done);
   });
 
-});
+});