Implement dictionary handling.

This adds the methods

- `deflateSetDictionary`
- `inflateSetDictionary`

as well as calling `setDictionary` at the right point when passing
the a `dictionary` option to one of

- `pako.deflate`
- `new pako.Deflate`
- `pako.inflate`
- `new pako.Inflate`

README.md

@@ -148,10 +148,9 @@ Notes
 Pako does not contain some specific zlib functions:
 
 - __deflate__ -  methods `deflateCopy`, `deflateBound`, `deflateParams`,
-  `deflatePending`, `deflatePrime`, `deflateSetDictionary`, `deflateTune`.
-- __inflate__ - `inflateGetDictionary`, `inflateCopy`, `inflateMark`,
-  `inflatePrime`, `inflateSetDictionary`, `inflateSync`, `inflateSyncPoint`,
-  `inflateUndermine`.
+  `deflatePending`, `deflatePrime`, `deflateTune`.
+- __inflate__ - methods `inflateCopy`, `inflateMark`,
+  `inflatePrime`, `inflateGetDictionary`, `inflateSync`, `inflateSyncPoint`, `inflateUndermine`.
 - High level inflate/deflate wrappers (classes) may not support some flush
   modes. Those should work: Z_NO_FLUSH, Z_FINISH, Z_SYNC_FLUSH.
 

lib/deflate.js

@@ -79,6 +79,7 @@ var Z_DEFLATED  = 8;
  * - `windowBits`
  * - `memLevel`
  * - `strategy`
+ * - `dictionary`
  *
  * [http://zlib.net/manual.html#Advanced](http://zlib.net/manual.html#Advanced)
  * for more information on these.
@@ -163,6 +164,27 @@ function Deflate(options) {
   if (opt.header) {
     zlib_deflate.deflateSetHeader(this.strm, opt.header);
   }
+
+  if (opt.dictionary) {
+    var dict;
+    // Convert data if needed
+    if (typeof opt.dictionary === 'string') {
+      // If we need to compress text, change encoding to utf8.
+      dict = strings.string2buf(opt.dictionary);
+    } else if (toString.call(opt.dictionary) === '[object ArrayBuffer]') {
+      dict = new Uint8Array(opt.dictionary);
+    } else {
+      dict = opt.dictionary;
+    }
+
+    status = zlib_deflate.deflateSetDictionary(this.strm, dict);
+
+    if (status !== Z_OK) {
+      throw new Error(msg[status]);
+    }
+
+    this._dict_set = true;
+  }
 }
 
 /**
@@ -309,6 +331,7 @@ Deflate.prototype.onEnd = function (status) {
  * - windowBits
  * - memLevel
  * - strategy
+ * - dictionary
  *
  * [http://zlib.net/manual.html#Advanced](http://zlib.net/manual.html#Advanced)
  * for more information on these.

lib/inflate.js

@@ -57,6 +57,7 @@ var toString = Object.prototype.toString;
  * on bad params. Supported options:
  *
  * - `windowBits`
+ * - `dictionary`
  *
  * [http://zlib.net/manual.html#Advanced](http://zlib.net/manual.html#Advanced)
  * for more information on these.
@@ -176,6 +177,7 @@ function Inflate(options) {
 Inflate.prototype.push = function (data, mode) {
   var strm = this.strm;
   var chunkSize = this.options.chunkSize;
+  var dictionary = this.options.dictionary;
   var status, _mode;
   var next_out_utf8, tail, utf8str;
 
@@ -208,6 +210,22 @@ Inflate.prototype.push = function (data, mode) {
 
     status = zlib_inflate.inflate(strm, c.Z_NO_FLUSH);    /* no bad return value */
 
+    if (status === c.Z_NEED_DICT && dictionary) {
+      var dict;
+
+      // Convert data if needed
+      if (typeof dictionary === 'string') {
+        dict = strings.string2buf(dictionary);
+      } else if (toString.call(dictionary) === '[object ArrayBuffer]') {
+        dict = new Uint8Array(dictionary);
+      } else {
+        dict = dictionary;
+      }
+
+      status = zlib_inflate.inflateSetDictionary(this.strm, dict);
+
+    }
+
     if (status === c.Z_BUF_ERROR && allowBufError === true) {
       status = c.Z_OK;
       allowBufError = false;

lib/zlib/deflate.js

@@ -183,6 +183,7 @@ function read_buf(strm, buf, start, size) {
 
   strm.avail_in -= len;
 
+  // zmemcpy(buf, strm->next_in, len);
   utils.arraySet(buf, strm.input, strm.next_in, len, start);
   if (strm.state.wrap === 1) {
     strm.adler = adler32(strm.adler, buf, len, start);
@@ -1745,6 +1746,94 @@ function deflateEnd(strm) {
 //
 //}
 
+
+/* =========================================================================
+ * Initializes the compression dictionary from the given byte
+ * sequence without producing any compressed output.
+ */
+function deflateSetDictionary(strm, dictionary) {
+  var dictLength = dictionary.length;
+
+  var s;
+  var str, n;
+  var wrap;
+  var avail;
+  var next;
+  var input;
+
+  if (!strm/*== Z_NULL*/ || !strm.state/*== Z_NULL*/) {
+    return Z_STREAM_ERROR;
+  }
+
+  s = strm.state;
+  wrap = s.wrap;
+
+  if (wrap === 2 || (wrap === 1 && s.status !== INIT_STATE) || s.lookahead) {
+    return Z_STREAM_ERROR;
+  }
+
+  /* when using zlib wrappers, compute Adler-32 for provided dictionary */
+  if (wrap === 1) {
+    /* adler32(strm->adler, dictionary, dictLength); */
+    strm.adler = adler32(strm.adler, dictionary, dictLength, 0);
+  }
+
+  s.wrap = 0;   /* avoid computing Adler-32 in read_buf */
+
+  /* if dictionary would fill window, just replace the history */
+  if (dictLength >= s.w_size) {
+    if (wrap === 0) {            /* already empty otherwise */
+      /*** CLEAR_HASH(s); ***/
+      zero(s.head); // Fill with NIL (= 0);
+      s.strstart = 0;
+      s.block_start = 0;
+      s.insert = 0;
+    }
+    /* use the tail */
+    // dictionary = dictionary.slice(dictLength - s.w_size);
+    var tmpDict = new utils.Buf8(s.w_size);
+    utils.arraySet(tmpDict, dictionary, dictLength - s.w_size, s.w_size, 0);
+    dictionary = tmpDict;
+    dictLength = s.w_size;
+  }
+  /* insert dictionary into window and hash */
+  avail = strm.avail_in;
+  next = strm.next_in;
+  input = strm.input;
+  strm.avail_in = dictLength;
+  strm.next_in = 0;
+  strm.input = dictionary;
+  fill_window(s);
+  while (s.lookahead >= MIN_MATCH) {
+    str = s.strstart;
+    n = s.lookahead - (MIN_MATCH - 1);
+    do {
+      /* UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); */
+      s.ins_h = ((s.ins_h << s.hash_shift) ^ s.window[str + MIN_MATCH - 1]) & s.hash_mask;
+
+      s.prev[str & s.w_mask] = s.head[s.ins_h];
+
+      s.head[s.ins_h] = str;
+      str++;
+    } while (--n);
+    s.strstart = str;
+    s.lookahead = MIN_MATCH - 1;
+    fill_window(s);
+  }
+  s.strstart += s.lookahead;
+  s.block_start = s.strstart;
+  s.insert = s.lookahead;
+  s.lookahead = 0;
+  s.match_length = s.prev_length = MIN_MATCH - 1;
+  s.match_available = 0;
+  strm.next_in = next;
+  strm.input = input;
+  strm.avail_in = avail;
+  s.wrap = wrap;
+  return Z_OK;
+}
+
+
 exports.deflateInit = deflateInit;
 exports.deflateInit2 = deflateInit2;
 exports.deflateReset = deflateReset;
@@ -1753,11 +1842,11 @@ exports.deflateSetHeader = deflateSetHeader;
 exports.deflate = deflate;
 exports.deflateEnd = deflateEnd;
 exports.deflateInfo = 'pako deflate (from Nodeca project)';
+exports.deflateSetDictionary = deflateSetDictionary;
 
 /* Not implemented
 exports.deflateBound = deflateBound;
 exports.deflateCopy = deflateCopy;
-exports.deflateSetDictionary = deflateSetDictionary;
 exports.deflateParams = deflateParams;
 exports.deflatePending = deflatePending;
 exports.deflatePrime = deflatePrime;

lib/zlib/inflate.js

@@ -1480,6 +1480,41 @@ function inflateGetHeader(strm, head) {
   return Z_OK;
 }
 
+function inflateSetDictionary(strm, dictionary) {
+  var dictLength = dictionary.length;
+
+  var state;
+  var dictid;
+  var ret;
+
+  /* check state */
+  if (!strm /* == Z_NULL */ || !strm.state /* == Z_NULL */) { return Z_STREAM_ERROR; }
+  state = strm.state;
+
+  if (state.wrap !== 0 && state.mode !== DICT) {
+    return Z_STREAM_ERROR;
+  }
+
+  /* check for correct dictionary identifier */
+  if (state.mode === DICT) {
+    dictid = 1; /* adler32(0, null, 0)*/
+    /* dictid = adler32(dictid, dictionary, dictLength); */
+    dictid = adler32(dictid, dictionary, dictLength, 0);
+    if (dictid !== state.check) {
+      return Z_DATA_ERROR;
+    }
+  }
+  /* copy dictionary to window using updatewindow(), which will amend the
+   existing dictionary if appropriate */
+  ret = updatewindow(strm, dictionary, dictLength, dictLength);
+  if (ret) {
+    state.mode = MEM;
+    return Z_MEM_ERROR;
+  }
+  state.havedict = 1;
+  // Tracev((stderr, "inflate:   dictionary set\n"));
+  return Z_OK;
+}
 
 exports.inflateReset = inflateReset;
 exports.inflateReset2 = inflateReset2;
@@ -1490,13 +1525,13 @@ exports.inflate = inflate;
 exports.inflateEnd = inflateEnd;
 exports.inflateGetHeader = inflateGetHeader;
 exports.inflateInfo = 'pako inflate (from Nodeca project)';
+exports.inflateSetDictionary = inflateSetDictionary;
 
 /* Not implemented
 exports.inflateCopy = inflateCopy;
 exports.inflateGetDictionary = inflateGetDictionary;
 exports.inflateMark = inflateMark;
 exports.inflatePrime = inflatePrime;
-exports.inflateSetDictionary = inflateSetDictionary;
 exports.inflateSync = inflateSync;
 exports.inflateSyncPoint = inflateSyncPoint;
 exports.inflateUndermine = inflateUndermine;

test/deflate.js

@@ -170,3 +170,31 @@ describe('Deflate RAW', function () {
   });
 
 });
+
+describe('Deflate dictionary', function () {
+  it('trivial dictionary', function (done) {
+    var dict = new Buffer('abcdefghijklmnoprstuvwxyz');
+    testSamples(zlib.createDeflate, pako.deflate, samples, { dictionary: dict }, done);
+  });
+
+  it('spdy dictionary', function (done) {
+    testSamples(zlib.createDeflate, pako.deflate, samples, { dictionary: helpers.spdyDict }, done);
+  });
+
+  it('handles multiple pushes', function () {
+    var dict = new Buffer('abcd');
+    var deflate = new pako.Deflate({ dictionary: dict });
+
+    deflate.push(new Buffer('hello'), false);
+    deflate.push(new Buffer('hello'), false);
+    deflate.push(new Buffer(' world'), true);
+
+    if (deflate.err) { throw new Error(deflate.err); }
+
+    var uncompressed = pako.inflate(new Buffer(deflate.result), { dictionary: dict });
+
+    if (!helpers.cmpBuf(new Buffer('hellohello world'), uncompressed)) {
+      throw new Error('Result not equal for p -> z');
+    }
+  });
+});

test/fixtures/spdy_dict.txt

@@ -0,0 +1 @@
+optionsgetheadpostputdeletetraceacceptaccept-charsetaccept-encodingaccept-languageauthorizationexpectfromhostif-modified-sinceif-matchif-none-matchif-rangeif-unmodifiedsincemax-forwardsproxy-authorizationrangerefererteuser-agent100101200201202203204205206300301302303304305306307400401402403404405406407408409410411412413414415416417500501502503504505accept-rangesageetaglocationproxy-authenticatepublicretry-afterservervarywarningwww-authenticateallowcontent-basecontent-encodingcache-controlconnectiondatetrailertransfer-encodingupgradeviawarningcontent-languagecontent-lengthcontent-locationcontent-md5content-rangecontent-typeetagexpireslast-modifiedset-cookieMondayTuesdayWednesdayThursdayFridaySaturdaySundayJanFebMarAprMayJunJulAugSepOctNovDecchunkedtext/htmlimage/pngimage/jpgimage/gifapplication/xmlapplication/xhtmltext/plainpublicmax-agecharset=iso-8859-1utf-8gzipdeflateHTTP/1.1statusversionurl\x0

test/helpers.js

@@ -165,8 +165,11 @@ function testInflate(samples, inflateOptions, deflateOptions, callback) {
   callback();
 }
 
+var spdyDict = new Buffer(fs.readFileSync(path.join(__dirname, 'fixtures', 'spdy_dict.txt')));
+
 
 exports.cmpBuf = cmpBuf;
 exports.testSamples = testSamples;
 exports.testInflate = testInflate;
 exports.loadSamples = loadSamples;
+exports.spdyDict = spdyDict;

test/inflate.js

@@ -13,7 +13,6 @@ var testInflate = helpers.testInflate;
 
 var samples = helpers.loadSamples();
 
-
 describe('Inflate defaults', function () {
 
   it('inflate, no options', function (done) {
@@ -28,6 +27,7 @@ describe('Inflate defaults', function () {
     var compressed_samples = helpers.loadSamples('samples_deflated_raw');
     helpers.testSamples(zlib.createInflateRaw, pako.inflateRaw, compressed_samples, {}, done);
   });
+
 });
 
 
@@ -162,3 +162,32 @@ describe('Inflate RAW', function () {
   });
 
 });
+
+describe('Inflate with dictionary', function () {
+  it('should throw on the wrong dictionary', function () {
+    // var zCompressed = helpers.deflateSync('world', { dictionary: new Buffer('hello') });
+    var zCompressed = new Buffer([ 120, 187, 6, 44, 2, 21, 43, 207, 47, 202, 73, 1, 0, 6, 166, 2, 41 ]);
+
+    try {
+      pako.inflate(zCompressed, { dictionary: new Buffer('world') });
+    } catch (err) {
+      if (err.message === 'stream error') {
+        return;
+      }
+
+      throw err;
+    }
+
+    throw new Error('Did not throw');
+  });
+
+  it('trivial dictionary', function (done) {
+    var dict = new Buffer('abcdefghijklmnoprstuvwxyz');
+    testInflate(samples, { dictionary: dict }, { dictionary: dict }, done);
+  });
+
+  it('spdy dictionary', function (done) {
+    testInflate(samples, { dictionary: helpers.spdyDict }, { dictionary: helpers.spdyDict }, done);
+  });
+
+});