From ffaf7c8252d99afa9e8a1f24ebdcae1c586028b8 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Sat, 26 Mar 2016 22:21:54 +0100 Subject: [PATCH] Implement dictionary handling. This adds the methods - `deflateSetDictionary` - `inflateSetDictionary` as well as calling `setDictionary` at the right point when passing the a `dictionary` option to one of - `pako.deflate` - `new pako.Deflate` - `pako.inflate` - `new pako.Inflate` --- README.md | 7 ++- lib/deflate.js | 23 ++++++++++ lib/inflate.js | 18 ++++++++ lib/zlib/deflate.js | 91 ++++++++++++++++++++++++++++++++++++- lib/zlib/inflate.js | 37 ++++++++++++++- test/deflate.js | 28 ++++++++++++ test/fixtures/spdy_dict.txt | 1 + test/helpers.js | 3 ++ test/inflate.js | 31 ++++++++++++- 9 files changed, 232 insertions(+), 7 deletions(-) create mode 100644 test/fixtures/spdy_dict.txt diff --git a/README.md b/README.md index 8cc9b3c..ef121e7 100644 --- a/README.md +++ b/README.md @@ -148,10 +148,9 @@ Notes Pako does not contain some specific zlib functions: - __deflate__ - methods `deflateCopy`, `deflateBound`, `deflateParams`, - `deflatePending`, `deflatePrime`, `deflateSetDictionary`, `deflateTune`. -- __inflate__ - `inflateGetDictionary`, `inflateCopy`, `inflateMark`, - `inflatePrime`, `inflateSetDictionary`, `inflateSync`, `inflateSyncPoint`, - `inflateUndermine`. + `deflatePending`, `deflatePrime`, `deflateTune`. +- __inflate__ - methods `inflateCopy`, `inflateMark`, + `inflatePrime`, `inflateGetDictionary`, `inflateSync`, `inflateSyncPoint`, `inflateUndermine`. - High level inflate/deflate wrappers (classes) may not support some flush modes. Those should work: Z_NO_FLUSH, Z_FINISH, Z_SYNC_FLUSH. diff --git a/lib/deflate.js b/lib/deflate.js index 1926786..11040d4 100644 --- a/lib/deflate.js +++ b/lib/deflate.js @@ -79,6 +79,7 @@ var Z_DEFLATED = 8; * - `windowBits` * - `memLevel` * - `strategy` + * - `dictionary` * * [http://zlib.net/manual.html#Advanced](http://zlib.net/manual.html#Advanced) * for more information on these. @@ -163,6 +164,27 @@ function Deflate(options) { if (opt.header) { zlib_deflate.deflateSetHeader(this.strm, opt.header); } + + if (opt.dictionary) { + var dict; + // Convert data if needed + if (typeof opt.dictionary === 'string') { + // If we need to compress text, change encoding to utf8. + dict = strings.string2buf(opt.dictionary); + } else if (toString.call(opt.dictionary) === '[object ArrayBuffer]') { + dict = new Uint8Array(opt.dictionary); + } else { + dict = opt.dictionary; + } + + status = zlib_deflate.deflateSetDictionary(this.strm, dict); + + if (status !== Z_OK) { + throw new Error(msg[status]); + } + + this._dict_set = true; + } } /** @@ -309,6 +331,7 @@ Deflate.prototype.onEnd = function (status) { * - windowBits * - memLevel * - strategy + * - dictionary * * [http://zlib.net/manual.html#Advanced](http://zlib.net/manual.html#Advanced) * for more information on these. diff --git a/lib/inflate.js b/lib/inflate.js index ff88df8..808dfc3 100644 --- a/lib/inflate.js +++ b/lib/inflate.js @@ -57,6 +57,7 @@ var toString = Object.prototype.toString; * on bad params. Supported options: * * - `windowBits` + * - `dictionary` * * [http://zlib.net/manual.html#Advanced](http://zlib.net/manual.html#Advanced) * for more information on these. @@ -176,6 +177,7 @@ function Inflate(options) { Inflate.prototype.push = function (data, mode) { var strm = this.strm; var chunkSize = this.options.chunkSize; + var dictionary = this.options.dictionary; var status, _mode; var next_out_utf8, tail, utf8str; @@ -208,6 +210,22 @@ Inflate.prototype.push = function (data, mode) { status = zlib_inflate.inflate(strm, c.Z_NO_FLUSH); /* no bad return value */ + if (status === c.Z_NEED_DICT && dictionary) { + var dict; + + // Convert data if needed + if (typeof dictionary === 'string') { + dict = strings.string2buf(dictionary); + } else if (toString.call(dictionary) === '[object ArrayBuffer]') { + dict = new Uint8Array(dictionary); + } else { + dict = dictionary; + } + + status = zlib_inflate.inflateSetDictionary(this.strm, dict); + + } + if (status === c.Z_BUF_ERROR && allowBufError === true) { status = c.Z_OK; allowBufError = false; diff --git a/lib/zlib/deflate.js b/lib/zlib/deflate.js index 99daf92..953a804 100644 --- a/lib/zlib/deflate.js +++ b/lib/zlib/deflate.js @@ -183,6 +183,7 @@ function read_buf(strm, buf, start, size) { strm.avail_in -= len; + // zmemcpy(buf, strm->next_in, len); utils.arraySet(buf, strm.input, strm.next_in, len, start); if (strm.state.wrap === 1) { strm.adler = adler32(strm.adler, buf, len, start); @@ -1745,6 +1746,94 @@ function deflateEnd(strm) { // //} + +/* ========================================================================= + * Initializes the compression dictionary from the given byte + * sequence without producing any compressed output. + */ +function deflateSetDictionary(strm, dictionary) { + var dictLength = dictionary.length; + + var s; + var str, n; + var wrap; + var avail; + var next; + var input; + + if (!strm/*== Z_NULL*/ || !strm.state/*== Z_NULL*/) { + return Z_STREAM_ERROR; + } + + s = strm.state; + wrap = s.wrap; + + if (wrap === 2 || (wrap === 1 && s.status !== INIT_STATE) || s.lookahead) { + return Z_STREAM_ERROR; + } + + /* when using zlib wrappers, compute Adler-32 for provided dictionary */ + if (wrap === 1) { + /* adler32(strm->adler, dictionary, dictLength); */ + strm.adler = adler32(strm.adler, dictionary, dictLength, 0); + } + + s.wrap = 0; /* avoid computing Adler-32 in read_buf */ + + /* if dictionary would fill window, just replace the history */ + if (dictLength >= s.w_size) { + if (wrap === 0) { /* already empty otherwise */ + /*** CLEAR_HASH(s); ***/ + zero(s.head); // Fill with NIL (= 0); + s.strstart = 0; + s.block_start = 0; + s.insert = 0; + } + /* use the tail */ + // dictionary = dictionary.slice(dictLength - s.w_size); + var tmpDict = new utils.Buf8(s.w_size); + utils.arraySet(tmpDict, dictionary, dictLength - s.w_size, s.w_size, 0); + dictionary = tmpDict; + dictLength = s.w_size; + } + /* insert dictionary into window and hash */ + avail = strm.avail_in; + next = strm.next_in; + input = strm.input; + strm.avail_in = dictLength; + strm.next_in = 0; + strm.input = dictionary; + fill_window(s); + while (s.lookahead >= MIN_MATCH) { + str = s.strstart; + n = s.lookahead - (MIN_MATCH - 1); + do { + /* UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); */ + s.ins_h = ((s.ins_h << s.hash_shift) ^ s.window[str + MIN_MATCH - 1]) & s.hash_mask; + + s.prev[str & s.w_mask] = s.head[s.ins_h]; + + s.head[s.ins_h] = str; + str++; + } while (--n); + s.strstart = str; + s.lookahead = MIN_MATCH - 1; + fill_window(s); + } + s.strstart += s.lookahead; + s.block_start = s.strstart; + s.insert = s.lookahead; + s.lookahead = 0; + s.match_length = s.prev_length = MIN_MATCH - 1; + s.match_available = 0; + strm.next_in = next; + strm.input = input; + strm.avail_in = avail; + s.wrap = wrap; + return Z_OK; +} + + exports.deflateInit = deflateInit; exports.deflateInit2 = deflateInit2; exports.deflateReset = deflateReset; @@ -1753,11 +1842,11 @@ exports.deflateSetHeader = deflateSetHeader; exports.deflate = deflate; exports.deflateEnd = deflateEnd; exports.deflateInfo = 'pako deflate (from Nodeca project)'; +exports.deflateSetDictionary = deflateSetDictionary; /* Not implemented exports.deflateBound = deflateBound; exports.deflateCopy = deflateCopy; -exports.deflateSetDictionary = deflateSetDictionary; exports.deflateParams = deflateParams; exports.deflatePending = deflatePending; exports.deflatePrime = deflatePrime; diff --git a/lib/zlib/inflate.js b/lib/zlib/inflate.js index 9ed0615..972f05e 100644 --- a/lib/zlib/inflate.js +++ b/lib/zlib/inflate.js @@ -1480,6 +1480,41 @@ function inflateGetHeader(strm, head) { return Z_OK; } +function inflateSetDictionary(strm, dictionary) { + var dictLength = dictionary.length; + + var state; + var dictid; + var ret; + + /* check state */ + if (!strm /* == Z_NULL */ || !strm.state /* == Z_NULL */) { return Z_STREAM_ERROR; } + state = strm.state; + + if (state.wrap !== 0 && state.mode !== DICT) { + return Z_STREAM_ERROR; + } + + /* check for correct dictionary identifier */ + if (state.mode === DICT) { + dictid = 1; /* adler32(0, null, 0)*/ + /* dictid = adler32(dictid, dictionary, dictLength); */ + dictid = adler32(dictid, dictionary, dictLength, 0); + if (dictid !== state.check) { + return Z_DATA_ERROR; + } + } + /* copy dictionary to window using updatewindow(), which will amend the + existing dictionary if appropriate */ + ret = updatewindow(strm, dictionary, dictLength, dictLength); + if (ret) { + state.mode = MEM; + return Z_MEM_ERROR; + } + state.havedict = 1; + // Tracev((stderr, "inflate: dictionary set\n")); + return Z_OK; +} exports.inflateReset = inflateReset; exports.inflateReset2 = inflateReset2; @@ -1490,13 +1525,13 @@ exports.inflate = inflate; exports.inflateEnd = inflateEnd; exports.inflateGetHeader = inflateGetHeader; exports.inflateInfo = 'pako inflate (from Nodeca project)'; +exports.inflateSetDictionary = inflateSetDictionary; /* Not implemented exports.inflateCopy = inflateCopy; exports.inflateGetDictionary = inflateGetDictionary; exports.inflateMark = inflateMark; exports.inflatePrime = inflatePrime; -exports.inflateSetDictionary = inflateSetDictionary; exports.inflateSync = inflateSync; exports.inflateSyncPoint = inflateSyncPoint; exports.inflateUndermine = inflateUndermine; diff --git a/test/deflate.js b/test/deflate.js index bb61eaa..5bb15d2 100644 --- a/test/deflate.js +++ b/test/deflate.js @@ -170,3 +170,31 @@ describe('Deflate RAW', function () { }); }); + +describe('Deflate dictionary', function () { + it('trivial dictionary', function (done) { + var dict = new Buffer('abcdefghijklmnoprstuvwxyz'); + testSamples(zlib.createDeflate, pako.deflate, samples, { dictionary: dict }, done); + }); + + it('spdy dictionary', function (done) { + testSamples(zlib.createDeflate, pako.deflate, samples, { dictionary: helpers.spdyDict }, done); + }); + + it('handles multiple pushes', function () { + var dict = new Buffer('abcd'); + var deflate = new pako.Deflate({ dictionary: dict }); + + deflate.push(new Buffer('hello'), false); + deflate.push(new Buffer('hello'), false); + deflate.push(new Buffer(' world'), true); + + if (deflate.err) { throw new Error(deflate.err); } + + var uncompressed = pako.inflate(new Buffer(deflate.result), { dictionary: dict }); + + if (!helpers.cmpBuf(new Buffer('hellohello world'), uncompressed)) { + throw new Error('Result not equal for p -> z'); + } + }); +}); diff --git a/test/fixtures/spdy_dict.txt b/test/fixtures/spdy_dict.txt new file mode 100644 index 0000000..ca55052 --- /dev/null +++ b/test/fixtures/spdy_dict.txt @@ -0,0 +1 @@ +optionsgetheadpostputdeletetraceacceptaccept-charsetaccept-encodingaccept-languageauthorizationexpectfromhostif-modified-sinceif-matchif-none-matchif-rangeif-unmodifiedsincemax-forwardsproxy-authorizationrangerefererteuser-agent100101200201202203204205206300301302303304305306307400401402403404405406407408409410411412413414415416417500501502503504505accept-rangesageetaglocationproxy-authenticatepublicretry-afterservervarywarningwww-authenticateallowcontent-basecontent-encodingcache-controlconnectiondatetrailertransfer-encodingupgradeviawarningcontent-languagecontent-lengthcontent-locationcontent-md5content-rangecontent-typeetagexpireslast-modifiedset-cookieMondayTuesdayWednesdayThursdayFridaySaturdaySundayJanFebMarAprMayJunJulAugSepOctNovDecchunkedtext/htmlimage/pngimage/jpgimage/gifapplication/xmlapplication/xhtmltext/plainpublicmax-agecharset=iso-8859-1utf-8gzipdeflateHTTP/1.1statusversionurl\x0 diff --git a/test/helpers.js b/test/helpers.js index e4e72c1..8626009 100644 --- a/test/helpers.js +++ b/test/helpers.js @@ -165,8 +165,11 @@ function testInflate(samples, inflateOptions, deflateOptions, callback) { callback(); } +var spdyDict = new Buffer(fs.readFileSync(path.join(__dirname, 'fixtures', 'spdy_dict.txt'))); + exports.cmpBuf = cmpBuf; exports.testSamples = testSamples; exports.testInflate = testInflate; exports.loadSamples = loadSamples; +exports.spdyDict = spdyDict; diff --git a/test/inflate.js b/test/inflate.js index 2cef723..d755a4e 100644 --- a/test/inflate.js +++ b/test/inflate.js @@ -13,7 +13,6 @@ var testInflate = helpers.testInflate; var samples = helpers.loadSamples(); - describe('Inflate defaults', function () { it('inflate, no options', function (done) { @@ -28,6 +27,7 @@ describe('Inflate defaults', function () { var compressed_samples = helpers.loadSamples('samples_deflated_raw'); helpers.testSamples(zlib.createInflateRaw, pako.inflateRaw, compressed_samples, {}, done); }); + }); @@ -162,3 +162,32 @@ describe('Inflate RAW', function () { }); }); + +describe('Inflate with dictionary', function () { + it('should throw on the wrong dictionary', function () { + // var zCompressed = helpers.deflateSync('world', { dictionary: new Buffer('hello') }); + var zCompressed = new Buffer([ 120, 187, 6, 44, 2, 21, 43, 207, 47, 202, 73, 1, 0, 6, 166, 2, 41 ]); + + try { + pako.inflate(zCompressed, { dictionary: new Buffer('world') }); + } catch (err) { + if (err.message === 'stream error') { + return; + } + + throw err; + } + + throw new Error('Did not throw'); + }); + + it('trivial dictionary', function (done) { + var dict = new Buffer('abcdefghijklmnoprstuvwxyz'); + testInflate(samples, { dictionary: dict }, { dictionary: dict }, done); + }); + + it('spdy dictionary', function (done) { + testInflate(samples, { dictionary: helpers.spdyDict }, { dictionary: helpers.spdyDict }, done); + }); + +});