0x5eal/rbxts-pako
1
0
Fork 0
mirror of https://github.com/0x5eal/rbxts-pako.git synced 2025-04-05 11:20:58 +01:00
Code Issues Projects Releases Packages Wiki Activity

Optimized utf8 decoder

Browse source
This commit is contained in:
Vitaly Puzrin 2014-04-14 00:46:30 +04:00
parent b949c39564
commit 554ffaf908
1 changed files with 82 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

111
lib/utils/strings.js
View file

@ -10,6 +10,13 @@ var STR_APPLY_OK = true;
try { String.fromCharCode.apply(null, [0]); } catch(__) { STR_APPLY_OK = false; }
// Table with utf8 lengths
var utf8len = new utils.Buf8(256);
for (var i=0; i<256; i++) {
utf8len[i] = (i >= 252 ? 6 : i >= 248 ? 5 : i >= 240 ? 4 : i >= 224 ? 3 : i >= 192 ? 2 : 1);
}
// convert string to array (typed, when possible)
// src: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Base64_encoding_and_decoding
exports.string2buf = function (str) {
@ -68,34 +75,6 @@ exports.string2buf = function (str) {
};
// convert array to string
// src: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Base64_encoding_and_decoding
exports.buf2string = function (buf, max) {
var str = '';
for (var part, len = max || buf.length, i = 0; i < len; i++) {
part = buf[i];
str += String.fromCharCode(
part > 251 && part < 254 && i + 5 < len ? /* six bytes */
/* (part - 252 << 32) is not possible in ECMAScript! So...: */
(part - 252) * 1073741824 + (buf[++i] - 128 << 24) + (buf[++i] - 128 << 18) + (buf[++i] - 128 << 12) + (buf[++i] - 128 << 6) + buf[++i] - 128
: part > 247 && part < 252 && i + 4 < len ? /* five bytes */
(part - 248 << 24) + (buf[++i] - 128 << 18) + (buf[++i] - 128 << 12) + (buf[++i] - 128 << 6) + buf[++i] - 128
: part > 239 && part < 248 && i + 3 < len ? /* four bytes */
(part - 240 << 18) + (buf[++i] - 128 << 12) + (buf[++i] - 128 << 6) + buf[++i] - 128
: part > 223 && part < 240 && i + 2 < len ? /* three bytes */
(part - 224 << 12) + (buf[++i] - 128 << 6) + buf[++i] - 128
: part > 191 && part < 224 && i + 1 < len ? /* two bytes */
(part - 192 << 6) + buf[++i] - 128
: /* part < 127 ? */ /* one byte */
part
);
}
return str;
};
// Convert byte array to binary string
exports.buf2binstring = function(buf) {
// use fallback for big arrays to avoid stack overflow
@ -121,7 +100,81 @@ exports.binstring2buf = function(str) {
};
// convert array to string
// src: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Base64_encoding_and_decoding
exports.buf2string = function (buf, max) {
/*jshint nonstandard:true*/
// That's not as fast as via String.fromCharCode.appy
/*return decodeURIComponent(escape(exports.buf2binstring(
(buf.length === max) ?
buf
:
buf.subarray ? buf.subarray(0, max) : buf.slice(0, max)
)));*/
var str = '', i, out, part, char_len;
var len = max || buf.length;
var out_len = 0;
var utf16buf;
// Calculate output length
for (i=0; i < len;) {
i += utf8len[buf[i]];
out_len++;
}
utf16buf = new utils.Buf32(out_len);
for (out=0, i=0; i<len; i++) {
part = buf[i];
char_len = utf8len[part];
// edge case - broken sequence
if (i + char_len > len) {
utf16buf[out++] = 0x7f;
break;
}
switch (char_len) {
case 1:
utf16buf[out++] = part;
break;
case 2:
utf16buf[out++] = ((part & 0x1f) << 6) | (buf[++i] & 0x7f);
break;
case 3:
utf16buf[out++] = ((part & 0x0f) << 12) | ((buf[++i] & 0x3f) << 6) | (buf[++i] & 0x3f);
break;
case 4:
utf16buf[out++] = ((part & 0x07) << 18) | ((buf[++i] & 0x3f) << 12) | ((buf[++i] & 0x3f) << 6) + (buf[++i] & 0x3f);
break;
// 5 & 6 bytes uticodes not supported in UTF16 (JS), so fill with dummy symbol
case 5:
i += 4;
utf16buf[out++] = 0x7f;
//utf16buf[out++] = (part - 248 << 24) + (buf[++i] - 128 << 18) + (buf[++i] - 128 << 12) + (buf[++i] - 128 << 6) + buf[++i] - 128;
break;
case 6:
i += 5;
utf16buf[out++] = 0x7f;
// (part - 252 << 32) is not possible in ECMAScript! So...:
//utf16buf[out++] = (part - 252) * 1073741824 + (buf[++i] - 128 << 24) + (buf[++i] - 128 << 18) + (buf[++i] - 128 << 12) + (buf[++i] - 128 << 6) + buf[++i] - 128;
break;
}
}
if (STR_APPLY_OK) {
return String.fromCharCode.apply(null, utf16buf);
}
// Fallback, when String.fromCharCode.apply not available
for (i=0, len=utf16buf.length; i<len; i++) {
str += String.fromCharCode(utf16buf[i]);
}
return str;
};
// calculate tail size of utf8 char by current byte value
exports.utf8tail = function(code) {
return code >= 252 ? 6 : code >= 248 ? 5 : code >= 240 ? 4 : code >= 224 ? 3 : code >= 192 ? 2 : 1;
return utf8len[code];
};