Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add byteLength method and hasState property #258

Open
wants to merge 19 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Apply ESLint to tests
  • Loading branch information
ashtuchkin committed Jul 17, 2020
commit 5da0746f46b3fa92d94eeedaedf2a7e53fb49ff6
1 change: 0 additions & 1 deletion .eslintignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,3 @@ package-lock.json

# Temporarily excluded
/generation/research
/test
58 changes: 30 additions & 28 deletions test/bom-test.js
Original file line number Diff line number Diff line change
@@ -1,83 +1,85 @@
var assert = require('assert'),
"use strict";

const assert = require('assert'),
Buffer = require('safer-buffer').Buffer,
iconv = require(__dirname+'/../');

var sampleStr = '<?xml version="1.0" encoding="UTF-8"?>\n<俄语>данные</俄语>';
const sampleStr = '<?xml version="1.0" encoding="UTF-8"?>\n<俄语>данные</俄语>',
strBOM = '\ufeff',
utf8BOM = Buffer.from([0xEF, 0xBB, 0xBF]),
utf16beBOM = Buffer.from([0xFE, 0xFF]),
utf16leBOM = Buffer.from([0xFF, 0xFE]);

describe("BOM Handling", function() {
it("strips UTF-8 BOM", function() {
var body = Buffer.concat([utf8BOM, Buffer.from(sampleStr)]);
const body = Buffer.concat([utf8BOM, Buffer.from(sampleStr)]);
assert.equal(iconv.decode(body, 'utf8'), sampleStr);
});

it("strips UTF-16 BOM", function() {
var body = Buffer.concat([utf16leBOM, iconv.encode(sampleStr, 'utf16le')]);
const body = Buffer.concat([utf16leBOM, iconv.encode(sampleStr, 'utf16le')]);
assert.equal(iconv.decode(body, 'utf16'), sampleStr);
assert.equal(iconv.decode(body, 'utf16le'), sampleStr);

var body = Buffer.concat([utf16beBOM, iconv.encode(sampleStr, 'utf16be')]);
assert.equal(iconv.decode(body, 'utf16'), sampleStr);
assert.equal(iconv.decode(body, 'utf16be'), sampleStr);
const body2 = Buffer.concat([utf16beBOM, iconv.encode(sampleStr, 'utf16be')]);
assert.equal(iconv.decode(body2, 'utf16'), sampleStr);
assert.equal(iconv.decode(body2, 'utf16be'), sampleStr);
});

it("doesn't strip BOMs when stripBOM=false", function() {
var body = Buffer.concat([utf8BOM, Buffer.from(sampleStr)]);
const body = Buffer.concat([utf8BOM, Buffer.from(sampleStr)]);
assert.equal(iconv.decode(body, 'utf8', {stripBOM: false}), strBOM + sampleStr);

var body = Buffer.concat([utf16leBOM, iconv.encode(sampleStr, 'utf16le')]);
assert.equal(iconv.decode(body, 'utf16', {stripBOM: false}), strBOM + sampleStr);
assert.equal(iconv.decode(body, 'utf16le', {stripBOM: false}), strBOM + sampleStr);
const body2 = Buffer.concat([utf16leBOM, iconv.encode(sampleStr, 'utf16le')]);
assert.equal(iconv.decode(body2, 'utf16', {stripBOM: false}), strBOM + sampleStr);
assert.equal(iconv.decode(body2, 'utf16le', {stripBOM: false}), strBOM + sampleStr);

var body = Buffer.concat([utf16beBOM, iconv.encode(sampleStr, 'utf16be')]);
assert.equal(iconv.decode(body, 'utf16', {stripBOM: false}), strBOM + sampleStr);
assert.equal(iconv.decode(body, 'utf16be', {stripBOM: false}), strBOM + sampleStr);
const body3 = Buffer.concat([utf16beBOM, iconv.encode(sampleStr, 'utf16be')]);
assert.equal(iconv.decode(body3, 'utf16', {stripBOM: false}), strBOM + sampleStr);
assert.equal(iconv.decode(body3, 'utf16be', {stripBOM: false}), strBOM + sampleStr);
});

it("adds/strips UTF-7 BOM", function() {
var bodyWithBOM = iconv.encode(sampleStr, 'utf7', {addBOM: true});
var body = iconv.encode(sampleStr, 'utf7');
const bodyWithBOM = iconv.encode(sampleStr, 'utf7', {addBOM: true});
const body = iconv.encode(sampleStr, 'utf7');
assert.notEqual(body.toString('hex'), bodyWithBOM.toString('hex'));
assert.equal(iconv.decode(body, 'utf7'), sampleStr);
});

it("adds UTF-8 BOM when addBOM=true", function() {
var body = Buffer.concat([utf8BOM, Buffer.from(sampleStr)]).toString('hex');
const body = Buffer.concat([utf8BOM, Buffer.from(sampleStr)]).toString('hex');
assert.equal(iconv.encode(sampleStr, 'utf8', {addBOM: true}).toString('hex'), body);
});

it("adds UTF-16 BOMs when addBOM=true", function() {
var body = Buffer.concat([utf16leBOM, iconv.encode(sampleStr, 'utf16le')]).toString('hex');
const body = Buffer.concat([utf16leBOM, iconv.encode(sampleStr, 'utf16le')]).toString('hex');
assert.equal(iconv.encode(sampleStr, 'utf16le', {addBOM: true}).toString('hex'), body);

var body = Buffer.concat([utf16beBOM, iconv.encode(sampleStr, 'utf16be')]).toString('hex');
assert.equal(iconv.encode(sampleStr, 'utf16be', {addBOM: true}).toString('hex'), body);
const body2 = Buffer.concat([utf16beBOM, iconv.encode(sampleStr, 'utf16be')]).toString('hex');
assert.equal(iconv.encode(sampleStr, 'utf16be', {addBOM: true}).toString('hex'), body2);
});

it("'UTF-16' encoding adds BOM by default, but can be overridden with addBOM=false", function() {
var body = Buffer.concat([utf16leBOM, iconv.encode(sampleStr, 'utf16le')]).toString('hex');
const body = Buffer.concat([utf16leBOM, iconv.encode(sampleStr, 'utf16le')]).toString('hex');
assert.equal(iconv.encode(sampleStr, 'utf16').toString('hex'), body);

var body = Buffer.concat([iconv.encode(sampleStr, 'utf16le')]).toString('hex');
assert.equal(iconv.encode(sampleStr, 'utf16', {addBOM: false}).toString('hex'), body);
const body2 = Buffer.concat([iconv.encode(sampleStr, 'utf16le')]).toString('hex');
assert.equal(iconv.encode(sampleStr, 'utf16', {addBOM: false}).toString('hex'), body2);
});


it("when stripping BOM, calls callback 'stripBOM' if provided", function() {
var bomStripped = false;
var stripBOM = function() { bomStripped = true; }
let bomStripped = false;
const stripBOM = function() { bomStripped = true; }

var body = Buffer.concat([utf8BOM, Buffer.from(sampleStr)]);
const body = Buffer.concat([utf8BOM, Buffer.from(sampleStr)]);
assert.equal(iconv.decode(body, 'utf8', {stripBOM: stripBOM}), sampleStr);
assert(bomStripped);

bomStripped = false;

body = Buffer.from(sampleStr);
assert.equal(iconv.decode(body, 'utf8', {stripBOM: stripBOM}), sampleStr);
const body2 = Buffer.from(sampleStr);
assert.equal(iconv.decode(body2, 'utf8', {stripBOM: stripBOM}), sampleStr);
assert(!bomStripped);
});

Expand Down
10 changes: 5 additions & 5 deletions test/dbcs-test.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
var fs = require('fs'),
assert = require('assert'),
var assert = require('assert'),
Buffer = require('safer-buffer').Buffer,
iconv = require(__dirname+'/../'),
Iconv = require('iconv').Iconv;
Expand Down Expand Up @@ -167,13 +166,14 @@ describe("Full DBCS encoding tests #full", function() {
var converter = new Iconv(aliases[enc] || enc, "utf-8");
var errors = [];
forAllChars(converter.convert.bind(converter), function(valid, inp, outp) {
var strActual = iconv.decode(inp, enc);
const strActual = iconv.decode(inp, enc);

if (0xE000 <= strActual.charCodeAt(0) && strActual.charCodeAt(0) < 0xF900) // Skip Private use area.
return;

let strExpected;
if (valid) {
var strExpected = outp.toString('utf-8');
strExpected = outp.toString('utf-8');
if (strActual === strExpected)
return;

Expand All @@ -184,7 +184,7 @@ describe("Full DBCS encoding tests #full", function() {
return;

} else {
var strExpected = "�";
strExpected = "�";
if (strActual[0] === "�")
return;

Expand Down
10 changes: 5 additions & 5 deletions test/gbk-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -53,19 +53,19 @@ describe("GBK tests", function() {

it("GB18030 findIdx works correctly", function() {
function findIdxAlternative(table, val) {
for (var i = 0; i < table.length; i++)
for (let i = 0; i < table.length; i++)
if (table[i] > val)
return i-1;
return table.length - 1;
}

var codec = iconv.getEncoder('gb18030');
const codec = iconv.getEncoder('gb18030');

for (var i = 0; i < 0x100; i++)
for (let i = 0; i < 0x100; i++)
assert.strictEqual(codec.findIdx(codec.gb18030.uChars, i), findIdxAlternative(codec.gb18030.uChars, i), i);

var tests = [0xFFFF, 0x10000, 0x10001, 0x30000];
for (var i = 0; i < tests.length; i++)
const tests = [0xFFFF, 0x10000, 0x10001, 0x30000];
for (let i = 0; i < tests.length; i++)
assert.strictEqual(codec.findIdx(codec.gb18030.uChars, tests[i]), findIdxAlternative(codec.gb18030.uChars, tests[i]), tests[i]);
});

Expand Down
47 changes: 24 additions & 23 deletions test/performance.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,46 +25,47 @@ for (var i = 0; i < 13; i++) {
console.log('\n' + encoding + ' charset performance test:');
console.log("\nEncoding "+str.length+" chars "+convertTimes+" times:");

var start = Date.now();
var converter = new iconv.Iconv("utf8", encoding);
for (var i = 0; i < convertTimes; i++) {
var b = converter.convert(str);
let start = Date.now();
let converter = new iconv.Iconv("utf8", encoding);
let b;
for (let i = 0; i < convertTimes; i++) {
b = converter.convert(str);
}
var duration = Date.now() - start;
var mbs = convertTimes*b.length/duration/1024;
let duration = Date.now() - start;
let mbs = convertTimes*b.length/duration/1024;

console.log("iconv: "+duration+"ms, "+mbs.toFixed(2)+" Mb/s.");

var start = Date.now();
for (var i = 0; i < convertTimes; i++) {
var b = iconv_lite.encode(str, encoding);
start = Date.now();
for (let i = 0; i < convertTimes; i++) {
b = iconv_lite.encode(str, encoding);
}
var duration = Date.now() - start;
var mbs = convertTimes*b.length/duration/1024;
duration = Date.now() - start;
mbs = convertTimes*b.length/duration/1024;

console.log("iconv-lite: "+duration+"ms, "+mbs.toFixed(2)+" Mb/s.");


// Test decoding.
var buf = iconv_lite.encode(str, encoding);
const buf = iconv_lite.encode(str, encoding);
console.log("\nDecoding "+buf.length+" bytes "+convertTimes+" times:");

var start = Date.now();
var converter = new iconv.Iconv(encoding, "utf8");
for (var i = 0; i < convertTimes; i++) {
var s = converter.convert(buf).toString();
start = Date.now();
converter = new iconv.Iconv(encoding, "utf8");
for (let i = 0; i < convertTimes; i++) {
converter.convert(buf).toString();
}
var duration = Date.now() - start;
var mbs = convertTimes*buf.length/duration/1024;
duration = Date.now() - start;
mbs = convertTimes*buf.length/duration/1024;

console.log("iconv: "+duration+"ms, "+mbs.toFixed(2)+" Mb/s.");

var start = Date.now();
for (var i = 0; i < convertTimes; i++) {
var s = iconv_lite.decode(buf, encoding);
start = Date.now();
for (let i = 0; i < convertTimes; i++) {
iconv_lite.decode(buf, encoding);
}
var duration = Date.now() - start;
var mbs = convertTimes*buf.length/duration/1024;
duration = Date.now() - start;
mbs = convertTimes*buf.length/duration/1024;

console.log("iconv-lite: "+duration+"ms, "+mbs.toFixed(2)+" Mb/s.");

8 changes: 3 additions & 5 deletions test/sbcs-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ var aliases = {

function iconvAlias(enc) {
var r;
if (r = /windows(\d+)/.exec(enc))
if ((r = /windows(\d+)/.exec(enc)))
return "WINDOWS-"+r[1];
if (r = /iso8859(\d+)/.exec(enc))
if ((r = /iso8859(\d+)/.exec(enc)))
return "ISO8859-"+r[1];
if (r = /koi8(\w+)/.exec(enc))
if ((r = /koi8(\w+)/.exec(enc)))
return "KOI8-"+r[1];
if (aliases[enc])
return aliases[enc];
Expand All @@ -49,8 +49,6 @@ function strToHex(str) { return spacify4(swapBytes(Buffer.from(str, 'ucs2')).toS
// Generate tests for all SBCS encodings.
iconv.encode('', 'utf8'); // Load all encodings.


var sbcsEncodingTests = {};
describe("Full SBCS encoding tests #full", function() {
if (!process.env.FULL_TEST_SUITE) return;
this.timeout(10000);
Expand Down
2 changes: 1 addition & 1 deletion test/streams-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ function checkStreamOutput(options) {
if (options.checkOutput) {
if (options.outputType)
var r;
if (r = /^buffer-?(.*)/.exec(options.outputType)) {
if ((r = /^buffer-?(.*)/.exec(options.outputType))) {
res = Buffer.concat(res);
if (r[1])
res = res.toString(r[1]); // Convert to string to make comparing buffers easier.
Expand Down