string_decoder: Add 'end' method, do base64 properly

doc/api/string_decoder.markdown

@@ -19,6 +19,10 @@ additional support for utf8.
 
 Accepts a single argument, `encoding` which defaults to `utf8`.
 
-### StringDecoder.write(buffer)
+### decoder.write(buffer)
 
-Returns a decoded string.
+Returns a decoded string.
+
+### decoder.end()
+
+Returns any trailing bytes that were left in the buffer.

lib/fs.js

@@ -1402,6 +1402,11 @@ ReadStream.prototype._read = function() {
  }
 
  if (bytesRead === 0) {
+ if (this._decoder) {
+ var ret = this._decoder.end();
+ if (ret)
+ this.emit('data', ret);
+ }
  self.emit('end');
  self.destroy();
  return;

lib/http.js

@@ -363,6 +363,11 @@ IncomingMessage.prototype._emitData = function(d) {
 
 IncomingMessage.prototype._emitEnd = function() {
  if (!this._endEmitted) {
+ if (this._decoder) {
+ var ret = this._decoder.end();
+ if (ret)
+ this.emit('data', ret);
+ }
  this.emit('end');
  }
 
@@ -1859,6 +1864,11 @@ Client.prototype.request = function(method, path, headers) {
  // but it will get removed when we remove this legacy interface.
  c.on('socket', function(s) {
  s.on('end', function() {
+ if (self._decoder) {
+ var ret = self._decoder.end();
+ if (ret)
+ self.emit('data', ret);
+ }
  self.emit('end');
  });
  });

lib/net.js

@@ -414,6 +414,11 @@ function onread(buffer, offset, length) {
  if (!self.writable) self._destroy();
 
  if (!self.allowHalfOpen) self.end();
+ if (self._decoder) {
+ var ret = self._decoder.end();
+ if (ret)
+ self.emit('data', ret);
+ }
  if (self._events && self._events['end']) self.emit('end');
  if (self.onend) self.onend();
  } else {

lib/string_decoder.js

@@ -32,6 +32,11 @@ var StringDecoder = exports.StringDecoder = function(encoding) {
  this.surrogateSize = 2;
  this.detectIncompleteChar = utf16DetectIncompleteChar;
  break;
+ case 'base64':
+ // Base-64 stores 3 bytes in 4 chars, and pads the remainder.
+ this.surrogateSize = 3;
+ this.detectIncompleteChar = base64DetectIncompleteChar;
+ break;
  default:
  this.write = passThroughWrite;
  return;
@@ -145,6 +150,21 @@ StringDecoder.prototype.detectIncompleteChar = function(buffer) {
  return i;
 };
 
+StringDecoder.prototype.end = function(buffer) {
+ var res = '';
+ if (buffer && buffer.length)
+ res = this.write(buffer);
+
+ if (this.charReceived) {
+ var cr = this.charReceived;
+ var buf = this.charBuffer;
+ var enc = this.encoding;
+ res += buf.slice(0, cr).toString(enc);
+ }
+
+ return res;
+};
+
 function passThroughWrite(buffer) {
  return buffer.toString(this.encoding);
 }
@@ -154,3 +174,9 @@ function utf16DetectIncompleteChar(buffer) {
  this.charLength = incomplete ? 2 : 0;
  return incomplete;
 }
+
+function base64DetectIncompleteChar(buffer) {
+ var incomplete = this.charReceived = buffer.length % 3;
+ this.charLength = incomplete ? 3 : 0;
+ return incomplete;
+}

lib/tls.js

@@ -967,6 +967,11 @@ SecurePair.prototype.destroy = function() {
  self.cleartext.writable = self.cleartext.readable = false;
 
  process.nextTick(function() {
+ if (self.cleartext._decoder) {
+ var ret = self.cleartext._decoder.end();
+ if (ret)
+ self.cleartext.emit('data', ret);
+ }
  self.cleartext.emit('end');
  self.encrypted.emit('close');
  self.cleartext.emit('close');

test/simple/test-string-decoder-end.js

@@ -0,0 +1,75 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// verify that the string decoder works getting 1 byte at a time,
+// the whole buffer at once, and that both match the .toString(enc)
+// result of the entire buffer.
+
+var assert = require('assert');
+var SD = require('string_decoder').StringDecoder;
+var encodings = ['base64', 'hex', 'utf8', 'utf16le', 'ucs2'];
+
+var bufs = [ '☃💩', 'asdf' ].map(function(b) {
+ return new Buffer(b);
+});
+
+// also test just arbitrary bytes from 0-15.
+for (var i = 1; i <= 16; i++) {
+ var bytes = new Array(i).join('.').split('.').map(function(_, j) {
+ return j + 0x78;
+ });
+ bufs.push(new Buffer(bytes));
+}
+
+encodings.forEach(testEncoding);
+
+console.log('ok');
+
+function testEncoding(encoding) {
+ bufs.forEach(function(buf) {
+ testBuf(encoding, buf);
+ });
+}
+
+function testBuf(encoding, buf) {
+ console.error('# %s', encoding, buf);
+
+ // write one byte at a time.
+ var s = new SD(encoding);
+ var res1 = '';
+ for (var i = 0; i < buf.length; i++) {
+ res1 += s.write(buf.slice(i, i + 1));
+ }
+ res1 += s.end();
+
+ // write the whole buffer at once.
+ var res2 = '';
+ var s = new SD(encoding);
+ res2 += s.write(buf);
+ res2 += s.end();
+
+ // .toString() on the buffer
+ var res3 = buf.toString(encoding);
+
+ console.log('expect=%j', res3);
+ assert.equal(res1, res3, 'one byte at a time should match toString');
+ assert.equal(res2, res3, 'all bytes at once should match toString');
+}