Single Buffer
If you have a single Buffer you can use its toString method that will convert all or part of the binary contents to a string using a specific encoding. It defaults to utf8 if you don't provide a parameter, but I've explicitly set the encoding in this example.
var req = http.request(reqOptions, function(res) {
...
res.on('data', function(chunk) {
var textChunk = chunk.toString('utf8');
// process utf8 text chunk
});
});
Streamed Buffers
If you have streamed buffers like in the question above where the first byte of a multi-byte UTF8-character may be contained in the first Buffer (chunk) and the second byte in the second Buffer then you should use a StringDecoder. :
var StringDecoder = require('string_decoder').StringDecoder;
var req = http.request(reqOptions, function(res) {
...
var decoder = new StringDecoder('utf8');
res.on('data', function(chunk) {
var textChunk = decoder.write(chunk);
// process utf8 text chunk
});
});
This way bytes of incomplete characters are buffered by the StringDecoder until all required bytes were written to the decoder.
Single Buffer
If you have a single Buffer you can use its toString method that will convert all or part of the binary contents to a string using a specific encoding. It defaults to utf8 if you don't provide a parameter, but I've explicitly set the encoding in this example.
var req = http.request(reqOptions, function(res) {
...
res.on('data', function(chunk) {
var textChunk = chunk.toString('utf8');
// process utf8 text chunk
});
});
Streamed Buffers
If you have streamed buffers like in the question above where the first byte of a multi-byte UTF8-character may be contained in the first Buffer (chunk) and the second byte in the second Buffer then you should use a StringDecoder. :
var StringDecoder = require('string_decoder').StringDecoder;
var req = http.request(reqOptions, function(res) {
...
var decoder = new StringDecoder('utf8');
res.on('data', function(chunk) {
var textChunk = decoder.write(chunk);
// process utf8 text chunk
});
});
This way bytes of incomplete characters are buffered by the StringDecoder until all required bytes were written to the decoder.
var fs = require("fs");
function readFileLineByLine(filename, processline) {
var stream = fs.createReadStream(filename);
var s = "";
stream.on("data", function(data) {
s += data.toString('utf8');
var lines = s.split("\n");
for (var i = 0; i < lines.length - 1; i++)
processline(lines[i]);
s = lines[lines.length - 1];
});
stream.on("end",function() {
var lines = s.split("\n");
for (var i = 0; i < lines.length; i++)
processline(lines[i]);
});
}
var linenumber = 0;
readFileLineByLine(filename, function(line) {
console.log(++linenumber + " -- " + line);
});
Buffer.toString('utf8') appears to use wtf-8
Buffer.toString('utf8') seems to change the output
Buffer to UTF8 String conversion DoS in node.js and io.js
NodeJS inserting unwanted characters into my string, stdout showing garbage!
» npm install utf8-buffer