The logic of encoding Unicode in UTF-8 is basically:
- Up to 4 bytes per character can be used. The fewest number of bytes possible is used.
- Characters up to U+007F are encoded with a single byte.
- For multibyte sequences, the number of leading 1 bits in the first byte gives the number of bytes for the character. The rest of the bits of the first byte can be used to encode bits of the character.
- The continuation bytes begin with 10, and the other 6 bits encode bits of the character.
Here's a function I wrote a while back for encoding a JavaScript UTF-16 string in UTF-8:
function toUTF8Array(str) {
var utf8 = [];
for (var i=0; i < str.length; i++) {
var charcode = str.charCodeAt(i);
if (charcode < 0x80) utf8.push(charcode);
else if (charcode < 0x800) {
utf8.push(0xc0 | (charcode >> 6),
0x80 | (charcode & 0x3f));
}
else if (charcode < 0xd800 || charcode >= 0xe000) {
utf8.push(0xe0 | (charcode >> 12),
0x80 | ((charcode>>6) & 0x3f),
0x80 | (charcode & 0x3f));
}
// surrogate pair
else {
i++;
// UTF-16 encodes 0x10000-0x10FFFF by
// subtracting 0x10000 and splitting the
// 20 bits of 0x0-0xFFFFF into two halves
charcode = 0x10000 + (((charcode & 0x3ff)<<10)
| (str.charCodeAt(i) & 0x3ff));
utf8.push(0xf0 | (charcode >>18),
0x80 | ((charcode>>12) & 0x3f),
0x80 | ((charcode>>6) & 0x3f),
0x80 | (charcode & 0x3f));
}
}
return utf8;
}
Answer from Joni on Stack OverflowThe logic of encoding Unicode in UTF-8 is basically:
- Up to 4 bytes per character can be used. The fewest number of bytes possible is used.
- Characters up to U+007F are encoded with a single byte.
- For multibyte sequences, the number of leading 1 bits in the first byte gives the number of bytes for the character. The rest of the bits of the first byte can be used to encode bits of the character.
- The continuation bytes begin with 10, and the other 6 bits encode bits of the character.
Here's a function I wrote a while back for encoding a JavaScript UTF-16 string in UTF-8:
function toUTF8Array(str) {
var utf8 = [];
for (var i=0; i < str.length; i++) {
var charcode = str.charCodeAt(i);
if (charcode < 0x80) utf8.push(charcode);
else if (charcode < 0x800) {
utf8.push(0xc0 | (charcode >> 6),
0x80 | (charcode & 0x3f));
}
else if (charcode < 0xd800 || charcode >= 0xe000) {
utf8.push(0xe0 | (charcode >> 12),
0x80 | ((charcode>>6) & 0x3f),
0x80 | (charcode & 0x3f));
}
// surrogate pair
else {
i++;
// UTF-16 encodes 0x10000-0x10FFFF by
// subtracting 0x10000 and splitting the
// 20 bits of 0x0-0xFFFFF into two halves
charcode = 0x10000 + (((charcode & 0x3ff)<<10)
| (str.charCodeAt(i) & 0x3ff));
utf8.push(0xf0 | (charcode >>18),
0x80 | ((charcode>>12) & 0x3f),
0x80 | ((charcode>>6) & 0x3f),
0x80 | (charcode & 0x3f));
}
}
return utf8;
}
JavaScript Strings are stored in UTF-16. To get UTF-8, you'll have to convert the String yourself.
One way is to mix encodeURIComponent(), which will output UTF-8 bytes URL-encoded, with unescape, as mentioned on ecmanaut.
var utf8 = unescape(encodeURIComponent(str));
var arr = [];
for (var i = 0; i < utf8.length; i++) {
arr.push(utf8.charCodeAt(i));
}
You don't need to write a full-on UTF-8 encoder; there is a much easier JS idiom to convert a Unicode string into a string of bytes representing UTF-8 code units:
unescape(encodeURIComponent(str))
(This works because the odd encoding used by escape/unescape uses %xx hex sequences to represent ISO-8859-1 characters with that code, instead of UTF-8 as used by URI-component escaping. Similarly decodeURIComponent(escape(bytes)) goes in the other direction.)
So if you want an Array out it would be:
function toUTF8Array(str) {
var utf8= unescape(encodeURIComponent(str));
var arr= new Array(utf8.length);
for (var i= 0; i<utf8.length; i++)
arr[i]= utf8.charCodeAt(i);
return arr;
}
You can use this function (gist):
function toUTF8Array(str) {
var utf8 = [];
for (var i=0; i < str.length; i++) {
var charcode = str.charCodeAt(i);
if (charcode < 0x80) utf8.push(charcode);
else if (charcode < 0x800) {
utf8.push(0xc0 | (charcode >> 6),
0x80 | (charcode & 0x3f));
}
else if (charcode < 0xd800 || charcode >= 0xe000) {
utf8.push(0xe0 | (charcode >> 12),
0x80 | ((charcode>>6) & 0x3f),
0x80 | (charcode & 0x3f));
}
else {
// let's keep things simple and only handle chars up to U+FFFF...
utf8.push(0xef, 0xbf, 0xbd); // U+FFFE "replacement character"
}
}
return utf8;
}
Example of use:
>>> toUTF8Array("中€")
[228, 184, 173, 226, 130, 172]
If you want negative numbers for values over 127, like Java's byte-to-int conversion does, you have to tweak the constants and use
utf8.push(0xffffffc0 | (charcode >> 6),
0xffffff80 | (charcode & 0x3f));
and
utf8.push(0xffffffe0 | (charcode >> 12),
0xffffff80 | ((charcode>>6) & 0x3f),
0xffffff80 | (charcode & 0x3f));
You can use TextEncoder which is part of the Encoding Living Standard. According to the Encoding API entry from the Chromium Dashboard, it shipped in Firefox and will ship in Chrome 38. There is also a text-encoding polyfill available.
The JavaScript code sample below returns a Uint8Array filled with the values you expect.
var s = "test.message";
var encoder = new TextEncoder();
encoder.encode(s);
// [116, 101, 115, 116, 46, 109, 101, 115, 115, 97, 103, 101]
JavaScript has no concept of character encoding for String, everything is in UTF-16. Most of time time the value of a char in UTF-16 matches UTF-8, so you can forget it's any different.
There are more optimal ways to do this but
function s(x) {return x.charCodeAt(0);}
"test.message".split('').map(s);
// [116, 101, 115, 116, 46, 109, 101, 115, 115, 97, 103, 101]
So what is unescape(encodeURIComponent(str)) doing? Let's look at each individually,
encodeURIComponentis converting every character instrwhich is illegal or has a meaning in URI Syntax into a URI escaped version so that there is no problem using it as a key or value in the search component of a URI, for exampleencodeURIComponent('&='); // "%26%3D"Notice how this is now a 6 character long String.unescapeis actually depreciated, but it does a similar job todecodeURIordecodeURIComponent(the reverse ofencodeURIComponent). If we look in the ES5 spec we can see11. Let c be the character whose code unit value is the integer represented by the four hexadecimal digits at positions k+2, k+3, k+4, and k+5 within Result(1).
So,4digits is2bytes is "UTF-8", however as I mentioned, all Strings are UTF-16, so it's really a UTF-16 string limiting itself to UTF-8.