The logic of encoding Unicode in UTF-8 is basically:
- Up to 4 bytes per character can be used. The fewest number of bytes possible is used.
- Characters up to U+007F are encoded with a single byte.
- For multibyte sequences, the number of leading 1 bits in the first byte gives the number of bytes for the character. The rest of the bits of the first byte can be used to encode bits of the character.
- The continuation bytes begin with 10, and the other 6 bits encode bits of the character.
Here's a function I wrote a while back for encoding a JavaScript UTF-16 string in UTF-8:
function toUTF8Array(str) {
var utf8 = [];
for (var i=0; i < str.length; i++) {
var charcode = str.charCodeAt(i);
if (charcode < 0x80) utf8.push(charcode);
else if (charcode < 0x800) {
utf8.push(0xc0 | (charcode >> 6),
0x80 | (charcode & 0x3f));
}
else if (charcode < 0xd800 || charcode >= 0xe000) {
utf8.push(0xe0 | (charcode >> 12),
0x80 | ((charcode>>6) & 0x3f),
0x80 | (charcode & 0x3f));
}
// surrogate pair
else {
i++;
// UTF-16 encodes 0x10000-0x10FFFF by
// subtracting 0x10000 and splitting the
// 20 bits of 0x0-0xFFFFF into two halves
charcode = 0x10000 + (((charcode & 0x3ff)<<10)
| (str.charCodeAt(i) & 0x3ff));
utf8.push(0xf0 | (charcode >>18),
0x80 | ((charcode>>12) & 0x3f),
0x80 | ((charcode>>6) & 0x3f),
0x80 | (charcode & 0x3f));
}
}
return utf8;
}
Answer from Joni on Stack OverflowThe logic of encoding Unicode in UTF-8 is basically:
- Up to 4 bytes per character can be used. The fewest number of bytes possible is used.
- Characters up to U+007F are encoded with a single byte.
- For multibyte sequences, the number of leading 1 bits in the first byte gives the number of bytes for the character. The rest of the bits of the first byte can be used to encode bits of the character.
- The continuation bytes begin with 10, and the other 6 bits encode bits of the character.
Here's a function I wrote a while back for encoding a JavaScript UTF-16 string in UTF-8:
function toUTF8Array(str) {
var utf8 = [];
for (var i=0; i < str.length; i++) {
var charcode = str.charCodeAt(i);
if (charcode < 0x80) utf8.push(charcode);
else if (charcode < 0x800) {
utf8.push(0xc0 | (charcode >> 6),
0x80 | (charcode & 0x3f));
}
else if (charcode < 0xd800 || charcode >= 0xe000) {
utf8.push(0xe0 | (charcode >> 12),
0x80 | ((charcode>>6) & 0x3f),
0x80 | (charcode & 0x3f));
}
// surrogate pair
else {
i++;
// UTF-16 encodes 0x10000-0x10FFFF by
// subtracting 0x10000 and splitting the
// 20 bits of 0x0-0xFFFFF into two halves
charcode = 0x10000 + (((charcode & 0x3ff)<<10)
| (str.charCodeAt(i) & 0x3ff));
utf8.push(0xf0 | (charcode >>18),
0x80 | ((charcode>>12) & 0x3f),
0x80 | ((charcode>>6) & 0x3f),
0x80 | (charcode & 0x3f));
}
}
return utf8;
}
JavaScript Strings are stored in UTF-16. To get UTF-8, you'll have to convert the String yourself.
One way is to mix encodeURIComponent(), which will output UTF-8 bytes URL-encoded, with unescape, as mentioned on ecmanaut.
var utf8 = unescape(encodeURIComponent(str));
var arr = [];
for (var i = 0; i < utf8.length; i++) {
arr.push(utf8.charCodeAt(i));
}
» npm install utf8-string-bytes
» npm install utf8-bytes
You don't need to write a full-on UTF-8 encoder; there is a much easier JS idiom to convert a Unicode string into a string of bytes representing UTF-8 code units:
unescape(encodeURIComponent(str))
(This works because the odd encoding used by escape/unescape uses %xx hex sequences to represent ISO-8859-1 characters with that code, instead of UTF-8 as used by URI-component escaping. Similarly decodeURIComponent(escape(bytes)) goes in the other direction.)
So if you want an Array out it would be:
function toUTF8Array(str) {
var utf8= unescape(encodeURIComponent(str));
var arr= new Array(utf8.length);
for (var i= 0; i<utf8.length; i++)
arr[i]= utf8.charCodeAt(i);
return arr;
}
You can use this function (gist):
function toUTF8Array(str) {
var utf8 = [];
for (var i=0; i < str.length; i++) {
var charcode = str.charCodeAt(i);
if (charcode < 0x80) utf8.push(charcode);
else if (charcode < 0x800) {
utf8.push(0xc0 | (charcode >> 6),
0x80 | (charcode & 0x3f));
}
else if (charcode < 0xd800 || charcode >= 0xe000) {
utf8.push(0xe0 | (charcode >> 12),
0x80 | ((charcode>>6) & 0x3f),
0x80 | (charcode & 0x3f));
}
else {
// let's keep things simple and only handle chars up to U+FFFF...
utf8.push(0xef, 0xbf, 0xbd); // U+FFFE "replacement character"
}
}
return utf8;
}
Example of use:
>>> toUTF8Array("中€")
[228, 184, 173, 226, 130, 172]
If you want negative numbers for values over 127, like Java's byte-to-int conversion does, you have to tweak the constants and use
utf8.push(0xffffffc0 | (charcode >> 6),
0xffffff80 | (charcode & 0x3f));
and
utf8.push(0xffffffe0 | (charcode >> 12),
0xffffff80 | ((charcode>>6) & 0x3f),
0xffffff80 | (charcode & 0x3f));
Using TextEncoder and TextDecoder
var uint8array = new TextEncoder("utf-8").encode("Plain Text");
var string = new TextDecoder().decode(uint8array);
console.log(uint8array ,string )
Run code snippetEdit code snippet Hide Results Copy to answer Expand
function stringToUint(string) {
var string = btoa(unescape(encodeURIComponent(string))),
charList = string.split(''),
uintArray = [];
for (var i = 0; i < charList.length; i++) {
uintArray.push(charList[i].charCodeAt(0));
}
return new Uint8Array(uintArray);
}
function uintToString(uintArray) {
var encodedString = String.fromCharCode.apply(null, uintArray),
decodedString = decodeURIComponent(escape(atob(encodedString)));
return decodedString;
}
I have done, with some help from the internet, these little functions, they should solve your problems! Here is the working JSFiddle.
EDIT:
Since the source of the Uint8Array is external and you can't use atob you just need to remove it(working fiddle):
function uintToString(uintArray) {
var encodedString = String.fromCharCode.apply(null, uintArray),
decodedString = decodeURIComponent(escape(encodedString));
return decodedString;
}
Warning: escape and unescape is removed from web standards. See this.