I'm trying to convert a unicode string to a hexadecimal representation in javascript.
This is what I have:
function convertFromHex(hex) {
var hex = hex.toString();//force conversion
var str = '';
for (var i = 0; i < hex.length; i += 2)
str += String.fromCharCode(parseInt(hex.substr(i, 2), 16));
return str;
}
function convertToHex(str) {
var hex = '';
for(var i=0;i<str.length;i++) {
hex += ''+str.charCodeAt(i).toString(16);
}
return hex;
}
But if fails on unicode characters, like chinese;
Input:
漢字
Output:
ªo"[W
Any ideas? Can this be done in javascript?
Remember that a JavaScript code unit is 16 bits wide. Therefore the hex string form will be 4 digits per code unit.
usage:
var str = "\u6f22\u5b57"; // "\u6f22\u5b57" === "漢字"
alert(str.hexEncode().hexDecode());
String to hex form:
String.prototype.hexEncode = function(){
var hex, i;
var result = "";
for (i=0; i<this.length; i++) {
hex = this.charCodeAt(i).toString(16);
result += ("000"+hex).slice(-4);
}
return result
}
Back again:
String.prototype.hexDecode = function(){
var j;
var hexes = this.match(/.{1,4}/g) || [];
var back = "";
for(j = 0; j<hexes.length; j++) {
back += String.fromCharCode(parseInt(hexes[j], 16));
}
return back;
}
Here is a tweak of McDowell's algorithm that doesn't pad the result:
function toHex(str) {
var result = '';
for (var i=0; i<str.length; i++) {
result += str.charCodeAt(i).toString(16);
}
return result;
}
A more up to date solution, for encoding:
// This is the same for all of the below, and
// you probably won't need it except for debugging
// in most cases.
function bytesToHex(bytes) {
return Array.from(
bytes,
byte => byte.toString(16).padStart(2, "0")
).join("");
}
// You almost certainly want UTF-8, which is
// now natively supported:
function stringToUTF8Bytes(string) {
return new TextEncoder().encode(string);
}
// But you might want UTF-16 for some reason.
// .charCodeAt(index) will return the underlying
// UTF-16 code-units (not code-points!), so you
// just need to format them in whichever endian order you want.
function stringToUTF16Bytes(string, littleEndian) {
const bytes = new Uint8Array(string.length * 2);
// Using DataView is the only way to get a specific
// endianness.
const view = new DataView(bytes.buffer);
for (let i = 0; i != string.length; i++) {
view.setUint16(i, string.charCodeAt(i), littleEndian);
}
return bytes;
}
// And you might want UTF-32 in even weirder cases.
// Fortunately, iterating a string gives the code
// points, which are identical to the UTF-32 encoding,
// though you still have the endianess issue.
function stringToUTF32Bytes(string, littleEndian) {
const codepoints = Array.from(string, c => c.codePointAt(0));
const bytes = new Uint8Array(codepoints.length * 4);
// Using DataView is the only way to get a specific
// endianness.
const view = new DataView(bytes.buffer);
for (let i = 0; i != codepoints.length; i++) {
view.setUint32(i, codepoints[i], littleEndian);
}
return bytes;
}
Examples:
bytesToHex(stringToUTF8Bytes("hello 漢字 👍"))
// "68656c6c6f20e6bca2e5ad9720f09f918d"
bytesToHex(stringToUTF16Bytes("hello 漢字 👍", false))
// "00680065006c006c006f00206f225b570020d83ddc4d"
bytesToHex(stringToUTF16Bytes("hello 漢字 👍", true))
// "680065006c006c006f002000226f575b20003dd84ddc"
bytesToHex(stringToUTF32Bytes("hello 漢字 👍", false))
// "00000068000000650000006c0000006c0000006f0000002000006f2200005b57000000200001f44d"
bytesToHex(stringToUTF32Bytes("hello 漢字 👍", true))
// "68000000650000006c0000006c0000006f00000020000000226f0000575b0000200000004df40100"
For decoding, it's generally a lot simpler, you just need:
function hexToBytes(hex) {
const bytes = new Uint8Array(hex.length / 2);
for (let i = 0; i !== bytes.length; i++) {
bytes[i] = parseInt(hex.substr(i * 2, 2), 16);
}
return bytes;
}
then use the encoding parameter of TextDecoder:
// UTF-8 is default
new TextDecoder().decode(hexToBytes("68656c6c6f20e6bca2e5ad9720f09f918d"));
// but you can also use:
new TextDecoder("UTF-16LE").decode(hexToBytes("680065006c006c006f002000226f575b20003dd84ddc"))
new TextDecoder("UTF-16BE").decode(hexToBytes("00680065006c006c006f00206f225b570020d83ddc4d"));
// "hello 漢字 👍"
Here's the list of allowed encoding names: https://www.w3.org/TR/encoding/#names-and-labels
You might notice UTF-32 is not on that list, which is a pain, so:
function bytesToStringUTF32(bytes, littleEndian) {
const view = new DataView(bytes.buffer);
const codepoints = new Uint32Array(view.byteLength / 4);
for (let i = 0; i !== codepoints.length; i++) {
codepoints[i] = view.getUint32(i * 4, littleEndian);
}
return String.fromCodePoint(...codepoints);
}
Then:
bytesToStringUTF32(hexToBytes("00000068000000650000006c0000006c0000006f0000002000006f2200005b57000000200001f44d"), false)
bytesToStringUTF32(hexToBytes("68000000650000006c0000006c0000006f00000020000000226f0000575b0000200000004df40100"), true)
// "hello 漢字 👍"
It depends on what encoding you use. If you want to convert utf-8 encoded hex to string, use this:
function fromHex(hex,str){
try{
str = decodeURIComponent(hex.replace(/(..)/g,'%$1'))
}
catch(e){
str = hex
console.log('invalid hex input: ' + hex)
}
return str
}
For the other direction use this:
function toHex(str,hex){
try{
hex = unescape(encodeURIComponent(str))
.split('').map(function(v){
return v.charCodeAt(0).toString(16)
}).join('')
}
catch(e){
hex = str
console.log('invalid text input: ' + str)
}
return hex
}
how do you get "\u6f22\u5b57" from 漢字 in JavaScript?
These are JavaScript Unicode escape sequences e.g. \u12AB. To convert them, you could iterate over every code unit in the string, call .toString(16) on it, and go from there.
However, it is more efficient to also use hexadecimal escape sequences e.g. \xAA in the output wherever possible.
Also note that ASCII symbols such as A, b, and - probably don’t need to be escaped.
I’ve written a small JavaScript library that does all this for you, called jsesc. It has lots of options to control the output.
Here’s an online demo of the tool in action: http://mothereff.in/js-escapes#1%E6%BC%A2%E5%AD%97
Your question was tagged as utf-8. Reading the rest of your question, UTF-8 encoding/decoding didn’t seem to be what you wanted here, but in case you ever need it: use utf8.js (online demo).
Here you go. :D
"漢字".split("").reduce((hex,c)=>hex+=c.charCodeAt(0).toString(16).padStart(4,"0"),"")
"6f225b57"
for non unicode
"hi".split("").reduce((hex,c)=>hex+=c.charCodeAt(0).toString(16).padStart(2,"0"),"")
"6869"
ASCII (utf-8) binary HEX string to string
"68656c6c6f20776f726c6421".match(/.{1,2}/g).reduce((acc,char)=>acc+String.fromCharCode(parseInt(char, 16)),"")
String to ASCII (utf-8) binary HEX string
"hello world!".split("").reduce((hex,c)=>hex+=c.charCodeAt(0).toString(16).padStart(2,"0"),"")
--- unicode ---
String to UNICODE (utf-16) binary HEX string
"hello world!".split("").reduce((hex,c)=>hex+=c.charCodeAt(0).toString(16).padStart(4,"0"),"")
UNICODE (utf-16) binary HEX string to string
"00680065006c006c006f00200077006f0072006c00640021".match(/.{1,4}/g).reduce((acc,char)=>acc+String.fromCharCode(parseInt(char, 16)),"")
Here is my take: these functions convert a UTF8 string to a proper HEX without the extra zeroes padding. A real UTF8 string has characters with 1, 2, 3 and 4 bytes length.
While working on this I found a couple key things that solved my problems:
str.split('') doesn't handle multi-byte characters like emojis correctly. The proper/modern way to handle this is with Array.from(str)
encodeURIComponent() and decodeURIComponent() are great tools to convert between string and hex. They are pretty standard, they handle UTF8 correctly.
(Most) ASCII characters (codes 0 - 127) don't get URI encoded, so they need to handled separately. But c.charCodeAt(0).toString(16) works perfectly for those
function utf8ToHex(str) {
return Array.from(str).map(c =>
c.charCodeAt(0) < 128 ? c.charCodeAt(0).toString(16) :
encodeURIComponent(c).replace(/\%/g,'').toLowerCase()
).join('');
},
function hexToUtf8: function(hex) {
return decodeURIComponent('%' + hex.match(/.{1,2}/g).join('%'));
}
Demo: https://jsfiddle.net/lyquix/k2tjbrvq/
UTF-8 Supported Convertion
Decode
function utf8ToHex(str) {
return Array.from(str).map(c =>
c.charCodeAt(0) < 128 ? c.charCodeAt(0).toString(16) :
encodeURIComponent(c).replace(/\%/g,'').toLowerCase()
).join('');
}
Encode
function hexToUtf8(hex) {
return decodeURIComponent('%' + hex.match(/.{1,2}/g).join('%'));
}
Sometimes from network transmissions/usdb devices you receive the data has a hexadimal string eg:
"12ADFF1345"
These type of string I want somehow to be converted into a binary equivalent into a buffer, in order to perform a some mathematical or binary operations on them.
Do you know how I can achieve that?
Use the builtin Buffer class :
let buf1 = Buffer.from('12ADFF1345', 'hex');
let value = buf1.readInt32LE(0);
let value2 = buf1.readInt16LE(2);
console.log(value,value2);
>> 335523090 5119
// '13ffad12' '13FF' (LE)
>> 313392915 -237
// '12ADFF13' 'ff13' (BE)
https://nodejs.org/api/buffer.html#buffer_class_method_buffer_from_string_encoding
Yes I know how to do that, the algorithm is simple (assuming that you have no escape characters):
Split the read string into a character.
Group each character pair.
Then generate the string 0x^first_character_pair^
parseInt the string above with base 16
In other words consult the following code:
const hexStringToBinaryBuffer = (string) => {
const subStrings = Array.from(string);
let previous = null;
const bytes = [];
_.each(subStrings, (val) => {
if (previous === null) { // Converting every 2 chars as binary data
previous = val;
} else {
const value = parseInt(`0x${previous}${val}`, 16);
bytes.push(value);
previous = null;
}
});
return Buffer.from(bytes);
};
This is usefull if you pass as string the result of a Buffer.toString('hex') or equivalent method via a network socket or a usb port and the other end received it.
I am reading the UID of my RFID card and storing it in a variable called myUID.
After that I am authorizing to the card with the factory key and read block number 4 (which has been written to earlier) and store it in a string readBlock.
On the Arduino, I print out the variables onto the serial interface like so.
Serial.println(myUID);
Serial.println(readBlock);
On the client side, I use a Java program that reads in serial data. My program uses the Processing Library.
Serial mySerial;
PrintWriter output;
void setup() {
output = createWriter( "data.txt" );
mySerial = new Serial( this, Serial.list()[0], 9600 );
mySerial.bufferUntil('\n');
}
void draw(){
while (mySerial.available() > 0) {
String inBuffer = mySerial.readString();
if (inBuffer != null)
output.println(inBuffer);
}
}
void keyPressed() { // Press a key to save the data
output.flush(); // Write the remaining data
output.close(); // Finish the file
exit(); // Stop the program
}
Now my data.txt is expected to look like
xxx xxx xxx xxx (uid of card)
00 00 00 00 00 00 00 00 ... (read block from card)
but looks like
237 63 58 1
07
37 37 95
37
97 98 50 54 37 5
4 55 102 55 52
45 98
I have tried several things like readStringUntil('\n'); in the Processing Library but without success.
For everyone interested, I have fixed the problem myself with many hours of searching Google, so maybe this will help someone in the future:
I could fix it with this code:
import processing.serial.*;
int count = 0;
String input = "";
String fileName = dataPath("SET FILEPATH HERE");
Serial mySerial;
import java.io.*;
void setup() {
mySerial = new Serial(this, Serial.list()[0], 9600);
mySerial.bufferUntil('\n');
File f = new File(fileName);
if (f.exists()) {
f.delete();
}
}
void draw(){}
// listen to serial events happening
void serialEvent(Serial mySerial){
input = mySerial.readStringUntil('\n');
write(input, count);
count++;
}
// function for writing the data to the file
void write(String inputString, int counter) {
// should new data be appended or replace any old text in the file?
boolean append = false;
// just for my purpose, because I have got two lines of serial which need to get written to the file
//(Line 1: UID of card, Line 2: Read block of card)
if(counter < 2){
append = true;
}
else{
count = 0;
}
try {
File file = new File("D:/xampp/htdocs/pizza/src/rfid/data.txt");
if (!file.exists()) {
file.createNewFile();
}
FileWriter fw = new FileWriter(file, append);
BufferedWriter bw = new BufferedWriter(fw);
PrintWriter pw = new PrintWriter(bw);
pw.write(inputString + '\n');
pw.close();
}
catch(IOException ioe) {
System.out.println("Exception ");
ioe.printStackTrace();
}
}
I want to generate string of given length, from given characters. The order of the characters matters, also I want to use multiple threads to generate it. Here are a few examples:
chars: a,b,c,d
length: 1
output:
a
b
c
d
chars: a,b,c,d
length: 2
output:
aa
ab
ac
ad
bb
ba
bc
bd
cc
ca
cb
cd
dd
da
db
dc
I've tried this algorithm:
Note: it's pseudo-code
func generate(set, str, k){
if (k == 0){
print str;
return;
}
for (c in set) {
newString = str + c;
generate(set, newString, k-1);
}
}
However I don't see how to use multiple threads. All other algorithms I've read about don't suite my needs.
No need of use multithreading for this algorithm. The for loop is only for generating string for required length. In java the code is like this.
import java.util.Random;
public class RandomString2 {
static String[] option = {"a","b","v","k"};
static String getRandomElement()
{
int idx = new Random().nextInt(option.length);
return option[idx];
}
static String getRandomString(int length)
{
String result="";
for(int i = 1; i<=length ; i++)
{
result += getRandomElement();
}
return result;
}
public static void main(String[] args) {
// TODO Auto-generated method stub
System.out.println(getRandomString(5)); // pass string length as parameter
}
}
I am trying to convert unicode[marathi] string in to byte array,but when I convert it and print, it shows " ? ? ? ? ? ? " like this.
using System;
using System.Text;
public class Example
{
public static void Main()
{
// Input string.
const string input = "पांडुरंग";
// Invoke GetBytes method.
// ... You can store this array as a field!
byte[] array = Encoding.ASCII.GetBytes(input);
// Loop through contents of the array.
foreach (byte element in array)
{
Console.WriteLine("{0} = {1}", element, (char)element);
}
Console.ReadLine();
}
}
.
Output:
63 = ?
63 = ?
63 = ?
63 = ?
63 = ?
63 = ?
63 = ?
63 = ?
if anyone have idea about this please help us.thanks in advance.
Try using Encoding.Unicode instead of Encoding.ASCII:
byte[] array = Encoding.Unicode.GetBytes(input);
Also, you are converting each byte to a separate char, whereas actually each character is represented by two bytes. The easiest way to get the chars from the byte array is to convert the bytes back to string using the same encoding, and calling GetCharArray() on the rsulting string:
foreach (byte element in Encoding.Unicode.GetString(array).ToCharArray()
{
Console.WriteLine("{0} = {1}", element, (char)element);
}