Proper hexadecimal conversion to text/ascii - javascript

Trying to figure out how to get my qr scanners to output text to be used in my website.
My qr reader is outputting text that needs to be rendered with event.target.value.getuintsomeintegerhere. I cannot get this to convert to text unless I plop it into some hex decoder online.
I do notice the comment about using TextDecoder to process the data, but it still sort of takes me back to the same place off not really understanding what's going on with this sort of thing...
For example
0x6d 0x6f 0x6f 0x0d should read 'moo'
I understand that the encoding is "utf-8" but not what that means or how to modify the code so that it outputs things correctly...
Here's a chunk of the code that seems to be where i need the most help:
function handleNotifications(event) {
let value = event.target.value;
let a = [];
// Convert raw data bytes to hex values just for the sake of showing something.
// In the "real" world, you'd use data.getUint8, data.getUint16 or even
// TextDecoder to process raw data bytes.
for (let i = 0; i < value.byteLength; i++) {
a.push('0x' + ('00' + value.getUint8(i).toString(16)).slice(-2));
}
log(hex2ascii('> ' + a.join(' ')));
}

It can all be vastly simplified
const h2a = str => str && str.replace(/ ?0x/g,"") // get rid of the space and 0x
.match(/[a-zA-Z0-9]{1,2}/g) // get the 00-FF parts
.map(v => String.fromCharCode(parseInt(v, 16))) // convert each to in using hex radix and get the ascii from the int
.join(''); // string them together
document.getElementById("hex")
.addEventListener("input", e => document.getElementById("ascii").value = h2a(e.target.value))
<input type="text" value="" id="hex" /> Hex (paste or scan - for example cut and paste 0x6d 0x6f 0x6f 0x0d)
<hr/>
<input type="text" value="" id="ascii" /> Ascii

function handleNotifications(event) {
let value = event.target.value;
let a = [];
// Convert raw data bytes to hex values just for the sake of showing something.
// In the "real" world, you'd use data.getUint8, data.getUint16 or even
// TextDecoder to process raw data bytes.
for (let i = 0; i < value.byteLength; i++) {
a.push('0x' + ('00' + value.getUint8(i).toString(16)).slice(-2));
}
log(hex2ascii('> ' + a.join(' ')));
}
function hex2ascii(hexString) {
const numberArray = hexString.slice(2) // Get the string without the first two characters (removes '> ')
.split(' ') // Split the string on spaces.
.map(hex => Number(hex)); // Create a new array where each element of the hex array is converted to a JavaScript Number
// Now we have an array of Numbers.
const byteArray = new Uint8Array(numberArray); // Make an array of bytes from the number array This is necessary to use the TextDecoder.
const string = (new TextDecoder()).decode(byteArray); // Create a TextDecoder, and decode the byte array to get a string.
return string;
}
console.log(hex2ascii("> 0x6d 0x6f 0x6f 0x0d"));
This snippet only logs the received data every 2 messages, concatenating the data received between them. For example, if the first message said moo, and the second message said I'm a cow!, the text moo I'm a cow! would be logged after receiving the second message.
let messageCount = 0;
let currentMessage = "";
function handleNotifications(event) {
let value = event.target.value;
let a = [];
// Convert raw data bytes to hex values just for the sake of showing something.
// In the "real" world, you'd use data.getUint8, data.getUint16 or even
// TextDecoder to process raw data bytes.
for (let i = 0; i < value.byteLength; i++) {
a.push('0x' + ('00' + value.getUint8(i).toString(16)).slice(-2));
}
currentMessage += hex2ascii('> ' + a.join(' '));
if (messageCount < 1) messageCount++;
else {
messageCount = 0;
log(currentMessage);
currentMessage = "";
}
}
function hex2ascii(hexString) {
const numberArray = hexString.slice(2) // Get the string without the first two characters (removes '> ')
.split(' ') // Split the string on spaces.
.map(hex => Number(hex)); // Create a new array where each element of the hex array is converted to a JavaScript Number
// Now we have an array of Numbers.
const byteArray = new Uint8Array(numberArray); // Make an array of bytes from the number array This is necessary to use the TextDecoder.
const string = (new TextDecoder()).decode(byteArray); // Create a TextDecoder, and decode the byte array to get a string.
return string;
}
console.log(hex2ascii("> 0x6d 0x6f 0x6f 0x0d"));

Related

How to use javascript (in Angular) to get bytes encoded by java.util.Base64? [duplicate]

I need to convert a base64 encode string into an ArrayBuffer.
The base64 strings are user input, they will be copy and pasted from an email, so they're not there when the page is loaded.
I would like to do this in javascript without making an ajax call to the server if possible.
I found those links interesting, but they didt'n help me:
ArrayBuffer to base64 encoded string
this is about the opposite conversion, from ArrayBuffer to base64, not the other way round
http://jsperf.com/json-vs-base64/2
this looks good but i can't figure out how to use the code.
Is there an easy (maybe native) way to do the conversion? thanks
Try this:
function _base64ToArrayBuffer(base64) {
var binary_string = window.atob(base64);
var len = binary_string.length;
var bytes = new Uint8Array(len);
for (var i = 0; i < len; i++) {
bytes[i] = binary_string.charCodeAt(i);
}
return bytes.buffer;
}
Using TypedArray.from:
Uint8Array.from(atob(base64_string), c => c.charCodeAt(0))
Performance to be compared with the for loop version of Goran.it answer.
For Node.js users:
const myBuffer = Buffer.from(someBase64String, 'base64');
myBuffer will be of type Buffer which is a subclass of Uint8Array. Unfortunately, Uint8Array is NOT an ArrayBuffer as the OP was asking for. But when manipulating an ArrayBuffer I almost always wrap it with Uint8Array or something similar, so it should be close to what's being asked for.
Goran.it's answer does not work because of unicode problem in javascript - https://developer.mozilla.org/en-US/docs/Web/API/WindowBase64/Base64_encoding_and_decoding.
I ended up using the function given on Daniel Guerrero's blog: http://blog.danguer.com/2011/10/24/base64-binary-decoding-in-javascript/
Function is listed on github link: https://github.com/danguer/blog-examples/blob/master/js/base64-binary.js
Use these lines
var uintArray = Base64Binary.decode(base64_string);
var byteArray = Base64Binary.decodeArrayBuffer(base64_string);
Async solution, it's better when the data is big:
// base64 to buffer
function base64ToBufferAsync(base64) {
var dataUrl = "data:application/octet-binary;base64," + base64;
fetch(dataUrl)
.then(res => res.arrayBuffer())
.then(buffer => {
console.log("base64 to buffer: " + new Uint8Array(buffer));
})
}
// buffer to base64
function bufferToBase64Async( buffer ) {
var blob = new Blob([buffer], {type:'application/octet-binary'});
console.log("buffer to blob:" + blob)
var fileReader = new FileReader();
fileReader.onload = function() {
var dataUrl = fileReader.result;
console.log("blob to dataUrl: " + dataUrl);
var base64 = dataUrl.substr(dataUrl.indexOf(',')+1)
console.log("dataUrl to base64: " + base64);
};
fileReader.readAsDataURL(blob);
}
Javascript is a fine development environment so it seems odd than it doesn't provide a solution to this small problem. The solutions offered elsewhere on this page are potentially slow. Here is my solution. It employs the inbuilt functionality that decodes base64 image and sound data urls.
var req = new XMLHttpRequest;
req.open('GET', "data:application/octet;base64," + base64Data);
req.responseType = 'arraybuffer';
req.onload = function fileLoaded(e)
{
var byteArray = new Uint8Array(e.target.response);
// var shortArray = new Int16Array(e.target.response);
// var unsignedShortArray = new Int16Array(e.target.response);
// etc.
}
req.send();
The send request fails if the base 64 string is badly formed.
The mime type (application/octet) is probably unnecessary.
Tested in chrome. Should work in other browsers.
Pure JS - no string middlestep (no atob)
I write following function which convert base64 in direct way (without conversion to string at the middlestep). IDEA
get 4 base64 characters chunk
find index of each character in base64 alphabet
convert index to 6-bit number (binary string)
join four 6 bit numbers which gives 24-bit numer (stored as binary string)
split 24-bit string to three 8-bit and covert each to number and store them in output array
corner case: if input base64 string ends with one/two = char, remove one/two numbers from output array
Below solution allows to process large input base64 strings. Similar function for convert bytes to base64 without btoa is HERE
function base64ToBytesArr(str) {
const abc = [..."ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"]; // base64 alphabet
let result = [];
for(let i=0; i<str.length/4; i++) {
let chunk = [...str.slice(4*i,4*i+4)]
let bin = chunk.map(x=> abc.indexOf(x).toString(2).padStart(6,0)).join('');
let bytes = bin.match(/.{1,8}/g).map(x=> +('0b'+x));
result.push(...bytes.slice(0,3 - (str[4*i+2]=="=") - (str[4*i+3]=="=")));
}
return result;
}
// --------
// TEST
// --------
let test = "Alice's Adventure in Wonderland.";
console.log('test string:', test.length, test);
let b64_btoa = btoa(test);
console.log('encoded string:', b64_btoa);
let decodedBytes = base64ToBytesArr(b64_btoa); // decode base64 to array of bytes
console.log('decoded bytes:', JSON.stringify(decodedBytes));
let decodedTest = decodedBytes.map(b => String.fromCharCode(b) ).join``;
console.log('Uint8Array', JSON.stringify(new Uint8Array(decodedBytes)));
console.log('decoded string:', decodedTest.length, decodedTest);
Caution!
If you want to decode base64 to STRING (not bytes array) and you know that result contains utf8 characters then atob will fail in general e.g. for character 💩 the atob("8J+SqQ==") will give wrong result . In this case you can use above solution and convert result bytes array to string in proper way e.g. :
function base64ToBytesArr(str) {
const abc = [..."ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"]; // base64 alphabet
let result = [];
for(let i=0; i<str.length/4; i++) {
let chunk = [...str.slice(4*i,4*i+4)]
let bin = chunk.map(x=> abc.indexOf(x).toString(2).padStart(6,0)).join('');
let bytes = bin.match(/.{1,8}/g).map(x=> +('0b'+x));
result.push(...bytes.slice(0,3 - (str[4*i+2]=="=") - (str[4*i+3]=="=")));
}
return result;
}
// --------
// TEST
// --------
let testB64 = "8J+SqQ=="; // for string: "💩";
console.log('input base64 :', testB64);
let decodedBytes = base64ToBytesArr(testB64); // decode base64 to array of bytes
console.log('decoded bytes :', JSON.stringify(decodedBytes));
let result = new TextDecoder("utf-8").decode(new Uint8Array(decodedBytes));
console.log('properly decoded string :', result);
let result_atob = atob(testB64);
console.log('decoded by atob :', result_atob);
Snippets tested 2022-08-04 on: chrome 103.0.5060.134 (arm64), safari 15.2, firefox 103.0.1 (64 bit), edge 103.0.1264.77 (arm64), and node-js v12.16.1
I would strongly suggest using an npm package implementing correctly the base64 specification.
The best one I know is rfc4648
The problem is that btoa and atob use binary strings instead of Uint8Array and trying to convert to and from it is cumbersome. Also there is a lot of bad packages in npm for that. I lose a lot of time before finding that one.
The creators of that specific package did a simple thing: they took the specification of Base64 (which is here by the way) and implemented it correctly from the beginning to the end. (Including other formats in the specification that are also useful like Base64-url, Base32, etc ...) That doesn't seem a lot but apparently that was too much to ask to the bunch of other libraries.
So yeah, I know I'm doing a bit of proselytism but if you want to avoid losing your time too just use rfc4648.
I used the accepted answer to this question to create base64Url string <-> arrayBuffer conversions in the realm of base64Url data transmitted via ASCII-cookie [atob, btoa are base64[with +/]<->js binary string], so I decided to post the code.
Many of us may want both conversions and client-server communication may use the base64Url version (though a cookie may contain +/ as well as -_ characters if I understand well, only ",;\ characters and some wicked characters from the 128 ASCII are disallowed). But a url cannot contain / character, hence the wider use of b64 url version which of course not what atob-btoa supports...
Seeing other comments, I would like to stress that my use case here is base64Url data transmission via url/cookie and trying to use this crypto data with the js crypto api (2017) hence the need for ArrayBuffer representation and b64u <-> arrBuff conversions... if array buffers represent other than base64 (part of ascii) this conversion wont work since atob, btoa is limited to ascii(128). Check out an appropriate converter like below:
The buff -> b64u version is from a tweet from Mathias Bynens, thanks for that one (too)! He also wrote a base64 encoder/decoder:
https://github.com/mathiasbynens/base64
Coming from java, it may help when trying to understand the code that java byte[] is practically js Int8Array (signed int) but we use here the unsigned version Uint8Array since js conversions work with them. They are both 256bit, so we call it byte[] in js now...
The code is from a module class, that is why static.
//utility
/**
* Array buffer to base64Url string
* - arrBuff->byte[]->biStr->b64->b64u
* #param arrayBuffer
* #returns {string}
* #private
*/
static _arrayBufferToBase64Url(arrayBuffer) {
console.log('base64Url from array buffer:', arrayBuffer);
let base64Url = window.btoa(String.fromCodePoint(...new Uint8Array(arrayBuffer)));
base64Url = base64Url.replaceAll('+', '-');
base64Url = base64Url.replaceAll('/', '_');
console.log('base64Url:', base64Url);
return base64Url;
}
/**
* Base64Url string to array buffer
* - b64u->b64->biStr->byte[]->arrBuff
* #param base64Url
* #returns {ArrayBufferLike}
* #private
*/
static _base64UrlToArrayBuffer(base64Url) {
console.log('array buffer from base64Url:', base64Url);
let base64 = base64Url.replaceAll('-', '+');
base64 = base64.replaceAll('_', '/');
const binaryString = window.atob(base64);
const length = binaryString.length;
const bytes = new Uint8Array(length);
for (let i = 0; i < length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
console.log('array buffer:', bytes.buffer);
return bytes.buffer;
}
made a ArrayBuffer from a base64:
function base64ToArrayBuffer(base64) {
var binary_string = window.atob(base64);
var len = binary_string.length;
var bytes = new Uint8Array(len);
for (var i = 0; i < len; i++) {
bytes[i] = binary_string.charCodeAt(i);
}
return bytes.buffer;
}
I was trying to use above code and It's working fine.
The result of atob is a string that is separated with some comma
,
A simpler way is to convert this string to a json array string and after that parse it to a byteArray
below code can simply be used to convert base64 to an array of number
let byteArray = JSON.parse('['+atob(base64)+']');
let buffer = new Uint8Array(byteArray);
Solution without atob
I've seen many people complaining about using atob and btoa in the replies. There are some issues to take into account when using them.
There's a solution without using them in the MDN page about Base64. Below you can find the code to convert a base64 string into a Uint8Array copied from the docs.
Note that the function below returns a Uint8Array. To get the ArrayBuffer version you just need to do uintArray.buffer.
function b64ToUint6(nChr) {
return nChr > 64 && nChr < 91
? nChr - 65
: nChr > 96 && nChr < 123
? nChr - 71
: nChr > 47 && nChr < 58
? nChr + 4
: nChr === 43
? 62
: nChr === 47
? 63
: 0;
}
function base64DecToArr(sBase64, nBlocksSize) {
const sB64Enc = sBase64.replace(/[^A-Za-z0-9+/]/g, "");
const nInLen = sB64Enc.length;
const nOutLen = nBlocksSize
? Math.ceil(((nInLen * 3 + 1) >> 2) / nBlocksSize) * nBlocksSize
: (nInLen * 3 + 1) >> 2;
const taBytes = new Uint8Array(nOutLen);
let nMod3;
let nMod4;
let nUint24 = 0;
let nOutIdx = 0;
for (let nInIdx = 0; nInIdx < nInLen; nInIdx++) {
nMod4 = nInIdx & 3;
nUint24 |= b64ToUint6(sB64Enc.charCodeAt(nInIdx)) << (6 * (3 - nMod4));
if (nMod4 === 3 || nInLen - nInIdx === 1) {
nMod3 = 0;
while (nMod3 < 3 && nOutIdx < nOutLen) {
taBytes[nOutIdx] = (nUint24 >>> ((16 >>> nMod3) & 24)) & 255;
nMod3++;
nOutIdx++;
}
nUint24 = 0;
}
}
return taBytes;
}
If you're interested in the reverse operation, ArrayBuffer to base64, you can find how to do it in the same link.

How to generate a new random string each time, in a given string text, a regex is satisfied?

I have splited the resulted text of a file, what I need to do is to rename the header of that file, so I have taken the first indice of the sp (the header) and for each word in this header, I want to replace them with a different random string. But I have the same random string for each satisfied regex.
var sp = reader.result.split("\n");
var randomString = Math.random().toString(36).substr(5, 5)
for ( let i in sp ){
if (i == 0) {
sp[i] = sp[i].replace(regex, randomString)
}
}
So you fetch that header text from your file and then you want to extract each word from it and then replace it with some random string. Here's the code for it.
On a side note Math.random() is not good enough, you need the crypto API
function generateHash (length = null) {
const array = new Uint8Array((length || 64) / 2)
window.crypto.getRandomValues(array)
return Array.from(array, dec => {
return dec.toString(16).padStart(2, '0')
}).join('')
}
const header = 'One word and another word'
const modified = header.split(' ').map(val => generateHash(12)).join(' ')
console.log(modified)

why does digest and digest('hex') result in different outputs?

I have 2 piece of codes.
1ST ONE
const hash1 = (data) => createHash('sha256').update(data).digest('hex');
var a1 = hash1("A");
var b1 = hash1("B");
console.log(hash1(a1+b1));
2ND ONE
const hash2 = (data) => createHash('sha256').update(data).digest();
var a2 = hash2("A");
var b2 = hash2("B");
console.log(hash2(Buffer.concat([a2,b2])).toString('hex'));
Why do they print the different results ?
digest('hex') and digest() are the same , but in different kind of format, but still the same. So, why do I get different results in console ? is it the + operator when I sum up the hexes versus when i sum up buffers ? why ?
The default encoding for hash.digest([encoding]) is utf-8. utf-8 is a variable-length encoding system. It will only use as many bytes as necessary to represent each character (anywhere between 1-4 bytes).
However, when you specify hex as the encoding, each character is stored as exactly 2 hexadecimal characters.
When you call hash.toString('hex') on a utf-8 encoded hash, the resulting hex representation is equivalent to hashing with hex encoding in the first place (as in hash.digest('hex')).
So, even though the hex representation is the same in each case, the actual data is different. i.e.:
hash.digest() != hash.digest('hex'), but
hash.digest().toString('hex') == hash.digest('hex').
digest('hex') and digest() technically different
Please try this code
var crypto = require('crypto');
const hash1 = (data) => crypto.createHash('sha256').update(data).digest('hex');
var a1 = hash1("A");
var b1 = hash1("B");
//console.log(a1)
//console.log(b1)
console.log(hash1(a1+b1));
const hash2 = (data) => crypto.createHash('sha256').update(data).digest();
var a2 = hash2("A");
var b2 = hash2("B");
//console.log(a2.toString("hex"))
//console.log(b2.toString("hex"))
console.log(hash2(a2.toString("hex") + b2.toString("hex") ).toString("hex"));
In the first you are appending two hex string and passing to hash1
In the second you are appending two non hex string and passing to hash2
Run Nodejs online

Working with memory to fetch string yields incorrect result

I am following the solutions from here:
How can I return a JavaScript string from a WebAssembly function
and here:
How to return a string (or similar) from Rust in WebAssembly?
However, when reading from memory I am not getting the desired results.
AssemblyScript file, helloWorldModule.ts:
export function getMessageLocation(): string {
return "Hello World";
}
index.html:
<script>
fetch("helloWorldModule.wasm").then(response =>
response.arrayBuffer()
).then(bytes =>
WebAssembly.instantiate(bytes, {imports: {}})
).then(results => {
var linearMemory = results.instance.exports.memory;
var offset = results.instance.exports.getMessageLocation();
var stringBuffer = new Uint8Array(linearMemory.buffer, offset, 11);
let str = '';
for (let i=0; i<stringBuffer.length; i++) {
str += String.fromCharCode(stringBuffer[i]);
}
debugger;
});
</script>
This returns an offset of 32. And finally yields a string that starts too early and has spaces between each letter of "Hello World":
However, if I change the array to an Int16Array, and add 8 to the offset (which was 32), to make an offset of 40. Like so:
<script>
fetch("helloWorldModule.wasm").then(response =>
response.arrayBuffer()
).then(bytes =>
WebAssembly.instantiate(bytes, {imports: {}})
).then(results => {
var linearMemory = results.instance.exports.memory;
var offset = results.instance.exports.getMessageLocation();
var stringBuffer = new Int16Array(linearMemory.buffer, offset+8, 11);
let str = '';
for (let i=0; i<stringBuffer.length; i++) {
str += String.fromCharCode(stringBuffer[i]);
}
debugger;
});
</script>
Then we get the correct result:
Why does the first set of code not work like its supposed to in the links I provided? Why do I need to change it to work with Int16Array to get rid of the space between "H" and "e" for example? Why do I need to add 8 bytes to the offset?
In summary, what on earth is going on here?
Edit: Another clue, is if I use a TextDecoder on the UInt8 array, decoding as UTF-16 looks more correct than decoding as UTF-8:
AssemblyScript uses utf-16: https://github.com/AssemblyScript/assemblyscript/issues/43
Additionally AssemblyScript stores the length of the string in the first 32 or 64 bits.
That's why my code behaves differently. The examples in the links at the top of this post were for C++ and Rust, which do string encoding differently

How can I convert an image file to base 2 binary?

I'm trying to do it like this:
function handleFiles(files) {
var selectedFile = files[0];
var reader = new FileReader();
reader.readAsBinaryString(selectedFile);
reader.onloadend = function() {
var result = reader.result;
convert(result);
};
}
function convert(data) {
var img = document.createElement("img");
var result = data.toString(2);
document.body.appendChild(img);
img.src = 'data:image/jpeg;base64,' + btoa(data); // this works
console.log("Base ?: " + data); // not sure, I think 16 or more likely 64,
// the MDN documentation doesn't specify what base of binary is produced by the readAsBinaryString method
console.log("Base 2: " + result); // not base 2 binary data as expected
}
<input type="file" id="input" onchange="handleFiles(this.files)">
This code will convert a jpeg into binary data, then render it, but it is not first converting to base 2 binary. If you run that code and look at the logs, it's something more (I'm naive on the subject of bases and binary, but my guess is base 16 or 64). I think toString(2) is supposed to convert to base 2, but it doesn't seem to be doing that. Before converting it back to base 64, I want to get the binary data to base 2 for experimentation.
Can't you just step through your binary string and convert each character to a binary representation?
function convert(data) {
var result = ''
for (let i = 0; i < data.length; i++){
result += data.charCodeAt(i).toString(2)
}
console.log(result)
}
[ I would test this with a small file first. It writes a lot to the console, as you might imagine ]
EDIT -- sorry, didn't realize you wanted to go back and forth. If you want to get from a binary string back to something else, you'll probably want to pad the binary with zeros so you can pull out 8 bit (or whatever) at a time:
function convert(data) {
var result = ''
for (let i = 0; i < data.length; i++){
// pad to make all binary string parts 8 bits long
result += data.charCodeAt(i).toString(2).padStart(8, 0)
}
return result
}
function makeBuffer(string){
buffer = ''
for(let i = 0; i < string.length/8; i++){
let stIdx = i*8
// maybe a way to do this without the slice? dunno.
buffer += String.fromCharCode(parseInt(string.slice(stIdx,stIdx+8), 2))
}
return buffer
}
Binary is always base 2, that's what the "bi" in "binary" means.
When you call fileReader.readAsBinaryString() it's returning binary; Strings can store binary because they're really just an array of characters.
Displaying the string will not be a series of zeros and ones, it will be a jumble of different ASCII characters (letters, numbers, symbols, etc.) because those are the equivalent characters that match the binary values.
Additionally, you don't need the following line since your data you're passing in is already a string.
var result = data.toString(2);

Categories