We Keep Coding

Unzipping encrypted compressed report from Amazon Selling Partner

I am using the getReport operation to fetch the documentId, which later I use to download the report document itself which is encrypted and compressed.
The code looks like this:
const documentData = await this.sellingPartner.callAPI({
operation: "getReportDocument",
endpoint: "reports",
path: { reportDocumentId: reportData.reportDocumentId }
})
const request = https.get(documentData.url, function(res) {
const data = [];
res.on("data", chunk => data.push(chunk));
res.on("end", () => {
const key = new Buffer.from(documentData.encryptionDetails.key, 'base64')
const initializationVector = new Buffer.from(documentData.encryptionDetails.initializationVector, 'base64')
const input = Buffer.concat(data)
let result;
try {
result = aes.decryptText(
aes.CIPHERS.AES_256,
key,
initializationVector,
input
)
} catch (e) {
console.log(e)
}
console.log(">>>>")
console.log(result)
zlib.gunzip(result, (err, unzipped) => {
debugger
});
});
}
The current error I am getting is from zlib:
Error: incorrect header check
at Zlib.zlibOnError [as onerror] (node:zlib:189:17)
I am getting the same even if I pass the unencrypted value directly to zlib.
There is a Sample Java code example in the docs, but I cannot understand very well where they do the decryption: before unzipping or after?
In any case, what is the right way to solve this: unzip and decrypt or decrypt and unzip? The former does not work at all, the latter almost works but fails at the unzipping part.
How can I solve the unzip problem?

Short Answer:
Decrypt first and then decompress if compression type is specified.
Longer answer:
After a little research I stumbled upon this example (sadly written in python not in javascript) but it covers the steps to do in more detail.
https://github.com/amzn/selling-partner-api-docs/issues/186#issuecomment-756108550
It contains both compressed or not compressed cases.
Specifically for compressed it looks like this:
import gzip
import requests
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
def ase_cbc_decryptor(key, iv, encryption):
cipher = Cipher(algorithms.AES(base64.b64decode(key)), modes.CBC(base64.b64decode(iv)))
decryptor = cipher.decryptor()
decrypted_text = decryptor.update(encryption)
unpadder = padding.PKCS7(algorithms.AES.block_size).unpadder()
unpaded_text = unpadder.update(decrypted_text)
return unpaded_text + unpadder.finalize()
def get_report_document_content(self, key, iv, url, compression_type=None):
resp = requests.get(url=url)
resp_content = resp.content
decrypted_content = ase_cbc_decryptor(key=key, iv=iv, encryption=resp_content)
if compression_type == 'GZIP':
decrypted_content = gzip.decompress(decrypted_content)
code = 'utf-8'
if 'cp1252' in resp.headers.get('Content-Type', '').lower():
code = 'Cp1252'
return decrypted_content.decode(code)
P.S. keep in mind you need to use AES in CBC mode
Short example here:
https://gist.github.com/manuks/5cef1e536ef791e97b39
var keyhex = "8479768f48481eeb9c8304ce0a58481eeb9c8304ce0a5e3cb5e3cb58479768f4"; //length 32
var blockSize = 16;
function encryptAES(input) {
try {
var iv = require('crypto').randomBytes(16);
//console.info('iv',iv);
var data = new Buffer(input).toString('binary');
//console.info('data',data);
key = new Buffer(keyhex, "hex");
//console.info(key);
var cipher = require('crypto').createCipheriv('aes-256-cbc', key, iv);
// UPDATE: crypto changed in v0.10
// https://github.com/joyent/node/wiki/Api-changes-between-v0.8-and-v0.10
var nodev = process.version.match(/^v(\d+)\.(\d+)/);
var encrypted;
if( nodev[1] === '0' && parseInt(nodev[2]) < 10) {
encrypted = cipher.update(data, 'binary') + cipher.final('binary');
} else {
encrypted = cipher.update(data, 'utf8', 'binary') + cipher.final('binary');
}
var encoded = new Buffer(iv, 'binary').toString('hex') + new Buffer(encrypted, 'binary').toString('hex');
return encoded;
} catch (ex) {
// handle error
// most likely, entropy sources are drained
console.error(ex);
}
}
function decryptAES(encoded) {
var combined = new Buffer(encoded, 'hex');
key = new Buffer(keyhex, "hex");
// Create iv
var iv = new Buffer(16);
combined.copy(iv, 0, 0, 16);
edata = combined.slice(16).toString('binary');
// Decipher encrypted data
var decipher = require('crypto').createDecipheriv('aes-256-cbc', key, iv);
// UPDATE: crypto changed in v0.10
// https://github.com/joyent/node/wiki/Api-changes-between-v0.8-and-v0.10
var nodev = process.version.match(/^v(\d+)\.(\d+)/);
var decrypted, plaintext;
if( nodev[1] === '0' && parseInt(nodev[2]) < 10) {
decrypted = decipher.update(edata, 'binary') + decipher.final('binary');
plaintext = new Buffer(decrypted, 'binary').toString('utf8');
} else {
plaintext = (decipher.update(edata, 'binary', 'utf8') + decipher.final('utf8'));
}
return plaintext;
}
var input="testing";
var encrypted = encryptAES(input);
console.info('encrypted:', encrypted);
var decryped = decryptAES(encrypted);
console.info('decryped:',decryped);

Why is writing the same data to IndexedDB a second time consistently slower?

I stored some jpeg files (exactly 350, same files same size. Total: 336.14 MB) as Blob in IndexedDB. It took around 1 second to complete the transaction. Then I read all the data from IndexedDB to an array and again sored to IndexedDB. But this time it takes around 15 Seconds. I observed this as a consistent behavior. Anything wrong here? I used performance.now() to get the time difference
Files: 350,
Size of each: 937 KB,
Browser: Chrome and Chromium Edge
//Open
var dbOpen = indexedDB.open(INDEXED_DB_NAME, INDEXED_DB_VERSION);
dbOpen.onupgradeneeded = function (e) {
console.log("onupgradeneeded");
var store = e.currentTarget.result.createObjectStore(
IMAGE_DATA_STORE, { autoIncrement: true });
};
dbOpen.onsuccess = function (e) {
image_data_db = dbOpen.result;
console.log("indexed DB opened");
};
//Initial Write
var inputFiles = document.getElementById('inputFiles');
for (var i = 0; i < inputFiles.files.length; i++) {
let file = inputFiles.files[i];
var b = new Blob([file], { type: file.type });
fileblobs.push(b);
}
StoreIdb(fileblobs); // < First write
//StoreIdb()
t0 = performace.now();
var trx = image_data_db.transaction(IMAGE_DATA_STORE, 'readwrite');
var imagestore = trx.objectStore(IMAGE_DATA_STORE);
for (i = 0; i < fileblobs.length; i++) {
request = imagestore.add(fileblobs[i]);
request.onsuccess = function (e) {
console.log('added');
};
request.onerror = function (e) {
console.error("Request Error", this.error);
};
}
trx.onabort = function (e) {
console.error("Exception:", this.error, this.error.name);
};
trx.oncomplete = function (e) {
console.log('completed');
t1 = performance.now();
timetaken = t1 - t0;
}
//Read
var objectStore = image_data_db.transaction(IMAGE_DATA_STORE).objectStore(IMAGE_DATA_STORE);
objectStore.openCursor().onsuccess = function (e) {
var cursor = e.target.result;
if (cursor) {
blobArray.push(cursor.value.blob);
cursor.continue();
}
else
{
// completed
}
}
// blobArray will be used for second time << Second Write

I figured it out. First time it was storing file instance blob.
I ve changed file instance blob to Array buffer just to want to ensure data type similar in both cases. Now it is taking same time.
for (var i = 0; i < inputFiles.files.length; i++) {
let file = inputFiles.files[i];
file.arrayBuffer().then((arrayBuffer) => {
let blob = new Blob([new Uint8Array(arrayBuffer)], {type: file.type });
blobs.push(blob);
if ( blobs.length == inputFiles.files.length){
callback(blobs);
}
});
}

js can I read a portion of a file with FileReader? [duplicate]

I have long file I need to parse. Because it's very long I need to do it chunk by chunk. I tried this:
function parseFile(file){
var chunkSize = 2000;
var fileSize = (file.size - 1);
var foo = function(e){
console.log(e.target.result);
};
for(var i =0; i < fileSize; i += chunkSize)
{
(function( fil, start ) {
var reader = new FileReader();
var blob = fil.slice(start, chunkSize + 1);
reader.onload = foo;
reader.readAsText(blob);
})( file, i );
}
}
After running it I see only the first chunk in the console. If I change 'console.log' to jquery append to some div I see only first chunk in that div. What about other chunks? How to make it work?

FileReader API is asynchronous so you should handle it with block calls. A for loop wouldn't do the trick since it wouldn't wait for each read to complete before reading the next chunk.
Here's a working approach.
function parseFile(file, callback) {
var fileSize = file.size;
var chunkSize = 64 * 1024; // bytes
var offset = 0;
var self = this; // we need a reference to the current object
var chunkReaderBlock = null;
var readEventHandler = function(evt) {
if (evt.target.error == null) {
offset += evt.target.result.length;
callback(evt.target.result); // callback for handling read chunk
} else {
console.log("Read error: " + evt.target.error);
return;
}
if (offset >= fileSize) {
console.log("Done reading file");
return;
}
// of to the next chunk
chunkReaderBlock(offset, chunkSize, file);
}
chunkReaderBlock = function(_offset, length, _file) {
var r = new FileReader();
var blob = _file.slice(_offset, length + _offset);
r.onload = readEventHandler;
r.readAsText(blob);
}
// now let's start the read with the first block
chunkReaderBlock(offset, chunkSize, file);
}

You can take advantage of Response (part of fetch) to convert most things to anything else blob, text, json and also get a ReadableStream that can help you read the blob in chunks 👍
var dest = new WritableStream({
write (str) {
console.log(str)
}
})
var blob = new Blob(['bloby']);
(blob.stream ? blob.stream() : new Response(blob).body)
// Decode the binary-encoded response to string
.pipeThrough(new TextDecoderStream())
.pipeTo(dest)
.then(() => {
console.log('done')
})
Old answer (WritableStreams pipeTo and pipeThrough was not implemented before)
I came up with a interesting idéa that is probably very fast since it will convert the blob to a ReadableByteStreamReader probably much easier too since you don't need to handle stuff like chunk size and offset and then doing it all recursive in a loop
function streamBlob(blob) {
const reader = new Response(blob).body.getReader()
const pump = reader => reader.read()
.then(({ value, done }) => {
if (done) return
// uint8array chunk (use TextDecoder to read as text)
console.log(value)
return pump(reader)
})
return pump(reader)
}
streamBlob(new Blob(['bloby'])).then(() => {
console.log('done')
})

The second argument of slice is actually the end byte. Your code should look something like:
function parseFile(file){
var chunkSize = 2000;
var fileSize = (file.size - 1);
var foo = function(e){
console.log(e.target.result);
};
for(var i =0; i < fileSize; i += chunkSize) {
(function( fil, start ) {
var reader = new FileReader();
var blob = fil.slice(start, chunkSize + start);
reader.onload = foo;
reader.readAsText(blob);
})(file, i);
}
}
Or you can use this BlobReader for easier interface:
BlobReader(blob)
.readText(function (text) {
console.log('The text in the blob is', text);
});
More information:
README.md
Docs

Revamped #alediaferia answer in a class (typescript version here) and returning the result in a promise. Brave coders would even have wrapped it into an async iterator…
class FileStreamer {
constructor(file) {
this.file = file;
this.offset = 0;
this.defaultChunkSize = 64 * 1024; // bytes
this.rewind();
}
rewind() {
this.offset = 0;
}
isEndOfFile() {
return this.offset >= this.getFileSize();
}
readBlockAsText(length = this.defaultChunkSize) {
const fileReader = new FileReader();
const blob = this.file.slice(this.offset, this.offset + length);
return new Promise((resolve, reject) => {
fileReader.onloadend = (event) => {
const target = (event.target);
if (target.error == null) {
const result = target.result;
this.offset += result.length;
this.testEndOfFile();
resolve(result);
}
else {
reject(target.error);
}
};
fileReader.readAsText(blob);
});
}
testEndOfFile() {
if (this.isEndOfFile()) {
console.log('Done reading file');
}
}
getFileSize() {
return this.file.size;
}
}
Example printing a whole file in the console (within an async context)
const fileStreamer = new FileStreamer(aFile);
while (!fileStreamer.isEndOfFile()) {
const data = await fileStreamer.readBlockAsText();
console.log(data);
}

Parsing the large file into small chunk by using the simple method:
//Parse large file in to small chunks
var parseFile = function (file) {
var chunkSize = 1024 * 1024 * 16; //16MB Chunk size
var fileSize = file.size;
var currentChunk = 1;
var totalChunks = Math.ceil((fileSize/chunkSize), chunkSize);
while (currentChunk <= totalChunks) {
var offset = (currentChunk-1) * chunkSize;
var currentFilePart = file.slice(offset, (offset+chunkSize));
console.log('Current chunk number is ', currentChunk);
console.log('Current chunk data', currentFilePart);
currentChunk++;
}
};

Calculate MD5 hash of a large file using javascript

How do you upload a 500mb file and get a MD5 hash with CryptoJS?
Here is my code:
$('#upload-file').change(function(){
var reader = new FileReader();
reader.addEventListener('load',function () {
var hash = CryptoJS.MD5(CryptoJS.enc.Latin1.parse(this.result));
window.md5 = hash.toString(CryptoJS.enc.Hex);
});
reader.readAsBinaryString(this.files[0]);
});
If the file is under 200mb, it works. Anything bigger, this.result is an empty "".
I've tried:
filereader api on big files
javascript FileReader - parsing long file in chunks
and almost got this to work , but console is complaining about .join("")
http://dojo4.com/blog/processing-huge-files-with-an-html5-file-input

CryptoJS has a progressive api for hash digests. The rest is taken form alediaferia's answer with slight modifications.
function process() {
getMD5(
document.getElementById("my-file-input").files[0],
prog => console.log("Progress: " + prog)
).then(
res => console.log(res),
err => console.error(err)
);
}
function readChunked(file, chunkCallback, endCallback) {
var fileSize = file.size;
var chunkSize = 4 * 1024 * 1024; // 4MB
var offset = 0;
var reader = new FileReader();
reader.onload = function() {
if (reader.error) {
endCallback(reader.error || {});
return;
}
offset += reader.result.length;
// callback for handling read chunk
// TODO: handle errors
chunkCallback(reader.result, offset, fileSize);
if (offset >= fileSize) {
endCallback(null);
return;
}
readNext();
};
reader.onerror = function(err) {
endCallback(err || {});
};
function readNext() {
var fileSlice = file.slice(offset, offset + chunkSize);
reader.readAsBinaryString(fileSlice);
}
readNext();
}
function getMD5(blob, cbProgress) {
return new Promise((resolve, reject) => {
var md5 = CryptoJS.algo.MD5.create();
readChunked(blob, (chunk, offs, total) => {
md5.update(CryptoJS.enc.Latin1.parse(chunk));
if (cbProgress) {
cbProgress(offs / total);
}
}, err => {
if (err) {
reject(err);
} else {
// TODO: Handle errors
var hash = md5.finalize();
var hashHex = hash.toString(CryptoJS.enc.Hex);
resolve(hashHex);
}
});
});
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/3.1.2/components/core.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/3.1.2/components/md5.js"></script>
<input id="my-file-input" type="file">
<button onclick="process()">Process</button>

You don't need to read the whole file at once and feed it all in one go to CryptoJS routines.
You can create the hasher object, and feed chunks as you read them, and then get the final result.
Sample taken from the CryptoJS documentation
var sha256 = CryptoJS.algo.SHA256.create();
sha256.update("Message Part 1");
sha256.update("Message Part 2");
sha256.update("Message Part 3");
var hash = sha256.finalize();

Emoticons and chars like дей are broken while UInt8 conversion

i have the problem that I get JS-Objects with fields like ... title: "действительный有効なដែលមាន... or for example containing this emote 😍. Now, due to encryption, I need to convert this information to UInt8.
The code that handles the conversion looks like this (and works properly in the general case ...)
public arrayBufferToString(arrayBuffer : Uint8Array) : string {
var str : string = String.fromCharCode.apply(null, new Uint8Array(arrayBuffer));
return str;
}
public stringToArrayBuffer(string:string) : Uint8Array {
if(typeof string === 'undefined' || string === null){
this.$log.warn('Cannot convert an undefined string');
return null;
}
var arrayBuffer : any = new ArrayBuffer(string.length);
var buffer : Uint8Array = new Uint8Array(arrayBuffer);
for (var i : number = 0, stringLength : number = string.length; i < stringLength; i++) {
buffer[i] = string.charCodeAt(i);
}
return buffer;
}
In these situations, however, I have an input like this:
"title":"действительный有効なដែលមានសុពលភាព有效有效માન્યમાבתוקף" and an output, after formatting it back to a string, like this
"title":"459AB28B5;L=K9 ¹j¶»¶ H H®¾¨Í¯®¾ÑêÕçã"
Do you have any idea why this happens? Thank you!

The following scripts encrypts and decrypts dataToEncrypt. It uses modern Encoding API.
"use strict";
var dataToEncrypt = "дей ... hopefully, it means 'day' :)";
var iv = window.crypto.getRandomValues(new Uint8Array(16));
var algorithm = {
name: "AES-CBC",
iv: iv
};
window.crypto.subtle.generateKey(
{
name: "AES-CBC",
length: 128
},
/* extractable */ true,
/*keyUsages */ ['encrypt', 'decrypt']
).
then(function (aesKey) {
var uint16array = new TextEncoder('utf-16').encode(dataToEncrypt);
window.crypto.subtle.encrypt(algorithm, aesKey, uint16array)
.then(function (encryptedArrayBuffer) {
console.log("encrypted", ab2hexstr(encryptedArrayBuffer));
window.crypto.subtle.decrypt(algorithm, aesKey, encryptedArrayBuffer)
.then(function (result) {
console.log("decrypted", new TextDecoder('utf-16').decode(result));
})
.catch(function (err) {
console.log("Decryption failed.", err);
});
})
.catch(function (err) {
console.log("Encryption failed.", err);
});
}
).
catch(function (err) {
console.log("generateKey - failed.", err);
});
function ab2hexstr(buf) {
var byteArray = new Uint8Array(buf);
var hexString = "";
var nextHexByte;
for (var i = 0; i < byteArray.byteLength; i++) {
nextHexByte = byteArray[i].toString(16); // Integer to base 16
if (nextHexByte.length < 2) {
nextHexByte = "0" + nextHexByte; // Otherwise 10 becomes just a instead of 0a
}
hexString += nextHexByte;
}
return hexString;
}
jsFiddle for the previous script
jsFiddle for a version without Encoding API
The scripts are based on crypto.js.
Note:
I did not use TypeScript but you can add types as you like.
Resources:
https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder/encode
https://googlechrome.github.io/samples/encoding-api/index.html
https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/encrypt