We Keep Coding

"Aw snap!" message on uploading a large sized directory in chrome

I am trying to upload a folder present on my local directory to my server on S3 using an API call.
This works fine for small folders(number of files: 100, size: 51mb), however, for large folders(number of files: 868, size: 180mb), the tab crashes after a few seconds giving the message " Aw snap! Something went wrong while displaying this webpage.".
Uploading is done by first selecting the folder on local, which gives me paths of all the files in it and then converting it into a tree structure to send in the body of an API. The folder structure I send at backend has a tree-like structure. The function used is given below:
// files are the paths of the files present in the folder
this.readFolder = function(files) {
return new Promise(function(resolve, reject) {
var tree = {};
for (var i = 0; i < files.length; i++) {
var file = files[i];
var loadCount = 0;
var reader = new FileReader();
if (file.type && file.type.indexOf('image') > -1) {
reader.readAsDataURL(file);
} else {
reader.readAsText(file);
}
(function(file, reader) {
reader.onload = function() {
var path = file.webkitRelativePath.split('/');
var folder = tree;
for (var i in path) {
if (i == path.length - 1) {
folder[path[i]] = reader.result;
} else {
folder[path[i]] = folder[path[i]] || {};
}
folder = folder[path[i]];
}
loadCount++;
if (loadCount == files.length) {
resolve(tree);
}
};
})(file, reader);
}
});
};
The tree structure returned by the function is then sent in the body of an API.
$http.post("/api/upload", {
data: tree
}).then(function(res) {
if (res.data.error) {
toaster.pop('error', '', res.data.reason);
} else {
toaster.pop('success', '', 'Uploaded successfully.');
}
}, function(rej) {
var error = (rej.data) ? rej.data.msg : "Some error occured.";
toaster.pop('error', '', error);
});
After some digging, I have found out that this might be caused because the body part of the API is converted to a string and sending large strings might create memory issues in chrome. Can somebody tell me exactly why this is not working and how to make it work?
PS: I couldn't find an error logged on the server or developer console. Also after the tab crashes, developer tools gets disconnected.
Also, this is working on Mozilla Firefox.

Multiple file upload to s3 node.js

I am working on MEAN stack application and I am using AWS SDK to upload multiple files to S3. I am using busboy and AWS SDK.
Code:
var inputObj = {};
var busboy = new Busboy({ headers: req.headers });
// The file upload has completed
busboy.on('finish', function() {
console.log('Upload finished.....');
var file = [];
const file1 = req.files.clogo;
const file2 = req.files.cbanner1;
const file3 = req.files.cbanner2;
const file4 = req.files.cbanner3;
const file5 = req.files.cbanner4;
const file6 = req.files.clongHeader;
file.push(file1);
file.push(file2);
file.push(file3);
file.push(file4);
file.push(file5);
file.push(file6);
multipleUploadToS3(req.body.cname, file, function(fileName) {
console.log("client file upload finished.....");
if(fileName.length == 6){
inputObj.clogo = fileName[0];
inputObj.cbanner1 = fileName[1];
inputObj.cbanner2 = fileName[2];
inputObj.cbanner3 = fileName[3];
inputObj.cbanner4 = fileName[4];
inputObj.clongHeader = fileName[5];
console.log(inputObj);
var clientObj = new client(inputObj);
clientObj.save(function(err, client) {
console.log("Client Saved.....");
if (err) {
return res.status(400).send({
message: errorHandler.getErrorMessage(err)
});
} else {
res.json(client);
}
});
}
});
});
req.pipe(busboy);
File Upload Method:
function multipleUploadToS3(client, file, callback) {
console.log("multipleUpload to S3");
console.log(client);
console.log(file);
let s3bucket = new AWS.S3({
accessKeyId: IAM_USER_KEY,
secretAccessKey: IAM_USER_SECRET,
Bucket: BUCKET_NAME,
});
var fileNames = [];
for(var i=0; i<file.length; i++){
s3bucket.createBucket(function () {
var params = {
Bucket: BUCKET_NAME,
Key: client+ '/' + file[i].name,
Body: file[i].data,
};
s3bucket.upload(params, function (err, data) {
if (err) {
console.log('error in callback');
console.log(err);
}
console.log('success');
//console.log(data.key);
fileNames.push(data.key);
if(i == file.length){ callback(fileNames);}
});
});
}
};
The issue: file upload is asynchronous so for example if file1 I am uploading is honest.jpg then I want multipleUploadToS3 method to return file name after its done uploading to S3. I am binding this in inputObj keys which will be saved to mongo db. so inputObj.logo should have logo.png in it not the banner image which is happening due to asynchronous call.
This is working for a single file but failing for multiple files.

The problem is because for loop is synchronous and file upload is asynchronous.
Take a look at this example below,
for(var i = 0; i<5;i++) {
setTimeout(function(){ console.log(i); }, 100);
}
The above loop will print 5 for 5 times i.e 55555.
This behaviour is because for loop gets executed immediately making i=5 and when timeout gets executed it prints "i" value 5 for 5 times. 5 times because setTimeout is pushed in the queue for five times.
There are two ways to solve the problem you are facing
You can use Recursion.
Use neo-async(async-parallel) lib to control the async flow of javascript(Nodejs). click here for neo-async lib
Hope this clears your doubt. Please comment for more info.

Running node-rdkafka code in server

I'm running the below node-rdkafka code in Eclipse as Node.js application. This is the sample code from https://blizzard.github.io/node-rdkafka/current/tutorial-producer_.html
I want to run this in a test server and call from iOS Mobile application.
I knew about running node.js app in AWS.
Question I: Is there any other options to run in a free test server environment like Tomcat?
Question II: Even If I am able to run this node.js app in a server, how do i call from a mobile application? Do I need to call producer.on('ready', function(arg) (or) What function i need to call from Mobile app?
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
//counter to stop this sample after maxMessages are sent
var counter = 0;
var maxMessages = 10;
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
}
//need to keep polling for a while to ensure the delivery reports are received
var pollLoop = setInterval(function() {
producer.poll();
if (counter === maxMessages) {
clearInterval(pollLoop);
producer.disconnect();
}
}, 1000);
});
/*
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});*/
//starting the producer
producer.connect();

First of all, you need an HTTP server. ExpressJS can be used. Then, just tack on the Express code basically at the end, but move the producer loop into the request route.
So, start with what you had
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
});
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});
//starting the producer
producer.connect();
Then, you can add this in the same file.
var express = require('express')
var app = express()
app.get('/', (req, res) => res.send('Ready to send messages!'))
app.post('/:maxMessages', function (req, res) {
if (req.params.maxMessages) {
var maxMessages = parseInt(req.params.maxMessages);
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
} // end for
} // end if
}); // end app.post()
app.listen(3000, () => console.log('Example app listening on port 3000!'))
I don't think the poll loop is necessary since you don't care about the counter anymore.
Now, connect your mobile app to http://<your server IP>:3000/ and send test messages with a POST request to http://<your server IP>:3000/10, for example, and adjust to change the number of messages to send

I might be late on this but this is how I did using promises and found it better than have a time out etc.
const postMessageToPublisher = (req, res) => {
return new Promise((resolve, reject) => {
producer.connect();
producer.setPollInterval(globalConfigs.producerPollingTime);
const actualBody = requestBody.data;
const requestBody = req.body;
const topicName = req.body.topicName;
const key = requestBody.key || uuid();
const partition = requestBody.partition || undefined;
const data = Buffer.from(JSON.stringify(udpatedBody));
/**
* Actual messages are sent here when the producer is ready
*/
producer.on(kafkaEvents.READY, () => {
try {
producer.produce(
topic,
partition,
message,
key // setting key user provided or UUID
);
} catch (error) {
reject(error);
}
});
// Register listener for debug information; only invoked if debug option set in driver_options
producer.on(kafkaEvents.LOG, log => {
logger.info('Producer event log notification for debugging:', log);
});
// Register error listener
producer.on(kafkaEvents.ERROR, err => {
logger.error('Error from producer:' + JSON.stringify(err));
reject(err);
});
// Register delivery report listener
producer.on(kafkaEvents.PUBLISH_ACKNOWLEDGMENT, (err, ackMessage) => {
if (err) {
logger.error(
'Delivery report: Failed sending message ' + ackMessage.value
);
logger.error('and the error is :', err);
reject({ value: ackMessage.value, error: err });
} else {
resolve({
teamName: globalConfigs.TeamNameService,
topicName: ackMessage.topic,
key: ackMessage.key.toString()
});
}
});
});
};
Please note that kafkaEvents contains my constants for the events we listen to and it is just a reference such as kafkaEvents.LOG is same as event.log
and also the calling function is expecting this to a promise and accordingly we user .then(data => 'send your response to user from here') and .catch(error => 'send error response to user
this is how I achieved it using promises

DocumentDB connection in js: "require is not defined"

I've been able to create the database and query it. Using Microsoft's tutorial on using node.js to query, I have been able to accomplish this with this code:
// Simple Query
"use strict";
var documentClient = require("documentdb").DocumentClient;
var config = require("./config");
var url = require('url');
// use the previously saved config.endpoint and config.primaryKey to create a new DocumentClient
var client = new documentClient(config.endpoint, { "masterKey": config.primaryKey });
// These urls are how the DocumentDB client will find the right database and collection.
var HttpStatusCodes = { NOTFOUND: 404 };
var databaseUrl = `dbs/${config.database.id}`;
var collectionUrl = `${databaseUrl}/colls/${config.collection.id}`;
// Query JSON document collection
function queryCollection() {
console.log(`Querying collection through index:\n${config.collection.id}`);
return new Promise((resolve, reject) => {
client.queryDocuments(
collectionUrl,
'SELECT VALUE gd.NFL FROM GamblersDenDB gd WHERE gd.id = "SanDiego"'
).toArray((err, results) => {
if (err) reject(err)
else {
for (var queryResult of results) {
let resultString = JSON.stringify(queryResult);
console.log(`\tQuery returned ${resultString}`);
}
console.log();
resolve(results);
}
});
});
};
queryCollection()
Running that js file in my command prompt works! It results in the output:
C:\Users\kenv\Desktop\DocDB Test>node SimpleQuery.js
Querying collection through index:
GamblersDenColl
Query returned {"ID":"SDC","name":"Chargers"}
Great. So now I've transferred my code to my project's folder and try to run in the app with
taco run android --device
When I pull up the console, the first error that sticks out to me is
Uncaught ReferenceError: require is not defined(…)
It's pointing to the line var documentClient = require("documentdb").DocumentClient; in my code.
Here is my complete js file code that's throwing the error:
(function () {
"use strict";
document.addEventListener( 'deviceready', onDeviceReady.bind( this ), false );
function onDeviceReady() {
navigator.splashscreen.hide();
console.log("Cordova is READY!");
// Handle the Cordova pause and resume events
document.addEventListener( 'pause', onPause.bind( this ), false );
document.addEventListener( 'resume', onResume.bind( this ), false );
$(".btnURL").on("click", function(){loadURL($(this))});
function loadURL(theObj) {
cordova.InAppBrowser.open(theObj.data("url"), "_blank", "location=yes");
}
//********* jQuery VARIABLES ***************//
var $elBtnSaveName= $("#btnSaveName"),
$elShowClients= $("#btnShowClients"),
$elDivShow= $("#divShow"),
$elFormClient= $("#formClient");
//********** EVENT HANDLERS *****************//
$elShowClients.on("click", queryCollection);
//********************* DOCUMENT DB SECTION *********************************************************/
var documentClient = require("documentdb").DocumentClient;
var config = require("./config");
var url = require('url');
// use the previously saved config.endpoint and config.primaryKey to create a new DocumentClient
var client = new documentClient(config.endpoint, { "masterKey": config.primaryKey });
// These urls are how the DocumentDB client will find the right database and collection.
var HttpStatusCodes = { NOTFOUND: 404 };
var databaseUrl = `dbs/${config.database.id}`;
var collectionUrl = `${databaseUrl}/colls/${config.collection.id}`;
// Query JSON document collection
function queryCollection() {
console.log(`Querying collection through index:\n${config.collection.id}`);
return new Promise((resolve, reject) => {
client.queryDocuments(
collectionUrl,
'SELECT VALUE gd.NFL FROM GamblersDenDB gd WHERE gd.id = "SanDiego"'
).toArray((err, results) => {
if (err) reject(err)
else {
for (var queryResult of results) {
let resultString = JSON.stringify(queryResult);
console.log(`\tQuery returned ${resultString}`);
}
console.log();
resolve(results);
fnShowClientsTable(result.rows);
}
});
});
};
function fnShowClientsTable(data){
var str = "<p><table id='tableResults'";
str += "<tr><th>ID</th><th>Name</th><th class='thEmpty'> </th></tr>" //added class to <th> for formatting
for(var i = 0; i < data.length; i++) { // For X number of times worth of data...
str += "<tr><td>" + data[i].doc.ID +
"</td><td>" + data[i].doc.name +
"</td><td class='btnPencil'>✎</td></tr>";
}
str += "</table></p>"; // END table
$elDivShow.html(str); //Show string as HTML on screen
} // END fnShowClientsTable
//************************* END DOCUMENT DB SECTION ******************************************************/
}; // END onDeviceReady()
function onPause() {
// TODO: This application has been suspended. Save application state here.
};
function onResume() {
// TODO: This application has been reactivated. Restore application state here.
};
} )();
I realize my other function to push it to a string for display in HTML is probably wrong (which I will most certainly have another post about that one :)) but right now I'm trying to determine how I can get past this first error.

Ionic how can I create an mp3 file?

I'm writing a web app with Ionic framework and I'm trying to manage a record and play sounds mechanism. I'm using the following snippet as a service:
.factory('MediaSrv', function ($q, $ionicPlatform, $window) {
var service = {
loadMedia: loadMedia,
getStatusMessage: getStatusMessage,
getErrorMessage: getErrorMessage
};
function loadMedia (src, onError, onStatus, onStop) {
var defer = $q.defer();
$ionicPlatform.ready(function () {
var mediaSuccess = function () {
if (onStop) { onStop(); }
};
var mediaError = function (err) {
_logError(src, err);
if (onError) { onError(err); }
};
var mediaStatus = function (status) {
if (onStatus) { onStatus(status); }
};
if ($ionicPlatform.is('android')) {
src = '/android_asset/www/' + src;
}
defer.resolve(new $window.Media(src, mediaSuccess, mediaError, mediaStatus));
});
return defer.promise;
}
...
return service;
});
I'm able to play an existing .mp3 file, but I cannot record on a non-existing file. I thought it would create the file by itself if the file wasn't found. How can I create an empty .mp3 file for recording?

Creating a file is a server function. You would need a node server using fs to create a file.
From Ionic's website:
Think of Ionic as the front-end UI framework that handles all of the look and feel and UI interactions your app needs in order to be compelling. Kind of like "Bootstrap for Native," but with support for a broad range of common native mobile components, slick animations, and beautiful design.

Ionic can use the Cordova plugins since its built on top of it.
You can use the media-capture plugin to capture audio, however I have found these record as AMR files.
From the [documentation][1]:
// capture callback
var captureSuccess = function(mediaFiles) {
var i, path, len;
for (i = 0, len = mediaFiles.length; i < len; i += 1) {
path = mediaFiles[i].fullPath;
// do something interesting with the file
}
};
// capture error callback
var captureError = function(error) {
navigator.notification.alert('Error code: ' + error.code, null, 'Capture Error');
};
// start audio capture
navigator.device.capture.captureAudio(captureSuccess, captureError, {limit:2});
[1]: http://docs.phonegap.com/en/edge/cordova_media_capture_capture.md.html