I have a very large array (10K) and i want to split it (i did according to this: https://stackoverflow.com/a/8495740/2183053 and it worked) but i need to pass the tempArray to request and wait for response which will be passed to savetodb.
Can someone help me out please. To be clear, I want to split the large Array and pass the separated arrays to request function then pass to save to db and continue this process until all the arrays are cleared.
the following is the code i did:
//i used waterfall because it worked in waiting to finish the first task before starting another
async.waterfall([
async.apply(handleFile, './jsonFile.json')
runRequest1,
savetoDb
], function(err) {
console.log('waterfall1 complete')
})
function handleFile(data, callback) {
var name
var authorNames = []
require('fs').readFile(data, 'utf8', function(err, data) {
if (err) throw err
var content = _.get(JSON.parse(data), [2, 'data'])
for (var x = 0; x < content.length; x++) {
authorNames.push(JSON.stringify(content[x]['author']))
}
//the large array is authorNames and im splitting it below:
for (i = 0, j = authorNames.length; i < j; i += chunk) {
temparray = authorNames.slice(i, i + chunk);
setTimeout(function() {
callback(null, temparray)
}, 2000);
}
})
}
You need promise to handle async things in nodejs
let findAllReceipts = function (accountArray) {
const a = [];
for (let i = 0; i < accountArray.length; i++) {
a.push(new Promise((resolve, reject) => {
receiptTable.find({accountNo: accountArray[i].accountNo}, function (err, data) {
if (!err) {
resolve(data);
} else {
reject(new Error('findPrice ERROR : ' + err));
}
});
}));
}
return Promise.all(a);
};
I added some promise.
const data = await handleFile('./jsonFile.json');
// save to db
async function handleFile(filePath) {
let arrayWillReturn = [];
var name
var authorNames = []
let data = await getFileData(filePath)
var content = _.get(JSON.parse(data), [2, 'data'])
for (var x = 0; x < content.length; x++) {
authorNames.push(JSON.stringify(content[x]['author']))
}
//the large array is authorNames and im splitting it below:
for (i = 0, j = authorNames.length; i < j; i += chunk) {
arrayWillReturn.push(authorNames.slice(i, i + chunk));
}
return arrayWillReturn;
}
async function getFileData(fileName) {
return new Promise(function (resolve, reject) {
fs.readFile(fileName, type, (err, data) => {
err ? reject(err) : resolve(data);
});
});
}
I'm answering my own question for future references. the only thing that I added to make my code work is Promise. It sounds easy but it took me a while to grasp the function and implementing it but it worked and it was worth it. Thank you so much for the responses.
Related
Trying to get values from an API using async functions. I need to be able to access the values in the array immediately.
componentDidMount() {
var devices = this.props.navigation.state.params.userInfoState.deviceIds
async function call_vstat(device, token) {
try {
let response = await fetch(
'http://app.yatis.io/api/admin/getDeviceStatus?api_access_token=' + token + '&deviceId=' + device,
);
let responseJson = await response.json();
return responseJson;
} catch (error) {
console.error(error);
}
}
var results = []
for (i = 0; i < devices.length; i++) {
var result = call_vstat(devices[i], my_token)
.then( (result) => (results.push(result) ) )
}
console.log(results.length)
console.log(results)
}
Here's the issue when I look at the logger, I get an array of length twenty one but when logging the length itself it shows zero.
If you are using async await you don't have to use the then(). What you can do is
for (i = 0; i < devices.length; i++) {
var result = await call_vstat(devices[i], my_token)
results.push(result)
}
Hope this works
As mentioned in my comments,
var promiseArray = [];
for (i = 0; i < devices.length; i++) {
promiseArray.push(call_vstat(devices[i], my_token));
}
var results = await Promise.all(promiseArray);
if (results.length > 0) {
//perform your business logic after this check to avoid any errors
}
I am modernizing some code. It has a piece to load database implemented as:
var customerQueue = async.queue(insertCustomer, DATABASE_PARALLELISM);
customerQueue.drain = function() {
logger.info('all customers loaded');
airportCodeMappingQueue.push(airportCodeMappings);
}
The function insertCustomer used to written with callbacks. I changed it to async/await, as a part of code modernization.
Now, think that I wrote an equivalent of async.queue as:
let customerQueueElements = [];
var customerQueue = {};
customerQueue.push = (customers) => {
customers.forEach(customer => {
customerQueueElements.push(insertCustomer(customer))
});
}
const processQueue = async (queue, parallelism) => {
for (let i = 0; i < queue.length; i += parallelism) {
for (let j = 0; j < parallelism; j++) {
let q = []
if (queue[i + j]) {
q.push(queue[i + j])
}
await Promise.all(q)
}
}
}
I am able now to do await ProcessQueue(customerQueue, DATABASE_PARALLELISM), but the syntax is bad, and I am keeping a visible named variable for each queue.
What would be a good way to handling this?
Also, drain() should be hooked-up to then, right ?
#Bergi is correct, as far as direction. I put together a work-in-process version:
module.exports = function () {
module.internalQueue = []
module.func = undefined
module.parallelism = 1
const process = async () => {
for (let i = 0; i < module.internalQueue.length; i += module.parallelism) {
for (let j = 0; j < module.parallelism; j++) {
let q = []
if (module.internalQueue[i + j]) {
q.push(module.func(module.internalQueue[i + j]))
}
await Promise.all(q)
}
}
module.internalQueue = []
module.drain()
}
module.queue = (func, parallelism = 1) => {
module.func = func
module.parallelism = parallelism
return module
}
module.push = async (customers) => {
module.internalQueue.push(customers)
await process()
}
module.drain = () => {}
return module
}
It is not exactly correct, yet. The signatures are similar to the async package, but my version shares the queue among all instances.
Need to figure out an easy way of creating an instance for each function with a separate e.g. "local" queue. Then, it will basically work like the original one.
Will update as I have progress.
I'm having a problem with my program at the moment in which I am running a promise in the .then of a promise, however it is running the 2nd promise before the first. This is not what I want since it relies on the 1st promise to work. Here is my code (The console.logs are random, but I just came up with stuff on the spot for checking when things run)
new Promise((resolve, reject) => {
for (i = 0; i < 10; i++) {
var query = {
class: result[`class${i}`]
};
dbo.collection("homeworks").find(query).toArray(function(err, result) {
if (err) throw err;
for (i = 0; i < result.length; i++) {
current = current + 1;
console.log(current)
allhomework[current] = result[i];
console.log(allhomework[current])
}
db.close();
});
}
console.log("HOLA");
resolve(allhomework);
})
.then((allhomework) => {
console.log("HOLA MIS AMIGOS")
new Promise((resolve, reject) => {
console.log("v");
for (i = 0; i < 10; i++) {
console.log("uaefshiudvhofdh")
console.log(allhomework[1]);
}
resolve();
}).then(() => {
response.render('index', {
username: req.session.user,
homeworks: allhomework
});
})
})
.catch(console.log)
Well, the best way to solve this would be to use the promise interface for your database and use Promise.all() to track when all the async database operations in your loop are done. But, if you want to code it manually with your existing loop, you just have to keep a counter for when they are all done.
new Promise((resolve, reject) => {
let cntr = 10;
for (let i = 0; i < 10; i++) {
var query = {
class: result[`class${i}`]
};
dbo.collection("homeworks").find(query).toArray(function(err, result) {
if (err) {
db.close();
return reject(err);
}
for (let i = 0; i < result.length; i++) {
current = current + 1;
console.log(current)
allhomework[current] = result[i];
console.log(allhomework[current])
}
--cntr;
if (cntr === 0) {
db.close();
resolve(allHomework);
}
});
}
console.log("HOLA");
});
Other notes about this code:
I'm also wondering why there's a db.close() in the middle of your outer loop. I would think that would cause a problem.
I also added a reject(err) to your if (err) check.
You have loop conflict with your i variable. Put a let for each one to make them separate or change one of them to a different letter.
Here's a version that uses the promise interface in your database. I was able to completely get rid of the inner loop and just let promises collect all the results in order for us. Monbodb's .toArray() returns a promise that resolves with the query result if you don't pass it a callback. We can use that with Promise.all() to collect all the results in order and to tell us when they are all done:
let promises = [];
for (let i = 0; i < 10; i++) {
let query = {
class: result[`class${i}`]
};
promises.push(dbo.collection("homeworks").find(query).toArray());
}
Promise.all(promises).then(allResults => {
// flatten the results into all one array - promises will have kept them in proper order
db.close();
let results = [].concat(...allResults);
// now do whatever you want with the final array of results
}).catch(err => {
db.close();
// handle error here
});
I have a solution where i have 2 loops inside each other it looks like this:
function myFunc(types, pages, callback) {
var pageResults = [];
for (var i = 0; i < types.length; i++) {
var type = types[i];
for (var j = 1; j < pages; j++) {
getBikes(type, j, function(err, results, settings){
if(err) throw err;
var obj = {
results: results,
settings: settings
};
pageResults.push(obj);
});
};
};
return callback(pageResults);
});
var types = ["string1", "string2", "string3", "string4"];
var pages = 15;
myFunc(types, pages, function(pageResults){
console.log("done");
});
But this ends right away it returns after 1 loop in the types loop, and if i log the obj that i return from the inner function without returning a callback it will run all the way through.
Yes getBikes is a async call similar to ajax, but in nodejs, with a module called request.js
you can not return the callback value but you can call the callback function like below
function myFunc(types, pages, callback) {
var pageResults = [];
for (var i = 0; i < types.length; i++) {
var type = types[i];
for (var j = 1; j < pages; j++) {
getBikes(type, j, function(err, results, settings){
if(err) throw err;
var obj = {
results: results,
settings: settings
};
pageResults.push(obj);
if((i === types.length-1) && (j === pages -1)){
callback(pageResults);
}
});
};
};
});
I have this code and I want to make the code wait until the asyncrnous query called prom has finessed before restarting the first for loop. So the array will be reset, before it starts the first for loop again.
items = [];
var promises = [];
for (var i = 0; i < userArray.length; i++) {
items.length = 0;
for (var i2 = 0; i2 < test.length; i2++) {
var UserFavourite = Parse.Object.extend("UserFavourite");
var queryUserFav = new Parse.Query(UserFavourite);
queryUserFav.equalTo('item', test[i2].get('item'));
queryUserFav.equalTo('school', test[i2].get('school'));
queryUserFav.equalTo('user1', userArray[i])
var prom = queryUserFav.find().then(function(res) {
for (var i3 = 0; i3 < res.length; i3++){
var item = res[i3];
var itemName = item.get('item');
items.push(itemName);
console.log(items)
}
return items;
});
promises.push(prom);
}
//return Parse.Promise.when.apply(Parse.Promise, promises); I have tried it here but
// this just stops the first for loop after its first loop
}
return Parse.Promise.when.apply(Parse.Promise, promises);
What you're trying to do is have a chain of promises, one for each item in an array.
If would be nice if javascript had an equivalent of .NET's await keyword, where you could go
await Parse.Promise.when(promises)
and then it allowed the promise code to run then returned back to running any code after the await. But Javascript doesn't give this to us.
Another approach is to maintain an index variable. After every set of queries is processed, you increment the index variable and process the next set of values.
function parseForUser(user) {
var promises = [];
for (var i2 = 0; i2 < test.length; i2++) {
var items = [];
var UserFavourite = Parse.Object.extend("UserFavourite");
var queryUserFav = new Parse.Query(UserFavourite);
queryUserFav.equalTo('item', test[i2].get('item'));
queryUserFav.equalTo('school', test[i2].get('school'));
queryUserFav.equalTo('user1', user)
var prom = queryUserFav.find().then(function(res) {
for (var i3 = 0; i3 < res.length; i3++){
var item = res[i3];
var itemName = item.get('item');
items.push(itemName);
console.log(items)
}
return items;
});
promises.push(prom);
}
return Parse.Promise.when(promises);
}
function parseUserArray(userArray) {
var returnPromise = new Parse.Promise(); // Do you have to call it via new?
//The documentation isn't clear.
var index = 0;
var doNext = function() {
if(index < userArray.length) {
var promise = parseForUser(userArray[index++]);
promise.done(doNext);
} else {
returnPromise.resolve();
}
}
doNext();
return returnPromise;
}
var parseUserArrayPromise = parseUserArray(userArray);
FWIW ...
This solution differs from #AndrewShepherd's chiefly in that here we take double advantage of the promise returned by asyncProcessUser().
firstly for flow control - the asynchronous sequencing of the inner loop
secondly for delivering an array of results, from which the final array of results is built, avoiding the need for an outer promises array.
function parseUserArray(userArray, test) {
// This function is the original inner for() loop, now expressed as a .map(),
// (plus peripheral wrapping and chaining).
// It asynchronously processes a single user against all test items,
// and returns a promise of an array of results.
// The promise resolves when all the individual .finds complete.
function asyncProcessUser(user) {
return Parse.Promise.when(test.map(function(dataItem) {
return (new Parse.Query(Parse.Object.extend("UserFavourite")))
.equalTo('item', dataItem.get('item'))
.equalTo('school', dataItem.get('school'))
.equalTo('user1', user)
.find().then(function(res) {
return res.map(function(r) {
return r.get('item');
});
});
})).then(function() {
return Array.prototype.slice.apply(arguments).reduce(function(arr, arr_) {
return arr.concat(arr_);
}, []);
});
}
// This is the original outer for() loop, now expressed as a .reduce(), which is
// a common pattern for performing a series of async tasks (ie what was the inner loop).
// Here, `userArray.reduce(...)` returns a promise of an array of
// the objects returned by `r.get('item')` above.
return userArray.reduce( function(p, user) {
return p.then(function(arr) {
return asyncProcessUser(user).then(function(arr_) {
return arr.concat(arr_);
});
});
}, Parse.Promise.as([]) );//† "returns a new promise that is resolved with a given value".
}
† : Documentation for Parse.Promise.as()
Without the comments, it's quite concise.
The concept is demonstrated here. Don't worry that the demo uses jQuery promises, it's the concept that matters.
Use this
function parseForUser(user) {
var promises = [];
for (var i2 = 0; i2 < test.length; i2++) {
var items = [];
var UserFavourite = Parse.Object.extend("UserFavourite");
var queryUserFav = new Parse.Query(UserFavourite);
queryUserFav.equalTo('item', test[i2].get('item'));
queryUserFav.equalTo('school', test[i2].get('school'));
queryUserFav.equalTo('user1', user)
var prom = queryUserFav.find().then(function(res) {
for (var i3 = 0; i3 < res.length; i3++){
var item = res[i3];
var itemName = item.get('item');
items.push(itemName);
console.log(items)
}
return items;
});
promises.push(prom);
}
return Parse.Promise.when(promises);
}
function parseUserArray(userArray) {
var returnPromise = new Parse.Promise(); // Do you have to call it via new?
//The documentation isn't clear.
var index = 0;
var doNext = function() {
if(index < userArray.length) {
var promise = parseForUser(userArray[index++]);
promise.done(doNext);
} else {
returnPromise.resolve();
}
}
doNext();
return returnPromise;
}
Just copy and paste