I am trying to do pagination with promise. Here's my workflow.
Step 1 : Total page in pagination is 50. The url will be like url?page=1
Step 2 : In each page i will get 50 products, which i need to call a seperate api.
The only condition is, during pagination, the second page should called only if first page's 50 api call is done. Once all the 50 pages are fetched, it should return the promise.
What i have so far is.
let promises = [];
paginate(50);
Promise.all(promises)
.catch(function(err) {
console.log('err',err);
})
.then(() => {
console.log('All Done!!!!!');
});
function paginate(loop){
promises.push(
axios(config)
.then(function (response) {
// Got 50 products
})
.catch(function (error) {
console.log('err',err);
})
)
In the place Got 50 products i need to still iterate the 50 products in axios. I am not sure whether it is possible to do promise inside promise.
As the server can't bear all sudden loads, the only condition is to iterate the second 50 products (or next page's products) only after first (or previous 50 api is called).
Edit :
// Here i have got 50 products
As i told, On each page i will get 50 products, i will call another api for all those 50 products. I have given the code below.
Only constraint is on first page response, the response 50's api should be called.. it's like product?product=1 . the next 50 should be called only after first 50's api called
for (let index = 0; index < response.length; index++) {
const element = response[index];
//Here call api for one item
axios(config)
.then(function (elemen t) {
// Here i have got 50 products
})
.catch(function (error) {
console.log('err',err);
})
}
You're probably not looking for Promise.all (which is meant for running things in parallel), but for recursion:
fetchPages(0, 49)
.then(console.log);
function fetchPages(from, to, results = []) {
if (from <= to) {
return fetchPage(from)
.then(res => results.push(...res))
.then(() => fetchPages(from + 1, to, results)); // calls itself
} else {
return results;
}
}
function fetchPage(n) {
console.log(`Fetching page ${n}`);
// Here, you would return axios()...
// But just for this demo, I fake that:
const results = new Array(50).fill(null)
.map((_,i) => `Product ${i} from page ${n}`);
return new Promise(resolve => setTimeout(() => resolve(results), 100));
}
Edit
The code above solves the problem of pages needing to be fetched only one after the other. We can keep the fetchPages function as it is. Now, since every page will contain products which will need to be fetched individually, we'll just edit fetchPage a bit.
There are multiple ways this could be done. Here are some:
Solution A: fetch every product in parallel
If the server can handle 50 requests going off at once, you could use Promise.all:
function fetchPage(n) {
console.log(`Fetching page ${n}`);
return axios(`/page?page=${n}`)
.then(productIds => {
return Promise.all(productIds.map(fetchProduct));
});
}
function fetchProduct(id) {
return axios(`/product?product=${id}`);
}
Solution B: fetch every product in sequence
If the server can't handle multiple requests going off at once, you could use recursion once again:
function fetchPage(n) {
console.log(`Fetching page ${n}`);
return axios(`/page?page=${n}`)
.then(fetchproducts);
}
function fetchProducts(productIds, results = []) {
if (productIds.length) {
const productId = productIds.shift();
return fetchProduct(productId)
.then(res => results.push(res))
.then(() => fetchProducts(productIds, results)); // calls itself
} else {
return results;
}
}
function fetchProduct(id) {
return axios(`/product?product=${id}`);
}
Solution C: fetch products X requests at a time
If the server can handle X requests going off at once, you could use a module like queue, which can help you with concurrency:
const queue = require('queue'); // Don't forget to $ npm install queue
const MAX_CONCURRENT_CALLS = 4; // 4 calls max at any given time
function fetchPage(n) {
console.log(`Fetching page ${n}`);
return axios(`/page?page=${n}`)
.then(fetchproducts);
}
function fetchProducts(productIds) {
const q = queue();
q.concurrency = MAX_CONCURRENT_CALLS;
const results = [];
q.push(
...productIds.map(productId => () => {
return fetchProduct(productId)
.then(product => results.push(product));
})
);
return new Promise((resolve, reject) => {
q.start(function (err) {
if (err) return reject(err);
resolve(results);
});
});
}
function fetchProduct(id) {
return axios(`/product?product=${id}`);
}
Related
What I'm trying to do in my endpoint, is:
Make an API call, which returns a JSON
for each item: search in our database for it
If it's found, skip it.
If it's not found, push it in an array "Response"
This is my code:
app.get("/test", (req,res) => {
spotifyApi.getUserPlaylists({ limit: 50 })
.then(function(data) {
let finalres = [];
const tbp = data.body.items;
// res.send('ok stop loading');
tbp.forEach(element => locateit(element,finalres));
console.log('This is the length of finalres, which should be 1:', finalres.length);
finalres.forEach(item =>{console.log(item)});
function locateit(item, finalres){
const thisplaylistid = item.id;
collection.find({ "id" : thisplaylistid }).toArray((error, result) => {
if(error) {
return res.status(500).send(error);
}
if(result.length == 0) { // if we don't find this playlist in our DB
console.log('This playlist is not in our database: ');
console.log(thisplaylistid);
finalres.push(thisplaylistid);
}
else{ //if it's already in our DB
console.log('This item is in our database.'); //This should be printed first, six times.
}
});
};
});
});
The content of data.body.items is 7 items, where only the first 6 of them are in our DB. This means, that the last item, should be pushed in finalres.
Therefore, the expected console outcome should be:
This item is in our database.
This item is in our database.
This item is in our database.
This item is in our database.
This item is in our database.
This playlist is not in our database:
3uDLmuYPeRUxXouxuTsWOe
This is the length of finalres, which should be 1: 1
3uDLmuYPeRUxXouxuTsWOe
But instead, I get this:
This is the length of finalres, which should be 1: 0
This should be displayed first, six times.
This should be displayed first, six times.
This should be displayed first, six times.
This should be displayed first, six times.
This should be displayed first, six times.
This should be displayed first, six times.
This playlist is not in our database:
3uDLmuYPeRUxXouxuTsWOe
It is obviously not executed in the right order. I tried to use async-wait, but I'm struggling to understand where/how it should be implemented. Any help?
This is the part where I tried it, but I get the same console outcome as before:
async function locateit(item, finalres){
const thisplaylistid = item.id;
await collection.find({ "id" : thisplaylistid }).toArray((error, result) => {
...
Update
After reading more about async-wait and promises, I tried to do it this way, but I'm still getting the same output.
app.get("/test", (req,res) => {
spotifyApi.getUserPlaylists({ limit: 50 })
.then(function(data) {
let finalres = [];
const tbp = data.body.items;
// res.send('ok stop loading');
for (const playlist of tbp) {
async function doWork() {
const found = await indb(playlist.id); //returns t/f if found or not found
if (!found){
finalres.push(playlist);
}
}
doWork();
}
console.log('This is the length of finalres and it should be 1: ',finalres.length);
})
});
and the indb function looks like that:
function indb(thisplaylistid){
return new Promise((resolve, reject) =>{
console.log('Searching in our DB...');
collection.find({ "id" : thisplaylistid }).toArray((error, result) => {
if(result.length == 0) { // if we don't find this playlist in our DB
console.log('This playlist is not in our database: ');
console.log(thisplaylistid);
resolve(false); //returns the id
}
else{ //if it's already in our DB
console.log('This item is in our database.'); //This should be printed first, six times.
resolve(true);
}
});
})
}
The problem here is that forEach resolves always resolves as void, no matter if you have async promises running within.
So, your code will return before executing the statements within the forEach
The correct would be wait for all promises to resolve using #Promise.all
Try this instead:
Updated
Using promise as suggested by Bergi instead of callback ( preferable )
app.get("/test", (req, res) => {
spotifyApi.getUserPlaylists({ limit: 50 })
.then((data) => {
// :refac: more meaningful variable names
const playlists = data.body.items
return Promise.all(
playlists.map(
// :refac: destructuring to get only the id, other ain't necessary
async({ id }) =>
collection.find({ id }).toArray()
)
)
.then(playlistsById =>
// :refac: no error occurred fetching playlists
const nonEmptyPlaylists = playlistsById.filter(playlistById => playlistById.length !== 0)
res.status(200).send(nonEmptyPlaylists)
)
.catch(error => {
// :refac: some error occurred at searching some playlist
console.log('error', error)
// :refac: if you might expect that is going to throw an error here, the code shouldn't be 500
return res.status(400).send(error)
})
})
})
As others mentioned, your usage of async/await is wrong. I believe this should work and do what you want, and as a bonus its shorter and easier to read. mastering async and await will simplify your life and save you from callback/promise hell, i highly recommend it
app.get("/test", async (req, res) => {
try {
const data = await spotifyApi.getUserPlaylists({ limit: 50 });
const tbp = data.body.items;
const results = [];
for(let item of tbp) {
const found = await indb(item.id);
if(!found){
results.push(item);
}
}
return res.status(200).send(results);
}
catch(err) {
return res.status(400).send(err);
}
});
I've written a Node.js script that uses the download, axios, and fs modules to extract urls from JSON provided by the Federal Register, and download the associated PDF files. However, the script routinely fails to download all of the PDFs.
For whatever reason, my script "stalls" before downloading all of the PDF files. Meaning, it starts off great (downloads maybe 70, 80 files) but then stalls. It doesn't fire my catch block, or fail in any way. It just stops downloading.
The number of files varies based on what wifi connection I'm on. However, I've never been able to get the code to finish, and fire the .then block in my code. Ideally, I would like to use the .then block to process the files once they are downloaded.
Here is the code:
// The callback function that writes the file...
function writeFile(path, contents, cb){
mkdirp(getDirName(path), function(err){
if (err) return cb(err)
fs.writeFile(path, contents, cb)
})
};
// The function that gets the JSON...
axios.get(`http://federalregister.gov/api/v1/public-inspection-documents.json?conditions%5Bavailable_on%5D=${today}`)
.then(downloadPDFS)
.catch((err) => {
console.log("COULD NOT DOWNLOAD FILES: \n", err);
});
// The function that downloads the data and triggers my write callback...
function downloadPDFS(res) {
const downloadPromises = res.data.results.map(item => (
download(item.pdf_url)
.then(data => new Promise((resolve, reject) => {
writeFile(`${__dirname}/${today}/${item.pdf_file_name}`, data, (err) => {
if(err) reject(err);
else resolve(console.log("FILE WRITTEN: ", item.pdf_file_name));
});
}))
))
return Promise.all(downloadPromises).then((res) => console.log("DONE"))
}
My project is on Github here, in case you'd like to install it and try for yourself. Here's a summary of what's going on, in plain English:
The script fetches JSON from a server, which contains the urls to all 126 PDFs. It then passes an array of these urls to the synchronous map function. Each of the urls is transformed into a promise, with the download module. That promise is implicitly returned, and stored in the Promise.all wrapper. When the download promise resolves (the document is done downloading) my custom writeFile function will trigger, writing the PDF file with the downloaded data. When all of the files have downloaded, the Promise.all wrapper should resolve. But that doesn't happen.
What is going wrong?
EDIT --
As you can see below, the script runs for a while, but then it just stalls and doesn't download any more files...
If it really is a rate issue then there's a few ways you can solve it (depending on how the API is rate limited)
Below there are 3 solutions in one
rateLimited ... this fires off requests limited to a given number of requests per second
singleQueue ... one request at a time, no rate limit, just all requests in series
multiQueue ... at most a given number of requests "in flight" at a time
const rateLimited = perSecond => {
perSecond = isNaN(perSecond) || perSecond < 0.0001 ? 0.0001 : perSecond;
const milliSeconds = Math.floor(1000 / perSecond);
let promise = Promise.resolve(Date.now);
const add = fn => promise.then(lastRun => {
const wait = Math.max(0, milliSeconds + lastRun - Date.now);
promise = promise.thenWait(wait).then(() => Date.now);
return promise.then(fn);
});
return add;
};
const singleQueue = () => {
let q = Promise.resolve();
return fn => q = q.then(fn);
};
const multiQueue = length => {
length = isNaN(length) || length < 1 ? 1 : length;
const q = Array.from({ length }, () => Promise.resolve());
let index = 0;
const add = fn => {
index = (index + 1) % length;
return q[index] = q[index].then(fn);
};
return add;
};
// uncomment one, and only one, of the three "fixup" lines below
let fixup = rateLimited(10); // 10 per second for example
//let fixup = singleQueue; // one at a time
//let fixup = multiQueue(6); // at most 6 at a time for example
const writeFile = (path, contents) => new Promise((resolve, reject) => {
mkdirp(getDirName(path), err => {
if (err) return reject(err);
fs.writeFile(path, contents, err => {
if (err) return reject(err);
resolve();
})
})
});
axios.get(`http://federalregister.gov/api/v1/public-inspection-documents.json?conditions%5Bavailable_on%5D=${today}`)
.then(downloadPDFS)
.catch((err) => {
console.log("COULD NOT DOWNLOAD FILES: \n", err);
});
function downloadPDFS(res) {
const downloadPromises = res.data.results.map(item => fixup(() =>
download(item.pdf_url)
.then(data => writeFile(`${__dirname}/${today}/${item.pdf_file_name}`, data))
.then(() => console.log("FILE WRITTEN: ", item.pdf_file_name))
));
return Promise.all(downloadPromises).then(() => console.log("DONE"));
}
I've also refactored the code a bit so downloadPDFS uses promises only - all the node-callback style code is put into writeFile
As Jaromanda pointed out, this is likely to do with the API limiting my access, not with an error in the script.
I added a filter to the script, to select less data, and it works. As follows:
axios.get(`http://federalregister.gov/api/v1/public-inspection-documents.json?conditions%5Bavailable_on%5D=${today}`)
.then(downloadPDFS)
.then(() => {
console.log("DONE")
})
.catch((err) => {
console.log("COULD NOT DOWNLOAD FILES: \n", err);
});
function downloadPDFS(res) {
const EPA = res.data.results.filter((item) => {
return item.agencies[0].raw_name === "ENVIRONMENTAL PROTECTION AGENCY"; //// THIS FILTER
});
const downloadPromises = EPA.map(item => ( //// ONLY DOWNLOADING SOME OF THE DATA
download(item.pdf_url)
.then(data => new Promise((resolve, reject) => {
writeFile(`${__dirname}/${today}/${item.pdf_file_name}`, data, (err) => {
if(err) reject(err);
else resolve(console.log("FILE WRITTEN: ", item.pdf_file_name));
});
}))
))
return Promise.all(downloadPromises)
}
In the following code, why doesn't the promise inside get_dbinfo resolve prior to executing .then(result) in the calling code block?
My understanding is that the code inside the new Promise will complete before returning to the .then part of the calling statement.
dbFuncs.get_dbinfo()
.then((result) => {
count = result.info.doc_count
if (count < 500){perPage = count};
});
function get_dbinfo() {
return new Promise((resolve, reject) => {
return db.info()
.then((result) => {
resolve(result)
}).catch((err) => {
console.log(err);
reject(err)
});
});
}
Figured this out. The first issue was not returning the first call to get_dbinfo, which I did not do because it was marking the post promise parts of the function as "unreachable, due to two returns within the same function", which clued me into the second part of the issue, which was trying to include two distinct promise chains in the same function.
The final solution was to bring everything under a single promise chain with a retun on the first call to dbFuncs, as shown below. No changes were made to get_dbinfo().
var count = 0;
var perPage = 500;
var page = req.query.p || 1;
return dbFuncs.get_dbinfo()
.then((result) => {
count = result.doc_count
if (count < 500){perPage = count};
return db.find({
selector: {
$and: [
{last_name: { '$gt': 1}},
]
},
use_index: ['patients'],
sort: [{'last_name': 'asc'}],
skip: ((perPage * page) - perPage),
limit: perPage
}).then((docs) => {
let pages = Math.ceil(count / perPage);
res.render("patients", {
pagination: {
currentpage: page,
page: pages
},
obj: docs,
current: page,
});
}).catch((err) => {
console.log('error in patients_controller', err);
});
});
};
For posterity, this complete block is an excellent way to paginate a dataset (in this case from pouchdb) to an hbs template.
I need to retrieve a list of collections using express and mongodb module.
First, ive retrieved a list of collection names which works, I then retrieve the data of those given collections in a loop. My problem is in getColAsync():
getColAsync() {
return new Promise((resolve, reject) => {
this.connectDB().then((db) => {
var allCols = [];
let dbase = db.db(this.databaseName);
dbase.listCollections().toArray((err, collectionNames) => {
if(err) {
console.log(err);
reject(err);
}
else {
for(let i = 0; i < collectionNames.length; i++) {
dbase.collection(collectionNames[i].name.toString()).find({}).toArray((err, collectionData) => {
console.log("current collection data: " + collectionData);
allCols[i] = collectionData;
})
}
console.log("done getting all data");
resolve(allCols);
}
})
})
})
}
connectDB() {
if(this.dbConnection) {
// if connection exists
return this.dbConnection;
}
else {
this.dbConnection = new Promise((resolve, reject) => {
mongoClient.connect(this.URL, (err, db) => {
if(err) {
console.log("DB Access: Error on mongoClient.connect.");
console.log(err);
reject(err);
}
else {
console.log("DB Access: resolved.");
resolve(db);
}
});
});
console.log("DB Access: db exists. Connected.");
return this.dbConnection;
}
}
In the forloop where i retrieve every collection, the console.log("done getting all data") gets called and the promise gets resolved before the forloop even begins. for example:
done getting all data
current collection data: something
current collection data: something2
current collection data: something3
Please help
The Problem
The problem in your code is this part:
for (let i = 0; i < collectionNames.length; i++) {
dbase.collection(collectionNames[i].name.toString()).find({}).toArray((err, collectionData) => {
console.log("current collection data: " + collectionData);
allCols[i] = collectionData;
})
}
console.log("done getting all data");
resolve(allCols);
You should notice that resolve(allCols); is called right after the for loop ends, but each iteration of the loop doesn't wait for the toArray callback to be called.
The line dbase.collection(collectionNames[i].name.toString()).find({}).toArray(callback) is asynchronous so the loop will end, you'll call resolve(allCols);, but the .find({}).toArray code won't have completed yet.
The Solution Concept
So, basically what you did was:
Initialize an array of results allCols = []
Start a series of async operations
Return the (still empty) array of results
As the async operations complete, fill the now useless results array.
What you should be doing instead is:
Start a series of async operations
Wait for all of them to complete
Get the results from each one
Return the list of results
The key to this is the Promise.all([/* array of promises */]) function which accepts an array of promises and returns a Promise itself passing downstream an array containing all the results, so what we need to obtain is something like this:
const dataPromises = []
for (let i = 0; i < collectionNames.length; i++) {
dataPromises[i] = /* data fetch promise */;
}
return Promise.all(dataPromises);
As you can see, the last line is return Promise.all(dataPromises); instead of resolve(allCols) as in your code, so we can no longer execute this code inside of a new Promise(func) constructor.
Instead, we should chain Promises with .then() like this:
getColAsync() {
return this.connectDB().then((db) => {
let dbase = db.db(this.databaseName);
const dataPromises = []
dbase.listCollections().toArray((err, collectionNames) => {
if (err) {
console.log(err);
return Promise.reject(err);
} else {
for (let i = 0; i < collectionNames.length; i++) {
dataPromises[i] = new Promise((res, rej) => {
dbase.collection(collectionNames[i].name.toString()).find({}).toArray((err, collectionData) => {
console.log("current collection data: " + collectionData);
if (err) {
console.log(err);
reject(err);
} else {
resolve(collectionData);
}
});
});
}
console.log("done getting all data");
return Promise.all(dataPromises);
}
});
})
}
Notice now we return a return this.connectDB().then(...), which in turn returns a Promise.all(dataPromises); this returning new Promises at each step lets us keep alive the Promise chain, thus getColAsync() will itself return a Promise you can then handle with .then() and .catch().
Cleaner Code
You can clean up your code a bit as fallows:
getColAsync() {
return this.connectDB().then((db) => {
let dbase = db.db(this.databaseName);
const dataPromises = []
// dbase.listCollections().toArray() returns a promise itself
return dbase.listCollections().toArray()
.then((collectionsInfo) => {
// collectionsInfo.map converts an array of collection info into an array of selected
// collections
return collectionsInfo.map((info) => {
return dbase.collection(info.name);
});
})
}).then((collections) => {
// collections.map converts an array of selected collections into an array of Promises
// to get each collection data.
return Promise.all(collections.map((collection) => {
return collection.find({}).toArray();
}))
})
}
As you see the main changes are:
Using mondodb functions in their promise form
Using Array.map to easily convert an array of data into a new array
Below I also present a variant of your code using functions with callbacks and a module I'm working on.
Promise-Mix
I'm recently working on this npm module to help get a cleaner and more readable composition of Promises.
In your case I'd use the fCombine function to handle the first steps where you select the db and fetch the list of collection info:
Promise.fCombine({
dbase: (dbURL, done) => mongoClient.connect(dbURL, done),
collInfos: ({ dbase }, done) => getCollectionsInfo(dbase, done),
}, { dbURL: this.URL })
This results in a promise, passing downstream an object {dbase: /* the db instance */, collInfos: [/* the list of collections info */]}. Where getCollectionNames(dbase, done) is a function with callback pattern like this:
getCollectionsInfo = (db, done) => {
let dbase = db.db(this.databaseName);
dbase.listCollections().toArray(done);
}
Now you can chain the previous Promise and convert the list of collections info into selected db collections, like this:
Promise.fCombine({
dbase: ({ dbURL }, done) => mongoClient.connect(dbURL, done),
collInfos: ({ dbase }, done) => getCollectionsInfo(dbase, done),
}, { dbURL: this.URL }).then(({ dbase, collInfos }) => {
return Promise.resolve(collInfos.map((info) => {
return dbase.collection(info.name);
}));
})
Now downstream we have a the list of selected collections from our db and we should fetch data from each one, then merge the results in an array with collection data.
In my module I have a _mux option which creates a PromiseMux which mimics the behaviour and composition patterns of a regular Promise, but it's actually working on several Promises at the same time. Each Promise gets in input one item from the downstream collections array, so you can write the code to fetch data from a generic collection and it will be executed for each collection in the array:
Promise.fCombine({
dbase: ({ dbURL }, done) => mongoClient.connect(dbURL, done),
collInfos: ({ dbase }, done) => getCollectionsInfo(dbase, done),
}, { dbURL: this.URL }).then(({ dbase, collInfos }) => {
return Promise.resolve(collInfos.map((info) => {
return dbase.collection(info.name);
}));
})._mux((mux) => {
return mux._fReduce([
(collection, done) => collection.find({}).toArray(done)
]).deMux((allCollectionsData) => {
return Promise.resolve(allCollectionsData);
})
});
In the code above, _fReduce behaves like _fCombine, but it accepts an array of functions with callbacks instead of an object and it passes downstream only the result of the last function (not a structured object with all the results). Finally deMux executes a Promise.all on each simultaneous Promise of the mux, putting together their results.
Thus the whole code would look like this:
getCollectionsInfo = (db, done) => {
let dbase = db.db(this.databaseName);
dbase.listCollections().toArray(done);
}
getCollAsync = () => {
return Promise.fCombine({
/**
* fCombine uses an object whose fields are functions with callback pattern to
* build consecutive Promises. Each subsequent functions gets as input the results
* from previous functions.
* The second parameter of the fCombine is the initial value, which in our case is
* the db url.
*/
dbase: ({ dbURL }, done) => mongoClient.connect(dbURL, done), // connect to DB, return the connected dbase
collInfos: ({ dbase }, done) => getCollectionsInfo(dbase, done), // fetch collection info from dbase, return the info objects
}, { dbURL: this.URL }).then(({ dbase, collInfos }) => {
return Promise.resolve(collInfos.map((info) => {
/**
* we use Array.map to convert collection info into
* a list of selected db collections
*/
return dbase.collection(info.name);
}));
})._mux((mux) => {
/**
* _mux splits the list of collections returned before into a series of "simultaneous promises"
* which you can manipulate as if they were a single Promise.
*/
return mux._fReduce([ // this fReduce here gets as input a single collection from the retrieved list
(collection, done) => collection.find({}).toArray(done)
]).deMux((allCollectionsData) => {
// finally we can put back together all the results.
return Promise.resolve(allCollectionsData);
})
});
}
In my module I tried to avoid most common anti-pattern thought there still is some Ghost Promise which I'll be working on.
Using the promises from mongodb this would get even cleaner:
getCollAsync = () => {
return Promise.combine({
dbase: ({ dbURL }) => { return mongoClient.connect(dbURL); },
collInfos: ({ dbase }) => {
return dbase.db(this.databaseName)
.listCollections().toArray();
},
}, { dbURL: this.URL }).then(({ dbase, collInfos }) => {
return Promise.resolve(collInfos.map((info) => {
return dbase.collection(info.name);
}));
}).then((collections) => {
return Promise.all(collections.map((collection) => {
return collection.find({}).toArray();
}))
});
}
Currently try to learn Node JS and getting my head around Async/Sync workflow.
Try to the follow:
Step 1:
- Get data 1 with function 1
- Get data 2 with function 2
- Get data 3 with function 3
Step2:
- Work out logic with data 1,2,3
Step 3
- Do final call
I been looking at Q and Async packages but still havent really find an example.
Can some one show me how they will go about this issue in Node JS?
Thanks
Not entirely clear on your implementation, but depending on how specific your ordering needs to be you could try something like this:
var data1 = null;
var data2 = null;
var data3 = null;
async.series([
function(httpDoneCallback){
async.parallel([
function(data1Callback){
$http(...).then(function(response){
// some logic here
data1 = response;
data1Callback();
})
},
function(data2Callback){
$http(...).then(function(response){
// some logic here
data2 = response;
data2Callback();
})
},
function(data3Callback){
$http(...).then(function(response){
// some logic here
data3 = response;
data3Callback();
})
}
], function(){
//all requests dome, move onto logic
httpDoneCallback();
})
},
function(logicDoneCallback){
// do some logic, maybe more asynchronous calls with the newly acquired data
logicDoneCallback();
}
], function(){
console.log('all done');
})
Do you want function 1, 2, and 3 to trigger at the same time? If so then this should help:
var async = require('async');
async.parallel([
function(cb1) {
cb1(null, "one")
},
function(cb2){
cb2(null, "two")
},
function(cb3){
cb3(null, "three")
}
], function(err, results) {
console.log(results); // Logs ["one", "two", "three"]
finalCall();
});
To explain, every function in the array submitted as the first param to the parallel method will also receive a callback function. Activating the callback function signifies that you're done fetching your data or doing whatever you need to do in said function. All three functions will trigger at the same time, and once all three callbacks are called, the final function is called. The callback accepts two parameters: "error", and "result." If everything's successful, pass "null" as the error parameter. The results will be given to the final function as an array containing each of the results for your individual functions.
You can setup a chain of Promises to do things sequentially:
var funcA = () => {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve('some data from A')
}, 1000)
});
}
var funcB = (dataFromA) => {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(dataFromA + ' data from B')
}, 2000)
})
}
var funcC = (dataFromB) => {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(dataFromB + ' data from C')
}, 500)
})
}
// Doing the functions on after another
funcA().then(funcB).then(funcC).then((data) => {
console.log(data);
})
Or if you want to do them all at the same time you can use Promise.all():
var promises = [];
promises.push(new Promise((resolve, reject) => {
setTimeout(() => {
resolve('some data from A')
}, 1000)
}));
promises.push(new Promise((resolve, reject) => {
setTimeout(() => {
resolve('some data from B')
}, 1000)
}));
promises.push(new Promise((resolve, reject) => {
setTimeout(() => {
resolve('some data from C')
}, 1000)
}));
// Execute the array of promises at the same time, and wait for them all to complete
Promise.all(promises).then((data) => {
console.log(data);
})
Probably the best thing to do is use Promises like #Tyler here states. However, for conceptual understanding it is best to first understand the node callback pattern.
Because some tasks take time, we give the task a function and say 'When you are done, put the data you retrieved into this function'. These functions that we give to other functions are called callbacks. They must be constructed to accept the data, and also an error in case there is a problem while fetching the data. In Node the error is the first callback parameter and the data is the second.
fs.readFile('/file/to/read.txt', function callback(error, data) {
if (error) console.log(error);
else console.log(data);
});
In this example, once node reads the file, it will feed that data into the callback function. In the callback we must account for the case that there was a problem and handle the error.
In your question you want to do multiple async tasks and use their results. Therefore you must take this pattern and nest several of them. So, continuing this example, if there is no error you will begin another async task.
fs.readFile('/file/to/read.txt', function callback(error, data) {
if (error) console.log(error);
else {
someOtherAsyncThing(function callback(error, data2) {
if (error) console.log(error);
else {
console.log(data + data2)
}
});
}
});