I want to perform an iteration over a mongoDB collection w/o numeric key(_id). The collection only has random strings as an _id, and the size of the collection is massive, thus loading up the whole documents on RAM using .toArray() is not a viable option. plus I want to perform asynchronous task on each element. the usage of .map() or .each(), .forEach() is limited because of the asynchronous nature of the task. I tried to run the task with those mentioned methods but it did of course conflicted with asynchronous task, returned pending promises instead of proper results.
example
async function dbanalyze(){
let cursor = db.collection('randomcollection').find()
for(;;){
const el = cursor.hasNext() ? loaded.next() : null;
if(!cursor) break
await performAnalyze(cursor) // <---- this doesn't return a document but just a cursor object
}
}
how can I iterate over mongoDB collection using only for()?
The Cursor.hasNext() method is also "asynchronous", so you need to await that as well. Same goes for Cursor.next(). Therefore the actual "loop" usage really should be a while:
async function dbanalyze(){
let cursor = db.collection('randomcollection').find()
while ( await cursor.hasNext() ) { // will return false when there are no more results
let doc = await cursor.next(); // actually gets the document
// do something, possibly async with the current document
}
}
As noted in the comments, eventually Cursor.hasNext() will return false when the cursor is actually depleted, and the Cursor.next() is the thing that is actually retrieving each value from the cursor. You could do other structures and break the loop when hasNext() is false, but it more naturally lends itself to a while.
These are still "async", so you need to await the promise resolution on each, and that was the main fact you were missing.
As for Cursor.map(), then you are probably missing the point that it can be marked with an async flag on the provided function as well:
cursor.map( async doc => { // We can mark as async
let newDoc = await someAsyncMethod(doc); // so you can then await inside
return newDoc;
})
But you still actually want to "iterate" that somewhere, unless you can get away with using .pipe() to some other output destination.
Also the async/await flags also make Cursor.forEach() "more practical again", as it's one common flaw was not being able to simply handle an "inner" asynchronous call, but with these flags you can now do so with ease, though admittedly since you must use a callback, you probably want to wrap this in a Promise :
await new Promise((resolve, reject) =>
cursor.forEach(
async doc => { // marked as async
let newDoc = await someAsyncMethod(doc); // so you can then await inside
// do other things
},
err => {
// await was respected, so we get here when done.
if (err) reject(err);
resolve();
}
)
);
Of course there has always been ways to apply this with either callbacks or plain Promise implementations, but it's the "sugar" of async/await than actually makes this look much cleaner.
NodeJS v10.x and MongoDB Node driver 3.1.x and up
And the favorite version uses AsyncIterator which is now enabled in NodeJS v10 and upwards. It's a much cleaner way to iterate
async function dbanalyze(){
let cursor = db.collection('randomcollection').find()
for await ( let doc of cursor ) {
// do something with the current document
}
}
Which "in a way" comes back to what the question originally asked as to using a for loop since we can do the for-await-of syntax here fore supporting iterable which supports the correct interface. And the Cursor does support this interface.
If you're curios, here's a listing I cooked up some time ago to demonstrate various cursor iteration techniques. It even includes a case for Async Iterators from a generator function:
const Async = require('async'),
{ MongoClient, Cursor } = require('mongodb');
const testLen = 3;
(async function() {
let db;
try {
let client = await MongoClient.connect('mongodb://localhost/');
let db = client.db('test');
let collection = db.collection('cursortest');
await collection.remove();
await collection.insertMany(
Array(testLen).fill(1).map((e,i) => ({ i }))
);
// Cursor.forEach
console.log('Cursor.forEach');
await new Promise((resolve,reject) => {
collection.find().forEach(
console.log,
err => {
if (err) reject(err);
resolve();
}
);
});
// Async.during awaits cursor.hasNext()
console.log('Async.during');
await new Promise((resolve,reject) => {
let cursor = collection.find();
Async.during(
(callback) => Async.nextTick(() => cursor.hasNext(callback)),
(callback) => {
cursor.next((err,doc) => {
if (err) callback(err);
console.log(doc);
callback();
})
},
(err) => {
if (err) reject(err);
resolve();
}
);
});
// async/await allows while loop
console.log('async/await while');
await (async function() {
let cursor = collection.find();
while( await cursor.hasNext() ) {
let doc = await cursor.next();
console.log(doc);
}
})();
// await event stream
console.log('Event Stream');
await new Promise((end,error) => {
let cursor = collection.find();
for ( let [k,v] of Object.entries({ end, error, data: console.log }) )
cursor.on(k,v);
});
// Promise recursion
console.log('Promise recursion');
await (async function() {
let cursor = collection.find();
function iterate(cursor) {
return cursor.hasNext().then( bool =>
(bool) ? cursor.next().then( doc => {
console.log(doc);
return iterate(cursor);
}) : Promise.resolve()
)
}
await iterate(cursor);
})();
// Uncomment if node is run with async iteration enabled
// --harmony_async_iteration
console.log('Generator Async Iterator');
await (async function() {
async function* cursorAsyncIterator() {
let cursor = collection.find();
while (await cursor.hasNext() ) {
yield cursor.next();
}
}
for await (let doc of cursorAsyncIterator()) {
console.log(doc);
}
})();
// This is supported with Node v10.x and the 3.1 Series Driver
await (async function() {
for await (let doc of collection.find()) {
console.log(doc);
}
})();
client.close();
} catch(e) {
console.error(e);
} finally {
process.exit();
}
})();
Related
I have a list from the request body that I want to parse all links in request async.
When all tasks are finished, I then want to return a response, but my response is always empty because none of the async tasks are finishing.
var index = 0;
let items = [];
exports.feed = async (req, res) => {
(async () => {
await getSiteData(req.body.sites)
})();
if(typeof items !== 'undefined' && items.length > 0){
res.status(200).json(items);
} else {
res.status(404).json({ error: "Something went wrong", success: false });
}
}
async function getSiteData(list){
try{
if(index == list.length-1){
return items;
}
await parser.parseURL(list[index], async function(err, feed){
if(err != null){
console.log(err);
return false;
}
items.push(feed.items)
index++
await getSiteData(list);
});
} catch(err){
console.log(err);
return false;
}
}
Firstly, there's no recursion required. It (almost?) never makes sense to recursively process a flat list (Array)
Your main issue is that parser.parseURL does NOT return a Promise, therefore awaiting it makes no sense, since await only waits for Promises to settle
So, let's fix that by creating a "promisified" parser.parseURL, which is easy with nodes util.promisify
const { promisify } = require('util');
const parseURL = promisify(parser.parseURL);
now you can await parseURL(url)
if you need to parse.parseURL in series, i.e. one at a time
async function getSiteData(list) {
const result = [];
for (const url of list) {
const { items } = await parseURL(url);
result.push(items);
}
return result;
}
to parse.parseURL in parallel
function getSiteData(list) {
return Promise.all(list.map(parseURL));
}
Some people think that this should be
async function getSiteData(list) {
return await Promise.all(list.map(parseURL));
}
I'm not one of those people. Their argument is, getSiteData returns a Promise, so should be marked as such, some malarkey about IDE hints or some such garbage. While the code is equivalent (not identical of course), it really isn't necessary in my opinion.
Note: now getSiteData (in either series or parallel form) returns a Promise that will resolve to an array of results, and, further more, neither version does any error handling. This is deliberate, so the function that calls getSiteData determine what to do with errors.
The other issus is in your feed export
(async () => {
await getSiteData(req.body.sites)
})();
//
// the code down here runs without waiting for getSiteData to complete
this runs that asynchronous IIFE, but the code afterwards does not wait for it to complete
You could
await (async () => {
await getSiteData(req.body.sites)
})();
But then, that's just
await getSiteData(req.body.sites);
Without the IIFE
Since the getSiteData now returns the list (rather than using a "global" or external array to hold the list), the code is now:
exports.feed = async (req, res) => {
try {
const items = await getSiteData(req.body.sites);
res.status(200).json(items);
} catch (err) {
res.status(404).json({error: "Something went wrong", success: false});
}
};
I fixed my issue with using for loop and promise.
exports.feed = async (req,res) => {
var items = [];
for(const elem of req.body.sites){
const newElement = await getSiteData(elem)
items.push(newElement)
}
if(typeof items !== 'undefined' && items.length > 0){
res.status(200).json(items);
}else{
res.status(404).json({ error: "Something went wrong", success: false });
}
}
function getSiteData(url) {
return new Promise((resolve) => {
setTimeout(()=>{
parser.parseURL(url, function(err, feed) {
if(err != null){
return false;
}
resolve(feed.items)
})
});
});
}
COIN LIST is an array of crypto coins(["BTCUSDT",...]). I try to get the price using getPriceAction and RSI from getRSI and these two functions are working when I try to console DATA. But when I try to print the response after the completion of the loop. It prints the empty array and the length is 0 of this array. I want to store the DATA object (consisting of SYMBOL, closing price and RSI) as an element in the response array
import { COIN_LIST } from "./COIN_LIST.js";
import { getPriceAction } from "./PRICE_ACTION.js";
import { getRSI } from "./RSI.js";
async function main() {
try {
let response = await [];
await COIN_LIST.forEach((element, i) => {
setTimeout(() => {
let data = { symbol: element };
getPriceAction(element, "4h").then((res) => {
data.closingPrice = res;
getRSI(res).then((res) => {
data.RSI = res.reverse();
data.closingPrice = data.closingPrice.reverse();
response.push(data);
console.log(data)
});
});
}, i * 1000);
});
console.log(response);
} catch (error) {
console.log(error.message);
}
}
main();
If you want to use async/await properly for your code, then use async/await, don't use .then/.catch as well
Some notable changes
there is no setTimeout of increasing seconds ... just waiting 1 second after one result before getting the next - far cleaner, and if one request happens to take a lot longer, you won't end up with two requests at once (which may be an issue if the API is rate limited)
no .then ... use async/await OR .then/.catch - very rare to need both in the one function
don't use forEach with async/await ... it never does what you want, and creating an array of Promises inside a .forEach is extremely naive, you may as well use .map instead! then you can await Promise.all(xxx.map(.....)) - but that's useful for concurrent requests, not so much for serial requests like your code does
import { COIN_LIST } from "./COIN_LIST.js";
import { getPriceAction } from "./PRICE_ACTION.js";
import { getRSI } from "./RSI.js";
async function main() {
try {
const wait = (ms) => new Promise(resolve => setTimeout(resolve, 1000));
let response = []; //---> don't need that `await`
for (let element of COIN_LIST) {
let data = { symbol: element };
data.closingPrice = await getPriceAction(element, "4h");
const res = await getRSI(data.closingPrice);
data.RSI = res.reverse();
data.closingPrice = data.closingPrice.reverse();
response.push(data);
console.log(data);
await wait(1000);
}
console.log(response);
} catch (error) {
console.log(error.message);
}
}
main();
the await wait(1000) could be tweaked depending on the rate limiting of the API ... if the rate limit applies to when the request is made, you could make a function that is smart about the delay between requests.
The code this way assumes the rate limit is based on the period between previous response to next request.
After the completion of the loop, the promises didn't get resolved yet, that's why it print an empty array.
One way to achieve what you need is using await for(...), or wait for all promises to be resolved, and then print the results.
import { COIN_LIST } from "./COIN_LIST.js";
import { getPriceAction } from "./PRICE_ACTION.js";
import { getRSI } from "./RSI.js";
async function main() {
try {
let response = []; //---> don't need that `await`
const promises = []; //---> array of promises
COIN_LIST.forEach((element, i) => {
setTimeout(() => {
let data = { symbol: element };
const promise = getPriceAction(element, "4h").then((res) => {
data.closingPrice = res;
getRSI(res).then((res) => {
data.RSI = res.reverse();
data.closingPrice = data.closingPrice.reverse();
response.push(data);
console.log(data)
});
});
promises.push(promise) //---> save the reference to a promise
}, i * 1000);
});
await Promise.all(promises) //---> await for all promises to be resolved, then print the result
console.log(response);
} catch (error) {
console.log(error.message);
}
}
main();
const crypto = require('crypto');
async function getKey(byteSize) {
let key = await crypto.randomBytes(byteSize);
return key;
}
async function g() {
let key = await getKey(12);
return key;
}
console.log(g());
console.log('hello - want this called after g() above');
I've been at this for an hour and I can't understand how to ensure that I get a key using async/await. I get a Promise-pending no matter what I do.
I've also tried this:
async function getKey(byteSize) {
let key = await crypto.randomBytes(byteSize);
return key;
}
getKey(12).then((result) => { console.log(result) })
console.log('hello');
... to no avail! Which was inspired by:
How to use await with promisify for crypto.randomBytes?
Can anyone help me with this?
All I'm trying to do is to get randomBytes async. using the async./await block but ensure that it fulfills the promise before I carry on in the code.
This is an extension of my comment on the question
Since you're not promisify'ing or passing a callback to crypto.randomBytes() it is synchronous so you can't await it. Additionally, you're not properly awaiting the promise returned by g() at the top level. That is why you always see the pending Promise in your console.log()
You can use util.promisify() to convert crypto.randomBytes() into a promise returning function and await that. There is no need for the async/await in your example because all that is doing is wrapping a promise with a promise.
const { promisify } = require('util')
const randomBytesAsync = promisify(require('crypto').randomBytes)
function getKey (size) {
return randomBytesAsync(size)
}
// This will print the Buffer returned from crypto.randomBytes()
getKey(16)
.then(key => console.log(key))
If you want to use getKey() within an async/await style function it would be used like so
async function doSomethingWithKey () {
let result
const key = await getKey(16)
// do something with key
return result
}
If the callback function is not provided, the random bytes are generated synchronously and returned as a Buffer
// Synchronous
const {
randomBytes
} = await import('node:crypto');
const buf = randomBytes(256);
console.log(
`${buf.length} bytes of random data: ${buf.toString('hex')}`);
const crypto = require('crypto');
async function getKey(byteSize) {
const buffer = await new Promise((resolve, reject) => {
crypto.randomBytes(byteSize, function(ex, buffer) {
if (ex) {
reject("error generating token");
}
resolve(buffer);
});
}
async function g() {
let key = await getKey(12);
return key;
}
const crypto = require("crypto");
async function getRandomBytes(byteSize) {
return await new Promise((resolve, reject) => {
crypto.randomBytes(byteSize, (err, buffer) => {
if (err) {
reject(-1);
}
resolve(buffer);
});
});
}
async function doSomethingWithRandomBytes(byteSize) {
if (!byteSize) return -1;
const key = await getRandomBytes(byteSize);
//do something with key
}
doSomethingWithRandomBytes(16);
I have a problème which I can't seem to figure out. I need this await call to happen only when it finishes one by one in the loop, but it seems like its just calling it twice in the same time quickly, which is not making the api post work correctly.
I've tried to use async, await, but the await doesn't seem to like it being in two loops.
Has anyone got a better solution?
async pushToCreate() {
const toCreate = this.json.toCreate
let counter = 0
// toCreate Sample = { en: [{}, {}] }, { fr: [{}, {}] }
Object.keys(toCreate).forEach((item) => {
toCreate[item].forEach((el) => {
const lang = item
const dataContent = el
await this.wp.products().param('lang', lang).create({
title: dataContent.dgn,
fields: dataContent,
status: 'publish'
}).then(function( response ) {
counter++
}).catch(function(err) {
console.log('error in create', err)
})
console.log('await finished')
})
})
// should console log when above is finished
console.log('END')
}
If you want to await for promises forEach will not help you. To make it work you should use:
for/for..in/for..of loops for the sequential processing and await for each Promise to become fulfilled on each iteration
map for the parallel processing which will return an array of promises and after that you can await for them to become fulfilled with Promise.all()
If you use async/await it looks more clear to use try/catch instead of then/catch chaining
Try this out if you need sequential processing:
...
let lang;
let dataContent;
for (const item in toCreate) {
lang = item;
for (let i = 0; i < toCreate[lang].length; i++) {
dataContent = toCreate[lang][i];
try {
await this.wp.products().param('lang', lang).create({
title: dataContent.dgn,
fields: dataContent,
status: 'publish',
});
counter++;
} catch (err) {
console.log('error in create', err);
}
console.log('await finished');
}
}
...
or this if you need parallel processing:
...
let lang;
let dataContent;
await Promise.all(
Object.keys(toCreate).map(async item => {
lang = item;
await Promise.all(
toCreate[lang].map(async el => {
dataContent = el;
try {
await this.wp.products().param('lang', lang).create({
title: dataContent.dgn,
fields: dataContent,
status: 'publish',
});
counter++;
} catch (err) {
console.log('error in create', err);
}
console.log('await finished');
}),
);
}),
);
...
Here is a good explanation of the sequence and parallel asynchronous calls handling using async/await: Using async/await with a forEach loop
I believe there are a few issues with your code.
The first is that await only works when a function returns a promise. Functions that return promises initially return a value that indicates that a promise is pending. If the function you have on the right side of await does not return a promise, it will not wait.
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await
The second is that you are also using the .then() method, which triggers it's callback argument when the promise is fulfilled. I do not think you can use both await and .then().https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/then
If you want to have a similar format as the .then() .catch() pattern, you could use a try/catch block with the async/await.
async function f() {
try {
let response = await fetch('/example');
let user = await response.json();
} catch(err) {
// catches errors both in fetch and response.json
alert(err);
}
}
https://javascript.info/async-await
I understand that the following code, will output resolved. My question is how is this useful, and when could async/await be useful when building real world application in react, node, etc ?
function foo() {
const resolveAfter2Seconds = () => {
return new Promise(resolve => {
setTimeout(() => {
resolve('resolved');
}, 2000);
});
}
async function asyncCall(){
console.log('calling');
var result = await resolveAfter2Seconds();
console.log(result);
}
return asyncCall;
}
const myFoo = foo()
myFoo();
The Network, Files, and Frequent Promises
I believe that the most common issues you'll run into that will make you want to switch your current function to async mode usually have to do with: network requests, operating on files, and the frequent use and/or nesting of promises.
Network Requests
When I write network requests I always use the async/await await combo. For me, it makes my code much more linear and readable. I also don't have to worry about returning the promise from fetch (or axios) when I'm done.
async function getPins(user, filterValue) {
const pins = await axios.get(...);
if (filterValue) {
return pins.filter(pin => pin.title.includes(filterValue));
}
return pins;
}
Files
async function handleFile(data, isEncrypted, isGzipped) {
if (isEncrypted) {
data = await decrypt(data, secret);
}
if (isGzipped) {
data = await ungzip(data);
}
return data;
}
Frequent Promises
async function init() {
const activeBoard = await getActiveBoard();
const boardMembers = await getBoardMembersFrom(activeBoard);
const allTasks = [];
for await (const tasks of boardMembers.map(getTasks)) {
allTasks.push(task);
this.setState({ tasks: [...allTasks] });
}
}
Note: you can use async/await with Promises. There's no need to limit yourself to one or the other.
const test = async props => {
const data = await fetch('...').then(res => res.json());
console.log(data) /* do what you want with the data now */
}