function Recursive_scan_and_Insert (path_dir) { //scanning path_dir recursively and insert filepath into temporary list
Recursive_Scan(path_dir, (err, files) => { //it's from npm recursive_readdir
if(err) {
console.log(err);
res.status(500).send('server error');
}
files.forEach(elements => {
let params = [elements]; //
DB("GET", "INSERT INTO filelist_t VALUES (null, ?, NOW(), 0, 0)", params).then(function(res) {
console.log('data input');
});
});
});
};
function Add_to_DB () { //moving temporal list to main list without duplicate
DB("GET", "INSERT INTO filelist (id, path, addeddate, isdeleted, ismodified) SELECT NULL, filelist_t.path, filelist_t.addeddate, filelist_t.isdeleted, filelist_t.ismodified FROM filelist_t LEFT JOIN filelist ON filelist.path = filelist_t.path WHERE filelist.id IS NULL; DELETE FROM filelist_t; ").then(function(res) {
console.log('data moving');
});
};
app.get('/db', (req, res) => { //PROBLEM PART
async function async_Two_Functions () {
var object_path = '/want/to/scan/path';
await Recursive_scan_and_Insert(object_path).then( () => {
return Add_to_DB()
})
}
async_Two_Functions();
res.send(res);
});
app.get('/dbp', (req, res) => { //show main list to my web
DB("GET", "SELECT * FROM filelist").then(function(res2) {
res.send(res2.row);
});
});
here's the thing.
there are 4 stage in my dream algorithm.
recursively scan all the path.
insert each data into temporary table.
moving temporal data on main table, without duplicate
present main table
it's very important to things get order. but I don't understand about async await exactly...
Well, here's a cleaned up version of the code with lots of changes.
const {promisify} = require('util');
const Recursive_ScanP = promisify(Recursive_Scan);
function Recursive_scan_and_Insert(path_dir) { //scanning path_dir recursively and insert filepath into temporary list
return Recursive_ScanP(path_dir).then(files => {
return Promise.all(files.map(elements => {
let params = [elements];
return DB("GET", "INSERT INTO filelist_t VALUES (null, ?, NOW(), 0, 0)", params).then(function(res) {
console.log('data input');
// what should the return value be here?
});
}));
});
};
function Add_to_DB () { //moving temporal list to main list without duplicate
return DB("GET", "INSERT INTO filelist (id, path, addeddate, isdeleted, ismodified) SELECT NULL, filelist_t.path, filelist_t.addeddate, filelist_t.isdeleted, filelist_t.ismodified FROM filelist_t LEFT JOIN filelist ON filelist.path = filelist_t.path WHERE filelist.id IS NULL; DELETE FROM filelist_t; ").then(function(res) {
console.log('data moving');
return res;
});
};
app.get('/db', async (req, res) => {
try {
let object_path = '/want/to/scan/path';
await Recursive_scan_and_Insert(object_path);
await Add_to_DB();
res.send(somethingHere); // you fill in what response you want to send here
} catch(e) {
console.log(e);
res.status(500).send("Server Error");
}
});
app.get('/dbp', (req, res) => { //show main list to my web
DB("GET", "SELECT * FROM filelist").then(function(res2) {
res.send(res2.row);
}).catch(err => {
console.log(err);
res.status(500).send("Server Error");
});
});
Changes:
Return a promise from every function that has an asynchronous operation in it
Return whatever you want the resolved value to be for the promise from every .then() handler
Promisify anything that uses a regular callback so you can do all control flow with promises
Use Promise.all() to know when multiple promises are done and to collect results in order from doing a set of asynchronous operation in parallel
Use async/await as desired, but particularly when you want to sequence multiple asynchronous operatoins
Use try/catch around any await to catch rejected promise that aren't being returned to a higher level where they will be caught
Use .catch() with any .then() that isn't being returned to a higher level where it will be caught
Open Questions:
Don't know what response you want to send from app.get('/db', ...). You will have to fill that in.
Are you expecting any resolved value from Recursive_scan_and_Insert()?
Does Add_to_DB() really accept no input? It just reorganizes things already in the database?
Related
This question already has answers here:
Why is my variable unaltered after I modify it inside of a function? - Asynchronous code reference
(7 answers)
How do I access previous promise results in a .then() chain?
(17 answers)
Closed 2 years ago.
I'm developing a script that connects with an API, then with the JSON reponse do some operations and then reformat the JSON to send it to another API.
But now I'm stuck in the first step as I can't deal with the first part as my Promises is not working as expected. How can I store the API's response into a variable? For development puropose I stored one API response into a JSON file. This is my code:
declare var require: any;
let url = './data/big buy/big-bui-product-list.json';
const fs = require('fs');
let counter = 0;
const getProductList = () => {
return new Promise((resolve, reject) => {
fs.readFile(url, 'utf8', (err, data) => {
if(err){
return reject (err);
}
else {
return resolve(JSON.parse(data));
}
})
})
}
const getProductStock = () => {
return new Promise((resolve, reject) => {
fs.readFile('./data/big buy/big-bui-product-stock.json', 'utf8', (err, data) => {
if(err) {
return reject(err);
}
else {
return resolve(JSON.parse(data));
}
})
})
}
try {
let products;
console.log('Products:');
Promise.all([getProductList()])
.then(function(result) {
products = result[0];
});
console.log('Stocks:');
const productStock = Promise.all([getProductStock()]);
console.log(products);
}
catch(e) {
console.log((`Ha ocurrido un error: ${e.message}`));
}
finally {
}
In this code, what I do is getting a list of products and then get the stocks of all the products, later I will add a new function that will filter by stock and get me just a list of products where stock is bigger than X units. Now when I launch it from the terminal I dont' get the response stored into products variable but if I add .then((data) => console.log(data)) into the Promise I see on screen the JSON but as I dont' have it stored it in any variable I don't see how can I work with the objects I'm retrieving.
Promises are asynchronous. They are quite literally promises that a value will be yielded in the future. When you do getProductList().then(x => products = x), you're saying that Javascript should fetch the product list in the background, and once it's finished, it should set the products variable to x. The key words there are "in the background", since the code afterwards keeps running while the product list is being fetched. The products variable is only guaranteed to be set after the .then portion is run. So, you can move things into the .then:
try {
let products;
getProductList().then(list => {
products = list;
return getProductStock(); // leverage promise chaining
}).then(stocks => {
console.log("Products:", products);
console.log("Stocks:", stocks);
});
}
catch(e) {
console.log((`Ha ocurrido un error: ${e.message}`));
}
finally {
}
You seem to be missing some fundamental knowledge about promises. I suggest reading through the MDN Using Promises guide to familiarize yourself a bit with them.
A structure like below never works.
let variable;
promise.then(data => variable = data);
console.log(variable);
This is because it is executed in the following manner.
Create the variable variable.
Add a promise handler to the promise.
Log variable.
Promise gets resolved.
Execute the promise handler.
Set variable variable to data.
If you are using Node.js 10 or higher you can use the promise API of file system to simplify your code quite a bit.
const fs = require('fs/promises');
const readJSON = (path) => fs.readFile(path, "utf8").then((json) => JSON.parse(json));
const getProductList = () => readJSON("./data/big buy/big-bui-product-list.json");
const getProductStock = () => readJSON("./data/big buy/big-bui-product-stock.json");
Promise.all([
getProductList(),
getProductStock(),
]).then(([products, stocks]) => {
console.log({ products, stocks });
}).catch((error) => {
console.log("Ha ocurrido un error:", error.message);
}).finally(() => {
// ...
});
i was reading about how to solve this problem, and everyone says that i should use "async" and "await", but i dont know how to put it propely on my code. (I'm making a new JSON, and i should send it to the front-end, but i need to wait to the JSON get ready inside the forEach loop first, then render on the screen).
router.get('/', (req, res) => {
Controle.find().lean().then( (controles) => {
var controles_cadastrados = []
controles.forEach((controle, index, array)=>{
Sensor.find({id_mac: controle.sensor}).lean().then((sensor)=>{
controle.sensor_nome = sensor[0].nome
Atuador.find({id_mac: controle.atuador}).lean().then((atuador)=>{
controle.atuador_nome = atuador[0].nome
controles_cadastrados.push(controle)
console.log(controles_cadastrados)
})
})
})
//wait to send the response
res.render('controle/controles', {controles: controles_cadastrados })
}).catch((erro) => {
console.log('Erro ao carregar controles')
})
})
I have try it in so many ways, but none seens to working good.
Sorry if i made some mistake.
You aren't properly waiting for all your asynchronous operations to be done before calling res.render() thus your array is empty or partially populated when you try to use it. So, you need to use the promises to be able to track when everything is done.
You have a couple choices. You can run all your request in parallel or in sequence. I'll show an example of both:
Here's processing all the database requests in parallel. This chains the various promises and use Promise.all() to know when they are all done. Results in the controles_cadastrados are not in any particular order, but this will probably run quicker than processing everything sequentially.
router.get('/', (req, res) => {
Controle.find()
.lean()
.then(controles => {
const controles_cadastrados = [];
Promise.all(controles.map(controle => {
return Sensor.find({ id_mac: controle.sensor })
.lean()
.then(sensor => {
controle.sensor_nome = sensor[0].nome;
return Atuador.find({ id_mac: controle.atuador })
.lean()
.then(atuador => {
controle.atuador_nome = atuador[0].nome;
controles_cadastrados.push(controle);
console.log(controles_cadastrados);
});
});
})).then(() => {
//wait to send the response
res.render('controle/controles', {
controles: controles_cadastrados,
});
});
}).catch(erro => {
console.log('Erro ao carregar controles');
res.sendStatus(500);
});
});
And, here's how you would sequence the operations using async/await:
router.get('/', async (req, res) => {
try {
let controles = await Controle.find().lean();
const controles_cadastrados = [];
for (let controle of controles) {
let sensor = await Sensor.find({ id_mac: controle.sensor }).lean();
controle.sensor_nome = sensor[0].nome;
let atuador = await Atuador.find({ id_mac: controle.atuador }).lean()
controle.atuador_nome = atuador[0].nome;
controles_cadastrados.push(controle);
console.log(controles_cadastrados);
}
//wait to send the response
res.render('controle/controles', {
controles: controles_cadastrados,
});
} catch(e) {
console.log(e, 'Erro ao carregar controles');
res.sendStatus(500);
}
});
Also, note that all possible rejected promises or other errors are captured here and a response is always sent to the incoming http request, even when there's an error.
This is a pseudo code of what I am trying to achieve. First I need to get a list of URLs from the request body then pass those URLs to request function (using request module) which will get the data from each url and then save those data to MongoDB. After all the requests are finished including saving data to the server only then it should send a response.
app.post('/', (req, resp) => {
const { urls } = req.body;
urls.forEach((url, i) => {
request(url, function (err, resp, body) {
if (err) {
console.log('Error: ', err)
} else {
// function to save data to MongoDB server
saveUrlData(body);
console.log(`Data saved for URL number - ${i+1}`)
}
})
});
// Should be called after all data saved from for loop
resp.send('All data saved')
})
I have tried this code and of course the resp.send() function will run without caring if the request has completed. Using this code I get a result on the console like this:
Data saved for URL number - 3
Data saved for URL number - 1
Data saved for URL number - 5
Data saved for URL number - 2
Data saved for URL number - 4
I could write them in nested form but the variable urlscan have any number of urls and that's why it needs to be in the loop at least from my understanding. I want the requests to run sequentially i.e. it should resolve 1st url and then second and so on and when all urls are done only then it should respond. Please help!
app.post('/', async (req, resp) => {
const {
urls
} = req.body;
for (const url of urls) {
try {
const result = await doRequest(url)
console.log(result)
} catch (error) {
// do error processing here
console.log('Error: ', err)
}
}
})
function doRequest(url) {
return new Promise((resolve, reject) => {
request(url, function(err, resp, body) {
err ? reject(err) ? resolve(body)
})
})
}
using async await
You should look at JavaScript Promises
Otherwise, you can do a recursive request like so:
app.post('/', (req, resp) => {
const { urls } = req.body;
sendRequest(urls, 0);
})
function sendRequest(urlArr, i){
request(urlArr[i], function (err, resp, body) {
if (err) {
console.log('Error: ', err)
}
else {
saveUrlData(body);
console.log(`Data saved for URL number - ${i+1}`)
}
i++;
if(i == urlArr.length) resp.send('All data saved') //finish
else sendRequest(urlArr, i); //send another request
})
}
All I had to do is create a separate function I can call over and over again, passing the url array and a base index 0 as arguments. Each success callback increments the index variable which I pass in the same function again. Rinse and repeat until my index hits the length of the url array, I'll stop the recursive loop from there.
You want to wait till all api response you get and stored in db, so you should do async-await and promisify all the response.
You can use Request-Promise module instead of request. So you will get promise on every requested api call instead of callback.
And use promise.all for pushing up all request(module) call inside array.
Using async-await you code execution will wait till all api call get response and stored in db.
const rp = require('request-promise');
app.post('/', async (req, res) => {
try{
const { urls } = req.body;
// completed all will have all the api resonse.
const completedAll = await sendRequest(urls);
// now we have all api response that needs to be saved
// completedAll is array
const saved = await saveAllData(completedAll);
// Should be called after all data saved from for loop
res.status(200).send('All data saved')
}
catch(err) {
res.status(500).send({msg: Internal_server_error})
}
})
function sendRequest(urlArr, i){
const apiCalls = [];
for(let i=0;i < urlArr.length; i++){
apiCalls.push(rp(urlArr[i]));
}
// promise.all will give all api response in order as we pushed api call
return Promise.all(apiCalls);
}
You can refer these links:
https://www.npmjs.com/package/request-promise
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all
Looking at the intention(a crawler) you can use Promise.all because the urls are not dependant upon each other.
app.post('/', (req, resp) => {
const { urls } = req.body;
const promises = urls.map((url, i) => {
return new Promise((resolve, rej)=>{
request(url, function (err, resp, body) {
if (err) {
rej(err);
} else {
resolve(body);
}
})
})
.then((body)=>{
//this should definitely be a promise as you are saving data to mongo
return saveUrlData(body);
})
});
// Should be called after all data saved from for loop
Promise.all(promises).then(()=>resp.send('All data saved'));
})
Note: Need to do error handling as well.
there are multiple ways to solve this.
you can use async/await
Promises
you can also use the async library
app.post('/', (req, res, next) => {
const { urls } = req.body;
async.each(urls, get_n_save, err => {
if (err) return next(err);
res.send('All data saved');
});
function get_n_save (url, callback) {
request(url, (err, resp, body) => {
if (err) {
return callback(err);
}
saveUrlData(body);
callback();
});
}
});
Reading some amazing tutorials about promises, I've discovered that, if I need to interate throw some promises, I can't use forEach or some other "traditional" iteration mechanisms, I have to use Q library for node, I've to "iterate" using Q.all.
I've written a simple example in Nodejs Express in order to be sure I've understood promises:
var form = [
{'name':'FORM_NAME_1.1',
'label2':'FORM_LABEL_1.2'
},
{'name':'FORM_NAME_2.1',
'label2':'FORM_LABEL_2.2'
}
];
var params = ['params1','params2'];
var meta = ['meta1','meta2'];
app.get('/', (req,res) => {
return Q.all([
form.map((currentValue,index,arr) => {
req.id = Math.random(); //Random ID to be used in the next promises
console.log(currentValue);
return Form.insert(currentValue,req);
}),
params.map((currentValue,index,arr) => {
console.log(req.id);
return Field.insert(currentValue,req.id);
}),
meta.map((currentValue,index,arr) => {
console.log(req.id);
return Meta.insert(currentValue,req.id);
})
])
.catch((err) => next(err))
.done(() => console.log('It\'s done'));
});
Form.insert code simply is a promise with a console.log call, the same for Field.insert and Meta.insert
var Form = {
insert: (param1,req) => {
var deferred = Q.defer();
console.log('Goes throw Form');
deferred.resolve();
return deferred.promise;
}
}
The problem is that seems to iterate right but the dynamicly generated id does not change along the promises, this is the console output:
Listening at port 3000...
{ name: 'FORM_NAME_1.1', label2: 'FORM_LABEL_1.2' }
Goes throw Form
{ name: 'FORM_NAME_2.1', label2: 'FORM_LABEL_2.2' }
Goes throw Form
0.3757301066790548
Goes throw Field
0.3757301066790548
Goes throw Field
0.3757301066790548
Goes throw Meta
0.3757301066790548
Goes throw Meta
It's done
Any ideas about what is going wrong? Thanks!!
the reason it is not working is because in first for loop, the req.id is set multiple times before other promises are started and and all of them end up using the last randomly generated value, change your code to:
app.get('/', (req,res) => {
let process = (currentValue,index,arr) => {
let reqCopy = {id: Math.random()}
for(let attr in req) // copy all the request attributes
if(attr && attr!='id')
reqCopy[attr] = req[attr]
return Q.all([
Form.insert(form[index],reqCopy),
Field.insert(params[index],reqCopy),
Meta.insert(meta[index],reqCopy)
])
}
return Q.all(form.map(process))
.catch(next)
.done(() => console.log('It\'s done'));
})
you would notice that I am copying all the attributes of req to clone reqCopy for I am not sure what attributes of req are required by the subsequent methods, but at the same time, single req.id would not work thanks to the async nature of code
I'm trying to do something like this
function retrieveUser(uname) {
var user = User.find({uname: uname}, function(err, users) {
if(err)
console.log(err);
return null;
else
return users[0];
return user;
But this returns a document instead of a user object. The parameter users is an array of user objects matching the query, so how would I store one of the objects into a variable that my function could return?
The function User.find() is an asynchronous function, so you can't use a return value to get a resultant value. Instead, use a callback:
function retrieveUser(uname, callback) {
User.find({uname: uname}, function(err, users) {
if (err) {
callback(err, null);
} else {
callback(null, users[0]);
}
});
};
The function would then be used like this:
retrieveUser(uname, function(err, user) {
if (err) {
console.log(err);
}
// do something with user
});
Updated on 25th Sept. 2019
Promise chaining can also be used for better readability:
Model
.findOne({})
.exec()
.then((result) => {
// ... rest of the code
return Model2.findOne({}).exec();
})
.then((resultOfModel2FindOne) => {
// ... rest of the code
})
.catch((error) => {
// ... error handling
});
I was looking for an answer to the same question.
Hopefully, MongooseJS has released v5.1.4 as of now.
Model.find({property: value}).exec() returns a promise.
it will resolve to an object if you use it in the following manner:
const findObject = (value) => {
return Model.find({property: value}).exec();
}
mainFunction = async => {
const object = await findObject(value);
console.log(object); // or anything else as per your wish
}
Basically, MongoDB and NodeJS have asynchronous functions so we have to make it to synchronous functions then after it will work properly as expected.
router.get('/', async function(req, res, next) {
var users = new mdl_users();
var userData = []; // Created Empty Array
await mdl_users.find({}, function(err, data) {
data.forEach(function(value) {
userData.push(value);
});
});
res.send(userData);
});
In Example, mdl_users is mongoose model and I have a user collection(table) for user's data in MongoDB database and that data storing on "userData" variable to display it.In this find function i have split all documents(rows of table) by function if you want just all record then use direct find() function as following code.
router.get('/', async function(req, res, next) {
var users = new mdl_users();
var userData = await mdl_users.find();
res.send(userData);
});