I'm trying to solve a problem i've been having with a nodejs cronjob of mine. So basically, This request grabs my tracks from soundcloud, I loop through the results and put the data in a mongodb collection. This all works great, but now i'm adding another section to the site, so I need to grab some additional info from another collection.
I have a tracks collection, and an inventory collection. The track id is in both collections to relate the additional track data to the newly pulled tracks. So my question is how can I get this additional track data in? Below i have tried to loop through it and inject the data using mongoose's query.find() but the loops do not work together. The callback of the inventory query will run all in one for loop it seems... I'm not exactly sure whats going on there.
I'm pretty sure you can also inject a document from another collection by referencing it in the schema... but i'm unsure of how to get this working. This would obviously be a better solution as it won't require more code like this.
if anybody has any suggestions for me that would be great!
request({
url: url,
json: true
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
var o = 1;
for(i=0; i < body.tracks.length; i++){
var last = (i + 1);
var track = body.tracks[i];
if( track.sharing == 'public'){
var invData;
var obj;
db.model('inventory').find({id:track.id}).exec(function(err,item){
//console.log(item[0]);
invData = item[0];
});
console.log(invData, obj);
obj = new TracksModel({
id: track.id,
sharing:track.sharing,
uri:track.uri,
description:track.description,
created_at:track.created_at,
duration:track.duration,
title:track.title,
description:track.description,
order: o,
inventory_data: invData
});
o++;
obj.save(function (err) {
if (!err) {
console.log('Track inserted successfully');
} else {
throw err;
}
});
if(last == body.length){
setTimeout(function(){
console.log("Automatically closing database connection after 5 seconds");
db.close();
}, 5000);
}
}
}
} else {
console.log('An error has occurred: ', error);
}
});
The way you are treating the query callback is wrong. You are assuming that the code starting from "console.log(invData, obj);" will be executed immediately after the db.model.find. That not the correct notion of Callback. How must put that code inside the exec callback function. You may have to use a closure.
Something like:
if (!error && response.statusCode === 200) {
var o = 1;
for(i=0; i < body.tracks.length; i++){
var last = (i + 1);
var track = body.tracks[i];
if( track.sharing == 'public'){
(function(track,last,o){
var invData;
var obj;
db.model('inventory').find({id:track.id}).exec(function(err,item){
//console.log(item[0]);
invData = item[0];
console.log(invData, obj);
obj = new TracksModel({
id: track.id,
sharing:track.sharing,
uri:track.uri,
description:track.description,
created_at:track.created_at,
duration:track.duration,
title:track.title,
description:track.description,
order: o,
inventory_data: invData
});
obj.save(function (err) {
if (!err) {
console.log('Track inserted successfully');
} else {
throw err;
}
});
if(last == body.length){
setTimeout(function(){
console.log("Automatically closing database connection after 5 seconds");
db.close();
}, 5000);
}
});
}(track,last,o);
o++;
}
}
}
Try this out:
var utils = require('restberry-utils');
var Inventory = mongoose.model('Inventory');
var Track = mongoose.model('Track');
request({
url: url,
json: true
}, function (error, response, body) {
if (error || response.statusCode !== 200) {
console.log('An error has occurred: ', error);
return;
}
utils.forEachAndDone(body.track, function(track, iter) {
if (track.sharing !== 'public') {
iter();
return;
}
Inventory.findOne({ id: track.id }, function(err, item) {
new Track({
id: track.id,
sharing: track.sharing,
uri: track.uri,
description: track.description,
created_at: track.created_at,
duration: track.duration,
title: track.title,
description: track.description,
order: o,
inventory_data: item,
}).save(function(err) {
if (err) {
throw err;
} else {
console.log('Track inserted successfully');
iter();
}
})
});
}, function() {
console.log('Done!');
setTimeout(function() {
console.log("Automatically closing database connection after 5 seconds");
db.close();
}, 5000);
})
});
Related
Firstly, I'm sorry if this has been posted before. I searched but couldn't find any credible solution.
So I'm working on this route in nodejs where I make an API call for a piece of information and then using that info in an if statement to check if it's the correct info(the server sometimes sends wrong info).
If I get the correct info then I use that in another API to get more info about it and render it into my template. Everything works fine.
But I want the first API call to take place again if the info doesn't match or it's wrong. How can I initiate the API call again from the start(like a loop) and it will break only if the info is correct. Please check the "comment" in the code below. That is where I don't know what to put. Your help would be highly appreciated.
PS. I am a beginner in nodejs and javascript.
Route
router.get("/check", (req, res) => {
if(req.query.search) {
var input = req.query.search;
var url = "http://firstapi.com/json/" + input + "?fields=query";
request(url, function(error, response, body) {
if(!error && response.statusCode === 200) {
var data = JSON.parse(body);
if(data.query.match(/((^|\.)((25[0-5])|(2[0-4]\d)|(1\d\d)|([1-9]?\d))){4}$/)){
var url = "https://secondapi.com/" + data.query + "?key=something";
request(url, function(error, response, body) {
if(!error && response.statusCode === 200) {
var Data = JSON.parse(body);
res.render("index", {data: Data});
}
});
}else{
//want to use the input at the top and check the firstapi again. All the code above should run again until its the correct one which I will use in my template.
}
}
});
}else{
res.render("index", {data: null});
}
});
I would probably do it this way:
router.get('/check', (req, res) => {
if (req.query.search) {
var input = req.query.search;
// Put this logic away in a `checkData` function
checkData(input)
.then(data => {
res.render('index', { data }); // Short version of {data: data}
})
.catch(err => {
console.error(err);
res.render('index', { data: null });
});
} else {
res.render('index', { data: null });
}
});
// Here, we have a `retries` parameter, set to 0 initially
function checkData (input, retries = 0) {
const maxRetries = 3;
// Return a promise (you could also use callbacks)
return new Promise((resolve, reject) => {
// Define a retry method
const retry = () => {
if (retries < maxRetries) {
// Increment the retries count and go for another try
checkData(input, retries + 1).then(resolve).catch(reject);
} else {
reject(`Could not get the data after ${retries} retries.`);
}
};
var url = `http://firstapi.com/json/${input}?fields=query`;
request(url, function (error, response, body) {
if (!error && response.statusCode === 200) {
var data = JSON.parse(body);
if (data.query.match(/((^|\.)((25[0-5])|(2[0-4]\d)|(1\d\d)|([1-9]?\d))){4}$/)) {
var url = 'https://secondapi.com/' + data.query + '?key=something';
request(url, function (error, response, body) {
if (!error && response.statusCode === 200) {
var Data = JSON.parse(body);
// If it worked, resolve with the data
resolve(Data);
} else {
retry();
}
});
} else {
retry();
}
}
});
});
}
I'm very new to nodejs's async.forEach and I'm having trouble aggregating the result of my nested forEach loop.
I have a dynamic range of dates and a number of screens that I would like to loop through and either create a schedule or update an existing one. That part works as designed. However, I can't get construct an array of all schedules that has been created and have been updated. I seem to only get the first one but not the rest.
I've tried many different ways of calling the callback but the most I've ever gotten is just one item in my output array.
I've tried different methods from this website but I haven't gotten the luck:
http://www.sebastianseilund.com/nodejs-async-in-practice
What is the best way of handling this scenario?
Below is my trimmed down loopback remoteMethod:
===========================
Schedule.Reservation = function(PostData, cb) {
var output = []; // <-- I would like to return this array ... which is report of all created and updated schedules
try {
// create all models
async.series([
function validateData(callback) {
callback();
},
function processReservation(callback) {
var screens = PostData.Screens;
var dates = getDateRangeArray(PostData);
async.forEach(dates, function(liveDate, callbackDate)
//for (var d = new Date(PostData.StartDate); d <= end; d.setDate(d.getDate() + 1))
{
async.forEach(screens, function(screen, callbackScreen)
//for (var s=0;s<screens.length;s++)
{
if (screen.details)
async.forEach(screen.details.BookingInformation, function(frame, callbackFrame) {
if ((frame.BlockedDays == 0) || (!isBlocked)) {
Schedule.findOne({
where: {
LiveDate: liveDate,
ScreenID: screen.id,
FrameID: frame.FrameID,
Remaining: {
gte: PostData.RequiredSlots
}
}
}, function(errSchedule, schedule) {
var scheduleLog = {}
scheduleLog.liveDate = liveDate;
scheduleLog.ScreenID = screen.id;
scheduleLog.FrameID = frame.FrameID;
if (!errSchedule) {
if (!schedule) {
var tempSchedule = {
LiveDate: liveDate,
Posts: "posts",
Remaining: remain
}
Schedule.create(tempSchedule,
function(err, result) {
if (err) {
output.push({
'Failed': scheduleLog,
'Error': err
});
//callbackFrame(output);
} else {
output.push({
'Created': scheduleLog,
'Success': result
});
//callbackFrame(output);
}
});
} else {
schedule.Remaining--;
schedule.save(function(err, result) {
if (err) {
output.push({
'Failed': scheduleLog,
'Error': err
});
//callbackFrame(output);
} else {
output.push({
'Updated': scheduleLog,
'Success': result
});
//callbackFrame(output);
}
});
} else {
output.push({
'Skipped': scheduleLog,
'Warning': 'Warning: Unable to update. Validation failed. ' + schedule
});
//callbackFrame(output);
}
}
} else {
output.push({
'Skipped': scheduleLog,
'Error': errSchedule
});
//callbackFrame(output);
}
}
);
}
},
function(result) {
if (output)
callback(output);
else
callbackScreen();
});
else {
throw new Error("Invalid Data");
return callbackScreen(output); //should throw an error.
}
},
function(result) {
if (output)
callbackDate(output);
else
callbackDate(output);
});
},
function(result) {
if (output)
callback(output);
else
callback();
});
//callback(output);
}
],
function(result) {
if (output) //also tried result, the outcome is the same.
{
cb(null, output);
} else
cb("Failed!!!");
});
} catch (ex) {
console.log(ex.message);
cb('!Error! ' + ex.message);
}
Do you use caolin's async library? See this link to get an idea on how to proceed: https://github.com/caolan/async#seriestasks-callback
Do not enclose your async code inside a try-catch as the async series/forEach provide their own way to handle any error. Typically any async callback accepts 2 parameters: error, result.
callbackScreen or callbackDate must call callback (second handler) to pass some result over in order to hook up your final series' callback (the one you declare at the same level of async.series).
async.series([
function A(callback){
// do some stuff ...
callback(null, 'abc'); //first arg is err. If not null you'll go to the final handler below
},
function B(callback){
// do some more stuff ...
async.forEach(dates, function(liveDate, callbackDate) {
//stuff to do with dates
callbackDate(null, 'your data'); //will go to the handler right below
}, function (err, data) {
if (err) console.error(err.message);
//here call callback of function B
callback(null, data'); //first arg is err
}))
}
],
// optional callback
function(err, results){
// results is now equal to ['abc', 'your data']
});
Been making great waves getting deeper into node.js and mongo but I've hit a bit of a wall.
I've got my data being pushing from a form into a collection with mongoose and I can see it being pulled into a JS file and calling the document with console.log below.
var fields = { 'cheap': 1, 'number': 1 };
var query = userModel.find({}).select(fields);
query.exec(function (err, docs) {
if (err) return handleError(err);
console.log(docs + 'first');
request(url, function(error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
//var $el = $("a:contains('Xbox')");
var $el = $('a:contains(' + docs + ')');
console.log(docs + 'second')
if ($el.length) {
client.messages.create({
to: "+ '+ cheap.number +'",
from: "+61418739508",
body: $el.text()
}, function(err, message) {
if(err){
console.log(err);
} else {
console.log(message.sid);
}
});
console.log($el.text());
} else {
console.log('hey');
}
}
});
});
Output:
[ { _id: 561650245b3d0d57ad7f3c72,
cheap: 'menus',
number: '0000000000' } ]
However, now I would like to take the values of the 'cheap' and 'number' fields and store them as a var that I can iterate through on the below function.
Can anyone point me in the right direction for what I need to work on? It very much seems like an elusive missing piece of a puzzle.
Found what I was looking for. Needed a for.. in construct to iterate through the properties of the object. From there I had to call the object and property type and it works.
Magic that fixed it:
var docs;
for (var key in docs) {
var value = docs[key];
console.log(key + ", " + value);
}
Full code:
var query = userModel.find({});
query.exec(function (err, docs) {
if (err) return handleError(err);
//console.log(docs);
var docs;
for (var key in docs) {
var value = docs[key];
console.log(key + ", " + value);
}
request(url, function(error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
//var $el = $("a:contains('Xbox')");
var $el = $('a:contains(' + value.cheap + ')');
//console.log($el);
if ($el.length) {
client.messages.create({
to: value.number,
from: "+61418739508",
body: $el.text()
}, function(err, message) {
if(err){
console.log(err);
} else {
console.log(message.sid);
}
});
console.log($el.text());
} else {
console.log('hey');
}
}
});
});
I think the mongoose is async, you should move the request function to exec something like this
` query.exec(function (err, cheap, number) {
if (err) return handleError(err);
console.log(cheap);
request(url, function(error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
async.each(cheap, function(iterateCheap, callback){
var $el = $("a:contains(" + cheap + ")");
}, function(err){
if(err) {
console.log(err);
} else {
console.log('We successfully iterate on all cheap')
}
})
......`
UPDATE1:
Or you should use async.js module and waterfall method for this situation
I am looking to do a get, run a function on the results which will do some manipulation by updating a field, and then put that doc back into the database. Really my issue is being able to chain together multiple DB calls. I have been struggling with this the past week or so. Any suggestions appreciated, thanks.
Here is what I have tried so far but I am receiving an error:
function geocode_cleanup(request, response, next) {
r.table('dealer_locations').filter(r.row('geodata').match('No geodata found.'))
.do(function(row) {
var geodata = opencage_geocoder.geocode(row.Address, function(error, response) {
if (error) {
console.log("Error.");
row.geodata = "No geodata found.";
row.active = true;
} else if (response.length == 0) {
console.log("Empty response.");
} else {
console.log("Success.");
console.log(response);
var latitude = response[0].latitude;
var longitude = response[0].longitude;
row.geodata = r.point(longitude, latitude);
row.active = true;
}
});
return r.table('dealer_locations').update({
geodata: geodata
})
}).run(conn, function(error, cursor) {
response.setHeader("Content-Type", "application/json");
if (error) {
handleError(response, error);
} else {
cursor.toArray(function(error, results) {
if (error) {
handleError(response, error);
} else {
response.send(results);
};
});
}
next();
})
};
Also, this gives the desired results returned in the response, but the second db action never happens because I am still inside of the same db connection I think:
function geocode_cleanup(request, response, next) {
var conn = request._rdbConn;
r.table('dealer_locations').filter({geodata: "No geodata found."}).run(conn, function(error, cursor) {
if (error) {
handleError(response, error);
} else {
cursor.toArray(function(error, results) {
if (error) {
handleError(response, error);
} else {
var i = 1;
async.forEach(results, function(item, callback) {
var address = (item.Address + " " + item.City).toString();
opencage_geocoder.geocode(address, function(err, res) {
if (err) {
console.log(i);
console.log("Error.");
item.id = i;
item.geodata = "No geodata found.";
item.active = true;
i++;
callback();
} else if (res.length == 0) {
console.log(i);
console.log("Empty response.");
i++;
callback();
} else {
console.log(i);
console.log("Success.");
console.log(res);
var latitude = res[0].latitude;
console.log(i + " " + latitude);
var longitude = res[0].longitude;
console.log(i + " " + longitude);
item.id = i;
item.geodata = r.point(longitude, latitude);
item.active = true;
i++;
callback();
}
});
}, function() {
r.table('dealer_locations').insert(results, {
conflict: "replace"
}).run(request._rdbConn, function(error, results) {
if (error) {
console.log("Data not inserted!");
} else {
console.log("Data inserted!");
}
});
console.log("Done!");
response.send(results);
});
}
})
}
})
}
Here's a possible solution which uses promises to organize the code a little bit.
// Guarantee support for promises and provide the `promisify` function
var Promise = require('bluebird');
// Promisify the geocode function to make it easier to use
var geocode = Promise.promisify(opencage_geocoder.geocode);
function geocode_cleanup(request, response, next) {
var conn = request._rdbConn;
r
.table('dealer_locations')
.filter(r.row('geodata').match('No geodata found.'))
.coerceTo('array')
.run(conn)
.then(function(rows) {
// This promise will be resolve when all rows have been geocoded and updated
// We map the rows into an array of promises, which is what Promise.all takes
return Promise.all(rows.map(function (row) {
return geocode(row.Address)
.then(function (response) {
console.log("Success.");
var latitude = response[0].latitude;
var longitude = response[0].longitude;
row.geodata = r.point(longitude, latitude);
row.active = true;
// Return the row
return row;
});
});
}));
})
.then(function (rows) {
// Now that all `dealer_locations` have been updated, re-query them
return r
.table('dealer_locations')
.insert(rows, {conflict: "update", return_changes: true})
.run(conn);
})
.then(function (results) {
// Send the response;
response.setHeader("Content-Type", "application/json");
response.send(results);
return;
})
.catch(function (err) {
return handleError(null, error);
})
};
Some problems I noticed with your code:
1. Use of do
r.table('dealer_locations').filter(r.row('geodata').match('No geodata found.'))
.do(function(row) {
var geodata = opencage_geocoder.geocode ...
})
In this code snippet, you use a JS function inside of do. You can't do that. Remember that what happens inside of do happens in the RethinkDB server (not in your Node.js server). Your RethinkDB server has no knowledge of your opencage_geocoder function and so this woudn't work.
Whatever do returns must be a valid ReQL query or ReQL expression. You can't execute arbitrary JavaScript inside of it.
If you want to run JavaScript with your query results, you have to .run the query and then do whatever you want to do inside the callback or .then function. At that point, that code will get executed in JavaScript and not in your RethinkDB server.
2. Use of update
return r.table('dealer_locations').update({
geodata: geodata
})
The update method can only update a single document. You can't pass it an array of documents. In this scenario you what have needed to do r.table().get().update() in order for this to work, because you have to be referencing a single document when you update something.
If you have an array of documents that you want to update, you can use the forEach method.
r.table('hello')
.merge({
'new_property': 'hello!'
})
.forEach(function (row) {
// Insert that property into the document
return r.table('hello').get(row.id).update(row);
})
You can also do this (which you are already doing):
r.table('hello')
.merge({
'new_property': 'hello!'
})
.do(function (rows) {
// Insert that property into the document
return r.table('hello')
.insert(rows, {conflict: "update", return_changes: true});
})
OK, I have a suggestion. This queries for the documents you're interested in, modifies them (on your app server, not in the db) and then reinserts them using the nifty conflict: 'update' option. It also uses promises because I think that's a bit cleaner.
function geocode_cleanup(request, response, next) {
r.table('dealer_locations')
.filter(r.row('geodata').match('No geodata found.'))
.run(conn).then(function(cursor) {
var to_update = [];
return cursor.toArray().then(function getGeocodes(rows) {
return rows.map(function getGeocode(row) {
row.geodata = opencage_geocoder.geocode(row.Address, function(error, response) {
if (error) {
console.log("Error.");
row.geodata = "No geodata found.";
row.active = true;
} else if (response.length == 0) {
console.log("Empty response.");
} else {
console.log("Success.");
console.log(response);
var latitude = response[0].latitude;
var longitude = response[0].longitude;
row.geodata = r.point(longitude, latitude);
row.active = true;
}
});
return row;
});
});
}).then(function doneGeocoding(modified_rows){
return r.table('dealer_locations')
.insert(modified_rows, {conflict: "update", return_changes: true})('changes')
.coerceTo('array')
.run(conn);
}).then(function finishResponse(changes){
response.setHeader("Content-Type", "application/json");
response.send(results);
next();
}).catch(function(err) {
// handle errors here
});
};
Caveat emptor, I haven't run this, so there may be syntax errors and things
I'm missing something about callbacks with Mongoose save function.
I am trying to insert a new transaction, then if that is successful, update a user account. The problem, I believe, This cause all people to be updated with the last person's amount. What am I trying to do is update a document after saving another document.
Here is the code. Please let me know what I am doing wrong.
Thanks in advance.
//find all schedules
SchedModel.find({ Day_Of_Week: day }, null, null, function (err, Sched) {
if (!err) {
//iterate through what we find
for (var key in Sched) {
if (Sched.hasOwnProperty(key)) {
var val = Sched[key];
console.log("val : " + val);
var Sched_Descr = day || ' Sched Trans';
var this_trans = new TransModel({
mID: val.K_Id,
mDate: today,
mDescr: Sched_Descr,
mAmt: val.mAmt
});
//insert the new trans
this_trans.save(function (err, trans) {
if (!err) {
//when we insert new trans, get the update model
MyModel.findById(val.K_Id, function (err, Model) {
Model.Balance = Model.Balance + val.mAmt;
//save model, this update to the last in the list
Model.save(function (err) {
if (!err) {
console.log("updated");
} else {
console.log(err);
}
});
});
} else {
return console.log(err);
}
});
}
}
} else {
console.log(err);
};
});
Update: ES6's let solves this pretty trivially, just replace var with let in your original code and it should work.
Your this_trans and such variables aren't unique in each iteration of the for-in loop. You might wanna wrap it in a self-executing anonymous function scope ((function(){})())
//find all schedules
SchedModel.find({ Day_Of_Week: day }, null, null, function (err, Sched) {
if (!err) {
//iterate through what we find
for (var key in Sched) {
(function(key){ // self-executing anonymous function scope
if (Sched.hasOwnProperty(key)) {
var val = Sched[key];
console.log("val : " + val);
var Sched_Descr = day || ' Sched Trans';
var this_trans = new TransModel({
mID: val.K_Id,
mDate: today,
mDescr: Sched_Descr,
mAmt: val.mAmt
});
//insert the new trans
this_trans.save(function (err, trans) {
if (!err) {
//when we insert new trans, get the update model
MyModel.findById(val.K_Id, function (err, Model) {
Model.Balance = Model.Balance + val.mAmt;
//save model, this update to the last in the list
Model.save(function (err) {
if (!err) {
console.log("updated");
} else {
console.log(err);
}
});
});
} else {
return console.log(err);
}
});
}
})(key);
}
} else {
console.log(err);
};
});