I'm currently fiddling around with Node.js and I stuck with this issue.
I'm using the csvtojson converter (https://github.com/Keyang/node-csvtojson) as a separate module that I can call in my other JS files as many times as I want.
Here is my tools.js:
module.exports = {
csvToJson: function (csvPath) {
var Converter = require('csvtojson').Converter;
var converter = new Converter({});
var transfer = "DEFAULT";
converter.fromFile(csvPath, function(err, result){
if (err) {
return console.log(err);
}
else {
transfer = result;
}
});
return transfer;
}
};
And here is how I call it:
var countriesCsvFile = path.join(__dirname, '..', 'testDataFiles', 'countries.csv');
//GRAB TOOLS
var tools = require('../app/tools');
console.log(tools.csvToJson(countriesCsvFile));
The result is always the "DEFAULT" value which indicates, that the converter is not touching it.
I want to pass it as the return value of the function, to further be able to process the data on the fly, without creating a file, and read that.
It is surely some scope issue, but after scratching my scalp for a few hours, and browsing the questions I couldn't retrieve anything remotely useful.
Also, another note: If I call console.log(result) instead of transfer = result, it shows me my precious and desired data.
You have to pass in a callback function because the csvToJson function is returning 'transfer' before any value is assigned to it. Like Sirko said, it's asynchronous. You can also use promises instead of callbacks but that's another topic in itself.
module.exports = {
csvToJson: function (csvPath, callback) {
var Converter = require('csvtojson').Converter;
var converter = new Converter({});
converter.fromFile(csvPath, function(err, result){
if (err) {
callback(err);
}
else {
callback(null, result);
}
});
}
};
Related
I did a couple of projects with node.js and I'm aware of the async behaviour and that one should usually use callback functions, etc. But one thing that bothers me ist the following.
I'm developing an Alexa skill and I have a function that handles the User intent:
'MyFunction': function() {
var toSay = ""; // Holds info what Alexa says
// Lot of checks and calculations what needs to be said by Alexa (nothing special)
if(xyz) {
toSay = "XYZ";
}else if(abc) {
toSay = "ABC";
}else{
toSay = "Something";
}
// Here is the "tricky" party
if(someSpecialEvent) {
toSay += " "+askDatabaseForInput(); // Add some information from database to string
}
this.emit(':ask', toSay, this.t('REPROMT_SPEECH')); // Gives the Info to Alexa (code execution stops here)
}
As mentioned in the code, there is some code which is usually used to find out what the output to Alexa should be.
Only on rare events, "someSpecialEvent", I need to query the database and add information to the String "toSay".
Querying the DB would look something like:
function askDatabaseForInput() { // The function to query the DB
var params = {
TableName: "MyTable",
OtherValues: "..."
};
// Do the Query
docClient.query(params, function(err, data) {
// Of course here are some checks if everything worked, etc.
var item = data.Items[0];
return item; // Item SHOULD be returned
});
return infoFromDocClient; // Which is, of course not possible
}
Now I know, that in the first function "'MyFunction'" I could just pass the variable "toSay" down to the DB Function and then to the DB Query and if everything is fine, I would do the "this.emit()" in the DB Query function. But for me, this looks very dirty and not much reusable.
So is there a way I can use "askDatabaseForInput()" to return DB information and just add it to a String? This means making the asynchronous call synchronous.
Making a synchronous call wouldn't affect the user experience, as the code isn't doing anything else anyway and it just creates the String and is (maybe) waiting for DB input.
Thanks for any help.
So you could do 2 things:
Like the person who commented says you could use a callback:
function askDatabaseForInput(callback) {
var params = {
TableName: "MyTable",
OtherValues: "..."
};
docClient.query(params, function(err, data) {
if (err) {
callback(err, null)
} else {
var item = data.Items[0];
callback(null, item);
}
});
}
or you could use promises:
function askDatabaseForInput() {
var params = {
TableName: "MyTable",
OtherValues: "..."
};
return new Promise(function (resolve, reject) {
docClient.query(params, function(err, data) {
if (err) {
reject(err)
} else {
var item = data.Items[0];
resolve(item);
}
});
});
}
you can then either put a function in where you call askDatabaseForInput or do askDatabaseForInput.then(....).
In the function or the .then you would add what you retrieved from the database to the variable toSay
hope this helps
I am trying to do maths on a number within a JSON object (the price of a stock ticker).
I want it to be a variable called 'btcusd_price', that I can then use to do arithmetic with.
How do I get a variable i can work with?
https://tonicdev.com/npm/bitfinex
var Bitfinex = require('bitfinex');
var bitfinex = new Bitfinex('your_key', 'your_secret');
var btcusd_price;
btcusd_price = bitfinex.ticker("btcusd", function(err, data) {
if(err) {
console.log('Error');
return;
}
console.log(data.last_price);
return data.last_price;
});
typeof btcusd_price;
console.log(btcusd_price); //i'm trying to work with the price, but it seems to be 'undefined'?
You have to set the values when they are available, the code is async and you were using the value before it is applied to btcusd_price. Note that the proper way to use the variable is when the callback executes its code.
Please, see the working code below:
Bitfinex = require('bitfinex');
var bitfinex = new Bitfinex('your_key', 'your_secret');
var btcusd_price;
bitfinex.ticker("btcusd", function(err, data) {
if(err) {
console.log('Error');
return;
}
btcusd_price = data.last_price;
console.log(typeof btcusd_price);
console.log(btcusd_price);
});
I'm trying to use the Steam Community (steamcommunity) npm package along with meteorhacks:npm Meteor package to retreive a user's inventory. My code is as follows:
lib/methods.js:
Meteor.methods({
getSteamInventory: function(steamId) {
// Check arguments for validity
check(steamId, String);
// Require Steam Community module
var SteamCommunity = Meteor.npmRequire('steamcommunity');
var community = new SteamCommunity();
// Get the inventory (730 = CSGO App ID, 2 = Valve Inventory Context)
var inventory = Async.runSync(function(done) {
community.getUserInventory(steamId, 730, 2, true, function(error, inventory, currency) {
done(error, inventory);
});
});
if (inventory.error) {
throw new Meteor.Error('steam-error', inventory.error);
} else {
return inventory.results;
}
}
});
client/views/inventory.js:
Template.Trade.helpers({
inventory: function() {
if (Meteor.user() && !Meteor.loggingIn()) {
var inventory;
Meteor.call('getSteamInventory', Meteor.user().services.steam.id, function(error, result) {
if (!error) {
inventory = result;
}
});
return inventory;
}
}
});
When trying to access the results of the call, nothing is displayed on the client or through the console.
I can add console.log(inventory) inside the callback of the community.getUserInventory function and receive the results on the server.
Relevant docs:
https://github.com/meteorhacks/npm
https://github.com/DoctorMcKay/node-steamcommunity/wiki/CSteamUser#getinventoryappid-contextid-tradableonly-callback
You have to use a reactive data source inside your inventory helper. Otherwise, Meteor doesn't know when to rerun it. You could create a ReactiveVar in the template:
Template.Trade.onCreated(function() {
this.inventory = new ReactiveVar;
});
In the helper, you establish a reactive dependency by getting its value:
Template.Trade.helpers({
inventory() {
return Template.instance().inventory.get();
}
});
Setting the value happens in the Meteor.call callback. You shouldn't call the method inside the helper, by the way. See David Weldon's blog post on common mistakes for details (section Overworked Helpers).
Meteor.call('getSteamInventory', …, function(error, result) {
if (! error) {
// Set the `template` variable in the closure of this handler function.
template.inventory.set(result);
}
});
I think the issue here is you're calling an async function inside your getSteamInventory Meteor method, and thus it will always try to return the result before you actually have the result from the community.getUserInventory call. Luckily, Meteor has WrapAsync for this case, so your method then simply becomes:
Meteor.methods({
getSteamInventory: function(steamId) {
// Check arguments for validity
check(steamId, String);
var community = new SteamCommunity();
var loadInventorySync = Meteor.wrapAsync(community.getUserInventory, community);
//pass in variables to getUserInventory
return loadInventorySync(steamId,730,2, false);
}
});
Note: I moved the SteamCommunity = Npm.require('SteamCommunity') to a global var, so that I wouldn't have to declare it every method call.
You can then just call this method on the client as you have already done in the way chris has outlined.
My code is supposed to download 5 files exported from our school's database, then run a query based on the first export (there will be queries for the other four files, and there are three schools, so my functions need to be scalable).
I have a function called "download" and another function called "updateSchedule." Both of these functions work separately.
var download = function(file){
var deferred = q.defer();
var i = 1;
var ftpGet = function(){
var number = i++;
toString(number);
filename = file+number+'.csv';
ftp.get(filename, filename, function(hadErr){
if (hadErr){
console.error('There was an error retrieving ' + filename);
}else{
console.log(filename + ' downloaded');
if(i <= 5){
ftpGet();
}else{
deferred.resolve();
}
}
});
}
ftpGet();
return deferred.promise;
}
var updateSchedule = function(school, school_id){
var deferred = q.defer();
console.log('');
connectionpool.getConnection(function(err, connection){
if(err){
console.error('CONNECTION error: ', err);
}else{
connection.query('DELETE FROM schedule WHERE school_id = "'+school_id+'"', function(err, rows){
if (err) console.error(err);
var path = './'+school+'_export1.csv';
var reader = csv.createCsvFileReader(path, {'separator': ',','quote': '"','excape': '"','comment': ''});
reader.addListener('data',function(data){
connection.query('INSERT INTO schedule SET section_id = "'+data[0]+'", student_id = "'+data[1]+'", course_number = "'+data[2]+'", period = "'+data[3]+'", teacher_id = "'+data[4]+'", school_id = "'+school_id+'"', function(err, rows){
if (err) console.error(err);
deferred.resolve();
});
});
});
connection.release();
console.log(school+' schedule updating...');
}
});
return deferred.promise;
}
When I call them using promises, however, I get an error saying it can't open the file. What am I doing wrong?
download('lca_export').then(updateSchedule('lca', '517'));
Change this:
download('lca_export').then(updateSchedule('lca', '517'));
to this:
download('lca_export').then(function() {
updateSchedule('lca', '517');
});
You were calling updateSchedule() immediately and passing the result to .then() rather than passing a function reference to .then() that can be called later. Thus, updateSchedule() was running before download() had done its work. This is a very common mistake. Just remember that if there are parens after the function name it will execute immediately. If it's just a function name or an anonymous declaration without parens, then you're passing a function reference.
Some other things to cleanup:
Add var in front of filename to make it a local variable rather than an implicit global.
Remove the toString(number). At best it isn't needed at all and isn't doing anything since you aren't assigning the result to anything. At worst, it's causing an error because there is no function toString(). FYI, numbers are automatically converted to strings when added to a string so you do not need to do that manually.
I'm using mongoose to insert some data into mongodb. The code looks like:
var mongoose = require('mongoose');
mongoose.connect('mongo://localhost/test');
var conn = mongoose.connection;
// insert users
conn.collection('users').insert([{/*user1*/},{/*user2*/}], function(err, docs) {
var user1 = docs[0], user2 = docs[1];
// insert channels
conn.collection('channels').insert([{userId:user1._id},{userId:user2._id}], function(err, docs) {
var channel1 = docs[0], channel2 = docs[1];
// insert articles
conn.collection('articles').insert([{userId:user1._id,channelId:channel1._id},{}], function(err, docs) {
var article1 = docs[0], article2 = docs[1];
}
});
};
You can see there are a lot of nested callbacks there, so I'm trying to use q to refactor it.
I hope the code will look like:
Q.fcall(step1)
.then(step2)
.then(step3)
.then(step4)
.then(function (value4) {
// Do something with value4
}, function (error) {
// Handle any error from step1 through step4
})
.end();
But I don't know how to do it.
You'll want to use Q.nfcall, documented in the README and the Wiki. All Mongoose methods are Node-style. I'll also use .spread instead of manually destructuring .then.
var mongoose = require('mongoose');
mongoose.connect('mongo://localhost/test');
var conn = mongoose.connection;
var users = conn.collection('users');
var channels = conn.collection('channels');
var articles = conn.collection('articles');
function getInsertedArticles() {
return Q.nfcall(users.insert.bind(users), [{/*user1*/},{/*user2*/}]).spread(function (user1, user2) {
return Q.nfcall(channels.insert.bind(channels), [{userId:user1._id},{userId:user2._id}]).spread(function (channel1, channel2) {
return Q.nfcall(articles.insert.bind(articles), [{userId:user1._id,channelId:channel1._id},{}]);
});
})
}
getInsertedArticles()
.spread(function (article1, article2) {
// you only get here if all three of the above steps succeeded
})
.fail(function (error) {
// you get here if any of the above three steps failed
}
);
In practice, you will rarely want to use .spread, since you usually are inserting an array that you don't know the size of. In that case the code can look more like this (here I also illustrate Q.nbind).
To compare with the original one is not quite fair, because your original has no error handling. A corrected Node-style version of the original would be like so:
var mongoose = require('mongoose');
mongoose.connect('mongo://localhost/test');
var conn = mongoose.connection;
function getInsertedArticles(cb) {
// insert users
conn.collection('users').insert([{/*user1*/},{/*user2*/}], function(err, docs) {
if (err) {
cb(err);
return;
}
var user1 = docs[0], user2 = docs[1];
// insert channels
conn.collection('channels').insert([{userId:user1._id},{userId:user2._id}], function(err, docs) {
if (err) {
cb(err);
return;
}
var channel1 = docs[0], channel2 = docs[1];
// insert articles
conn.collection('articles').insert([{userId:user1._id,channelId:channel1._id},{}], function(err, docs) {
if (err) {
cb(err);
return;
}
var article1 = docs[0], article2 = docs[1];
cb(null, [article1, article2]);
}
});
};
}
getInsertedArticles(function (err, articles) {
if (err) {
// you get here if any of the three steps failed.
// `articles` is `undefined`.
} else {
// you get here if all three succeeded.
// `err` is null.
}
});
With alternative deferred promise implementation, you may do it as following:
var mongoose = require('mongoose');
mongoose.connect('mongo://localhost/test');
var conn = mongoose.connection;
// Setup 'pinsert', promise version of 'insert' method
var promisify = require('deferred').promisify
mongoose.Collection.prototype.pinsert = promisify(mongoose.Collection.prototype.insert);
var user1, user2;
// insert users
conn.collection('users').pinsert([{/*user1*/},{/*user2*/}])
// insert channels
.then(function (users) {
user1 = users[0]; user2 = users[1];
return conn.collection('channels').pinsert([{userId:user1._id},{userId:user2._id}]);
})
// insert articles
.match(function (channel1, channel2) {
return conn.collection('articles').pinsert([{userId:user1._id,channelId:channel1._id},{}]);
})
.done(function (articles) {
// Do something with articles
}, function (err) {
// Handle any error that might have occurred on the way
});
Considering Model.save instead of Collection.insert (quite the same in our case).
You don't need to use Q, you can wrap yourself the save method and return directly a Mongoose Promise.
First create an utility method to wrap the save function, that's not very clean but something like:
//Utility function (put it in a better place)
var saveInPromise = function (model) {
var promise = new mongoose.Promise();
model.save(function (err, result) {
promise.resolve(err, result);
});
return promise;
}
Then you can use it instead of save to chain your promises
var User = mongoose.model('User');
var Channel = mongoose.model('Channel');
var Article = mongoose.model('Article');
//Step 1
var user = new User({data: 'value'});
saveInPromise(user).then(function () {
//Step 2
var channel = new Channel({user: user.id})
return saveInPromise(channel);
}).then(function (channel) {
//Step 3
var article = new Article({channel: channel.id})
return saveInPromise(article);
}, function (err) {
//A single place to handle your errors
});
I guess that's the kind of simplicity we are looking for.. right? Of course the utility function can be implemented with better integration with Mongoose.
Let me know what you think about that.
By the way there is an issue about that exact problem in the Mongoose Github:
Add 'promise' return value to model save operation
I hope it's gonna be solved soon. I think it takes some times because they are thinking of switching from mpromise to Q: See here and then here.
Two years later, this question just popped up in my RSS client ...
Things have moved on somewhat since May 2012 and we might choose to solve this one in a different way now. More specifically, the Javascript community has become "reduce-aware" since the decision to include Array.prototype.reduce (and other Array methods) in ECMAScript5. Array.prototype.reduce was always (and still is) available as a polyfill but was little appreciated by many of us at that time. Those who were running ahead of the curve may demur on this point, of course.
The problem posed in the question appears to be formulaic, with rules as follows :
The objects in the array passed as the first param to conn.collection(table).insert() build as follows (where N corresponds to the object's index in an array):
[ {}, ... ]
[ {userId:userN._id}, ... ]
[ {userId:userN._id, channelId:channelN._id}, ... ]
table names (in order) are : users, channels, articles.
the corresopnding object properties are : user, channel, article (ie the table names without the pluralizing 's').
A general pattern from this article by Taoofcode) for making asynchronous call in series is :
function workMyCollection(arr) {
return arr.reduce(function(promise, item) {
return promise.then(function(result) {
return doSomethingAsyncWithResult(item, result);
});
}, q());
}
With quite light adaptation, this pattern can be made to orchestrate the required sequencing :
function cascadeInsert(tables, n) {
/*
/* tables: array of unpluralisd table names
/* n: number of users to insert.
/* returns promise of completion|error
*/
var ids = []; // this outer array is available to the inner functions (to be read and written to).
for(var i=0; i<n; i++) { ids.push({}); } //initialize the ids array with n plain objects.
return tables.reduce(function (promise, t) {
return promise.then(function (docs) {
for(var i=0; i<ids.length; i++) {
if(!docs[i]) throw (new Error(t + ": returned documents list does not match the request"));//or simply `continue;` to be error tolerant (if acceptable server-side).
ids[i][t+'Id'] = docs[i]._id; //progressively add properties to the `ids` objects
}
return insert(ids, t + 's');
});
}, Q());
}
Lastly, here's the promise-returning worker function, insert() :
function insert(ids, t) {
/*
/* ids: array of plain objects with properties as defined by the rules
/* t: table name.
/* returns promise of docs
*/
var dfrd = Q.defer();
conn.collection(t).insert(ids, function(err, docs) {
(err) ? dfrd.reject(err) : dfrd.resolve(docs);
});
return dfrd.promise;
}
Thus, you can specify as parameters passed to cascadeInsert, the actual table/property names and the number of users to insert.
cascadeInsert( ['user', 'channel', 'article'], 2 ).then(function () {
// you get here if everything was successful
}).catch(function (err) {
// you get here if anything failed
});
This works nicely because the tables in the question all have regular plurals (user => users, channel => channels). If any of them was irregular (eg stimulus => stimuli, child => children), then we would need to rethink - (and probably implement a lookup hash). In any case, the adaptation would be fairly trivial.
Today we have mongoose-q as well. A plugin to mongoose that gives you stuff like execQ and saveQ which return Q promises.