We Keep Coding

Why nodejs process lost data?

I'm really confused at this point with losing data and can't figure out why. So I write a service to send GPS info from a device to an endpoint.
I'm using pm2 to launch my processes but the problem is this service sometimes don't send the info to the endpoint, and the device is sending data. The solution until now was restarting the instance in pm2. But this sometimes is not viable because I create a crontab to restart the GPS instance in pm2 every 45 minutes but it happens to lose information in a time window < 45 min this will not work...
I can't figure this out. Why I lose the data and restarting everything is okay? I saw a post in stack overflow about lost data in node.js process when the child sends the data to the parent and I read about 2 possible causes:
The child is not reading the GPS data info quickly enough in time to make a post and send info.
The child needs to make a JSON.stringify of the content sent to parent and parent needs to make a JSON.parse of the info received.
Here's my code:
var child_process = require("child_process");
var argv = require('minimist')(process.argv.slice(2));
//ex: nsGPSService.js -d 1111-11-11-111
var deviceId = argv.d;
var processDevices = [];
function runParent() {
setTimeout(function() {
return Database.Devices.getDevices().then(function(devices) {
return new Promise(function(resolve, reject) {
async.each(devices, function(device, callback) {
var result = _.filter(processDevices, { "id": device.id });
if (result.length == 0) {
var process = child_process.fork(__dirname + '/nsGPSService.js', ["-d", device.id]);
processDevices.push({ "process": process, "id": device.id });
process.on('message', function(data) {
//receber mensagens do filho
if (data.reason == "deleted") {
//child end process and alerts parent to remove from the list
var index = _.findIndex(processDevices, { "id": data.deviceId });
processDevices.splice(index, 1);
}
});
process.on('exit', function(code) {});
process.on("uncaughtException", function(error) {
process.exit(1);
});
}
callback();
}, function(error) {
error ? reject(error) : resolve();
});
}).then(function() {
runParent()
}).catch(function(error) {
runParent()
});
});
}, 5000);
}
if (!deviceId) {
return runParent();
}
function runChild(id) {
setTimeout(function() {
return Database.Devices.getDeviceById(id).then(function(device) {
if (!device) {
proccess.send({
"deviceId": id,
"reason": "deleted"
});
process.exit();
return;
}
return Controllers.Gps.getRadioInfo('gps', 'info', {}, device).then(function(data) {
return Controllers.Gps.sendDeviceInfo(data, device);
}).then(function() {
return runChild(id);
}).catch(function(e) {
return runChild(id);
});
});
}, 5000);
}
I really need to figure this out because I never know when I need to restart the service because I'm not getting info when in reality I'm receiving...
Which solution is really viable in my scenario and anyone can figure this problem?

Trouble chaining HTTP requests in Node/Javascript

I'm attempting to recreate a Python script I wrote in Node/js and I'm having trouble wrapping my head around the asynchronous/callback way of it all.
The script is fairly simple and uses two basic HTTP requests to eBay's API. The first request gets a list of resulting item ids, then the second request gets specific information on each item (description/listing information etc). In Python this was fairly simple and I used a simple loop. It wasn't fast by any means, but it worked.
In javascript, however, I'm struggling to get the same functionality. My code right now is as follows:
var ebay = require('ebay-api');
var params ={};
params.keywords = ["pS4"];
var pages = 2;
var perPage = 2;
ebay.paginateGetRequest({
serviceName: 'FindingService',
opType: 'findItemsAdvanced',
appId: '',
params: params,
pages: pages,
perPage: perPage,
parser: ebay.parseItemsFromResponse
},
function allItemsCallback(error,items){
if(error) throw error;
console.log('FOUND', items.length, 'items from', pages, 'pages');
for (var i=0; i<items.length; i++){
getSingle(items[i].itemId);
}
}
);
function getSingle(id){
console.log("hello");
ebay.ebayApiGetRequest({
'serviceName': 'Shopping',
'opType': 'GetSingleItem',
'appId': '',
params: {
'ItemId': id ,
'includeSelector': 'Description'
}
},
function(error, data) {
if (error) throw error;
console.dir(data); //single item data I want
}
);
}
This is one attempt of many, but I'm recieving "possible EventEmitter memory leak detected" warnings and it eventually breaks with a "Error:Bad 'ack' code undefined errorMessage? null". I'm fairly sure this just has to do with proper utilization of callbacks but I'm unsure how to properly go about it. Any answers or help would be greatly appreciated. I apologize if this isn't a good question, if so please let me know how to correctly go about asking.

Node.js's asynchronous event chain is built on callbacks. Rather than:
getSingle(items[i].itemId);
You'll need to write a callback into that function that executes once the parent function is complete:
getSingle(items[i].itemId, function(err, data) {
// now you can access the data
});
And because ebay.ebayApiGetRequest is a lengthy function, the callback that tells its parent function that it's done must be called after that completes, like so:
ebay.ebayApiGetRequest({
//
},
function(error, data) {
callback(error, data);
}
);
But of course, if the parent function getSingle doesn't support a callback, then it won't go anywhere. So you'll need to support a callback param there as well. Here's the full script, rewritten using the event-driven callback model:
var ebay = require('ebay-api');
var async = require('async');
var params = {};
params.keywords = ["pS4"];
var pages = 2;
var perPage = 2;
ebay.paginateGetRequest({
serviceName: 'FindingService',
opType: 'findItemsAdvanced',
appId: '',
params: params,
pages: pages,
perPage: perPage,
parser: ebay.parseItemsFromResponse
},
function allItemsCallback(error, items) {
if (error) throw error;
console.log('FOUND', items.length, 'items from', pages, 'pages');
async.each(items, function(item, callback) {
getSingle(item.itemId, function(err, data) {
callback(err, data);
});
}, function(err, results) {
// now results is an array of all the data objects
});
}
);
function getSingle(id, callback) {
console.log("hello");
ebay.ebayApiGetRequest({
'serviceName': 'Shopping',
'opType': 'GetSingleItem',
'appId': '',
params: {
'ItemId': id,
'includeSelector': 'Description'
}
},
function(error, data) {
if (error) throw error;
console.dir(data); //single item data I want
callback(error, data);
}
);
}

angular push adds operator that that results in MongoError

I am implementing the tutorial on the mean stack https://www.youtube.com/watch?v=AEE7DY2AYvI
I am adding a delete feature to remove items from the database on a button click
My client side controller has the following 2 functions to add to db and remove
$scope.createMeetup = function() {
var meetup = new Meetup();
meetup.name = $scope.meetupName;
meetup.$save(function (result) {
$scope.meetups.push(result);
$scope.meetupName = '';
});
}
$scope.deleteMeetup = function() {
item = $scope.meetups[0];
console.log("deleting meetup: " + item["name"]);
Meetup.delete(item);
scope.meetups.shift();
}
My server side has the following code
module.exports.create = function (req, res) {
var meetup = new Meetup(req.body);
meetup.save(function (err, result) {
res.json(result);
});
}
module.exports.remove = function(req, res) {
console.log("GOING TO REMOVE!!!");
console.log(req.query);
item = req.query;
Meetup.remove(item, function (err, results) {
console.log("done");
console.log(err);
console.log(results);
});
}
When I run my code and if I delete an already loaded item in the list, it is removed from Mongodb just fine. But if I add an item to the list and I do not refresh the page, it results in an error at my server that appears as
GOING TO REMOVE!!!
{ '$resolved': 'true',
__v: '0',
_id: '54ec04e70398fab504085178',
name: 'j' }
done
{ [MongoError: unknown top level operator: $resolved]
name: 'MongoError',
code: 2,
err: 'unknown top level operator: $resolved' }
null
I if I refresh the page, the it gets deleted fine. But if I added the entry, angular seems to be adding a new variable $resolved. Why is that happening?
Also another question, What is the proper way to call delete? I call it now but I am not able to put a callback. I want a callback which returns and then I shift the list of items. I tried adding a callback but the code never reaches it.
ie I tried the following
/*
Meetup.delete(item, function () {
console.log("In callback!!");
console.log(returnValue);
console.log(responseHeaders);
$scope.meetups.splice(item);
});
*/
/*Meetup.delete(item,
function (returnValue, responseHeaders) {
console.log("In callback!!");
console.log(returnValue);
console.log(responseHeaders);
$scope.meetups.splice(item);
},
function (httpResponse){
// error handling here
console.log("Need to handle errors");
});
*/
I am very new to node and am confused. Any help is very, very appreciated

Looks like it possible to call item.delete instead of Meetup.delete(item). You can call same methods on model instance. It prevent sending angular properties to server.
But better to make a rest API with delete method
DELETE /meetups/:id
and send just a _id
Meetup.remove({id: item._id});

Using Streaming Data from Twitter with Meteor

So far I have been able to pull down streaming real time data from Twitter. How do I use this data? I am trying to insert it into collection but I am getting this Error:
Error: Meteor code must always run within a Fiber. Try wrapping callbacks that you pass to non-Meteor libraries with Meteor.bindEnvironment.
I tried wrapping my code with a fiber but it didn't work/or I am not wrapping the right part of the code. Also, I'm not sure if this is the proper way to use streaming data in Meteor.
Posts = new Meteor.Collection('posts');
if (Meteor.isClient) {
Meteor.call("tweets", function(error, results) {
console.log(results); //results.data should be a JSON object
});
}
if (Meteor.isServer) {
Meteor.methods({
tweets: function(){
Twit = new TwitMaker({
consumer_key: '...',
consumer_secret: '...',
access_token: '...',
access_token_secret: '...'
});
sanFrancisco = [ '-122.75', '36.8', '-121.75', '37.8' ];
stream = Twit.stream('statuses/filter', { locations: sanFrancisco });
stream.on('tweet', function (tweet) {
userName = tweet.user.screen_name;
userTweet = tweet.text;
console.log(userName + " says: " + userTweet);
Posts.insert({post: tweet})
})
}
})
}

Code that mutates the database needs to be run in a fiber, which is what the error is about. Code that runs in a callback from a library other than Meteor is not (necessarily) run in a fiber, so you'll need to wrap the callback function to make sure it gets run in a fiber, or at least the part of it that interacts with the database.
Meteor.bindEnvironment is not currently documented, but it is generally considered the most reliable method of wrapping callbacks. Meteor.bindEnvironment, which the error talks about, is defined here for reference:
https://github.com/meteor/meteor/blob/master/packages/meteor/dynamics_nodejs.js#L63
Something like this is probably the easiest way of making this work:
tweets: function() {
...
// You have to define this wrapped function inside a fiber .
// Meteor.methods always run in a fiber, so we should be good here.
// If you define it inside the callback, it will error out at the first
// line of Meteor.bindEnvironment.
var wrappedInsert = Meteor.bindEnvironment(function(tweet) {
Posts.insert(tweet);
}, "Failed to insert tweet into Posts collection.");
stream.on('tweet', function (tweet) {
var userName = tweet.user.screen_name;
var userTweet = tweet.text;
console.log(userName + " says: " + userTweet);
wrappedInsert(tweet);
});
}

This works for me. Essential is to call Meteor.bindEnvironment from inside the Twit callback.
Meteor.methods({
consumeTwitter: function () {
var Twit = Meteor.npmRequire('twit');
var T = new Twit({
consumer_key: 'xxx', // API key
consumer_secret: 'yyy', // API secret
access_token: 'xxx',
access_token_secret: 'xxx'
});
// search twitter for all tweets containing the word 'banana'
var now = new Date().getTime();
var wrappedInsert = Meteor.bindEnvironment(function(tweet) {
Tweets.insert(tweet);
}, "Failed");
T.get('search/tweets',
{
q: 'banana since:2011-11-11',
count: 4
},
function(err, data, response) {
var statuses = data['statuses'];
for(var i in statuses) {
wrappedInsert(statuses[i]);
}
}
)}
});

I had written a lengthy post about Building Twitter Monitoring Apps with MeteorJS from Scratch, including the Meteor.bindEnvironment part, extract as below.
var Twit = Meteor.npmRequire(‘twit’);
var conf = JSON.parse(Assets.getText(‘twitter.json’));
var T = new Twit({
consumer_key: conf.consumer.key,
consumer_secret: conf.consumer.secret,
access_token: conf.access_token.key,
access_token_secret: conf.access_token.secret
//
// filter the twitter public stream by the word ‘iwatch’.
//
var stream = T.stream(‘statuses/filter’, { track: conf.keyword })
stream.on(‘tweet’, Meteor.bindEnvironment(function (tweet) {
console.log(tweet);
Tweets.insert(tweet);
}))
There are only two functions added:
Meteor.bindEnvironment()
This function helps us to bind a function to the current value of all the environment variables.
Have fun!

REST API testing using vows, tobi and node.js

I am trying to combine the examples here, here to write a vows test for my node.js / express app that:
Creates a new user object
Checks the response was sane
Uses the returned _id to test looking up the newly created user
Again uses the _id to test updating the user
Item 1 and 2 work fine, but there is something wrong with my sub-context 'GET /users/:id'. It errors and I cannot figure out why. Tried Googling and using the debugger, but I still can't see what it is, I am probably just overlooking something obvious.
···✗ Errored » 3 honored ∙ 1 errored
Can anyone tell me why the 4th vow errors?
Here's my vows code:
var vows = require('vows')
, assert = require('assert')
, tobi = require('tobi')
var suite = vows.describe('Users API')
, now = new Date().getTime()
, newUser = { name: now + '_test_user', email: now + '#test.com' }
, browser = tobi.createBrowser(3000, 'localhost')
, defaultHeaders = { 'Content-Type': 'application/json' }
function assertStatus(code) {
return function (res, $) {
res.should.have.status(code)
}
}
var client = {
get: function(path) {
return function() {
browser.get(path, { headers: defaultHeaders }, this.callback)
}
},
post: function(path, data) {
return function() {
browser.post(path, { body: JSON.stringify(data), headers: defaultHeaders }, this.callback)
}
}
}
suite.addBatch({
'GET /users': {
topic: client.get('/users'),
'should respond with a 200 ok': assertStatus(200)
},
'POST /users': {
topic: client.post('/users', newUser),
'should respond with a 200 ok': assertStatus(200),
'should return the new user': function(res, $){
assert.isNotNull(res.body._id)
assert.isNotNull(res.body.created_at)
assert.isTrue(res.body._id.length > 0)
assert.equal(newUser.name, res.body.name)
assert.equal(newUser.email, res.body.email)
},
'GET /users/:id': { // Sub-context of POST /users
topic: function(res) { return client.get('/users/' + res.body._id) },
'should respond with a 200 ok': assertStatus(200)
}
}
})
suite.export(module)
EDIT
I tried simplifying the code as follows to help see if this.callback was the problem, but the error is still there:
'GET /users/:id': { // Sub-context of POST /users
topic: function(res) {
console.log('About to request /users/' + res.body._id)
browser.get('/users/' + res.body._id, { headers: defaultHeaders }, this.callback)
},
'should respond with a 200 ok': assertStatus(200)
}

How are you populating res for the fourth tes?? It wouldn't be visible outside the line
'should return the new user'
Try creating the id variable outside the addBatch call, and set it in the third test. then call
client.get('/users/' + id)
EDIT:
Better yet, put it back into newUser in the third test:
'should return the new user': function(res, $){
newUser.id = res.body._id
....
and then do:
client.get('/users/' + newUser.id)