We Keep Coding

Find Email-adresses in the mailbody with Mailparser

I'm quite new to the topic and i'm still having some issues with my mailparser. Though searching and finding emails in the email header (mail.from) does work, it doesn't work in the email body. Does anybody have some experience with that and is willing to help? You can find the function i'm talking about under the "// Check for other addresses in Mail-Body (Doesn't work yet)"-comment. I think, that my Regex is correct. Also if the matchAll-Function give back an array and it can't be saved in the the subscriber.email-object, it shall be at least logged to the console. Also i checked manually in the inbox if there are mails with email adresses in the mail body. There are at least two, which shall be found..
The part of the App.js, that does the mailparsing:
const simpleParser = require('mailparser').simpleParser;
//const htmlparser = require("htmlparser2");
var fs = require('fs');
var config = require('./config');
var Imap = require('imap');
var imap = new Imap(config.imap);
var blacklistString = '';
String.prototype.matchAll = function(regexp) {
var matches = [];
this.replace(regexp, function() {
var arr = ([]).slice.call(arguments, 0);
var extras = arr.splice(-2);
arr.index = extras[0];
arr.input = extras[1];
matches.push(arr);
});
return matches.length ? matches : null;
};
function openInbox(subbox,cb) {
imap.openBox('INBOX.'+subbox, true, cb);
}
function getBoxes(cb) {
imap.getBoxes(cb);
}
function showBoxes(boxes) {
imap.end();
}
function logArrayElements(element) {
if(element[1].indexOf('placeholder.de')==-1){
addToBlacklistString(element[1]);
}
}
function addToBlacklistString(str) {
blacklistString += str+"\n";
}
function writeBlacklistFile() {
fs.appendFile('data/data.csv', blacklistString, function (err) {
if (err) throw err;
console.log('Saved!');
});
}
function search(searchArray, regex){
imap.search(searchArray, function(err, results) {
if (err) throw err;
var temp = 0;
var mailtemp = [];
var f = imap.fetch(results, { bodies: '' });
f.on('message', function(msg, seqno) {
console.log('Message #%d', seqno);
var prefix = '(#' + seqno + ') ';
msg.on('body', function(stream, info) {
simpleParser(stream, (err, mail)=>{
//console.log(temp);
//console.log(mail.subject);
/*fs.writeFile('data/'+seqno+'.txt',mail.text, function(err){
console.log(err);
});*/
//var text = mail.text;
// New Subscriber Object
var subscr = new Subscriber({nr: '', mailIdent: '', from: '', emails: '', text:'', uLink: '', anwalt: false });
subscr.nr = seqno;
//Check for From-Address
if(!!mail.from) {
//console.log(mail.from.value);
for(var i = 0; i < mail.from.value.length; i++) {
mailtemp = mail.from.value[i].address.matchAll(regex);
mailtemp.forEach(function(element){
/*fs.appendFile('data/data.csv', element[0] + "\n", function(error){
console.log(error);
});*/
subscr.from = element[0];
});
if(!!mailtemp) {
mailtemp.forEach(logArrayElements);
}
}
}else{
//console.log(mail.text);
}
// Message-ID
if(!!mail.messageId) {
subscr.mailIdent = mail.messageId;
}
console.log(mail.messageId);
// Check for other addresses in Mail-Body (Doesn't work yet)
var regexEmails = new RegExp('/([\w\.\-\_\#\+]+#[\w\.\-\_äüö]+\.[a-zA-Z]+)/g');
if(!!mail.text){
if(mail.text.matchAll(regexEmails)!=null) {
subscr.emails = mail.text.matchAll(regexEmails);
console.log(subscr.emails);
}
}
/* Split mail.text at substrings in substr-array. Extend if necessary..
*
* Also check for 'Anwalt'-Expression in splitted Substring
*
* If mail.text doesn't exist -> Check for html body and convert it to text-format
*/
//var regexLink = new RegExp('\.de\/(unsubscribe|austragen)\/([^\"]+)');
var regexAnwalt = new RegExp('nwalt|echtsanwalt|rechtlicher');
if(!!mail.text) {
var substr = ["schrieb pplaceholder.de", "Von: \"placeholder.de", "Von: pplaceholder.de", "From: placeholder.de", "Ursprüngliche Nachricht"];
for (var i = 0; i<substr.length; i++) {
if(mail.text.indexOf(substr[i]) > -1) {
var textTemp = mail.text;
var arr = textTemp.split(substr[i]);
if(arr[0].matchAll(regexAnwalt)!=null) {
subscr.anwalt = true;
};
subscr.text = arr[0];
break;
} else {
subscr.text = mail.text;
}
}
//console.log(arr);
}
else
{
var html = mail.html;
var text = htmlToText.fromString(html, {
noLinkBrackets: true,
ignoreImage: true,
uppercaseHeadings: false,
preserveNewlines: false,
wordwrap:130,
format: {
heading: function (node, fn, options) {
var h = fn(node.children, options);
return '\n==== ' + h + ' ====\n\n';
}
}
});
subscr.text = text;
}
mail.headers.forEach(function(value, key) {
//console.log(value);
});
subscr.save();
//console.log(subscr);
temp++;
});
});
msg.once('end', function() {
console.log(prefix + 'Finished');
});
});
f.once('error', function(err) {
console.log('Fetch error: ' + err);
});
f.once('end', function() {
console.log('Done fetching all messages!');
//writeBlacklistFile();
imap.end();
});
});
}
imap.once('ready', function() {
openInbox('Test',function(err, box) {
var searchArray = [['FROM', '#']];
search(searchArray,/([\w\.\-\_\#\+]+#[\w\.\-\_äüö]+\.[a-zA-Z]+)/g);
});
});
imap.once('error', function(err) {
console.log(err);
});
imap.once('end', function() {
console.log('Connection ended');
});
imap.connect();
app.listen(2700, function(){
console.log("Listening on Port 2700")
});
module.exports = app;
subscriber.js
const mongoose = require('mongoose');
var subscriberSchema = mongoose.Schema({
nr: Number,
mailIdent: String,
from: String,
emails: String,
text: String,
uLink: String,
anwalt: Boolean
});
var Subscriber = module.exports = mongoose.model('Subscriber', subscriberSchema);
//get Subscriber
module.exports.getSubscribers = function(callback, limit){
Subscriber.find(callback).limit(limit);
};
module.exports.getSubscriberByID = function(_id, callback){
Subscriber.findById(_id, callback);
};

The Regex for the Emails was a little bit wrong.
Also i didn't noticed that the matchAll-Fct. is giving back a two-dimensional Array. Here is the changed part of the code:
var regexEmails = new RegExp("([\\w\\.\\-\\_\\#\\+]+#[\\w\\.\\-\\_äüö]+\\.[a-zA-Z]+)");
var temp1 = mail.text.matchAll(regexEmails);
if(!!temp1){
//console.log(temp1);
for(var i =0; i<temp1.length; i++) {
if(temp1[0][i]!=='info#service.placeholder.de' && temp1[0][i] !== "info#placeholder.de"){
subscr.emails += temp1[0][i];
}
}
}

Trouble using nested callbacks in NodeJS

I'm writing a program that scrapes a site for links, then scrapes these links for information. In order to scrape the site, it is necessary to log in first. And so the order is: Log in -> Scrape the index for links -> Scrape the links for info
The callback to the login function prints an empty array { results: [], hasMore: true }, so something is wrong with my code (the scraping part works):
var request = require('request');
var request = request.defaults({jar: true}); // necessary for persistent login
var cheerio = require('cheerio');
var url1 = "https://example.org/torrents/browse/index/";
var loginUrl = "https://example.org/user/account/login/";
var credentials = {
username: 'user1',
password: 'passpass'
};
login(function (result) {
console.log(result);
});
function login(callback) {
request.post({
uri: loginUrl,
headers: { 'content-type': 'application/x-www-form-urlencoded' },
body: require('querystring').stringify(credentials)
}, function(err, res, body){
if(err) {
console.log("Login error");
return;
}
scrapeTorrents(url1, function (result) {
callback(result);
});
});
}
function scrapeTorrents(url, callback) {
request(url, function(err, res, body) {
if(err) {
console.log("Main scrape error");
return;
}
var links = []
var $ = cheerio.load(body);
$('span.title').each(function(i, element){
var title = $(this);
var a = $(this).children().eq(0);
var detailsUrl = a.attr('href');
//console.log(detailsUrl);
links.push(detailsUrl);
});
scrapeTorrentDetails(links, function (result) {
callback(result);
});
});
}
function scrapeTorrentDetails(links, callback) {
var results = [];
function getDetails(url) {
request(url, function(err, res, body) {
if(err) {
console.log("Detail scrape error");
return;
}
console.log("Scraping: " + url);
var $ = cheerio.load(body);
var tds = $('td');
var title = $(tds).get(1).firstChild.data;
var hash = $(tds).get(3).firstChild.data.trim();
var size = $(tds).get(9).firstChild.data;
// console.log(tds.length);
if (tds.length > 23) {
var rlsDate = $(tds).get(23).firstChild.data || '';;
var genres = $(tds).get(27).firstChild.data || '';;
var runtime = $(tds).get(31).firstChild.data || '';;
if ( $(tds).get(33).firstChild != null) {
var plot = $(tds).get(33).firstChild.data || '';;
}
var rating = $('#imdb_rating').parent().next().text() || '';; // of 10
var imdb_id = $('[name=imdbID]').get(0).attribs.value || '';;
var cover = $('#cover').children().eq(0).get(0).attribs.href || '';;
var thumb = $('[alt=Cover]').get(0).attribs.src || '';;
if (typeof cover == 'undefined') {
cover = thumb;
}
} else {
var rlsDate = "notfound";
var genres = "notfound";
var runtime = "notfound";
var plot = "notfound";
var rating = "notfound"; // of 10
var imdb_id = "notfound";
var cover = "notfound";
var thumb = "notfound";
}
var movie = {
type: 'movie',
imdb_id: imdb_id,
title: title,
year: rlsDate,
genre: genres,
rating: rating,
runtime: runtime,
image: thumb,
cover: cover,
synopsis: plot,
torrents: {
magnet: 'magnet:?xt=urn:btih:' + hash + '&tr=http://tracker.example.org:2710/a/announce',
filesize: size
}
};
results.push(movie);
});
}
for (var i=0; i<links.length; i++){
getDetails("https://example.org" + links[i]);
}
callback( {
results: results,
hasMore: true
});
}
Maybe Q promises would be better. How would I implement that in the code above?
If you're wondering what the code is for, I'm planning to modify Popcorn-time to use another torrent-tracker (without an API).
Thanks

A main problem is with this code:
for (var i=0; i<links.length; i++){
getDetails("https://example.org" + links[i]);
}
callback( {
results: results,
hasMore: true
});
getDetails() is async, but you just call it links.length times and move on - acting like they have all completed. So, none of the requests in getDetails() is done before you call the callback and try to pass the results. But, none of the results have yet been filled in so they will be empty.
You have all these other nested callbacks everywhere through your code (as required), yet you dropped the ball in this one place. You need to know when all the getDetails() calls are done before you call the final callback with the results.
In addition, you also have to decide if you're OK calling all the getDetails() calls in parallel (all in flight at once) or if what you really want to do is to call one, wait for it to finish, then call the next, etc... Right now you are putting them all in-flight at once which can work if the destination server doesn't object to that many requests all at once.
There are several potential strategies for fixing this.
Add a callback to getDetails() and then keep a count of when you've gotten links.length callbacks from getDetails() and only when the entire count has finished so you call the final callback.
Change getDetails() to return a promise. You can then use something like links.map(getDetails) to create an array of promises that you can then use Promise.all() with to know when they are all done.
Personally, I would change all of your code to use promises and I'd use the Bluebird promises library for it's extra features such as Promise.map() to make this even simpler.
Here's a fix that adds a callback to getDetails() and then counts how many are done:
function scrapeTorrentDetails(links, callback) {
var results = [];
function getDetails(url, done) {
request(url, function(err, res, body) {
if(err) {
console.log("Detail scrape error");
done(err);
return;
}
console.log("Scraping: " + url);
var $ = cheerio.load(body);
var tds = $('td');
var title = $(tds).get(1).firstChild.data;
var hash = $(tds).get(3).firstChild.data.trim();
var size = $(tds).get(9).firstChild.data;
// console.log(tds.length);
if (tds.length > 23) {
var rlsDate = $(tds).get(23).firstChild.data || '';;
var genres = $(tds).get(27).firstChild.data || '';;
var runtime = $(tds).get(31).firstChild.data || '';;
if ( $(tds).get(33).firstChild != null) {
var plot = $(tds).get(33).firstChild.data || '';;
}
var rating = $('#imdb_rating').parent().next().text() || '';; // of 10
var imdb_id = $('[name=imdbID]').get(0).attribs.value || '';;
var cover = $('#cover').children().eq(0).get(0).attribs.href || '';;
var thumb = $('[alt=Cover]').get(0).attribs.src || '';;
if (typeof cover == 'undefined') {
cover = thumb;
}
} else {
var rlsDate = "notfound";
var genres = "notfound";
var runtime = "notfound";
var plot = "notfound";
var rating = "notfound"; // of 10
var imdb_id = "notfound";
var cover = "notfound";
var thumb = "notfound";
}
var movie = {
type: 'movie',
imdb_id: imdb_id,
title: title,
year: rlsDate,
genre: genres,
rating: rating,
runtime: runtime,
image: thumb,
cover: cover,
synopsis: plot,
torrents: {
magnet: 'magnet:?xt=urn:btih:' + hash + '&tr=http://tracker.example.org:2710/a/announce',
filesize: size
}
};
results.push(movie);
done();
});
}
var doneCnt = 0;
for (var i=0; i<links.length; i++){
getDetails("https://example.org" + links[i], function() {
++doneCnt;
if (doneCnt === links.length) {
callback( {
results: results,
hasMore: true
});
}
});
}
}

The following is the given sample code rewritten to use bind, a custom this object and a count of the requests that have yet to complete (I think promises obscure the execution path).
The reason that the callback is returning an empty array seems to be that there are no spans in the document with a title attribute, so as a result no further requests are triggered.
var
request = require('request').defaults({
jar: true
}), // necessary for persistent login
cheerio = require('cheerio'),
process = require('process'),
url1 = "https://example.org/torrents/browse/index/",
loginUrl = "https://example.org/user/account/login/",
login = function(callback) {
request.post({
uri: loginUrl,
headers: {
'content-type': 'application/x-www-form-urlencoded'
},
body: require('querystring').stringify({
username: 'user1',
password: 'passpass'
})
}, fna.bind({
callback: callback
}));
},
fna = function(err, res, body) {
if (err) {
console.log("Login error");
return;
}
request(url1, fnb.bind(this));
},
fnb = function(err, res, body) {
if (err) {
console.log("Main scrape error");
return;
}
var
$ = cheerio.load(body),
links = [],
fnd = fne.bind(this);
$('span.title').each(function() {
links.push($(this).children().first().attr('href'));
});
this.results = [];
this.resultCount = links.length;
if (this.resultCount) {
fnd = fnc.bind(this);
for (var i = 0; i < links.length; i++) {
request("https://example.org" + links[i], fnd);
}
} else {
process.nextTick(fnd);
}
},
fnc = function(err, res, body) {
if (err) {
console.log("Detail scrape error");
return;
}
console.log("Scraping: " + url);
var
$ = cheerio.load(body),
tds = $('td'),
title = $(tds).get(1).firstChild.data,
hash = $(tds).get(3).firstChild.data.trim(),
size = $(tds).get(9).firstChild.data,
rlsDate = "notfound",
genres = "notfound",
runtime = "notfound",
plot = "notfound",
rating = "notfound", // of 10
imdb_id = "notfound",
cover = "notfound",
thumb = "notfound";
if (tds.length > 23) {
rlsDate = $(tds).get(23).firstChild.data || '';
genres = $(tds).get(27).firstChild.data || '';
runtime = $(tds).get(31).firstChild.data || '';
if ($(tds).get(33).firstChild != null) {
plot = $(tds).get(33).firstChild.data || '';
}
rating = $('#imdb_rating').parent().next().text() || ''; // of 10
imdb_id = $('[name=imdbID]').get(0).attribs.value || '';
cover = $('#cover').children().eq(0).get(0).attribs.href || '';
thumb = $('[alt=Cover]').get(0).attribs.src || '';
if (typeof cover == 'undefined') {
cover = thumb;
}
}
this.results.push({
type: 'movie',
imdb_id: imdb_id,
title: title,
year: rlsDate,
genre: genres,
rating: rating,
runtime: runtime,
image: thumb,
cover: cover,
synopsis: plot,
torrents: {
magnet: 'magnet:?xt=urn:btih:' + hash + '&tr=http://tracker.example.org:2710/a/announce',
filesize: size
}
});
this.resultCount--;
if (this.resultCount === 0) {
this.callback({
results: this.results,
hasMore: true
});
}
},
fne = function() {
this.callback({
results: this.results,
hasMore: true
});
};
login(function(result) {
console.log(result);
});

Extract common code from sails.js / mongodb

I'm current trying to use sails.js with mongodb, I need some custom mapReduce function to group data.
Now I could achieve what I want by using waterline's native function, but have some questions.
These function has only small variation actually, but I found myself keep repeating codes like the following one:
function getSomeData() {
// First-query
Log.native(function(err, logCollection) {
var mapFunction = function() {
function dateFormatter(date) {
return date.getFullYear() + "-" + (date.getMonth() + 1)
}
//! Generate Grouping Key
emit(dateFormatter(this.emb_date), this.bad_qty)
}
var reduceFunction = function (key, values) {
return Array.sum(values);
}
var outputControl = {
out: {inline: 1},
//! Filters
query: {order_type: product}
}
logCollection.mapReduce(mapFunction, reduceFunction, outputControl, function (err, result) {
if (err) {
callback(err);
return;
}
var resultSet = [];
//! post-processing
for (var i = 0; i < result.length; i++) {
//.....
}
callback(err, resultSet);
});
});
}
Second-query:
function getAnotherData() {
Log.native(function(err, logCollection) {
var mapFunction = function() {
//! Generate Grouping Key
emit(dateFormatter(this.product), this.bad_qty)
}
var reduceFunction = function (key, values) {
return Array.sum(values);
}
var outputControl = {
out: {inline: 1},
//! Filters
query: {order_type: product}
}
logCollection.mapReduce(mapFunction, reduceFunction, outputControl, function (err, result) {
if (err) {
callback(err);
return;
}
var resultSet = [];
//! post-processing
for (var i = 0; i < result.length; i++) {
//......
}
callback(err, resultSet);
});
});
}
As you can see, these two snippet shares lots of common code, only has difference in three place (Generate grouping key, filters, post-process).
So I would really like to extract the common part to make my code cleaner, but have no success.
I first try to make dateFromatter is provided by a callback instead of hard-coding like the following:
function dateFormatter(data) {
return data.emb_date.getFullYear() + "-" + (data.emb_date.getMonth() + 1)
}
function getSomeData(groupingKey) {
// First-query
Log.native(function(err, logCollection) {
var mapFunction = function() {
//! Generate Grouping Key
emit(groupingKey(this.emb_date), this.bad_qty)
}
var reduceFunction = function (key, values) {
return Array.sum(values);
}
var outputControl = {
out: {inline: 1},
//! Filters
query: {order_type: product}
}
logCollection.mapReduce(mapFunction, reduceFunction, outputControl, function (err, result) {
if (err) {
callback(err);
return;
}
var resultSet = [];
//! post-processing
for (var i = 0; i < result.length; i++) {
//.....
}
callback(err, resultSet);
});
});
}
But without any luck, I keep getting error like the following one:
MongoError: exception: ReferenceError: groupingKey is not defined near 'emit(groupingKey(this), this.bad_qty' (line 3)
at Object.toError (/home/brianhsu/zh800/dashboard/node_modules/sails-mongo/node_modules/mongodb/lib/mongodb/utils.js:114:11)
What should I do if I would like to reduce those duplicate part of code?

Finally I found that I need pass the option called 'scope' to mongodb, I come up with the following solution which works quite well.
exports.defineOn = function(options) {
var model = options.model
var groupingFunction = options.groupingFunction
var mongoFilters = options.mongoFilters
var customFilter = options.customFilter
var converter = options.converter
var sorting = options.sorting
return function(callback) {
model.native(function(err, collection) {
var mapFunction = function() { emit(groupingFunction(this), this.bad_qty) }
var reduceFunction = function(key, values) { return Array.sum(values); }
var mapReduceOptions = {
out: {inline: 1},
query: mongoFilters,
scope: {
groupingFunction: groupingFunction,
mongoFilters: mongoFilters,
customFilter: customFilter,
converter: converter
}
}
var processCallback = function (err, result) {
if (err) {
callback(err);
return;
}
if (sorting) {
result.sort(sorting);
}
var resultSet = [];
for (var i = 0; i < result.length; i++) {
if (customFilter && customFilter(result[i])) {
resultSet.push(converter(result[i]));
} else if (!customFilter) {
resultSet.push(converter(result[i]));
}
}
callback(err, resultSet);
}
collection.mapReduce(mapFunction, reduceFunction, mapReduceOptions, processCallback);
});
}
}
Usage:
function machineDetail (year, month, date, machine, callback) {
var startDate = new Date(+year, +(month-1), +date);
var endDate = new Date(+year, +(month-1), (+date) + 1);
var mapReducer = MapReducer.defineOn({
model: Log,
groupingFunction: function(data) {
return {date: data.emb_date, error: data.defact_id};
},
mongoFilters: {
mach_id: machine,
emb_date: {$gte: startDate, $lt: endDate}
},
converter: function (data) {
return {
name: data._id,
value: data.value,
};
}
});
mapReducer(callback);
}

Meteor JS: Use subset of same subscribed data

I built out a custom pagination script to display data for my app. It works wonderfully. However, I am having a slight problem when it comes to trying to figure out how to grab a subset of the same paginated subscription.
Meteor.startup(function(){
Session.setDefault('page', 1);
Session.setDefault('recordCount', 0);
Session.setDefault('recordsPerPage', 10);
Session.setDefault('currentIndustry', null);
Session.setDefault('currentMapArea', null);
Session.setDefault('gmapLoaded', false);
});
Deps.autorun(function () {
Meteor.call('getJobsCount', Session.get('currentIndustry'), Session.get('currentMapArea'), function (err, count) {
Session.set('recordCount', count);
});
Meteor.subscribe('pagedRecords', Session.get('page'), Session.get('recordsPerPage'), Session.get('currentIndustry'), Session.get('currentMapArea'));
});
Template.gmap.rendered = function() {
if(!Session.get('gmapLoaded'))
gmaps.initialize();
}
var templateName = "jobs";
function plotCities(jobs) {
var addresses = _.chain(jobs)
.countBy('address')
.pairs()
.sortBy(function(j) {return -j[1];})
.map(function(j) {return j[0];})
.slice(0, 99)
.value();
gmaps.clearMap();
$.each(_.uniq(addresses), function(k, v){
var addr = v.split(', ');
Meteor.call('getCity', addr[0].toUpperCase(), addr[1], function(error, city){
if(city) {
var opts = {};
opts.lng = city.loc[1];
opts.lat = city.loc[0];
opts.population = city.pop;
opts._id = city._id;
gmaps.addMarker(opts);
}
});
});
}
Template[templateName].helpers({
selected: function(){
return Session.get('recordsPerPage');
}
});
Template[templateName].pages = function() {
var numPages = Math.ceil(Session.get('recordCount') / Session.get('recordsPerPage'));
var currentPage = Session.get('page');
var totalPages = Session.get('recordCount');
var prevPage = Number(currentPage) - 1;
var nextPage = Number(currentPage) + 1;
var html = '<div class="pagination-cont"><ul class="pagination">';
if (numPages !== 1) {
if (currentPage > 1) {
html += '<li>«</li>';
}
for (var i = currentPage; (i <= numPages) && (i - currentPage < 4); i++) {
if (i < 1) continue;
if (i !== currentPage)
html += '<li>' + i + '</li>';
else
html += '<li class="active">' + i + '</li>';
}
if (currentPage < numPages) {
html += '<li>»</li>';
}
}
html += '</ul></div>';
return html;
}
Template[templateName].jobs = function() {
var options = {};
var cursor;
if(!Session.get('currentMapArea')) {
cursor = Jobs.find({}, {limit: 500});
plotCities(cursor.fetch());
}
return Jobs.find({}, { limit: Session.get('recordsPerPage') });
}
Template[templateName].rendered = function(){
var select = $('#perPage');
var option = select.attr('_val');
$('option[value="' + option + '"]').attr("selected", "selected");
select.selectpicker({
style: 'btn-info col-md-4',
menuStyle: 'dropdown-inverse'
});
}
Template[templateName].events({
'click div.select-block ul.dropdown-menu li': function(e){
var selectedIndex = $(e.currentTarget).attr("rel");
var val = $('select#perPage option:eq(' + selectedIndex + ')').attr('value');
var oldVal = Session.get('recordsPerPage');
if(val != oldVal)
Session.set('recordsPerPage', Number(val));
},
'click .pageNum': function(e){
e.preventDefault();
var num = $(e.currentTarget).data('page');
Session.set('page', Number(num));
}
});
Currently, by default, only 10 records per page show up (unless the user selects from a drop-down a different amount). I have a plotCities function that I am using to try to plot the top 100 cities from the subset that is returned, however, I can't grab the top 100 because only 10 at a time show up.
Is there anyway to do what I am describing?

Ok, so the jobsPerCity and jobs are two totally different things, so I would use a separate on-fly-collection for the first one. Nothing will be stored in the database but the client will "think" that there is actually a jobsPerCity collection, which you can use to plot your map. The way you can achieve this is to define another named collection:
// only on the client !!!
var jobsPerCity = new Meteor.Collection("jobsPerCity");
On the server you will need to define a custom publish method:
Meteor.publish('jobsPerCity', function (options) {
var self = this;
var cities = new Meteor.Collection(null);
var jobToCity = {};
handle1 = Jobs.find({/* whatever condition you want */}).observeChanges({
added: function (id, fields) {
jobToCity[id] = fields.address.split(',')[0].toUpper();
cities.upsert({ _id: jobToCity[id] }, { $inc: { jobsCount: 1 } });
},
removed: function (id) {
cities.upsert({ _id: jobToCity[id] }, { $inc: { jobsCount: -1 } });
delete jobToCity[id];
},
changed: function (id, fields) {
// left as an exercise ;)
},
});
handle2 = cities.find({}, {sort: {jobsCount: -1}, limit: 100}).observeChanges({
added: function (id, fields) {
self.added('jobsPerCity', id, fields);
},
changed: function (id, fields) {
self.changed('jobsPerCity', id, fields);
},
removed: function (id) {
self.removed('jobsPerCity', id);
},
});
self.ready();
self.onStop(function () { handle1.stop(); handle2.stop(); });
});
and your good to go :)
EDIT (simple solution for more static data)
If the data is not going to be updated very often (as #dennismonsewicz suggested in one of his comments), the publish method can be implemented in a much simpler way:
Meteor.publish('jobsPerCity', function (options) {
var self = this, jobsPerCity = {};
Jobs.find({/* whatever condition you want */}).forEach(function (job) {
var city = job.address.split(',')[0].toUpper();
jobsPerCity[city] = jobsPerCity[city] !== undefined ? jobsPerCity[city] + 1 : 1;
});
_.each(jobsPerCity, function (jobsCount, city) {
self.added('jobsPerCity', city, { jobsCount: jobsCount });
});
self.ready();
});

async callback for loop response out of order

I do a async call in a for loop and i know that the response is coming async but how can i get my response always in the same order. Here's my code:
setInterval(function () {
callback = function (response)
{
var temp2 = '';
var str = "";
test = [];
console.log('STATUS: ' + response.statusCode);
response.setEncoding('utf8');
response.on('data', function (chunk)
{
str += chunk;
});
response.on('end', function ()
{
console.log("end found");
temp2 = JSON.parse(str);
for (var i in temp2['build'])
{
test.push(temp2['build'][i]['id']);
var req3 = http.request({
host: host, // here only the domain name
auth: auth,
port: 8111,
path: '/httpAuth/app/rest/builds/id:' + test[i] + '/statistics/', // the rest of the url with parameters if needed
method: 'GET', // do GET
headers: { "Accept": "application/json" }
}, callback2);
req3.end();
}
});
}
var req4 = http.request(options4, callback);
req4.end();
callback2 = function (response) {
//console.log('STATUS: ' + response.statusCode);
//console.log('HEADERS: ' + JSON.stringify(response.headers));
response.setEncoding('utf8');
var str2 = "";
response.on('data', function (chunk) {
str2 += chunk;
});
response.on('end', function () {
points.push(parseInt(JSON.parse(str2)["property"][2]["value"]));
});
j++;
if (j == test.length) {
var sumTotal = 0;
var sumThree = 0;
var status = '';
for (var i in points) {
sumTotal += points[i];
}
var averageTotal = parseInt(Math.round(sumTotal / points.length));
for (var i = 0; i < 3; i++) {
sumThree += points[i];
}
var averageThree = parseInt(Math.round(sumThree / 3));
/*if(averageThree>averageTotal)
{
status='warning';
}
else
{
status='ok';
}*/
console.log('average: ' + averageThree + ' average 100 ' + averageTotal + ' status ' + status);
//send_event('speed', {current: averageThree/*, status: status*/, last: averageTotal});
j = 0;
points = [];
}
}
}, 15 * 1000);
so my question is how can i be sure my response 'point's' have always the same order. I've tried sending the var i to the callback function but can't get it to work
edit:
changed formatting.
The output of the first callback:
{
"count":100,
"nextHref":"/httpAuth/app/rest/builds/?locator=buildType:bt2,count:100,status:SUCCESS,start:100",
"build":[
{
"id":17469,
"number":"5075",
"status":"SUCCESS",
"buildTypeId":"bt2",
"startDate":"20140224T183152+0100",
"href":"/httpAuth/app/rest/builds/id:17469",
"webUrl":"http://x.x.x.x:8111/viewLog.html?buildId=17469&buildTypeId=bt2"
},
{
"id":17464,
"number":"5074",
"status":"SUCCESS",
"buildTypeId":"bt2",
"startDate":"20140224T165758+0100",
"href":"/httpAuth/app/rest/builds/id:17464",
"webUrl":"http://x.x.x.x:8111/viewLog.html?buildId=17464&buildTypeId=bt2"
},
{
"id":17461,
"number":"5073",
"status":"SUCCESS",
"buildTypeId":"bt2",
"startDate":"20140224T161852+0100",
"href":"/httpAuth/app/rest/builds/id:17461",
"webUrl":"http://x.x.x.x:8111/viewLog.html?buildId=17461&buildTypeId=bt2"
},
This output contains 100 items. From this output I take the id number and make a new request with this array of id's. This new callback gives me the build duration but the problem is because this happens asynchronously the response I get is not from the latest build but from the first response. So my question is how can i get these build speed array in the right order

It's not recommended to use anonymous function in a for loop.
The best way (for me), it's to use the async library.
A simple exemple to answer your question :
var objectList = [
{"name":"Doe", "firstname":"John", "position":1},
{"name":"Foo", "firstname":"Bar", "position":2},
{"name":"Gates", "firstname":"Bill", "position":3},
{"name":"Jobs", "firstname":"Steve", "position":4},
];
var arr = [];
async.each(objectList, function(person, callback) {
arr.push(person); // async.each is asynchronous, so at the end, the order will be bad
}, function(err) {
async.sortBy(arr, function(p, callback) { // Reorder
callback(err, p.position);
}, function(err, results) {
callback(null, results); // Send the result
});
});
This example will work for your issue.