I have an array of links that use link parametr for function that scraped data by PhantomJS.
How to serilize this function? This for statemant runs paralely 3 function in one time and i recive an event error.
In this case its proper to use async, but how it use in series? Time of running the functions are always different, but how async should understood that it's done and start with new URL?
var phantom = require('phantom')
, async = require('async');
var urls = [
'http://en.wikipedia.org/wiki/Main_Page',
'http://es.wikipedia.org/wiki/Wikipedia:Portada',
'http://de.wikipedia.org/wiki/Wikipedia:Hauptseite'
];
async.mapSeries(urls, getTitle, function(err, result){
console.log(result);
})
function getTitle (link, callback) {
phantom.create(function(ph) {
return ph.createPage(function(page) {
return page.open(link, function(status) {
return page.evaluate((function() {
return document.title;
}), function(result) {
callback(null, result);
return ph.exit();
});
});
});
});
};
I'd try something like:
var links = []
var _ph
function init(cb) {
phantom.create(function(ph) {
//for each link in links call doStuff()
_ph = ph
doStuff(ph, link, cb)
})
}
function doStuff(ph, link, cb) {
ph.createPage(function(page) { //does things in parallel?
page.open(link, function(status) {
page.evaluate((function() {
document.title;
}), function(result) {
cb(null, result);
page.close();
});
});
}
var counter = links.length
var titles;
function results(err, res) {
titles.push(res)
if(--counter == 0) {
//done
_ph.exit()
}
}
init(results)
Probably not working code (I wrote it here), but I hope you get the idea. If you want to only use 1 page, something like:
var links = []
var _ph
var _page
function init(cb) {
phantom.create(function(ph) {
_ph = ph
ph.createPage(function(page) {
_page = page
doStuff(link, cb)
}
})
}
function doStuff(page, link, cb) {
page.open(link, function(status) {
page.evaluate((function() {
document.title;
}), function(result) {
cb(null, result);
page.close();
});
});
}
var counter = links.length
var titles;
function results(err, res) {
titles.push(res)
if(--counter == 0) {
//done
_ph.exit()
return
}
doStuff(links[counter], results)
}
init(results)
Related
Been trying everything to get this to work specifically the async each method. for (const element of resultsHistory) didn't work either.
I'm trying to modify the result array from a previous query by running a foreach over it and doing a mysql query.
However this needs to wait for the query to complete.
Is there a way to access these results without the setTimeout(function() { } I put manually in order to wait for the query to finish?
function getUserLikes(params, callback) {
var usersArrayCat = [];
console.log(`length of array ${params.length}`)
// 1 here means 1 request at a time
async.eachLimit(params, 1, function (element, cb) {
element.liked = 0;
var queryLiked = `SELECT * from users_likes WHERE user_id = \"${req.body.userid}\" AND product_id = \"${element.product_id}\"`;
connectionPromise.query(queryLiked, function (err, result) {
if (!result) {
} else if (result.length == 0) {
} else {
element.liked = result[0].userlike;
usersArrayCat.push(element);
// console.log(usersArrayCat);
cb();
}
})
}, function (err) {
if (err) return callback(err);
callback(null, usersArrayCat)
});
};
getUserLikes(resultsHistory, function (e) {
console.log(e);
});
if(!res.headersSent) {
setTimeout(function() {
res.send(JSON.stringify({"status": 200 ,"error": null, "top3":resultsHistory}));
}, 150);
}
Managed to get it working like this, but still not sure if this is the right way.
async function getUserLikes(resultsHistory) {
for (const element of resultsHistory) {
element.liked = 0;
let queryLiked = `SELECT * from users_likes WHERE user_id = \"${req.body.userid}\" AND product_id = \"${element.product_id}\"`;
let liked = await conn2.query(queryLiked);
if (liked[0]) {
element.liked = liked[0].userlike;
}
}
if(!res.headersSent) {
conn2.release();
let newres = resultsHistory.sort(
firstBy(function (v1, v2) { return v2.rating - v1.rating; })
.thenBy(function (v1, v2) { return v2.dranktimes - v1.dranktimes; })
).filter( function(history) {
return history.event === eventName;
}).slice(0, 3);
res.send({"status": 200 ,"error": null, "top3":newres});
}
}
getUserLikes(resultsHistory);
See my answer:
function getUserLikes(params, callback) {
var usersArrayCat = [];
console.log(`length of array ${params.length}`)
// 1 here means 1 request at a time
async.eachLimit(params, 1, function (element, cb) {
element.liked = 0;
var queryLiked = `SELECT * from users_likes WHERE user_id = \"${req.body.userid}\" AND product_id = \"${element.product_id}\"`;
connectionPromise.query(queryLiked, function (err, result) {
if (!result) {
} else if (result.length == 0) {
} else {
element.liked = result[0].userlike;
usersArrayCat.push(element);
// console.log(usersArrayCat);
}
cb();
})
}, function (err) {
if (err) return callback(err);
callback(null, usersArrayCat)
});
};
getUserLikes(resultsHistory, function (e, usersArrayCat) {
if (e) {
console.log(e);
return res.send({status: 400, error: e}); // your error response
}
console.log(usersArrayCat); // your usersArrayCat with liked property
res.send(JSON.stringify({ // why your need return a string intead of json object ???
"status": 200,
"error": null,
"top3": usersArrayCat // I think return `usersArrayCat` is a right way
}));
});
So I'm trying to send my data from my server code to my client code, yet it's returning undefined. Not sure what to do as I've been stuck here for a while.
I used these packages:
https://github.com/meteorhacks/npm
https://www.npmjs.com/package/wiki-infobox
Rappers = new Mongo.Collection(null)
var page = 'Tupac'
var language = 'en'
var rappers = null
var texts
var rappers2
if (Meteor.isClient) {
getGists = function getGists(user, callback) {
Meteor.call('getGists', user, callback);
}
Meteor.startup(function() {
rappers2 = []
function call(text, callback) {
Meteor.call('getWikiStuff', rappers2, function(err, result) {
console.log(result)
})
var timer = setTimeout(function() {
callback()
}, 4000)
}
function consoleit() {
console.log(rappers2)
}
call('hello', consoleit)
})
}
if (Meteor.isServer) {
Meteor.startup(function() {
Meteor.methods({
getWikiStuff: function(rappers3) {
var infobox = Meteor.npmRequire('wiki-infobox')
var bound = Meteor.bindEnvironment(function(callback) {
callback()
});
console.log("HERE")
bound(function() {
infobox(page, language, function(err, data) {
if (err) {
return
}
rappers = data.associated_acts
for (var x = 0; x < rappers.length; x++)
if (rappers[x].text != undefined) {
var yo = rappers[x].text
rappers3.push(yo)
}
for (var value of rappers3)
console.log(value)
})
})
return rappers3
}
})
})
}
How can I return a object of data returned by asynchronous function called multiple times from within a asynchronous function.
I'm trying to implement like this :
var figlet = require('figlet');
function art(dataToArt, callback)
{
var arry[];
figlet(dataToArt, function(err, data) {
if (err) {
console.log('Something went wrong...');
console.dir(err);
return callback('');
}
arry[0] = data;
callback(arry);
});
figlet(dataToArt, function(err, data) {
if (err) {
console.log('Something went wrong...');
console.dir(err);
return callback('');
}
arry[1] = data;
callback(arry);
});
}
art('Hello World', function (data){
console.log(data);
});
How can I do it correctly, I searched and searched but couldn't find a solution.
Ps. I'm using Figlet.js
I don't know if you're ok using an external module, but you can use tiptoe.
Install it using npm install tiptoe like any regular module and it basically goes like this:
var tiptoe = require('tiptoe')
function someAsyncFunction(obj, callback) {
// something something
callback(null, processedData);
}
tiptoe(
function() {
var self = this;
var arr = ['there', 'are', 'some', 'items', 'here'];
arr.forEach(function(item) {
someAsyncFunction(item, self.parallel());
});
},
function() {
var data = Array.prototype.slice.call(arguments);
doSomethingWithData(data, this);
},
function(err) {
if (err) throw (err);
console.log('all done.');
}
);
the someAsyncFunction() is the async function you want to call does something and calls the callback parameter as a function with the parameters error and data. The data parameter will get passed as an array item to the following function on the tiptoe flow.
Did it Myself :) Thanks to mostafa-samir's post
var figlet = require('figlet');
function WaterfallOver(list, iterator, callback) {
var nextItemIndex = 1;
function report() {
nextItemIndex++;
if(nextItemIndex === list.length)
callback();
else
iterator([list[0],list[nextItemIndex]], report);
}
iterator([list[0],list[1]], report);
}
var FinalResult = [];
WaterfallOver(["hello","Standard","Ghost"], function(path, report) {
figlet.text(path[0], { font: path[1] }, function(err, data) {
if (err) {
FinalResult.push("Font name error try help");
report();
return;
}
data = '<pre>.\n' + data + '</pre>';
FinalResult.push(data);
report();
});
}, function() {
console.log(FinalResult[0]);
console.log(FinalResult[1]);
});
I am writing an service, where I retrieve a list of items from a another service, then iterate over result performing keystone.list operation(s).
I am loosing the return status in the find/exec operation. I have tried promises, async, etc.
If someone could point out the correct way to implement this, I would appreciate it.
general implementation:
exports = module.exports = function (req, res) {
var rtn = {
added: 0,
count: 0
}
service(params)
.then(function(svcResult) {
svcResult.forEach(function(item) {
rtn.count++; // <-- correctly seen in apiresponse
Artifact.model.find()
.where({ artifactId: item.id})
.exec(function(err, result) {
if (result.length == 0) {
result = new Artifact.model({
... populate from item ....
});
result.save();
rtn.added++; // <-- not seen in api response
});
});
res.apiResponse(rtn);
});
}
for starters, exec is an async call, which you are ignoring in your res.apiResponse and thus count is incremented and not added, to make life easy, I am moving the exec call outside and wrapping it with promise:
function pExec(id){
return new Promise(function(resolve, reject){
Artifact.model.find()
.where({ artifactId: id})
.exec(function(err, result){
console.log('result: ', result); // there is a possibility that this is not empty array, which seems to be the only case when you increment added value
err? reject(err): resolve(result);
});
});
}
exports = module.exports = function(req, res){ // I think it is 'exports' not 'exposts'
service(params)
.then(function(svcResult) {
var promises = svcResult.map(function(item){
rtn.count++;
return pExec(item.id).then(function(result){
if (result.length == 0) {
result = new Artifact.model({
//... populate from item ....
});
result.save(); // again this might be an async call whose response you might need before incrementing added...
rtn.added++; // <-- not seen in api response
};
});
});
Promise.all(promises).then(function(){
res.apiResponse(rtn);
});
});
}
Thanks... Here is what I have come up with so far....
function getArtifact(id) {
return new Promise(function (resolve, reject) {
Artifact.model.findOne()
.where({artifactId: id})
.exec(function (err, artifact) {
err ? resolve(null) : resolve(artifact);
});
});
}
function createArtifact(item) {
return new Promise(function (resolve, reject) {
var artifact = new Artifact.model({
// ... populate from item ....
});
artifact.save(function (err, artifact) {
err ? resolve(null) : resolve(artifact);
});
});
}
exports = module.exports = function (req, res) {
var rtn = {
success: false,
count: 0,
section: '',
globalLibrary: {
Added: 0,
Matched: 0
},
messages: [],
};
if (!req.user || !req.user._id) {
rtn.messages.push("Requires Authentication");
return res.apiResponse(rtn);
}
if (!req.params.section) {
rtn.messages.push("Invalid parameters");
return res.apiResponse(rtn);
}
var userId = req.user._id;
var section = req.params.section;
rtn.section = section;
service(section)
.then(function (svcResult) {
if (svcResult.length == 0 || svcResult.items.length == 0) {
rtn.messages.push("Retrieved empty collection");
return;
}
rtn.messages.push("Retrieved collection");
var artifacts = svcResult.items(function (item) {
rtn.count++;
return getArtifact(item.objectid)
.then(function (artifact) {
if (!artifact || artifact.length == 0) {
rtn.messages.push("Global Library Adding: " + item.name['$t']);
rtn.globalLibrary.Added++;
artifact = createArtifact(item);
} else {
rtn.globalLibrary.Matched++;
}
return artifact;
})
});
Promise.all(artifacts)
.then(function () {
rtn.success = true;
res.apiResponse(rtn);
});
});
}
I am trying to stub several ajax calls, but I want to have both beforeSend and success executed, is this possible?
I want something like this:
var stub = sinon.stub(jQuery, "ajax");
stub.onCall(0).yieldsTo("beforeSend").yieldsTo("success", {some: 'data'});
stub.onCall(1).yieldsTo("beforeSend").yieldsTo("success", {other: 'stuff'});
But this skips the 'beforeSend' method.
I know it would be easier to allow ajax to do it's stuff and use sinon's fakeServer, but I can't as I'm testing in Node with a fake browser and it just doesn't work
You could use yieldTo after the call:
var stub = sinon.stub();
stub({
foo: function() {
console.log('foo');
},
bar: function() {
console.log('bar');
}
});
stub.yieldTo('foo');
stub.yieldTo('bar');
I was able to work around this by adding some additional code:
var responses = {};
var executionComplete;
beforeEach(function () {
executionComplete = $.Deferred();
sinon.stub($, "ajax", function (options) {
if (options.beforeSend) {
options.beforeSend();
}
completeAjaxCall(options);
});
});
afterEach(function () {
$.ajax.restore();
});
var completeAjaxCall = function (options) {
var response = findResponse(options.url, options.type);
setTimeout(function () {
if (response.code < 400) {
if (options.dataFilter && response.data) {
response.data = options.dataFilter(JSON.stringify(response.data));
}
if (options.success) {
options.success(response.data);
}
} else {
if (options.error) {
options.error(response.data);
}
}
if (options.complete) {
options.complete();
}
if (response.completeExecution) {
executionComplete.resolve();
}
}, response.serverResponseTime);
};
var findResponse = function (url, type) {
var response = responses[url];
expect(response, url + ' should have a response').to.exist;
expect(response.type).to.equal(type);
delete responses[url];
if (Object.keys(responses).length === 0) {
response.completeExecution = true;
}
return response;
};
var givenResponse = function (response) {
responses[response.url] = response;
};
Then in my test I can use it like this:
it('should do some stuff', function (done) {
//given
givenResponse({serverResponseTime: 4, code: 200, url: '/saveStuff', type: 'POST'});
givenResponse({serverResponseTime: 1, code: 200, url: '/getStuff', type: 'GET'});
//when
$('button').click();
//then
executionComplete.then(function () {
expect(something).to.be.true;
done();
});
});