I've read several posts on the subject, but unfortunately struggling with my situation.
I am pulling some urls from an array of urls and then using those links to obtain (let's say) the title of the page from those subsites. At the end I want a list of original urls and an array of titles (from the sublinks). i.e., go into a site/domain, find some links, curl those links to find page titles.
Problem is that my titleArray just returns a Promise and not the actual data. I'm totally not getting closures right and promises. Code runs in node as is. I'm using personal sites in my real code, but substituted common sites to show an example.
const popsicle = require('popsicle');
var sites = ['http://www.google.com','http://www.cnn.com'];
// loop through my sites (here I'm just using google and cnn as a test
for(var i=0;i<sites.length;i++) {
// call function to pull subsites and get titles from them
var titleArray = processSites(sites[i]);
console.log(sites[i] + ": " + titleArray);
}
// get request on site and then get subsites
function processSites(url) {
return popsicle.get(url)
.then(function(res) {
var data = res.body;
// let's assume I get another collection of URLs
// that I pull from the main site
var subUrls = ['http://www.yahoo.com','http://www.espn.com'];
var titleArray = [];
for(var j=0;j<subUrls.length;i=j++) {
var title = processSubSites(subUrls[j])
titleArray.push(title);
}
return titleArray;
});
}
function processSubSites(url) {
return popsicle.get(url)
.then(function(res) {
var data = res.body;
// let's say I pull the title of the site somehow
var title = "The Title for " + url;
console.log(title);
return title;
});
}
The result after running this is:
http://www.google.com: [object Promise]
http://www.cnn.com: [object Promise]
The Title for http://www.espn.com
The Title for http://www.espn.com
The Title for http://www.yahoo.com
The Title for http://www.yahoo.com
whereas it should be:
http://www.google.com: ['The Title for http://www.yahoo.com', 'The Title for http://www.espn.com']
http://www.cnn.com: ['The Title for http://www.yahoo.com', 'The Title for http://www.espn.com']
...
You cannot return normal data from inside a Promise. You need to return another Promise to make it chainable. To process multiple Promise objects in loop, you need to push them in an array and call Promise.all(array);
const popsicle = require('popsicle');
var sites = ['http://www.google.com','http://www.cnn.com'];
var titleArrayPromises = [];
// loop through my sites (here I'm just using google and cnn as a test
for(var i=0;i<sites.length;i++) {
titleArrayPromises.push(processSites(sites[i]));
}
var titleArray = [];
Promise.all(titleArrayPromises).then(function (titleArrays) {
for(var i=0; i<titleArrays.length; i++) {
titleArray.concat(titleArrays[i])
}
// You now have all the titles from your site list in the titleArray
})
function processSites(url) {
return popsicle.get(url)
.then(function(res) {
var data = res.body;
// let's assume I get another collection of URLs
// that I pull from the main site
var subUrls = ['http://www.yahoo.com','http://www.espn.com'];
var titlePromiseArray = [];
for(var j=0;j<subUrls.length;j++) {
titlePromiseArray.push(processSubSites(subUrls[j]));
}
return Promise.all(titlePromiseArray);
});
}
function processSubSites(url) {
return popsicle.get(url)
.then(function(res) {
var data = res.body;
// let's say I pull the title of the site somehow
var title = "The Title for " + url;
return Promise.resolve(title);
});
}
Related
I want to retrieve tweets and mentions from the twitter api. After doing this I want gather the tweets and mentions and store it in one json array. So far when I store both tweets and mentions, the system replaces the tweets in the object with the mentions?
jsonx = {};
var tweets = function(){
client.get('search/tweets', {q:"#"+string.teamname, count:1},
function (err,data){
for(var index in data.statuses){
var tweet = data.statuses[index];
jsonx[index] = tweet
}
})
}
var mentions = function(){
client.get('statuses/user_timeline', {screen_name:"#"+string.teamname, count:1},
function(err,data) {
for(var index in data){
var tweet = data[index];
jsonx[index] = tweet
}
})
}
var doSometing = function(tweets, mentions){
if (string.checktweet1 == 'on'){
tweets()
}
if (string.checkmentions1 == 'on'){
mentions()
}
}
doSometing(tweets,mentions)
Look into Object.assign() which is like merging to Objects (but in reality it creates a new Object). If you combine it with Array.concat, you can do something like this...
let jsonx = {
tweets: [],
mentions: []
};
// ....
let tweet = data.statuses[index];
jsonx = Object.assign({}, jsonx, {
tweets: jsonx.tweets.concat([tweet])
});
Your example involves mutation which is undesirable and leads to your mentions overriding tweets. This example is constantly creating a new object based on old data (same with Array.concat which creates a new array).
I just downloaded and installed phantom-crawler for nodejs. I copy and pasted the following script into a file called crawler.js:
var Crawler = require('phantom-crawler');
// Can be initialized with optional options object
var crawler = new Crawler();
// queue is an array of URLs to be crawled
crawler.queue.push('https://google.com/');
// Can also do `crawler.fetch(url)` instead of pushing it and crawling it
// Extract plainText out of each phantomjs page
Promise.all(crawler.crawl())
.then(function(pages) {
var texts = [];
for (var i = 0; i < pages.length; i++) {
var page = pages[i];
// suffix Promise to return promises instead of callbacks
var text = page.getPromise('plainText');
texts.push(text);
text.then(function(p) {
return function() {
// Pages are like tabs, they should be closed
p.close()
}
}(page));
}
return Promise.all(texts);
})
.then(function(texts) {
// texts = array of plaintext from the website bodies
// also supports ajax requests
console.log(texts);
})
.then(function () {
// kill that phantomjs bridge
crawler.phantom.then(function (p) {
p.exit();
});
})
I'd like to print the complete html source (in this case from the google page) to the console.
I searched a lot, but I haven't found anything similar, so how do I do this?
get the content instead of the plainText promise.
The module phantom-crawler uses the module node-phantom-simple, which uses phantomjs.
You can find the list of properties you can call in the phantomjs wiki.
var Crawler = require('phantom-crawler');
// Can be initialized with optional options object
var crawler = new Crawler();
// queue is an array of URLs to be crawled
crawler.queue.push('https://google.com/');
// Can also do `crawler.fetch(url)` instead of pushing it and crawling it
// Extract plainText out of each phantomjs page
Promise.all(crawler.crawl())
.then(function(pages) {
var allHtml = [];
for (var i = 0; i < pages.length; i++) {
var page = pages[i];
// suffix Promise to return promises instead of callbacks
var html = page.getPromise('content');
allHtml.push(html);
html.then(function(p) {
return function() {
// Pages are like tabs, they should be closed
p.close()
}
}(page));
}
return Promise.all(allHtml);
})
.then(function(allHtml) {
// allHtml = array of plaintext from the website bodies
// also supports ajax requests
console.log(allHtml);
})
.then(function () {
// kill that phantomjs bridge
crawler.phantom.then(function (p) {
p.exit();
});
})
I have been working around this for a really long while and I think I'm sort of giving up and coming here to ask now. I have a cloud code function that first lets me get a list of categories. Every category has multiple products. This one-to-many relationship is carried out using pointers, where every product points to the category it belongs to. However I'm failing to retrieve any products.
This my code,
Parse.Cloud.define("GetCategories", function(request, response) {
var _ = require('underscore.js')
var MSTCATEGORY_CLASS = "MSTCategory";
var MSTPRODUCT_CLASS = "MSTProduct";
var CATEGORY_TAG = "Category";
var SHOP_TAG = "Shop";
var VIEWS_TAG = "Views";
var CAT_LIMIT = 10;
var PROD_LIMIT = 5;
var productList = [];
var CatAndProd = [];
var MSTCategory = Parse.Object.extend(MSTCATEGORY_CLASS);
var query = new Parse.Query(MSTCategory);
query.limit(CAT_LIMIT);
query.find().then(function(results) {
var promise = Parse.Promise.as();
_.each(results, function(result) {
promise = promise.then(function() {
var MSTProduct = Parse.Object.extend(MSTPRODUCT_CLASS);
var ProdQuery = new Parse.Query(MSTProduct);
ProdQuery.equalTo(CATEGORY_TAG, result);
ProdQuery.include(CATEGORY_TAG);
ProdQuery.include(SHOP_TAG);
ProdQuery.descending(VIEWS_TAG);
ProdQuery.limit(PROD_LIMIT);
var category = result;
ProdQuery.find().then(function(ProdResults){
ProdResults.forEach(function(product) {
productList.push(product);
});
var singleItem = {
"category" : category,
"products" : productList
};
CatAndProd.push(singleItem);
return Parse.Promise.as("Hello!");
});
});
});
return promise;
}).then(function(hello) {
var jsonObject = {
"categoryAndProducts": CatAndProd
};
response.success(jsonObject);
});
});
What I am trying to do is after getting the category, I'd fetch products in it, add them to a jsonObject. And once I'm done with all categories. I'll create an array to carry all those json objects and send it as response. This is really basic, I'm pretty sure it's become my logic is incorrect. I'm new to Javascript and Parse.
There are few errors:
in your _.each, you are overwriting the same promise object in each loop
you can return directly your singleItem via the promise resolving, not need for an extra array
you're using a global productList array so the last category will have ALL products.
Try like this, I put few comments in the code:
query.find().then(function(results) {
// an array to store all our promises
var promises = [];
results.forEach(function(result) {
// one promise for each result
var promise = new Parse.Promise();
var MSTProduct = Parse.Object.extend(MSTPRODUCT_CLASS);
var ProdQuery = new Parse.Query(MSTProduct);
ProdQuery.equalTo(CATEGORY_TAG, result);
ProdQuery.include(CATEGORY_TAG);
ProdQuery.include(SHOP_TAG);
ProdQuery.descending(VIEWS_TAG);
ProdQuery.limit(PROD_LIMIT);
var category = result;
ProdQuery.find().then(function(ProdResults) {
// remove this
// ProdResults.forEach(function(product) {
// productList.push(product);
// });
var singleItem = {
"category" : category,
"products" : ProdResults
};
// pass singleItem directly when resolving the promises
promise.resolve(singleItem);
});
promises.push(promise);
});
// return a promise resolving when ALL Promises are resolved
return Parse.Promise.when(promises);
}).then(function(allProducts) {
// allProducts is an array of singleItem
var jsonObject = {
"categoryAndProducts": allProducts
};
response.success(jsonObject);
});
Also, try to not mix Array.forEach and _.each, use one of them but not both at the same time.
I have been working to make a small app to get the images and info from the pokemon api at http://pokeapi.co/ , I am following course using promises however I can get a group (example: water type) which give me 78 objects , each one with a resource_uri which is the data which give me the information of each pokemon.
Until now I have make this and I can get all the objects, however, now how can I make in order to get for each object push in the console.log (or later use them) all the 78pokemons with the data of each.
My code until now is this>
var $ = window.jQuery
var base = 'http://pokeapi.co/api/v1/egg/'
Promise.resolve($.get(base + '2'))
.then((results) =>
.then((results) => {
var pokechara = results.pokemon
var basechara = 'http://pokeapi.co'
var promises = []
for (let chara of pokechara){
var pokech = pokechara[i]
'${something}'
var pokechurl = basechara + pokechara[0].resource_uri
promises.push(Promise.resolve($.get(pokechurl)))
}
debugger
return Promise.all(promises)
})
.then((poke) => {
console.log(poke)
})
.catch((err) => {
debugger
})
I really wanted to make something with the api of yugioh.wikia but I dont see how I can get work because the problem with the Cros-server or header. To see a preview of what I am doing you can see at http://www.kengreg.com/yugiohapp/
Thanks in advance
Actually, in the code used in your reply to Alex, the string variables you use to call the api are the only thing wrong, the 'http://' part at the start is needed. Apart from that, your code works fine.
var $ = window.jQuery;
var base = 'http://pokeapi.co/api/v1/egg/';
Promise.resolve($.get(base + '2'))
.then((results) => {
var pokechara = results.pokemon;
var basechara = 'http://pokeapi.co/';
var promises = [];
for (var i in pokechara){
var pokech = pokechara[i];
var pokechurl = basechara + pokechara[0].resource_uri;
promises.push(Promise.resolve($.get(pokechurl)));
}
debugger
return Promise.all(promises);
})
.then((poke) => {
debugger;
console.log(poke);
})
.catch((err) => { debugger })
I left the debugger so you can checkout the results of the promises. The poke variable at the end should have the objects array you wanted.
I am currently, trying to National Library of Australia's API to find pictures on a specific search term. Trove API I have the following functions which should send a query from an input form to the api and receive images back, however I am not receiving the majority of the images. In a particular example, if is search for 'sydney' I am only receiving 3 images back when there is in fact way more. For instance, this is the json, that is returned. I know that you will not be familiar with this api, but in my code below, is there anything that you can see, that would be causing it not to return all the images? I have changed a few things around to try and find the problem as well as put a few console.log statements but it is still not being kind to me.
var availableImages = {
"nla": {
"numImages":0,
"url_pattern":"nla.gov.au",
"images":[]
},
};
var url_patterns = ["nla.gov.au"];
$(document).ready(function(){
$("form#searchTrove").submit();
$("form#searchTrove").submit(function() {
resetImageData();
//get input values
var searchTerm = $("#searchTerm").val().trim();
searchTerm = searchTerm.replace(/ /g,"%20");
var sortBy = $("#sortBy").val();
//create searh query
var url = "http://api.trove.nla.gov.au/result?key="
+ apiKey + "&l-availability=y%2Ff&encoding=json&zone=picture"
+ "&sortby=relevance&n=100&q=" + searchTerm + "&callback=?";
//print JSON object
console.log(url);
//get the JSON information we need to display the images
$.getJSON(url, function(data) {
$('#output').empty();
$.each(data.response.zone[0].records.work, processImages);
//console.log(data);
printImages();
});
});
});
function processImages(index, troveItem){
console.log("av"+ availableImages);
for(var i in availableImages){
//console.log(availableImages[i].url_pattern)
if(troveItem.identifier[0].value.indexOf(availableImages[i].url_pattern) >= 0){
console.log("Trove URL "+troveItem.identifier[0].value+" Pattern: "+availableImages[i]["url_pattern"]);
availableImages[i].numImages++;
availableImages.totalimages++;
availableImages[i]["images"].push(troveItem.identifier[0].value);
}
}
}
function printImages(){
$("#output").append("<h3>Image Search Results</h3>");
for(var i in availableImages){
if(availableImages[i]["url_pattern"]=="nla.gov.au" && availableImages[i]["numImages"]>0){
printNLAImages();
console.log(availableImages);
}
}
}
function printNLAImages(){
$("#output").append("<h3>National Library of Australia</h3><p>"
+availableImages["nla"]["numImages"]+" images found from <a href='http://"
+availableImages["nla"]["url_pattern"]+"'>"
+availableImages["nla"]["url_pattern"]+"</a></p>");
for (var i in availableImages["nla"]["images"]){
$("#output").append("<img src='"+availableImages["nla"]["images"][i]+"-v'>");
}
console.log(availableImages);
}
function resetImageData(){
availableImages.totalimages = 0;
for (var i in availableImages){
availableImages[i].numImages = 0;
availableImages[i]["images"] = [];
}
console.log(availableImages); //displaying hee
}