find cities and country name from string

find cities and country name from string - javascript

I have cities and countires list in a json file. For a given string, I need to check the city name and country name is present or not. If present I have to capitalize the word. what is the best way to acheive this in node JS
Please consider json from this link.
https://raw.githubusercontent.com/russ666/all-countries-and-cities-json/master/countries.json
my input is "united states to play davis cup in bratislava"
output should be "United States to play davis cup in Bratislava"
Hint: First letter of city and country name should be capital.
I am expecting code something like this
var myString="united states to play davis cup in bratislava";
var data=myjson;
var i=0;
myString=myString.split("");
for(i=0;i<myString.length;i++){
var output="";
//help this line
var check=myString[i].match(data)
if(check){
output+=myString[i].charAt(0).toUpperCase() + myString[i].slice(1);
}
else{
output+=myString[i]}
}

It all starts from function start() at the bottom. For displaying purpose i've used a small dataset but you can require the data from the json file by using const data = require('data.json'). I've tested for large dataset also, works like a charm. Hope it helps.
const data = {
"United States":[
"Washington","Bratislava","Hard","Going"],
"Afghanistan": [
"Herat",
"Kabul",
"Kandahar",
"Molah",
"Rana",
"Shar",
"Sharif",
"Wazir Akbar Khan"
]};
Array.prototype.myJoin = function(start,end){
if(!start) start = 0;
if(!end) end = this.length - 1;
end++;
return this.slice(start,end);
};
const getCityData = async (country) => {
return country;
}
const changeFormat = async () => {
try {
let countries = Object.keys(data).map( (country, index) => {
return country;
})
let citiesData = [];
await countries.map( (country, index) => {
citiesData = citiesData.concat(data[country]);
})
return countries.concat(citiesData);
} catch (err) {
return err;
}
}
const checkSentence = (text, text_original, number, modified_data) => {
return new Promise((resolve, reject)=>{
try {
if( !text || !text.length ){
throw new Error('empty text');
}
// console.log('started ' + number);
// console.log('number ' + number +' started')
let upperCase = [];
const number_const = number;
let temp1 = new Array(text.length);
temp1.fill(2);
temp1.map( (v, i) => {
// let temp = text;
let temp = [...text_original, ...[]];
// console.log('i' + i);
// console.log('number' + number);
if(i + number <= text.length ) {
// console.log('inside 1st if');
temp = temp.slice(i, i + number)
// console.log(text + ' 1');
temp = temp.join(' ')
// console.log(text + ' 2');
temp = temp.toLowerCase();
// console.log(text + ' 3');
if(modified_data.indexOf(temp) != -1){
upperCase.push({ start: i, end: i + number - 1 })
}
}
})
let toBeChanged = [];
if(upperCase.length){
upperCase.map( (v, i) => {
// console.log(v);
let arr = range( v.start, v.end )
toBeChanged = toBeChanged.concat(arr);
})
}
// console.log('ended number' + number);
// console.log(toBeChanged);
return resolve(toBeChanged);
} catch (err) {
return reject(err);
// console.error(err);
// return false;
}
})
}
const range = (start, end) => {
// console.log(start);
// console.log(end);
return Array(end - start + 1).fill().map((_, idx) => start + idx)
}
const start = async() => {
try {
excludeWords.map( (word, index) => {
excludeWords[index] = excludeWords[index].toLowerCase();
});
let modified_data_1 = await changeFormat();
let maximum = 1;
modified_data = modified_data_1.map( (v, i) => {
if(v.split(' ').length > maximum){
maximum = v.split(' ').length
}
if(excludeWords.indexOf(v.toLowerCase()) == -1) {
return v.toLowerCase();
}
});
text = text.split(' ');
if(maximum > text.length){
maximum = text.length;
}
// console.log(maximum);
let temp = new Array(maximum);
temp.fill(2);
let answer = await temp.map( (v, i) => {
let tempArray = [...text, ...[]];
let tempArray1 = [...text, ...[]];
return checkSentence(tempArray, tempArray1, (maximum - i), modified_data);
})
return Promise.all(answer).then( (results) => {
let merged = [].concat.apply([], results);
// console.log('merged');
merged = new Set(merged);
merged = [...merged];
// console.log(merged);
merged.map((v, i) => {
if(v == undefined || v == null){
return;
}
let temp1 = text[v].split('');
temp1[0] = temp1[0].toUpperCase();
text[v] = temp1.join('');
})
// console.log(text.join(' '));
return text.join(' ');
}).catch((err)=>{
console.log(err);
})
} catch (err) {
// console.error('here ERROR');
console.error(err);
return false;
}
}
let excludeWords = ['Hard', 'Going'];
let text = 'united states to davis cup hard wazir Akbar Khan in bratislava';
( async () => {
let answer = await start();
console.log(answer);
})();

Hi I have done in simple way it is also working. My problem is in the string the word "davis" also present as a city in json. How to not capitalize that word. For ex: "Hard", "Going" these words also have city name. but these words not be considered as city in my program.
Case 1:
Input: taiwan hit hard by sars outbreak.
Output should be: Taiwan hit hard by sars outbreak.
My output: Taiwan hit Hard by sars outbreak.
Please install capitalize npm and use data.json folder in your root folder to execute below code
var myData=require("./data");
var countriesArray=Object.keys(myData.data);
var citiesArray=Object.values(myData.data);
var capitalize=require('capitalize');
var citiesFlatten = [].concat.apply([], citiesArray);
var countryCities=countriesArray.concat(citiesFlatten);
var str = 'russia ponders space tourism deal';
var pattern = new RegExp("\\b("+countryCities.join("|")+")\\b","ig");
var matchArray=str.match(pattern);
if(!!matchArray){
matchArray.forEach(function(item) {
str=str.replace(item,capitalize.words(item));
});
console.log( str.replace(/^\w/, c => c.toUpperCase()));
}

Related

Fetch and display json data with javascript using reduce group sorting

I have a multi array json data:
https://marialaustsen.com/foo.json
I need to return top 5 apdex by host, this format:
https://marialaustsen.com/grid-mode.jpg
So fare I have found top 5 apdex and sorted decending.
I think I will need to create a for each function and display apdex and name for each host.
But I am not aware how to do this. Please see fiddle:
https://jsfiddle.net/marialaustsen/8uqafbs4/
fetch('https://marialaustsen.com/foo.json')
.then(function(response) {
return response.json();
})
.then(function(data) {
appendData(data);
console.log('data: ' + data);
})
.catch(function(err) {
console.log('error: ' + err);
});
Array.prototype.groupBy = function(prop) {
return this.reduce(function(groups, item) {
var val = item[prop]
groups[val] = groups[val] || []
groups[val].push(item)
return groups
}, {})
}
function appendData(data) {
function getTopN(data, prop, n) {
// dublicate array before sorting, to preserve the original array
var clone = data.slice(0);
// sort descending
clone.sort(function(x, y) {
if (x[prop] == y[prop]) return 0;
else if (parseInt(x[prop]) < parseInt(y[prop])) return 1;
else return -1;
});
// array.slice(start, end)
return clone.slice(0, n);
}
// top 5 first instances of the descending sorted by apdex
var n = 5;
var topScorers = getTopN(data, "apdex", n);
console.log("Top " + n + " apdex:");
console.log('topScorers' + JSON.stringify(topScorers));
topScorers.forEach(function(topScorers) {
if (topScorers === 'host') {
const groupedByHost = topScorers.groupBy('host')
console.log('groupedByHost' + JSON.stringify(groupedByHost));
}
});

if you have the data and want to render to the dom then do this
var root = document.querySelector('#root');
var a = data.map(item => {
return (
`<div>
Name: ${item.name}
<br />
Contributors: ${item.contributors.map(cont => ` <span>${cont}</span>`)}
<br />
Host: ${item.host.map(ht => ` <span>${ht}</span>`)}
<br />
</div><br />`
)
});
a = a.splice(',').join('');
root.innerHTML += a;

Firebase cloud function Deadline exceeded error in nested promises

I have a firebase cloud function that is unable to finish executing. I suspect my code can be dramatically improved but I'm not quite sure how.
I've made the query as specific as possible to try and reduce the number of documents required to iterate through but that didn't solve the issue.
I get a Deadline Exceeded error which I suspect is due to the fact that I'm iterating through so many documents and trying to update them.
I increased the timeout (9 minutes)and memory allocation (2GB) in Google cloud console but that didn't help either.
exports.updatePollWinner = functions.runWith(runtimeOpts).firestore.document('triggerAccuracyCalculation/{id}').onCreate(trigger => {
const week = trigger.get('week');
const scoringTags = ["STD", "0.25PPR", "0.5PPR", "PPR", "0.10PPC", "0.25PPC", "0.5PPC", "4PTPASS", "5PTPASS", "6PTPASS", "-2INT", "TEPREMIUM"]
let winningChoiceIds = [];
let totalPollIds = [];
return db.collection("polls").where("sport", "==", 1).where("week", "==", week).where("pollType", "==", "WDIS").get()
.then((querySnapshot) => {
console.log("A");
querySnapshot.forEach((doc) => {
totalPollIds.push(doc.id);
let pollData = doc.data();
// extract relevant scoring tags
let tags = pollData.tags.filter(tag => scoringTags.includes(tag.code)).map(tag => tag.code);
// if no scoring setting is tagged, then use STD - determine what STD is
// extract player from each option
let winner = {score: 0, choice: {}, choiceId: null};
let cnt = 0;
pollData.choices.forEach((choice) => {
let choiceId = choice.id
let mappedChoices = choice.players.map(player => {
return { displayName: player.displayName, playerId: player.playerId, position: player.position, team: player.team }
});
// ToDo: What happens if someone posts a poll with two players in one option? This poll should be ignoree from accuracy calculation
// ignmore if option has more than one player
// if (mappedChoices.length > 1) return;
const player = mappedChoices[0]
// We can't score defense
if (player.position === "DEF") {
return;
}
const playerId = player.playerId;
// Make FFN API call to retrieve stats for that player in that weekconst statsEndpoint = `https://www.fantasyfootballnerd.com/service/player/json/${functions.config().ffnerd.key}${req.url}`;
const statsEndpoint = `https://www.fantasyfootballnerd.com/service/player/json/${functions.config().ffnerd.key}/${playerId}`;
const json = {"Stats": {"2019": ""}, "Player": {}};
https.get(statsEndpoint, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
const weekString = week.toString();
const fetchedStats = JSON.parse(data).Stats
if (!fetchedStats) return;
const response = fetchedStats["2019"]
if (!response) return;
// TODO SCORE KICKERS AND DEFENSES
const stats = response[weekString];
let score = 0;
stats["recYards"] ? score += parseInt(stats["recYards"]) / 10 : false
stats["recTD"] ? score += parseInt(stats["recTD"]) * 6 : false
stats["rushYards"] ? score += parseInt(stats["rushYards"]) / 10 : false
stats["rushTD"] ? score += parseInt(stats["rushTD"]) * 6 : false
stats["xpMade"] ? score += parseInt(stats["xpMade"]) : false
stats["fgMade"] ? score += parseInt(stats["fgMade"]) * 3 : false
stats["kickoffRet"] ? score += parseInt(stats["kickoffRet"]) / 10 : false
stats["SackYards"] ? score -= parseInt(stats["SackYards"]) / 10 : false
stats["fumbleLost"] ? score -= parseInt(stats["fumbleLost"]) * 2 : false
// Determine winner
// ToDo: handle ties
if (score > winner.score) {
winner.score = score;
winner.choiceId = choiceId;
winner.choice = choice;
}
if (cnt>=pollData.choices.length-1){
// Save player object on the poll Document (include choice ID)
winningChoiceIds.push(winner.choiceId);
const pollDoc = db.doc(`polls/${doc.id}`);
pollDoc.update({winner: winner});
}
cnt++;
});
}).on("error", (err) => {
console.log("Error: ", err.message);
});
});
});
console.log("B");
return false;
}).then(() => {
console.log("C");
let dateToQueryAfter = new Date(new Date("08/22/19").setHours(0,0,0,0))
return db.collection("users").where("recentVote", ">", dateToQueryAfter).get()
})
.then((querySnapshot) => {
console.log("D");
const promises = [];
querySnapshot.forEach((doc) => {
const p = db.collection("votes").where("uid", "==", doc.id).where("week", "==", week).where("pollType", "==", "WDIS").get()
promises.push(p)
});
return Promise.all(promises)
})
.then((querySnapshots) => {
console.log("E");
querySnapshots.forEach((querySnapshot) => {
if (querySnapshot.docs.length <= 0) return;
const uid = querySnapshot.docs[0].data().uid
const retrieveUserDoc = db.doc(`users/${uid}`);
let correctVotes = 0;
let cnt = 0;
let totalVotes = 0;
let pollVoteIds = [];
let pollVoteIdsCorrect = [];
querySnapshot.docs.forEach((doc) => {
const voteData = doc.data();
if (totalPollIds.includes(voteData.poll)) {
pollVoteIds.push(voteData.poll)
totalVotes++;
if (winningChoiceIds.includes(voteData.choice)) {
pollVoteIdsCorrect.push(voteData.poll)
correctVotes++;
}
}
if (cnt>=querySnapshot.size-1){
console.log("Updating user ID: ", uid);
retrieveUserDoc.update({
['accuracyWeeks.week'+week]: true,
['accuracy.week'+week]: {
totalVotes: totalVotes,
correct: correctVotes,
accuracy: correctVotes/totalVotes,
correctVoteIds: pollVoteIdsCorrect,
totalVoteIds: pollVoteIds
}
});
}
cnt++;
})
});
console.log("F");
return false;
})
.catch((error) => {
console.log("Error getting documents: ", error);
});
});

i want to make a search in which i can search upper case and lower case both at same time..?

the problem statement is i have to make a search function in which i can search lower can upper case element both even if i type lower case either upper case
i tried upper case search and lower case but its not working at all please suggest me as soon as possible
search(searchValue) {
if (searchValue != null && searchValue != "") {
var searchItem = searchValue;
var allOppData = this.stagesWiseOpportunitiesData;
var filtered = _.mapValues(allOppData, statuses =>
_.filter(statuses, statusT =>
_.some(statusT, T => _.includes(T, searchItem))
)
);
this.stagesWiseOpportunitiesData = filtered;
let stages = this.opportunitiesStateReason;
stages.forEach(element => {
let num = this.stagesWiseOpportunitiesData[element.orderData].reduce(
function(sum, value) {
return sum + value.expected_revenue;
},
0
);
element.totalExpectedRevenue = num.toFixed(2);
});
} else {
this.stagesWiseOpportunitiesData = this.stagesWiseOpportunitiesDataCopy;
let stages = this.opportunitiesStateReason;
stages.forEach(element => {
let num = this.stagesWiseOpportunitiesData[element.orderData].reduce(
function(sum, value) {
return sum + value.expected_revenue;
},
0
);
element.totalExpectedRevenue = num.toFixed(2);
});
}
}
}

Try like this. you have missed to return the for the search function.
NOTE: you have only returned for a function inside search function
function search(searchValue) {
if (searchValue != null && searchValue != "") {
var searchItem = searchValue;
var allOppData = this.stagesWiseOpportunitiesData;
var filtered = _.mapValues(allOppData, statuses =>
_.filter(statuses, statusT =>
_.some(statusT, T => _.includes(T, searchItem))
)
);
this.stagesWiseOpportunitiesData = filtered;
let stages = this.opportunitiesStateReason;
return stages.forEach(element => {
let num = this.stagesWiseOpportunitiesData[element.orderData].reduce(
function(sum, value) {
return sum + value.expected_revenue;
},
0
);
element.totalExpectedRevenue = num.toFixed(2);
});
} else {
this.stagesWiseOpportunitiesData = this.stagesWiseOpportunitiesDataCopy;
let stages = this.opportunitiesStateReason;
return stages.forEach(element => {
let num = this.stagesWiseOpportunitiesData[element.orderData].reduce(
function(sum, value) {
return sum + value.expected_revenue;
},
0
);
element.totalExpectedRevenue = num.toFixed(2);
});
}
}
console.log(search("Test"))

ReplaceAll causing issues in array.reduce

I am still pretty new to this, so forgive me if I dont' say this correctly. We have an array.reduce that calls a method with a returning promise that iterates through a list of files and post results to the db. Everything was working great, until it ran into a field that had an apostrophe in it and then the db insert fails. This is the field value. 'Expected 100002822' to be 100002822.'
I tried adding a replaceAll on the field and now get an error in the array.reduce.
Here is the .reduce
console.log('Found test results in ' + matches.length + ' files. Parsing and posting to the database now...');
var startTime = moment();
var parser = new Parser();
matches.reduce(function (p, val) {
return p.then(function () {
return parser.parseResults(val);
});
}, Promise.resolve()).then(function (finalResult) {
var endTime = moment();
var testDuration = moment.duration(endTime.diff(startTime));
console.log(chalk.blue('*** File parsing time: ' + testDuration.humanize() + ' ***'));
if (finalResult.insertSuccess == matches.length) {
var publishOut = {
totalFiles: matches.length,
totalTests: 0,
totalTestsSuccess: 0,
totalTestsFailed: 0
}
publishOut.totalTests += finalResult.totalTests;
publishOut.totalTestsSuccess += finalResult.testPassedCount;
publishOut.totalTestsFailed += finalResult.testFailedCount;
console.log(`Successfully inserted ${finalResult.insertSuccess} of ${publishOut.totalTests} test results.`);
// for (var i = 0; i < matches.length; i++) {
// var currentFile = `./testing/results/${matches[i]}`;
// fs.unlinkSync(currentFile);
// }
resolve(publishOut);
} else {
reject('Only ' + finalResult.insertSuccess + ' of ' + matches.length + ' successfully posted to the database');
}
}, function (err) {
reject('error in reduce', err);
});
I have tried several different ways of using the replaceAll with the same failure. It hits this code from the array.reduce
}, function (err) {
reject('error in reduce', err);
});
And this is the called method. The added code causing the failure in the .reduce is this Message = expectation.message.replaceAll("'", "");
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
//console.log('In parseresults', fileName);
var currentFile = './testing/results/' + fileName
json.readFile(currentFile, function (err, obj) {
if (err != null) {
console.log('error reading file', err);
reject(err);
} else {
resolve(obj);
}
});
}).then(function (obj) {
var results = [];
for (var suite in obj) {
var specs = obj[suite].specs;
for (let i = 0; i < specs.length; i++) {
const assert = specs[i];
const tcR = /TC[\d]+/;
const tc = assert.description.match(tcR);
let Passed = 1;
let Message = '';
let Stack = '';
testResults.totalTests++;
if (assert.failedExpectations.length) {
const expectation = assert.failedExpectations[assert.failedExpectations.length - 1];
Passed = 0;
Message = expectation.message.replaceAll("'", "");
Stack = expectation.stack.split('\n')[1].trim();
testResults.testFailedCount++
} else {
testResults.testPassedCount++
}
if (tc != null) {
const time = moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss');
const promise = utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild, 'P', Message, Stack, 0, time, '');
results.push(promise.then(() => {
//fs.unlinkSync(currentFile);
testResults.insertSuccess++;
//console.log('insertSuccess', testResults.insertSuccess);
},
err => { console.log('… failed', err); throw err; }
));
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
// I don't think you want to `throw err` here, right?
}
}
}
return Promise.all(results).then(() => testResults);
});
};

Callback function Node JS

I'm recently diving into nodejs (and using nightmare.js) to parse a website and am having issues with the callback functions and displaying the returned results. I'm trying to call a separate function in another function, but can't seem to return any results. They all return undefined. Any help on this is greatly appreciated.
function getDetails(loadURL, callback){
nightmare.goto(loadURL)
.wait(2000)
.evaluate(function(){
var gigs = [];
$('.hidden-xs .used-vehicle').each(function(){
item = {}
item["year"] = $(this).attr('data-year')
item["make"] = $(this).attr('data-make')
item["model"] = $(this).attr('data-model')
item["body"] = $(this).attr('data-body')
item["color"] = $(this).attr('data-ext-color')
item["trim"] = $(this).attr('data-trim')
item["mileage"] = $(this).attr('data-mileage')
item["transmission"] = $(this).attr('data-transmission')
item["vin"] = $(this).find(".vehicle-overview").attr('id')
item["title"] = $(this).find(".vehicle-overview h2 a").text()
item["link"] = $(this).find(".vehicle-overview h2 a").attr('href')
item["price"] = $(this).find(".vehicle-content .price").text()
gigs.push(item)
})
return gigs
})
.end()
.then(function(result){
var returnString = '';
for(gig in result){
returnString = returnString + result[gig].title + " " + result[gig].link + " " + result[gig].year + " " + result[gig].make + " " + result[gig].model + " " + result[gig].body + " " + result[gig].color + " " + result[gig].trim + " " + result[gig].transmission + " " + result[gig].vin + " " + result[gig].price + "\n"
}
callback(returnString)
})
}
// We will need to get the total amount of pages that we need to parse
function getInventory(sURL, callback){
nightmare.goto(sURL)
.wait(2000)
.evaluate(function(){
totals = [];
items = {}
totalCars = $('.total-found .count').text()
carsOnPage = $('.hidden-xs .used-vehicle').size()
items['carTotal'] = totalCars
items['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
items['tPages'] = Math.ceil(pageCalc)
totals.push(items)
return totals
})
.end()
.then(function(result){
var totalCars = '';
var totalPages = '';
for (item in result){
totalPages = result[item].tPages
totalCars = result[item].carTotal
}
counter = 0;
newURL = '';
returnDetails = '';
for (i =0; i < totalPages; i++){
if (i == 0){
newURL = sURL;
} else {
counter = i + 1;
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
//console.log(newURL)
getINV = getDetails(newURL, function(returnString){
callback(returnString)
})
returnDetails = returnDetails + getINV
}
callback(returnDetails)
})
}
getInventory(startURL, function(result){
console.log(result)
})

I won't bother telling you that you should not mix callbacks with promises like that. But let's see the problem for now.
Case 1
How about you check for errors too? Maybe your script is throwing errors. I can see you are calling the callback on .then but nothing on .catch. Maybe then is never getting any data.
Case 2
Let's check your functions. You are calling .end every time. Are you creating new Nightmare instance everytime too?
On the getInventory function, you should not call .end. On the getDetails function, you should not call .end. It's ending the nightmare instances and you are losing your data.
Call nightmare.end() after you are done with all of your functions and works. To do this properly you will need to learn more about Promises check case 3 below.
Case 3
Learn how promises works. On the line below, you are never waiting for the function to finish.
getINV = getDetails(newURL, function(returnString){
callback(returnString)
})
You should wait for the promises to finish. Also, make sure nightmare is not trying to browse two links at same time.
So go ahead and learn about Promises and async await stuff.
How would I solve your code?
I would use Promise.all, .map and bunch of other new stuff. Here are some sample code done for you, don't copy paste or run the code directly, try to understand why it's different from your code and what can be the result of it.
const pLimit = require("promise-limit")(2);
function getDetails(loadURL) {
return nightmare
.goto(loadURL)
.wait(2000)
.evaluate(() => {
const gigs = [];
$(".hidden-xs .used-vehicle").each(function() {
item = {};
item["year"] = $(this).attr("data-year");
item["make"] = $(this).attr("data-make");
item["model"] = $(this).attr("data-model");
item["body"] = $(this).attr("data-body");
item["color"] = $(this).attr("data-ext-color");
item["trim"] = $(this).attr("data-trim");
item["mileage"] = $(this).attr("data-mileage");
item["transmission"] = $(this).attr("data-transmission");
item["vin"] = $(this)
.find(".vehicle-overview")
.attr("id");
item["title"] = $(this)
.find(".vehicle-overview h2 a")
.text();
item["link"] = $(this)
.find(".vehicle-overview h2 a")
.attr("href");
item["price"] = $(this)
.find(".vehicle-content .price")
.text();
gigs.push(item);
});
return gigs;
})
.then(result => {
let returnString = "";
for (gig in result) {
returnString =
`${returnString +
result[gig].title} ${result[gig].link} ${result[gig].year} ${result[gig].make} ${result[gig].model} ${result[gig].body} ${result[gig].color} ${result[gig].trim} ${result[gig].transmission} ${result[gig].vin} ${result[gig].price}\n`;
}
return returnString;
})
.catch(error => {
throw new Error(error);
});
}
// We will need to get the total amount of pages that we need to parse
function getInventory(sURL) {
return nightmare
.goto(sURL)
.wait(2000)
.evaluate(() => {
totals = [];
items = {};
totalCars = $(".total-found .count").text();
carsOnPage = $(".hidden-xs .used-vehicle").size();
items["carTotal"] = totalCars;
items["onPage"] = carsOnPage;
const pageCalc = totalCars / carsOnPage;
items["tPages"] = Math.ceil(pageCalc);
totals.push(items);
return totals;
})
.then(result => {
let totalCars = "";
let totalPages = "";
for (item in result) {
totalPages = result[item].tPages;
totalCars = result[item].carTotal;
}
counter = 0;
newURL = "";
urls = [];
returnDetails = [];
for (i = 0; i < totalPages; i++) {
if (i == 0) {
newURL = sURL;
} else {
counter = i + 1;
newURL =
`${sURL}#action=im_ajax_call&perform=get_results&_post_id=5&page=${counter}&show_all_filters=false`;
}
// push to the url array
// use .map for cleaner code
urls.push(newURL);
}
// return a new promise with concurrency limit
return Promise.all(
urls.map(url => {
return limit(() => getDetails(newURL));
})
);
})
.catch(error => {
throw new Error(error);
});
}
getInventory(startURL)
.then(result => {
console.log(result);
})
.catch(error => {
console.err(error);
});
Resources:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
https://javascript.info/async-await
https://ponyfoo.com/articles/understanding-javascript-async-await

Try using callback instead of return
function getDetails(loadURL, callback){
nightmare.goto(loadURL)
.wait(2000)
.evaluate(function(callback){
var gigs = [];
$('.hidden-xs .used-vehicle').each(function(){
item = {}
item["year"] = $(this).attr('data-year')
item["make"] = $(this).attr('data-make')
item["model"] = $(this).attr('data-model')
item["body"] = $(this).attr('data-body')
item["color"] = $(this).attr('data-ext-color')
item["trim"] = $(this).attr('data-trim')
item["mileage"] = $(this).attr('data-mileage')
item["transmission"] = $(this).attr('data-transmission')
item["vin"] = $(this).find(".vehicle-overview").attr('id')
item["title"] = $(this).find(".vehicle-overview h2 a").text()
item["link"] = $(this).find(".vehicle-overview h2 a").attr('href')
item["price"] = $(this).find(".vehicle-content .price").text()
gigs.push(item)
})
callback(gigs)
})
.end()
.then(function(result){
var returnString = '';
for(gig in result){
returnString = returnString + result[gig].title + " " + result[gig].link + " " + result[gig].year + " " + result[gig].make + " " + result[gig].model + " " + result[gig].body + " " + result[gig].color + " " + result[gig].trim + " " + result[gig].transmission + " " + result[gig].vin + " " + result[gig].price + "\n"
}
callback(returnString)
})
}
// We will need to get the total amount of pages that we need to parse
function getInventory(sURL, callback){
nightmare.goto(sURL)
.wait(2000)
.evaluate(function(){
totals = [];
items = {}
totalCars = $('.total-found .count').text()
carsOnPage = $('.hidden-xs .used-vehicle').size()
items['carTotal'] = totalCars
items['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
items['tPages'] = Math.ceil(pageCalc)
totals.push(items)
return totals
})
.end()
.then(function(result){
var totalCars = '';
var totalPages = '';
for (item in result){
totalPages = result[item].tPages
totalCars = result[item].carTotal
}
counter = 0;
newURL = '';
returnDetails = '';
for (i =0; i < totalPages; i++){
if (i == 0){
newURL = sURL;
} else {
counter = i + 1;
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
//console.log(newURL)
getINV = getDetails(newURL, function(returnString){
callback(returnString)
})
returnDetails = returnDetails + getINV
}
callback(returnDetails)
})
}
getInventory(startURL, function(result){
console.log(result)
})

Maybe the following will work, changed the loops to reduce and map, took out jQuery and made some small changes.
The most important ones are:
Getting an attribute from a html element returns a string, you should convert it to a number.
Got rid of the callbacks and have the functions return promises.
Here is the code:
const getDetails = loadURL =>
nightmare.goto(loadURL)//return promise here
.wait(2000)
.evaluate(
()=>
Array.from(document.querySelectorAll('.hidden-xs .used-vehicle'))
.reduce(
(all,item)=>
all.concat(
[
element.getAttribute('data-year'),
element.getAttribute('data-make'),
element.getAttribute('data-model'),
element.getAttribute('data-body'),
element.getAttribute('data-ext-color'),
element.getAttribute('data-trim'),
element.getAttribute('data-mileage'),
element.getAttribute('data-transmission'),
element.querySelector(".vehicle-overview").getAttribute('id'),
element.querySelector(".vehicle-overview h2 a").innerText,
element.querySelector(".vehicle-overview h2 a").getAttribute('href'),
element.querySelector(".vehicle-content .price").innerText
].join(" ")
),
[]//the all array
)
);
// We will need to get the total amount of pages that we need to parse
const getInventory = sURL =>
nightmare.goto(sURL)
.wait(2000)
.evaluate(
()=> {
//there is only one item here, not sure why you push it into totals
// and call it items
const item = {}
//getAttribute returns a string, parse it to number
totalCars = parseInt(document.querySelector('.total-found .count').innerText,10);
carsOnPage = document.querySelectorAll('.hidden-xs .used-vehicle').length;
item['carTotal'] = totalCars
item['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
item['tPages'] = Math.ceil(pageCalc)
return item;
}
)
.then(
totalItem =>{
var totalCars = '';
var totalPages = '';
totalPages = totalItem.tPages
totalCars = totalItem.carTotal
newURL = '';
returnDetails = '';
return Array.from(new Array(totalPages),(_,index)=>index+1)
.reduce(
(p,counter)=>
p.then(
results=>{
if (counter === 1) {
newURL = sURL;
} else {
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
return getDetails(newURL)
.then(
result=>results.concat(result)
);
}
),
Promise.resolve([])
);
}
);
getInventory(startURL)
.then(
result=>
console.log(result)
).catch(
err=>
console.warn("Something went wrong:",err)
);

Learning promises and callbacks with asynchronous functions is quite an undertaking. I was able to get this to work with the following. Thank you all for your help and direction. Each answer sent me down a different rabbit hole to ultimately find the solution.
function getInventory(sURL){
nightmare.goto(sURL)
.wait(2000)
.evaluate(function(){
totals = [];
items = {}
totalCars = $('.total-found .count').text()
carsOnPage = $('.hidden-xs .used-vehicle').size()
items['carTotal'] = totalCars
items['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
items['tPages'] = Math.ceil(pageCalc)
totals.push(items)
return totals
})
.then(result => {
var totalCars = '';
var totalPages = '';
for (item in result){
totalPages = result[item].tPages
totalCars = result[item].carTotal
}
counter = 0;
let links = [];
let returnLinks = '';
newURL = '';
for (i = 0; i < totalPages; i++){
if (i == 0){
newURL = sURL;
} else {
counter = i + 1;
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
links.push(newURL);
}
return links;
})
.then(results => {
var arrayLinks = results;
arrayLinks.reduce(function(accumulator, url){
return accumulator.then(function(newResults){
return nightmare.goto(url)
.wait(5000)
.evaluate(() => {
const gigs = [];
$(".hidden-xs .used-vehicle").each(function() {
item = {};
item["year"] = $(this).attr("data-year");
item["make"] = $(this).attr("data-make");
item["model"] = $(this).attr("data-model");
item["body"] = $(this).attr("data-body");
item["color"] = $(this).attr("data-ext-color");
item["trim"] = $(this).attr("data-trim");
item["mileage"] = $(this).attr("data-mileage");
item["transmission"] = $(this).attr("data-transmission");
item["vin"] = $(this).find(".vehicle-overview").attr("id");
item["title"] = $(this).find(".vehicle-overview h2 a").text();
item["link"] = $(this).find(".vehicle-overview h2 a").attr("href");
item["price"] = $(this).find(".vehicle-content .price").text();
gigs.push(item);
});
return gigs;
})
.then(detail => {
for (gig in detail) {
try {
var carVin = detail[gig].vin;
var carTitle = detail[gig].title;
var carDescrip = detail[gig].year + " " + detail[gig].make + " " + detail[gig].model;
var carURL = detail[gig].link;
var carMake = detail[gig].make;
var carModel = detail[gig].model;
var carYear = detail[gig].year;
var carMileageFull = detail[gig].mileage;
var carMileage = carMileageFull.replace(',', '');
var carTransmission = detail[gig].transmission;
var carBody = detail[gig].body;
var carPriceFull = detail[gig].price;
var carPriceFull = carPriceFull.replace('$', '');
var carPriceFull = carPriceFull.replace('*', '');
var carPriceFull = carPriceFull.replace(',', '');
var carPrice = carPriceFull.trim();
var dealerAddress = "{addr1: '"+ addressFull.addr1 +"', city: '"+ addressFull.city +"', region: '"+ addressFull.region +"', postal_code: '"+ addressFull.postal_code +"', country: '"+ addressFull.country +"'}";
var dealerLat = latLongFull.latitude;
var dealerLong = latLongFull.longitude;
var carColor = detail[gig].color;
arrSetup = [carVin, carTitle, carDescrip, carURL, carMake, carModel, carYear, carMileage, 'MI', '', '', 'AUTOMATIC', 'GASOLINE', 'OTHER', 'Other', carVin, 'OTHER', carPrice + " USD", dealerAddress, carColor, carPrice + " USD", 'AVAILABLE', 'USED', dealerLat, dealerLong];
newResults.push(arrSetup);
}
catch(error){
returnString += error;
}
}
return newResults;
})
.catch(error => {
throw new Error(error);
});
});
}, Promise.resolve([]))
.then(function(finalCall){
/*
We need to get the 3rd image on every vdp in the array. We will need to create a loop, go to the page, get the image and properly insert it into the proper array index
*/
finalCall.reduce(function(accumulator, resultArray){
return accumulator.then(function(finalResults){
var vdp = resultArray[3];
return nightmare.goto(vdp)
.wait(500)
.evaluate(() => {
var thirdIMG = $('.gallery-thumbs .owl-item:nth-of-type(3) img').attr('src');
return thirdIMG;
})
.then(imgResult => {
// 9
resultArray.splice(9, 1, imgResult);
console.log(resultArray);
finalResults.push(resultArray);
return finalResults;
})
.catch(error => {
throw new Error(error);
});
});
}, Promise.resolve([]))
.then(finalInsert => {
const csvWriter = createCsvWriter({
header: ["vehicle_id", "title", "description", "url", "make", "model", "year", "mileage.value", "mileage.unit", "image[0].url", "image[0].tag[0]", "transmission", "fuel_type", "body_style", "drivetrain", "vin", "condition", "price", "address", "exterior_color", "sale_price", "availability", "state_of_vehicle", "latitude", "longitude"],
path: 'test.csv'
});
var records = finalInsert;
console.log(records)
csvWriter.writeRecords(records)
.then(() => {
nightmare.end();
console.log('...Done');
});
})
});
})
.catch(function(error){
return error;
})
}
getInventory(startURL, function(response){
try {
console.log("This is the response" + response);
}
catch(error){
console.log(error)
}
});

We Keep Coding

JavaScript is the programming language of the Web.

find cities and country name from string - javascript

Related

Fetch and display json data with javascript using reduce group sorting

Firebase cloud function Deadline exceeded error in nested promises

i want to make a search in which i can search upper case and lower case both at same time..?

ReplaceAll causing issues in array.reduce

Callback function Node JS

Categories

Resources