For loops breaks when using await

For loops breaks when using await - javascript

I have a problem when using await in a for loop. Every time it hits the await funtion it executes it fine, but it stops loping through the rest of the array that is looping through. I'm using nodejs with axios to send http request to a restAPI. The function that is in the api is big. So I thought maby that is the problem but it wasn't. Also the api uses its own await functions to, but that wasn't the problem either(As far as I know).
Here is my code for the request
var files = await globPromise("assets/series/**/*.json");
var json = null;
var file = null;
var series = [];
for (i = 0; i < files.length; i++) {
file = files[i];
var raw = fs.readFileSync(file);
json = JSON.parse(raw);
if (json.type === "series") {
try {
var userId = null;
if (req.user == null) {
userId = req.query.userid;
} else {
userId = req.user.id;
}
const response = await axios.get("http://" + host + "/series/info/" + json.id + "?userid=" + userId);
series.push(JSON.parse(response.data));
} catch (error) {
console.log(error);
series.push(json);
}
}
}
res.json(JSON.stringify(series));
});
And also the api side:
app.get('/series/info/:id', async(req, res) => {
var files = await globPromise("assets/series/**/*.json");
var json = null;
var file = null;
for (i = 0; i < files.length; i++) {
file = files[i];
var raw = fs.readFileSync(file);
json = JSON.parse(raw);
if (json.type === "series" && json.id === req.params.id) {
break;
}
json = null;
}
let path = pathFs.dirname(file) + "/";
try {
var seriesFiles = await fsPromise.readdir(path);
var latestWatchedVideo = null;
var latestWatchedTime = null;
var latestWatchTime = null;
var latestWatchDuration = null;
var seasonCount = 0;
var seasons = [];
for (i = 0; i < seriesFiles.length; i++) {
seriesFile = seriesFiles[i];
if (fs.lstatSync(path + "/" + seriesFile).isDirectory()) {
if (!seriesFile.startsWith("s")) {
continue;
}
seasonCount++;
try {
var videoFiles = await fsPromise.readdir(path + "/" + seriesFile + "/");
var videos = [];
for (let i = 0; i < videoFiles.length; i++) {
const video = videoFiles[i];
if (video.endsWith(".json")) {
var rawVideo = fs.readFileSync(path + "/" + seriesFile + "/" + video);
videoJson = JSON.parse(rawVideo);
const query = util.promisify(con.query).bind(con);
var userId = null;
if (req.user == null) {
userId = req.query.userid;
} else {
userId = req.user.id;
}
var results = await query(`SELECT * FROM watched WHERE video_id = "${videoJson.id}" AND user_id = "${userId}"`);
if (results.length > 0) {
var updated = JSON.parse(JSON.stringify(results[0].updated));
var duration = JSON.parse(JSON.stringify(results[0].duration));
var time = JSON.parse(JSON.stringify(results[0].time));
if (latestWatchedVideo == null) {
latestWatchedVideo = videoJson.id;
latestWatchedTime = updated;
latestWatchTime = time;
latestWatchDuration = duration;
} else {
if (latestWatchedTime < updated) {
latestWatchedVideo = videoJson.id;
latestWatchedTime = updated;
latestWatchTime = time;
latestWatchDuration = duration;
}
}
}
videos.push(videoJson);
}
}
function compare(a, b) {
if (a.episode < b.episode) {
return -1;
}
if (a.episode > b.episode) {
return 1;
}
return 0;
}
videos.sort(compare);
seasons.push({
season: seasonCount,
title: seriesFile.replace("s" + seasonCount, ""),
videos: videos
});
} catch (error) {
console.log(error);
}
}
}
json.seasonCount = seasonCount;
json.seasons = seasons;
json.latestWatchDuration = latestWatchDuration;
json.latestWatchTime = latestWatchTime;
json.latestWatchedVideo = latestWatchedVideo;
json.latestWatchedTime = latestWatchedTime;
res.json(JSON.stringify(json));
} catch (error) {
console.log(error);
}
});
Is there something (important) about await and async that I've missed?
Edit: my problem is that is loops fine through the first item and the await is working fine too, but it stops the loop and executes the next lines of code like there are no other items in my array.
Solved: I tried using the for/of and it works now. I don't know whats is so different between de default for and this one but it works!

Related

JSON.stringify not working with nested array of objects

I know this question has been asked but none of the solutions are working for me and I can't figure out what's wrong. I have an object with a nested array of objects I am stringifying but I get a blank array of the nested array when I use JSON.stringify on it.
This is a simplified version of the way I'm constructing the object. The main difference is that there is a for loop iterating through all the rows, here I am just manually creating 2 rows
// JAVASCRIPT
let obj = {};
obj['type'] = 'Setting';
obj['id'] = 1;
obj['import'] = parseCSV();
function parseCSV() {
let jsonData = [];
let row1 = {};
let row2 = {};
row1['date'] = '2022-01-01';
row1['amount'] = '30';
row2['date'] = '2022-01-02';
row2['amount'] = '50';
jsonData.push(row1);
jsonData.push(row2);
return jsonData;
}
console.log('RAW DATA', obj);
console.log('STRINGIFIED', JSON.stringify(obj));
The above outputs the correct stringified JSON
But the full version of my code gives me a blank array for import.
Both objects look identical to me. The culprit is somewhere in my parseCSV function, because when I use the simplified version above I get the correct stringified data, but I can't pinpoint where I'm wrong. Below is my full function.
function parseCSV(file) {
let filename = file.name;
let extension = filename.substring(filename.lastIndexOf('.')).toUpperCase();
if(extension == '.CSV') {
try {
let reader = new FileReader();
let jsonData = [];
let headers = [];
reader.readAsBinaryString(file);
reader.onload = function(e) {
let rows = e.target.result.split('\n');
for(let i = 0; i < rows.length; i++) {
let cells = rows[i].split(',');
let rowData = {};
for(let j = 0; j < cells.length; j++) {
if(i == 0) headers.push(cells[j].trim());
else {
if(headers[j]) rowData[headers[j]] = cells[j].trim();
}
}
if(i != 0 && rowData['date'] != '') jsonData.push(rowData);
}
}
return jsonData;
} catch(err) {
console.error('!! ERROR READING CSV FILE', err);
}
} else alert('PLEASE UPLOAD A VALID CSV FILE');*/
}
Thanks for the help!
EDIT
When I add await before parseCSV as #BJRINT's answer suggests I get a syntax error await is only valid in async function
async function submitForm(event) {
event.preventDefault();
let newItem = await gatherFormData(event.target);
return fetch('server.php', {
method: "POST",
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json'
},
body: JSON.stringify(newItem)
})
.then(checkError)
.then(data => parseData(data))
.catch(err => console.error('>> ERROR READING JSON DATA', err));
}
function gatherFormData(target) {
const inputs = target.querySelectorAll('input');
let obj = {};
inputs.forEach(function(input) {
if(intKeys.indexOf(input.name) >= 0) obj[input.name] = parseInt(input.value);
else if(curKeys.indexOf(input.name) >= 0) obj[input.name] = parseInt(parseFloat(input.value) * 100);
else if(chkKeys.indexOf(input.name) >= 0) input.checked ? obj[input.name] = 1 : obj[input.name] = 0;
else if(fileKeys.indexOf(input.name) >= 0 && input.files.length > 0) obj[input.name] = parseCSV(input.files[0]);
else obj[input.name] = input.value;
});
return obj;
}

The problem does not come from the stringify function. Since you are filling your array asynchronously (when the reader callback is executed) but returning your data first, it is empty.
You could wrap your function with a Promise that resolves when the reader callback function is finally executed, like so:
function parseCSV(file) {
return new Promise((resolve, reject) => {
let filename = file.name;
let extension = filename.substring(filename.lastIndexOf('.')).toUpperCase();
if(extension !== '.CSV')
return reject('PLEASE UPLOAD A VALID CSV FILE')
try {
let reader = new FileReader();
let jsonData = [];
let headers = [];
reader.readAsBinaryString(file);
reader.onload = function(e) {
let rows = e.target.result.split('\n');
for(let i = 0; i < rows.length; i++) {
let cells = rows[i].split(',');
let rowData = {};
for(let j = 0; j < cells.length; j++) {
if(i == 0) headers.push(cells[j].trim());
else {
if(headers[j]) rowData[headers[j]] = cells[j].trim();
}
}
if(i != 0 && rowData['date'] != '') jsonData.push(rowData);
}
return resolve(jsonData);
}
} catch(err) {
return reject('!! ERROR READING CSV FILE', err);
}
})
}
// calling the function
const data = await parseCSV(file)

The solution that worked for my specific case was to use a combination of BJRINT's answer and a timer to keep checking if the data had finished loading which I found here.
async function parseCSV(file) {
return await new Promise((resolve, reject) => {
let extension = file.name.substring(file.name.lastIndexOf('.')).toUpperCase();
if(extension !== '.CSV') reject('PLEASE UPLOAD A VALID CSV FILE');
try {
let reader = new FileReader();
reader.readAsText(file);
reader.onload = function(e) {
let jsonData = [];
let headers = [];
let rows = e.target.result.split(/\r\n|\r|\n/);
for(let i = 0; i < rows.length; i++) {
let cells = rows[i].split(',');
let rowData = {};
for(let j = 0; j < cells.length; j++) {
if(i == 0) headers.push(cells[j].trim());
else {
if(headers[j]) rowData[headers[j]] = cells[j].trim();
}
}
if(i != 0 && rowData['date'] != '') jsonData.push(rowData);
}
resolve(jsonData);
}
} catch(err) {
reject(err);
}
});
}
function submitForm(event) {
event.preventDefault();
showForm(false);
loading.classList.remove('hidden');
let ready = true;
const inputs = event.target.querySelectorAll('input');
let newItem = {};
let check = function() {
if(ready === true) {
console.log(newItem);
console.log(JSON.stringify(newItem));
return fetch('server.php', {
method: "POST",
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json'
},
body: JSON.stringify(newItem)
})
.then(checkError)
.then(data => parseData(data))
.catch(err => console.error('>> ERROR READING JSON DATA', err));
}
setTimeout(check, 1000);
}
inputs.forEach(function(input) {
if(intKeys.indexOf(input.name) >= 0) newItem[input.name] = parseInt(input.value);
else if(curKeys.indexOf(input.name) >= 0) newItem[input.name] = parseInt(parseFloat(input.value) * 100);
else if(chkKeys.indexOf(input.name) >= 0) input.checked ? newItem[input.name] = 1 : newItem[input.name] = 0;
else if(fileKeys.indexOf(input.name) >= 0 && input.files.length > 0) {
ready = false;
parseCSV(input.files[0]).then(data => {
ready = true;
newItem[input.name] = data;
});
}
else newItem[input.name] = input.value;
});
check();
}

Async/Await + Ajax .done: Returning undefined

Sorry if this is extremely simple, I can't get my head around it. I've seen similar questions but nothing which gets to the heart of my problem, I think. I have a simple async function:
async function isOpen() {
var open;
var data = $.ajax({
//Working code here
});
data.done(function(dat) {
var obj = JSON.parse(dat);
var msg = "";
for (var i = 0; i < obj.length; i++) {
let closingDate = Date.parse(obj[i].close);
let openingDate = Date.parse(obj[i].open);
if (closingDate > Date.now() && openingDate < Date.now()) {
open = true;
} else {
open = false;
}
}
});
return open;
}
I know that this code is all working - using console.log I have seen that open is always successfully assigned to true or false. So I call this async function from another async function:
async function testFunction(){
const open1 = await isOpen();
//More code here...
}
But open1 (in testFunction) is always undefined - even though I use await.
Any ideas what this could be?

async function isOpen() {
var open;
var data = await $.ajax({
//Working code here
});
var obj = JSON.parse(data);
var msg = "";
for (var i = 0; i < obj.length; i++) {
let closingDate = Date.parse(obj[i].close);
let openingDate = Date.parse(obj[i].open);
if (closingDate > Date.now() && openingDate < Date.now()) {
open = true;
} else {
open = false;
}
}
return open;
}
of course, the value of open will be determined by the last obj[i]
So you want
async function isOpen() {
const data = await $.ajax({
//Working code here
});
const obj = JSON.parse(data);
let msg = "";
for (var i = 0; i < obj.length; i++) {
const closingDate = Date.parse(obj[i].close);
const openingDate = Date.parse(obj[i].open);
if (closingDate > Date.now() && openingDate < Date.now()) {
return true;
}
}
return false;
}
Or even
async function isOpen() {
const data = await $.ajax({
//Working code here
});
const obj = JSON.parse(data);
let msg = "";
return obj.some(({open, close}) => {
const closingDate = Date.parse(close);
const openingDate = Date.parse(open);
const now = Date.now();
return closingDate > now && openingDate < now();
});
}

This is happening because you are using a callback for ajax request and not waiting for ajax request to complete, which is going to set open in isOpen. You can return a Promise to resolve this,
function isOpen() {
return new Promise((resolve, reject) => {
var open;
var data = $.ajax({
//Working code here
});
data.done(function(dat) {
var obj = JSON.parse(dat);
var msg = "";
for (var i = 0; i < obj.length; i++) {
let closingDate = Date.parse(obj[i].close);
let openingDate = Date.parse(obj[i].open);
if (closingDate > Date.now() && openingDate < Date.now()) {
open = true;
} else {
open = false;
}
}
resolve(open);
});
}
}
Note: For the above code, you need to handle the logic for errors in the callback, using reject otherwise it will hang forever.

node js await buffer

I am adding data to buffer with getOdds
getOdds = async(data) => {
send({"Command":"GetMatchMarkets","Params":data});
var message = JSON.stringify({"Command":"GetMatchMarkets","Params":data});
var length = Buffer.byteLength(message),
buffer = new Buffer(4 + Buffer.byteLength(message));
buffer.writeUInt32LE(length, 0);
buffer.write(message, 4);
client.write(buffer);
I WANT TO GET DATA FROM receiveCB getOdds
}
Read data with receiveBuff.
after.
I'm sending to receiveCB
function receiveBuff (buf) {
var offset = 0;
if (bytesToReceive === 0) {
if(buf.length < 4){ return; }
bytesToReceive = buf.readUInt32LE(0);
offset = 4;
}
var currentCommandBytes = Math.min(bytesToReceive, buf.length - offset);
receivedData += buf.slice(offset, offset + currentCommandBytes);
bytesToReceive -= currentCommandBytes;
if (bytesToReceive === 0) {
bytesToReceive = 0;
receiveCB(JSON.parse(receivedData));
receivedData = "";
}
if (currentCommandBytes < buf.length - offset) {
receiveBuff(buf.slice(currentCommandBytes+offset))
}
}
receiveCB = async(response) => {
return response;
}
I want to get data from receiveCB getOdds
Please Click image

How pagination works which is fetch from api in Nativescript

I am not getting the concepts of how pagination works in load more requested data from json api. Even i am getting the data of first page but for remaining pages not getting. If I am using the two api for paging and fetching data but still not getting. Any one help me clear this concepts please..
main-view-model.js
const httpModule = require("tns-core-modules/http");
var observableModule = require("tns-core-modules/data/observable");
function RegisterViewModel() {
var viewModel = observableModule.fromObject({
_sourceDataItems: [],
initDataItems: function () {
//This url for the pagination
var url1 = "https://example.org/api.php?action=query&gcmtitle=Category:xyz&pilimit=max&prop=pageimages&pithumbsize=200&generator=categorymembers&format=json&gcmcontinue=";
httpModule.getJSON(url1).then((r) => {
id = r.continue.gcmcontinue
gcm.push(id)
console.log(gcm)
var pageid = [];
for (var id in r.query.pages) {
pageid.push(id);
}
for (var i = 0; i < pageid.length; i++) {
if (r.query.pages[pageid[i]].thumbnail == null) {
let abc = {
ttt: r.query.pages[pageid[i]].title,
path1: "~/images/xyz.png"
}
this._sourceDataItems.push(abc)
}
else {
let aaa = {
ttt: r.query.pages[pageid[i]].title,
path1: r.query.pages[pageid[i]].thumbnail.source
}
this._sourceDataItems.push(aaa)
}
}
}, (e) => {
});
var gcm = [];
//This url for the fetching the data on scroll
for (var i = 0; i < gcm.length; i++) {
var url2 = "https://example.org/api.php?action=query&gcmtitle=Category:xyz&pilimit=max&prop=pageimages&pithumbsize=200&generator=categorymembers&format=json&gcmcontinue=" + gcm[i];
httpModule.getJSON(url2).then((r) => {
id= r.continue.gcmcontinue)
gcm.push(id)
console.log(r.continue.gcmcontinue)
var pageid = [];
for (var id in r.query.pages) {
pageid.push(id);
}
for (var i = 0; i < pageid.length; i++) {
//placeholder Image
if (r.query.pages[pageid[i]].thumbnail == null) {
let abc = {
ttt: r.query.pages[pageid[i]].title,
path1: "~/images/xyz.png"
}
this._sourceDataItems.push(abc)
}
else {
let aaa = {
ttt: r.query.pages[pageid[i]].title,
path1: r.query.pages[pageid[i]].thumbnail.source
}
this._sourceDataItems.push(aaa)
}
}
}, (e) => {
});
}
},
_DataItems: [],
addMoreItemsFromSource: function (chunkSize) {
let newItems = this._sourceDataItems.splice(0, chunkSize);
this._DataItems = this._DataItems.concat(newItems);
},
onLoadMoreItemsRequested: function (args) {
console.log("---load item called----");
const that = new WeakRef(this);
const listView = args.object;
if (this._sourceDataItems.length > 0) {
setTimeout(function () {
that.get().addMoreItemsFromSource(5);
listView.notifyLoadOnDemandFinished();
}, 100);
args.returnValue = true;
} else {
args.returnValue = false;
}
},
});
return viewModel;
}
exports.RegisterViewModel = RegisterViewModel;
Even loading of the data is working fine for page one. While passing the gcm continue id it wont take that please help me to clear this concept

Callback function Node JS

I'm recently diving into nodejs (and using nightmare.js) to parse a website and am having issues with the callback functions and displaying the returned results. I'm trying to call a separate function in another function, but can't seem to return any results. They all return undefined. Any help on this is greatly appreciated.
function getDetails(loadURL, callback){
nightmare.goto(loadURL)
.wait(2000)
.evaluate(function(){
var gigs = [];
$('.hidden-xs .used-vehicle').each(function(){
item = {}
item["year"] = $(this).attr('data-year')
item["make"] = $(this).attr('data-make')
item["model"] = $(this).attr('data-model')
item["body"] = $(this).attr('data-body')
item["color"] = $(this).attr('data-ext-color')
item["trim"] = $(this).attr('data-trim')
item["mileage"] = $(this).attr('data-mileage')
item["transmission"] = $(this).attr('data-transmission')
item["vin"] = $(this).find(".vehicle-overview").attr('id')
item["title"] = $(this).find(".vehicle-overview h2 a").text()
item["link"] = $(this).find(".vehicle-overview h2 a").attr('href')
item["price"] = $(this).find(".vehicle-content .price").text()
gigs.push(item)
})
return gigs
})
.end()
.then(function(result){
var returnString = '';
for(gig in result){
returnString = returnString + result[gig].title + " " + result[gig].link + " " + result[gig].year + " " + result[gig].make + " " + result[gig].model + " " + result[gig].body + " " + result[gig].color + " " + result[gig].trim + " " + result[gig].transmission + " " + result[gig].vin + " " + result[gig].price + "\n"
}
callback(returnString)
})
}
// We will need to get the total amount of pages that we need to parse
function getInventory(sURL, callback){
nightmare.goto(sURL)
.wait(2000)
.evaluate(function(){
totals = [];
items = {}
totalCars = $('.total-found .count').text()
carsOnPage = $('.hidden-xs .used-vehicle').size()
items['carTotal'] = totalCars
items['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
items['tPages'] = Math.ceil(pageCalc)
totals.push(items)
return totals
})
.end()
.then(function(result){
var totalCars = '';
var totalPages = '';
for (item in result){
totalPages = result[item].tPages
totalCars = result[item].carTotal
}
counter = 0;
newURL = '';
returnDetails = '';
for (i =0; i < totalPages; i++){
if (i == 0){
newURL = sURL;
} else {
counter = i + 1;
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
//console.log(newURL)
getINV = getDetails(newURL, function(returnString){
callback(returnString)
})
returnDetails = returnDetails + getINV
}
callback(returnDetails)
})
}
getInventory(startURL, function(result){
console.log(result)
})

I won't bother telling you that you should not mix callbacks with promises like that. But let's see the problem for now.
Case 1
How about you check for errors too? Maybe your script is throwing errors. I can see you are calling the callback on .then but nothing on .catch. Maybe then is never getting any data.
Case 2
Let's check your functions. You are calling .end every time. Are you creating new Nightmare instance everytime too?
On the getInventory function, you should not call .end. On the getDetails function, you should not call .end. It's ending the nightmare instances and you are losing your data.
Call nightmare.end() after you are done with all of your functions and works. To do this properly you will need to learn more about Promises check case 3 below.
Case 3
Learn how promises works. On the line below, you are never waiting for the function to finish.
getINV = getDetails(newURL, function(returnString){
callback(returnString)
})
You should wait for the promises to finish. Also, make sure nightmare is not trying to browse two links at same time.
So go ahead and learn about Promises and async await stuff.
How would I solve your code?
I would use Promise.all, .map and bunch of other new stuff. Here are some sample code done for you, don't copy paste or run the code directly, try to understand why it's different from your code and what can be the result of it.
const pLimit = require("promise-limit")(2);
function getDetails(loadURL) {
return nightmare
.goto(loadURL)
.wait(2000)
.evaluate(() => {
const gigs = [];
$(".hidden-xs .used-vehicle").each(function() {
item = {};
item["year"] = $(this).attr("data-year");
item["make"] = $(this).attr("data-make");
item["model"] = $(this).attr("data-model");
item["body"] = $(this).attr("data-body");
item["color"] = $(this).attr("data-ext-color");
item["trim"] = $(this).attr("data-trim");
item["mileage"] = $(this).attr("data-mileage");
item["transmission"] = $(this).attr("data-transmission");
item["vin"] = $(this)
.find(".vehicle-overview")
.attr("id");
item["title"] = $(this)
.find(".vehicle-overview h2 a")
.text();
item["link"] = $(this)
.find(".vehicle-overview h2 a")
.attr("href");
item["price"] = $(this)
.find(".vehicle-content .price")
.text();
gigs.push(item);
});
return gigs;
})
.then(result => {
let returnString = "";
for (gig in result) {
returnString =
`${returnString +
result[gig].title} ${result[gig].link} ${result[gig].year} ${result[gig].make} ${result[gig].model} ${result[gig].body} ${result[gig].color} ${result[gig].trim} ${result[gig].transmission} ${result[gig].vin} ${result[gig].price}\n`;
}
return returnString;
})
.catch(error => {
throw new Error(error);
});
}
// We will need to get the total amount of pages that we need to parse
function getInventory(sURL) {
return nightmare
.goto(sURL)
.wait(2000)
.evaluate(() => {
totals = [];
items = {};
totalCars = $(".total-found .count").text();
carsOnPage = $(".hidden-xs .used-vehicle").size();
items["carTotal"] = totalCars;
items["onPage"] = carsOnPage;
const pageCalc = totalCars / carsOnPage;
items["tPages"] = Math.ceil(pageCalc);
totals.push(items);
return totals;
})
.then(result => {
let totalCars = "";
let totalPages = "";
for (item in result) {
totalPages = result[item].tPages;
totalCars = result[item].carTotal;
}
counter = 0;
newURL = "";
urls = [];
returnDetails = [];
for (i = 0; i < totalPages; i++) {
if (i == 0) {
newURL = sURL;
} else {
counter = i + 1;
newURL =
`${sURL}#action=im_ajax_call&perform=get_results&_post_id=5&page=${counter}&show_all_filters=false`;
}
// push to the url array
// use .map for cleaner code
urls.push(newURL);
}
// return a new promise with concurrency limit
return Promise.all(
urls.map(url => {
return limit(() => getDetails(newURL));
})
);
})
.catch(error => {
throw new Error(error);
});
}
getInventory(startURL)
.then(result => {
console.log(result);
})
.catch(error => {
console.err(error);
});
Resources:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
https://javascript.info/async-await
https://ponyfoo.com/articles/understanding-javascript-async-await

Try using callback instead of return
function getDetails(loadURL, callback){
nightmare.goto(loadURL)
.wait(2000)
.evaluate(function(callback){
var gigs = [];
$('.hidden-xs .used-vehicle').each(function(){
item = {}
item["year"] = $(this).attr('data-year')
item["make"] = $(this).attr('data-make')
item["model"] = $(this).attr('data-model')
item["body"] = $(this).attr('data-body')
item["color"] = $(this).attr('data-ext-color')
item["trim"] = $(this).attr('data-trim')
item["mileage"] = $(this).attr('data-mileage')
item["transmission"] = $(this).attr('data-transmission')
item["vin"] = $(this).find(".vehicle-overview").attr('id')
item["title"] = $(this).find(".vehicle-overview h2 a").text()
item["link"] = $(this).find(".vehicle-overview h2 a").attr('href')
item["price"] = $(this).find(".vehicle-content .price").text()
gigs.push(item)
})
callback(gigs)
})
.end()
.then(function(result){
var returnString = '';
for(gig in result){
returnString = returnString + result[gig].title + " " + result[gig].link + " " + result[gig].year + " " + result[gig].make + " " + result[gig].model + " " + result[gig].body + " " + result[gig].color + " " + result[gig].trim + " " + result[gig].transmission + " " + result[gig].vin + " " + result[gig].price + "\n"
}
callback(returnString)
})
}
// We will need to get the total amount of pages that we need to parse
function getInventory(sURL, callback){
nightmare.goto(sURL)
.wait(2000)
.evaluate(function(){
totals = [];
items = {}
totalCars = $('.total-found .count').text()
carsOnPage = $('.hidden-xs .used-vehicle').size()
items['carTotal'] = totalCars
items['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
items['tPages'] = Math.ceil(pageCalc)
totals.push(items)
return totals
})
.end()
.then(function(result){
var totalCars = '';
var totalPages = '';
for (item in result){
totalPages = result[item].tPages
totalCars = result[item].carTotal
}
counter = 0;
newURL = '';
returnDetails = '';
for (i =0; i < totalPages; i++){
if (i == 0){
newURL = sURL;
} else {
counter = i + 1;
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
//console.log(newURL)
getINV = getDetails(newURL, function(returnString){
callback(returnString)
})
returnDetails = returnDetails + getINV
}
callback(returnDetails)
})
}
getInventory(startURL, function(result){
console.log(result)
})

Maybe the following will work, changed the loops to reduce and map, took out jQuery and made some small changes.
The most important ones are:
Getting an attribute from a html element returns a string, you should convert it to a number.
Got rid of the callbacks and have the functions return promises.
Here is the code:
const getDetails = loadURL =>
nightmare.goto(loadURL)//return promise here
.wait(2000)
.evaluate(
()=>
Array.from(document.querySelectorAll('.hidden-xs .used-vehicle'))
.reduce(
(all,item)=>
all.concat(
[
element.getAttribute('data-year'),
element.getAttribute('data-make'),
element.getAttribute('data-model'),
element.getAttribute('data-body'),
element.getAttribute('data-ext-color'),
element.getAttribute('data-trim'),
element.getAttribute('data-mileage'),
element.getAttribute('data-transmission'),
element.querySelector(".vehicle-overview").getAttribute('id'),
element.querySelector(".vehicle-overview h2 a").innerText,
element.querySelector(".vehicle-overview h2 a").getAttribute('href'),
element.querySelector(".vehicle-content .price").innerText
].join(" ")
),
[]//the all array
)
);
// We will need to get the total amount of pages that we need to parse
const getInventory = sURL =>
nightmare.goto(sURL)
.wait(2000)
.evaluate(
()=> {
//there is only one item here, not sure why you push it into totals
// and call it items
const item = {}
//getAttribute returns a string, parse it to number
totalCars = parseInt(document.querySelector('.total-found .count').innerText,10);
carsOnPage = document.querySelectorAll('.hidden-xs .used-vehicle').length;
item['carTotal'] = totalCars
item['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
item['tPages'] = Math.ceil(pageCalc)
return item;
}
)
.then(
totalItem =>{
var totalCars = '';
var totalPages = '';
totalPages = totalItem.tPages
totalCars = totalItem.carTotal
newURL = '';
returnDetails = '';
return Array.from(new Array(totalPages),(_,index)=>index+1)
.reduce(
(p,counter)=>
p.then(
results=>{
if (counter === 1) {
newURL = sURL;
} else {
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
return getDetails(newURL)
.then(
result=>results.concat(result)
);
}
),
Promise.resolve([])
);
}
);
getInventory(startURL)
.then(
result=>
console.log(result)
).catch(
err=>
console.warn("Something went wrong:",err)
);

Learning promises and callbacks with asynchronous functions is quite an undertaking. I was able to get this to work with the following. Thank you all for your help and direction. Each answer sent me down a different rabbit hole to ultimately find the solution.
function getInventory(sURL){
nightmare.goto(sURL)
.wait(2000)
.evaluate(function(){
totals = [];
items = {}
totalCars = $('.total-found .count').text()
carsOnPage = $('.hidden-xs .used-vehicle').size()
items['carTotal'] = totalCars
items['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
items['tPages'] = Math.ceil(pageCalc)
totals.push(items)
return totals
})
.then(result => {
var totalCars = '';
var totalPages = '';
for (item in result){
totalPages = result[item].tPages
totalCars = result[item].carTotal
}
counter = 0;
let links = [];
let returnLinks = '';
newURL = '';
for (i = 0; i < totalPages; i++){
if (i == 0){
newURL = sURL;
} else {
counter = i + 1;
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
links.push(newURL);
}
return links;
})
.then(results => {
var arrayLinks = results;
arrayLinks.reduce(function(accumulator, url){
return accumulator.then(function(newResults){
return nightmare.goto(url)
.wait(5000)
.evaluate(() => {
const gigs = [];
$(".hidden-xs .used-vehicle").each(function() {
item = {};
item["year"] = $(this).attr("data-year");
item["make"] = $(this).attr("data-make");
item["model"] = $(this).attr("data-model");
item["body"] = $(this).attr("data-body");
item["color"] = $(this).attr("data-ext-color");
item["trim"] = $(this).attr("data-trim");
item["mileage"] = $(this).attr("data-mileage");
item["transmission"] = $(this).attr("data-transmission");
item["vin"] = $(this).find(".vehicle-overview").attr("id");
item["title"] = $(this).find(".vehicle-overview h2 a").text();
item["link"] = $(this).find(".vehicle-overview h2 a").attr("href");
item["price"] = $(this).find(".vehicle-content .price").text();
gigs.push(item);
});
return gigs;
})
.then(detail => {
for (gig in detail) {
try {
var carVin = detail[gig].vin;
var carTitle = detail[gig].title;
var carDescrip = detail[gig].year + " " + detail[gig].make + " " + detail[gig].model;
var carURL = detail[gig].link;
var carMake = detail[gig].make;
var carModel = detail[gig].model;
var carYear = detail[gig].year;
var carMileageFull = detail[gig].mileage;
var carMileage = carMileageFull.replace(',', '');
var carTransmission = detail[gig].transmission;
var carBody = detail[gig].body;
var carPriceFull = detail[gig].price;
var carPriceFull = carPriceFull.replace('$', '');
var carPriceFull = carPriceFull.replace('*', '');
var carPriceFull = carPriceFull.replace(',', '');
var carPrice = carPriceFull.trim();
var dealerAddress = "{addr1: '"+ addressFull.addr1 +"', city: '"+ addressFull.city +"', region: '"+ addressFull.region +"', postal_code: '"+ addressFull.postal_code +"', country: '"+ addressFull.country +"'}";
var dealerLat = latLongFull.latitude;
var dealerLong = latLongFull.longitude;
var carColor = detail[gig].color;
arrSetup = [carVin, carTitle, carDescrip, carURL, carMake, carModel, carYear, carMileage, 'MI', '', '', 'AUTOMATIC', 'GASOLINE', 'OTHER', 'Other', carVin, 'OTHER', carPrice + " USD", dealerAddress, carColor, carPrice + " USD", 'AVAILABLE', 'USED', dealerLat, dealerLong];
newResults.push(arrSetup);
}
catch(error){
returnString += error;
}
}
return newResults;
})
.catch(error => {
throw new Error(error);
});
});
}, Promise.resolve([]))
.then(function(finalCall){
/*
We need to get the 3rd image on every vdp in the array. We will need to create a loop, go to the page, get the image and properly insert it into the proper array index
*/
finalCall.reduce(function(accumulator, resultArray){
return accumulator.then(function(finalResults){
var vdp = resultArray[3];
return nightmare.goto(vdp)
.wait(500)
.evaluate(() => {
var thirdIMG = $('.gallery-thumbs .owl-item:nth-of-type(3) img').attr('src');
return thirdIMG;
})
.then(imgResult => {
// 9
resultArray.splice(9, 1, imgResult);
console.log(resultArray);
finalResults.push(resultArray);
return finalResults;
})
.catch(error => {
throw new Error(error);
});
});
}, Promise.resolve([]))
.then(finalInsert => {
const csvWriter = createCsvWriter({
header: ["vehicle_id", "title", "description", "url", "make", "model", "year", "mileage.value", "mileage.unit", "image[0].url", "image[0].tag[0]", "transmission", "fuel_type", "body_style", "drivetrain", "vin", "condition", "price", "address", "exterior_color", "sale_price", "availability", "state_of_vehicle", "latitude", "longitude"],
path: 'test.csv'
});
var records = finalInsert;
console.log(records)
csvWriter.writeRecords(records)
.then(() => {
nightmare.end();
console.log('...Done');
});
})
});
})
.catch(function(error){
return error;
})
}
getInventory(startURL, function(response){
try {
console.log("This is the response" + response);
}
catch(error){
console.log(error)
}
});

We Keep Coding

JavaScript is the programming language of the Web.

For loops breaks when using await - javascript

Related

JSON.stringify not working with nested array of objects

Async/Await + Ajax .done: Returning undefined

node js await buffer

How pagination works which is fetch from api in Nativescript

Callback function Node JS

Categories

Resources