How can I do an async function over and over? I have tried doing it inside a while loop but it only does the very first line which is a console.log and nothing else.
import fs from 'fs-extra'
import fetch from 'node-fetch'
function wait(milliseconds) {
const date = Date.now();
let currentDate = null;
do {
currentDate = Date.now();
} while (currentDate - date < milliseconds);
}
async function gasFee() {
console.log("fetching ETH Price")
var ethprice = await fetch('https://api.coingecko.com/api/v3/simple/price?ids=ethereum&vs_currencies=usd')
var ethPriceJSON = await ethprice.json()
console.log("fetching Ethermine GWEI")
var etherminegwei = await fetch('https://api.ethermine.org/poolStats')
var ethermineGweiJSON = await etherminegwei.json()
var ethPrice = ethPriceJSON.ethereum.usd
var ethermineGwei = ethermineGweiJSON.data.estimates.gasPrice
var gweiPrice = ethPrice/1000000000
var price = ethermineGwei * gweiPrice * 21000 .toFixed(2)
var timeNow = new Date()
if (price > 5) {
console.log("Gas Price Logged")
fs.appendFileSync('gasPrice.txt', '$' + price + ' | ' + timeNow + '\r\n')
}
else {return}
if (price <= 5) {
console.log(`Gas Price is $${price} at ${timeNow}`)
fs.appendFileSync('lowGasPrice.txt', '$' + price + ' | ' + timeNow + '\r\n')
}
else {return}
}
while (true) {
gasFee()
wait(1500)
}
Your wait function is not a promise-based async function and you need to change it.
Also, you need to await your getFee() function to make an async execution.
import fs from "fs-extra";
import fetch from "node-fetch";
const wait = ms => new Promise((resolve, reject) => setTimeout(resolve, ms));
async function gasFee() {
console.log("fetching ETH Price");
var ethprice = await fetch(
"https://api.coingecko.com/api/v3/simple/price?ids=ethereum&vs_currencies=usd"
);
var ethPriceJSON = await ethprice.json();
console.log("fetching Ethermine GWEI");
var etherminegwei = await fetch("https://api.ethermine.org/poolStats");
var ethermineGweiJSON = await etherminegwei.json();
var ethPrice = ethPriceJSON.ethereum.usd;
var ethermineGwei = ethermineGweiJSON.data.estimates.gasPrice;
var gweiPrice = ethPrice / 1000000000;
var price = ethermineGwei * gweiPrice * (21000).toFixed(2);
var timeNow = new Date();
if (price > 5) {
console.log("Gas Price Logged");
fs.appendFileSync("gasPrice.txt", "$" + price + " | " + timeNow + "\r\n");
} else {
return;
}
if (price <= 5) {
console.log(`Gas Price is $${price} at ${timeNow}`);
fs.appendFileSync(
"lowGasPrice.txt",
"$" + price + " | " + timeNow + "\r\n"
);
} else {
return;
}
}
(async function run() {
while (true) {
await gasFee();
await wait(1500);
}
})();
To compliment the accepted answer, you might consider using the built-in javascript function setInterval().
This takes a function as a callback, which is executed each x milliseconds. The function returns an ID, that can be used to cancel the interval later:
var gasFee = function () {
console.log("fetching ETH Price");
// ... Rest of function
}
// Call gasFee() every 1500 MS
var gasFeeIntervalID = setInterval(gasFee, 1500);
// Cancel execution if needed
clearInterval(gasFeeIntervalID);
I'm working on setting cookies for a website and I've been having trouble getting it right. When I check DevTools on my browser (Chrome), I'm always getting messages about the SameSite attribute being unspecified. Would anyone be able to help out?
const dropCookie = true;
const cookieDays = 14;
const cookieName = "Compliance";
const cookieState = "on";
const banner = document.getElementById("cookie-banner");
const displayBanner = () => {
const main = document.getElementsByTagName("main")[0];
banner.style.display = "flex";
main.className += " cookie-banner";
createCookie(window.cookieName, window.cookieState, window.cookieDays);
}
const createCookie = (name, value, days) => {
let cookie = name + "=" + encodeURIComponent(value);
if (typeof days === "number") {
const date = new Date();
date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000));
const expires = "expires=" + date.toUTCString();
}
if (window.dropCookie) {
cookie = `${name}=${value}; ${expires}; path=/; secure; samesite=none`
document.cookie = cookie;
}
}
function checkCookie(name) {
const nameEq = `${name}=`;
const cookieArr = document.cookie.split(";");
for (let i = 0; i < cookieArr.length; i++) {
let c = cookieArr[i];
while (c.charAt(0) == " ") c = c.substring(1, c.length);
if (c.indexOf(nameEq) == 0) return c.substring(nameEq.length, c.length);
}
return null;
}
function eraseCookie(name) {
createCookie(name, "", -1);
}
window.onload = () => {
if (checkCookie(window.cookieName) != window.cookieState) {
displayBanner();
}
}
I found that there were several reference errors being thrown from my code. I had to correct the instances where I referenced variables as properties of window, and also fix the Unix to UTC time conversion in createCookie.
const dropCookie = true;
const cookieDays = 14;
const cookieName = "compliance";
const cookieState = "on";
const banner = document.getElementById("cookie-banner");
const displayBanner = () => {
const main = document.getElementsByTagName("main")[0];
banner.style.display = "flex";
main.className += " cookie-banner";
createCookie(cookieName, cookieState, cookieDays);
}
const closeBanner = () => {
banner.style.display = "none";
}
const createCookie = (name, value, days) => {
let cookie = name + "=" + encodeURIComponent(value);
if (typeof days === "number") {
let date = new Date();
let unixDate = date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000));
date = new Date(unixDate);
const expires = "expires=" + date.toUTCString();
if (dropCookie) {
document.cookie = `${name}=${value}; ${expires}; path=/; secure; samesite=lax`
}
}
}
Lastly, I fixed the window.onload statement by saving the function to a variable windowInit, then calling it in window.onload.
const windowInit = () => {
if (checkCookie(cookieName) != cookieState) {
displayBanner();
}
}
window.onload = windowInit();
I am trying to access my API with a loop as this is one of my requirements. The code i have written is below.
currentWeekStartDate = (new Date(requestStartDate));
currentWeekEndDate = (new Date(requestEndDate));
let currentWeekStartDateNew = (new Date(requestStartDate));
let currentWeekEndDateNew = (new Date(requestEndDate));
console.log("testing ZONE");
/*test zone */
let outerEventArray = [];
while (currentWeekStartDateNew.getDate() !==currentWeekEndDateNew.getDate()){
console.log(currentWeekStartDateNew);
let startTIme = new Date(currentWeekStartDateNew);
let endTime = new Date(currentWeekEndDateNew);
startTIme.setHours(startHour,startMinute,0,0);
endTime.setHours(endHour,endMinute,0,0)
while(startTIme.getHours() !== endTime.getHours()){
let newStartTime = new Date(startTIme);
newStartTime.setHours(startTIme.getHours() + 1);
let firstTimeArg = startTIme.toISOString().replace("Z", "0000");
let secondTimeArg = newStartTime.toISOString().replace("Z", "0000");
let returnVal = getEventArray(access_token, firstTimeArg,secondTimeArg).then(res => {
return res;
});
if(returnVal !== null){
console.log("returnVal");
console.log(returnVal);
outerEventArray.push(returnVal);
console.log("returnVal");
}
/*
function
*/
startTIme = new Date(newStartTime);
}
currentWeekStartDateNew.setDate(currentWeekStartDateNew.getDate() + 1);
}
console.log("outerEventArray");
console.log(outerEventArray);
console.log("outerEventArray");
and my function for the API call is
getEventArray = async(access_token, startArguement,endArguement) => {
let eventVal;
let urlVal = "https://graph.microsoft.com/v1.0/users/" + user + "/calendarview?startdatetime=" + startArguement + "&enddatetime=" + endArguement + "";
request.get({
url: urlVal,
headers: {
"Authorization": "Bearer " + access_token
}
}, function (err, response, body) {
let bodyJSON = JSON.parse(body);
eventVal = bodyJSON.value;
if(eventVal == null){
console.log("eventVal is null");
}
else{
let eventArray = [];
for (let valueVal of Object.values(eventVal)) {
let eventStartTime = valueVal.start.dateTime;
let eventEndTime = valueVal.end.dateTime;
if (!((eventStartTime[eventStartTime.length - 1] === 'Z') || (eventStartTime[eventStartTime.length - 1] === 'z'))) {
eventStartTime = eventStartTime + "Z";
}
if (!((eventEndTime[eventEndTime.length - 1] === 'Z') || (eventEndTime[eventEndTime.length - 1]) === 'z')) {
eventEndTime = eventEndTime + "z"
}
eventArray.push({ start: new Date(eventStartTime).toLocaleString(), end: new Date(eventEndTime).toLocaleString() });
}
console.log(eventArray);
return eventArray;
}
});
}
However i am not able to get the outerEventArray value outside. My requirement is to access is outside with all the data populated. Is there any way this is possible?
I'm recently diving into nodejs (and using nightmare.js) to parse a website and am having issues with the callback functions and displaying the returned results. I'm trying to call a separate function in another function, but can't seem to return any results. They all return undefined. Any help on this is greatly appreciated.
function getDetails(loadURL, callback){
nightmare.goto(loadURL)
.wait(2000)
.evaluate(function(){
var gigs = [];
$('.hidden-xs .used-vehicle').each(function(){
item = {}
item["year"] = $(this).attr('data-year')
item["make"] = $(this).attr('data-make')
item["model"] = $(this).attr('data-model')
item["body"] = $(this).attr('data-body')
item["color"] = $(this).attr('data-ext-color')
item["trim"] = $(this).attr('data-trim')
item["mileage"] = $(this).attr('data-mileage')
item["transmission"] = $(this).attr('data-transmission')
item["vin"] = $(this).find(".vehicle-overview").attr('id')
item["title"] = $(this).find(".vehicle-overview h2 a").text()
item["link"] = $(this).find(".vehicle-overview h2 a").attr('href')
item["price"] = $(this).find(".vehicle-content .price").text()
gigs.push(item)
})
return gigs
})
.end()
.then(function(result){
var returnString = '';
for(gig in result){
returnString = returnString + result[gig].title + " " + result[gig].link + " " + result[gig].year + " " + result[gig].make + " " + result[gig].model + " " + result[gig].body + " " + result[gig].color + " " + result[gig].trim + " " + result[gig].transmission + " " + result[gig].vin + " " + result[gig].price + "\n"
}
callback(returnString)
})
}
// We will need to get the total amount of pages that we need to parse
function getInventory(sURL, callback){
nightmare.goto(sURL)
.wait(2000)
.evaluate(function(){
totals = [];
items = {}
totalCars = $('.total-found .count').text()
carsOnPage = $('.hidden-xs .used-vehicle').size()
items['carTotal'] = totalCars
items['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
items['tPages'] = Math.ceil(pageCalc)
totals.push(items)
return totals
})
.end()
.then(function(result){
var totalCars = '';
var totalPages = '';
for (item in result){
totalPages = result[item].tPages
totalCars = result[item].carTotal
}
counter = 0;
newURL = '';
returnDetails = '';
for (i =0; i < totalPages; i++){
if (i == 0){
newURL = sURL;
} else {
counter = i + 1;
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
//console.log(newURL)
getINV = getDetails(newURL, function(returnString){
callback(returnString)
})
returnDetails = returnDetails + getINV
}
callback(returnDetails)
})
}
getInventory(startURL, function(result){
console.log(result)
})
I won't bother telling you that you should not mix callbacks with promises like that. But let's see the problem for now.
Case 1
How about you check for errors too? Maybe your script is throwing errors. I can see you are calling the callback on .then but nothing on .catch. Maybe then is never getting any data.
Case 2
Let's check your functions. You are calling .end every time. Are you creating new Nightmare instance everytime too?
On the getInventory function, you should not call .end. On the getDetails function, you should not call .end. It's ending the nightmare instances and you are losing your data.
Call nightmare.end() after you are done with all of your functions and works. To do this properly you will need to learn more about Promises check case 3 below.
Case 3
Learn how promises works. On the line below, you are never waiting for the function to finish.
getINV = getDetails(newURL, function(returnString){
callback(returnString)
})
You should wait for the promises to finish. Also, make sure nightmare is not trying to browse two links at same time.
So go ahead and learn about Promises and async await stuff.
How would I solve your code?
I would use Promise.all, .map and bunch of other new stuff. Here are some sample code done for you, don't copy paste or run the code directly, try to understand why it's different from your code and what can be the result of it.
const pLimit = require("promise-limit")(2);
function getDetails(loadURL) {
return nightmare
.goto(loadURL)
.wait(2000)
.evaluate(() => {
const gigs = [];
$(".hidden-xs .used-vehicle").each(function() {
item = {};
item["year"] = $(this).attr("data-year");
item["make"] = $(this).attr("data-make");
item["model"] = $(this).attr("data-model");
item["body"] = $(this).attr("data-body");
item["color"] = $(this).attr("data-ext-color");
item["trim"] = $(this).attr("data-trim");
item["mileage"] = $(this).attr("data-mileage");
item["transmission"] = $(this).attr("data-transmission");
item["vin"] = $(this)
.find(".vehicle-overview")
.attr("id");
item["title"] = $(this)
.find(".vehicle-overview h2 a")
.text();
item["link"] = $(this)
.find(".vehicle-overview h2 a")
.attr("href");
item["price"] = $(this)
.find(".vehicle-content .price")
.text();
gigs.push(item);
});
return gigs;
})
.then(result => {
let returnString = "";
for (gig in result) {
returnString =
`${returnString +
result[gig].title} ${result[gig].link} ${result[gig].year} ${result[gig].make} ${result[gig].model} ${result[gig].body} ${result[gig].color} ${result[gig].trim} ${result[gig].transmission} ${result[gig].vin} ${result[gig].price}\n`;
}
return returnString;
})
.catch(error => {
throw new Error(error);
});
}
// We will need to get the total amount of pages that we need to parse
function getInventory(sURL) {
return nightmare
.goto(sURL)
.wait(2000)
.evaluate(() => {
totals = [];
items = {};
totalCars = $(".total-found .count").text();
carsOnPage = $(".hidden-xs .used-vehicle").size();
items["carTotal"] = totalCars;
items["onPage"] = carsOnPage;
const pageCalc = totalCars / carsOnPage;
items["tPages"] = Math.ceil(pageCalc);
totals.push(items);
return totals;
})
.then(result => {
let totalCars = "";
let totalPages = "";
for (item in result) {
totalPages = result[item].tPages;
totalCars = result[item].carTotal;
}
counter = 0;
newURL = "";
urls = [];
returnDetails = [];
for (i = 0; i < totalPages; i++) {
if (i == 0) {
newURL = sURL;
} else {
counter = i + 1;
newURL =
`${sURL}#action=im_ajax_call&perform=get_results&_post_id=5&page=${counter}&show_all_filters=false`;
}
// push to the url array
// use .map for cleaner code
urls.push(newURL);
}
// return a new promise with concurrency limit
return Promise.all(
urls.map(url => {
return limit(() => getDetails(newURL));
})
);
})
.catch(error => {
throw new Error(error);
});
}
getInventory(startURL)
.then(result => {
console.log(result);
})
.catch(error => {
console.err(error);
});
Resources:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
https://javascript.info/async-await
https://ponyfoo.com/articles/understanding-javascript-async-await
Try using callback instead of return
function getDetails(loadURL, callback){
nightmare.goto(loadURL)
.wait(2000)
.evaluate(function(callback){
var gigs = [];
$('.hidden-xs .used-vehicle').each(function(){
item = {}
item["year"] = $(this).attr('data-year')
item["make"] = $(this).attr('data-make')
item["model"] = $(this).attr('data-model')
item["body"] = $(this).attr('data-body')
item["color"] = $(this).attr('data-ext-color')
item["trim"] = $(this).attr('data-trim')
item["mileage"] = $(this).attr('data-mileage')
item["transmission"] = $(this).attr('data-transmission')
item["vin"] = $(this).find(".vehicle-overview").attr('id')
item["title"] = $(this).find(".vehicle-overview h2 a").text()
item["link"] = $(this).find(".vehicle-overview h2 a").attr('href')
item["price"] = $(this).find(".vehicle-content .price").text()
gigs.push(item)
})
callback(gigs)
})
.end()
.then(function(result){
var returnString = '';
for(gig in result){
returnString = returnString + result[gig].title + " " + result[gig].link + " " + result[gig].year + " " + result[gig].make + " " + result[gig].model + " " + result[gig].body + " " + result[gig].color + " " + result[gig].trim + " " + result[gig].transmission + " " + result[gig].vin + " " + result[gig].price + "\n"
}
callback(returnString)
})
}
// We will need to get the total amount of pages that we need to parse
function getInventory(sURL, callback){
nightmare.goto(sURL)
.wait(2000)
.evaluate(function(){
totals = [];
items = {}
totalCars = $('.total-found .count').text()
carsOnPage = $('.hidden-xs .used-vehicle').size()
items['carTotal'] = totalCars
items['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
items['tPages'] = Math.ceil(pageCalc)
totals.push(items)
return totals
})
.end()
.then(function(result){
var totalCars = '';
var totalPages = '';
for (item in result){
totalPages = result[item].tPages
totalCars = result[item].carTotal
}
counter = 0;
newURL = '';
returnDetails = '';
for (i =0; i < totalPages; i++){
if (i == 0){
newURL = sURL;
} else {
counter = i + 1;
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
//console.log(newURL)
getINV = getDetails(newURL, function(returnString){
callback(returnString)
})
returnDetails = returnDetails + getINV
}
callback(returnDetails)
})
}
getInventory(startURL, function(result){
console.log(result)
})
Maybe the following will work, changed the loops to reduce and map, took out jQuery and made some small changes.
The most important ones are:
Getting an attribute from a html element returns a string, you should convert it to a number.
Got rid of the callbacks and have the functions return promises.
Here is the code:
const getDetails = loadURL =>
nightmare.goto(loadURL)//return promise here
.wait(2000)
.evaluate(
()=>
Array.from(document.querySelectorAll('.hidden-xs .used-vehicle'))
.reduce(
(all,item)=>
all.concat(
[
element.getAttribute('data-year'),
element.getAttribute('data-make'),
element.getAttribute('data-model'),
element.getAttribute('data-body'),
element.getAttribute('data-ext-color'),
element.getAttribute('data-trim'),
element.getAttribute('data-mileage'),
element.getAttribute('data-transmission'),
element.querySelector(".vehicle-overview").getAttribute('id'),
element.querySelector(".vehicle-overview h2 a").innerText,
element.querySelector(".vehicle-overview h2 a").getAttribute('href'),
element.querySelector(".vehicle-content .price").innerText
].join(" ")
),
[]//the all array
)
);
// We will need to get the total amount of pages that we need to parse
const getInventory = sURL =>
nightmare.goto(sURL)
.wait(2000)
.evaluate(
()=> {
//there is only one item here, not sure why you push it into totals
// and call it items
const item = {}
//getAttribute returns a string, parse it to number
totalCars = parseInt(document.querySelector('.total-found .count').innerText,10);
carsOnPage = document.querySelectorAll('.hidden-xs .used-vehicle').length;
item['carTotal'] = totalCars
item['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
item['tPages'] = Math.ceil(pageCalc)
return item;
}
)
.then(
totalItem =>{
var totalCars = '';
var totalPages = '';
totalPages = totalItem.tPages
totalCars = totalItem.carTotal
newURL = '';
returnDetails = '';
return Array.from(new Array(totalPages),(_,index)=>index+1)
.reduce(
(p,counter)=>
p.then(
results=>{
if (counter === 1) {
newURL = sURL;
} else {
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
return getDetails(newURL)
.then(
result=>results.concat(result)
);
}
),
Promise.resolve([])
);
}
);
getInventory(startURL)
.then(
result=>
console.log(result)
).catch(
err=>
console.warn("Something went wrong:",err)
);
Learning promises and callbacks with asynchronous functions is quite an undertaking. I was able to get this to work with the following. Thank you all for your help and direction. Each answer sent me down a different rabbit hole to ultimately find the solution.
function getInventory(sURL){
nightmare.goto(sURL)
.wait(2000)
.evaluate(function(){
totals = [];
items = {}
totalCars = $('.total-found .count').text()
carsOnPage = $('.hidden-xs .used-vehicle').size()
items['carTotal'] = totalCars
items['onPage'] = carsOnPage
var pageCalc = (totalCars / carsOnPage)
items['tPages'] = Math.ceil(pageCalc)
totals.push(items)
return totals
})
.then(result => {
var totalCars = '';
var totalPages = '';
for (item in result){
totalPages = result[item].tPages
totalCars = result[item].carTotal
}
counter = 0;
let links = [];
let returnLinks = '';
newURL = '';
for (i = 0; i < totalPages; i++){
if (i == 0){
newURL = sURL;
} else {
counter = i + 1;
newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
}
links.push(newURL);
}
return links;
})
.then(results => {
var arrayLinks = results;
arrayLinks.reduce(function(accumulator, url){
return accumulator.then(function(newResults){
return nightmare.goto(url)
.wait(5000)
.evaluate(() => {
const gigs = [];
$(".hidden-xs .used-vehicle").each(function() {
item = {};
item["year"] = $(this).attr("data-year");
item["make"] = $(this).attr("data-make");
item["model"] = $(this).attr("data-model");
item["body"] = $(this).attr("data-body");
item["color"] = $(this).attr("data-ext-color");
item["trim"] = $(this).attr("data-trim");
item["mileage"] = $(this).attr("data-mileage");
item["transmission"] = $(this).attr("data-transmission");
item["vin"] = $(this).find(".vehicle-overview").attr("id");
item["title"] = $(this).find(".vehicle-overview h2 a").text();
item["link"] = $(this).find(".vehicle-overview h2 a").attr("href");
item["price"] = $(this).find(".vehicle-content .price").text();
gigs.push(item);
});
return gigs;
})
.then(detail => {
for (gig in detail) {
try {
var carVin = detail[gig].vin;
var carTitle = detail[gig].title;
var carDescrip = detail[gig].year + " " + detail[gig].make + " " + detail[gig].model;
var carURL = detail[gig].link;
var carMake = detail[gig].make;
var carModel = detail[gig].model;
var carYear = detail[gig].year;
var carMileageFull = detail[gig].mileage;
var carMileage = carMileageFull.replace(',', '');
var carTransmission = detail[gig].transmission;
var carBody = detail[gig].body;
var carPriceFull = detail[gig].price;
var carPriceFull = carPriceFull.replace('$', '');
var carPriceFull = carPriceFull.replace('*', '');
var carPriceFull = carPriceFull.replace(',', '');
var carPrice = carPriceFull.trim();
var dealerAddress = "{addr1: '"+ addressFull.addr1 +"', city: '"+ addressFull.city +"', region: '"+ addressFull.region +"', postal_code: '"+ addressFull.postal_code +"', country: '"+ addressFull.country +"'}";
var dealerLat = latLongFull.latitude;
var dealerLong = latLongFull.longitude;
var carColor = detail[gig].color;
arrSetup = [carVin, carTitle, carDescrip, carURL, carMake, carModel, carYear, carMileage, 'MI', '', '', 'AUTOMATIC', 'GASOLINE', 'OTHER', 'Other', carVin, 'OTHER', carPrice + " USD", dealerAddress, carColor, carPrice + " USD", 'AVAILABLE', 'USED', dealerLat, dealerLong];
newResults.push(arrSetup);
}
catch(error){
returnString += error;
}
}
return newResults;
})
.catch(error => {
throw new Error(error);
});
});
}, Promise.resolve([]))
.then(function(finalCall){
/*
We need to get the 3rd image on every vdp in the array. We will need to create a loop, go to the page, get the image and properly insert it into the proper array index
*/
finalCall.reduce(function(accumulator, resultArray){
return accumulator.then(function(finalResults){
var vdp = resultArray[3];
return nightmare.goto(vdp)
.wait(500)
.evaluate(() => {
var thirdIMG = $('.gallery-thumbs .owl-item:nth-of-type(3) img').attr('src');
return thirdIMG;
})
.then(imgResult => {
// 9
resultArray.splice(9, 1, imgResult);
console.log(resultArray);
finalResults.push(resultArray);
return finalResults;
})
.catch(error => {
throw new Error(error);
});
});
}, Promise.resolve([]))
.then(finalInsert => {
const csvWriter = createCsvWriter({
header: ["vehicle_id", "title", "description", "url", "make", "model", "year", "mileage.value", "mileage.unit", "image[0].url", "image[0].tag[0]", "transmission", "fuel_type", "body_style", "drivetrain", "vin", "condition", "price", "address", "exterior_color", "sale_price", "availability", "state_of_vehicle", "latitude", "longitude"],
path: 'test.csv'
});
var records = finalInsert;
console.log(records)
csvWriter.writeRecords(records)
.then(() => {
nightmare.end();
console.log('...Done');
});
})
});
})
.catch(function(error){
return error;
})
}
getInventory(startURL, function(response){
try {
console.log("This is the response" + response);
}
catch(error){
console.log(error)
}
});
I have made an Azure function with 3 calls to db. But as they work async only the first item gets added.
My code below. So to be exact Context.Done but first, the calls have to be finished before going to the next call hope someone can help me.
It seems it has to do with the Azure function context as promises did not work either.
const rp = require('request-promise');
const azure = require("azure-storage");
var feed = require('feed-read');
const env = require('dotenv').config();
const con = "storageconnection";
function guid() {
return s4() + s4() + '-' + s4() + '-' + s4() + '-' +
s4() + '-' + s4() + s4() + s4();
}
function CreateEntitie(item) {
var link = item.link;
var team = link.substr(39, link.length);
team = team.substr(0, team.indexOf('/'));
var newlink = link.substr(0, link.lastIndexOf('#'));
var pubdate = item.published;
var titel = item.title;
var Entity = {
PartitionKey: { '_': '' },
RowKey: { '_': '0' },
Gelezen: { "_": false },
Team: { "_": team },
Titel: { "_": item.title },
Com_url: { "_": newlink },
pubdate: { "_": item.published, '$': 'Edm.DateTime' }
}
return Entity;
}
function s4() {
return Math.floor((1 + Math.random()) * 0x10000)
.toString(16)
.substring(1);
}
function getrownummer(azure, tableSvc, entitie) {
for (var x = 0; x < entitie.length; x++) {
var query = new azure.TableQuery()
.where('com_url eq ?', entitie[x].Com_url._);
//check if exists and update
tableSvc.queryEntities('Posts', query, null, function (error, posts, response) {
if (!error) {
//update
var rownummer = "0";
for (var a = 0; a < posts.length; a++) {
rownummer = posts[a].RowKey._;
}
if (rownummer === "0") {
rownummer = guid();
}
entitie[x].RowKey._ = rownummer;
}
});
}
return entitie;
}
function InsertorReplaceItem(tableSvc, entitie) {
var returnvalue;
tableSvc.insertOrReplaceEntity('posts', entitie, function (error, inputed, response) {
if (!error) {
console.log(entitie.Com_url._ + "rownummer=" + entitie.RowKey._);
// Entity updated
return true;
}
else {
return false;
}
});
}
module.exports = function (context, myTimer) {
var timeStamp = new Date().toISOString();
if (myTimer.isPastDue) {
context.log('JavaScript is running late');
}
tableSvc = azure.createTableService(con);
tableSvc.queryEntities('rssfeeds', null, null, function (error, result, response) {
if (error) {
context.log('well that didn\'t work: ' + err.stack);
context.done();
} else {
context.log(result)
};
if (result) {
var i = 1;
var lastupdates = []
feed(result.entries[i].rssurl._, function (err, articles) {
if (err) throw err;
var newdate = new Date(articles[0].published).getTime();
for (var x = 0; x < articles.length; x++) {
//console.log(articles[x].title)
var from = new Date(articles[x].published).getTime();
var to = new Date(result.entries[i].Lastupdate._).getTime();
if (from >= to) {
var entitie = null;
var entitie = CreateEntitie(articles[x]);
lastupdates.push(entitie);
// context.log(entitie.titel);
}
if (newdate <= from) {
newdate = from;
}
}
context.log(lastupdates);
var completeArticle = [];
completeArticle = getrownummer(azure, tableSvc, lastupdates);
context.log(completeArticle);
if (completeArticle) {
for (var x = 0; x < completeArticle.length; x++) {
var endresult = InsertorReplaceItem(tableSvc, completeArticle[x]);
// context.log(endresult);
}
if (endresult) {
context.done;
}
}
//update date to latest date
});
// }
}
});
}