We Keep Coding

Javascript fetch to variable issues

Mediocre javascript developer here and need some help..
I want to make a GET call to a url several times in a for loop.
I am trying to use fetch, but because of the promise/timing situation, I can't figure out how to make it work.
Below is the code and it's entirely possible that the fetch approach doesn't make sense for what I am trying to do. Would appreciate your help either helping me with code or telling me I am idiot and advising an alternative :)
var fromAmt = 100;
var fromOOP = 50;
var fromGM = 50;
var fromCur = "USD"
var toCur = ["USD","EUR","INR","GBP","SGD"];
var adjAmt = [];
async function getConversionAsync(fcur,tcur,amt)
{
let response = await fetch('https://data.fixer.io/api/convert?access_key=xyxyxyxyxyxyxy&from=' + fcur + '&to=' + tcur + '&amount=' + amt);
let data = await response.json()
return data;
}
for (i = 0; i < toCur.length; i++) {
getConversionAsync(fromCur,toCur[0].toString(),fromAmt)
.then(data => display(data));
}
function display(thing){
adjAmt.push(thing.result);
}
document.getElementById("something").innerHTML = adjAmt[0].toString();

In your example, document.getElementById("something").innerHTML = adjAmt[0].toString(); is executed before anything is pushed to adjAmt. You need to wait for the loop calls to finish before displaying a result, and for this you could wrap everything inside an async function.
const fromAmt = 100;
const fromOOP = 50;
const fromGM = 50;
const fromCur = 'USD';
const toCur = ['USD', 'EUR', 'INR', 'GBP', 'SGD'];
const adjAmt = [];
const getConversionAsync = async (fcur, tcur, amt) => {
const response = await fetch(`https://data.fixer.io/api/convert?access_key=xyxyxyxyxyxyxy&from=${fcur}&to=${tcur}&amount=${amt}`);
return response.json();
}
function display(thing) {
adjAmt.push(thing.result);
}
(async () => {
for (i = 0; i < toCur.length; i += 1) {
const data = await getConversionAsync(fromCur, toCur[0], fromAmt);
display(data);
}
document.getElementById('something').innerHTML = adjAmt[0].toString();
})();

Some small changes to make it work without the API call, but you'll want to access the correct index in your loop. I don't know what the exact output you're wanting here but in this case I just joined all the values in the array.
Additionally, the setting of innerHTML needs to be done once all the values are retrieved from the API, so I would even suggest doing that when the loop terminates, or some other "done" type event.
Additionally, you can use Promise.all instead of a loop, which is what I would go with personally.
var fromAmt = 100;
var fromOOP = 50;
var fromGM = 50;
var fromCur = "USD"
var toCur = ["USD", "EUR", "INR", "GBP", "SGD"];
var adjAmt = [];
async function getConversionAsync(fcur, tcur, amt) {
let response = await sampleRequest()
/* let data = await response.json() */
return response;
}
for (i = 0; i < toCur.length; i++) {
const data = getConversionAsync(fromCur, toCur[i].toString(), fromAmt).then(data => {
display(data)
})
}
function display(thing) {
adjAmt.push(thing);
document.getElementById("something").innerHTML = adjAmt.join(', ')
}
function sampleRequest() {
return new Promise((resolve, reject) => {
resolve(Math.round(Math.random() * 1000))
})
}
<div id="something"></div>

How to get child Snapshots inside snapshots on Google Cloud functions (Firebase)?

I'm trying to get some for Loops running inside a google cloud functions everytime I delete my /users node.
This is the code I'm using
exports.deleteUserAssets = functions.database.ref('/users/{userId}').onWrite((change, context) => {
const beforeData = change.before.val();
const afterData = change.after.val();
const userBuildings = Object.keys(beforeData.isAdmin); // get the buildings of the user stored in the user/userId/isAdmin node .. so far so good
const userId = beforeData.userIDforCloudFunctions; // I'm getting this from a /users/userid/userIDforCloudFucntions node ...so far so good (i've been logging it to confirm)
// making sure it was a delete operation ... so far so good
if (afterData !== null) {
return 0;
}
else {
// on each building
for (var i = 0; i < userBuildings.length; i++) {
let eachBuilding = [userBuildings[i]]
// HERE IS WERE THE PROBLEM IS: Trying to delete all depts + rooms + doors
admin.database().ref('/buildings/' + eachBuilding)
.child("hasDepts")
.once("value")
.then(function(snapshot) { // This is where it goes south – snapshot is returning null
snapshot.forEach(function(childSnapshot) {
var deptKeyString = childSnapshot.key; // will try to get the keys of the departments stored under this space
var deptsOnNode = admin.database().ref('/depts/' + deptKeyString);
deptsOnNode.remove(); // and use the keys to delete each of the depts on depts
});
});
admin.database().ref('/buildings/' + eachBuilding).set({}); // this is working
admin.database().ref('/buildingsUserUid/' + userId + '/' + eachBuilding).remove(); // this is working
}
}
return 0;
});
The snapshot of admin.database().ref('/buildings/' + eachBuilding).child("hasDepts") is returning null.
How can I get to it? Besides admin.database().ref() I've tried to reach it with firebase.database().ref() which is the command/object i use to get this running on frontend functions. I've also tried functions.database() with no result.

Taking in consideration what Doug Stevenson mentioned in his second comment:
exports.deleteUserAssets = functions.database.ref('/users/{userId}').onDelete((change, context, event) => {
const beforeData = change.before.val(); // data before the write (data of all the doors child nodes)
const afterData = change.after.val(); // data before the write (data of all the doors child nodes)
const userBuildings = Object.keys(beforeData.isAdmin); // get the buildings of the user
const userId = beforeData.userIDforCloudFunctions;
// make sure user was deleted
if (afterData !== null) {
return 0;
}
else {
// on each building
for (var i = 0; i < userBuildings.length; i++) {
let eachBuilding = [userBuildings[i]]
// Need to RETURN the whole chain of promises
return admin.database().ref('/buildings/' + eachBuilding)
.child("hasDepts")
.once("value")
.then(function(snapshot) {
console.log(snapshot.val()) // this now works
snapshot.forEach(function(childSnapshot) {
console.log(childSnapshot.val()) // this works as well
var deptKeyString = childSnapshot.key; // get the keys of the departments stored under this space
var deptsOnNode = admin.database().ref('/depts/' + deptKeyString);
// and you can keep on going deeper if you return promises
return deptsOnNode
.child('hasRooms')
.once('value')
.then(function(grandchildSnapshot){
console.log(grandchildSnapshot.val())
grandchildSnapshot.forEach(function(grandGrandchildSnapshot){
var roomKeyString = grandGrandchildSnapshot.key;
var roomsOnDepts = admin.database().ref('/rooms/' + roomKeyString);
admin.database().ref('/roomOwners/' + userId + '/' + roomKeyString).remove();
// and return again here...
return roomsOnDepts
.child('hasDoors')
.once('value')
.then(function(grandgrandGrandchildSnapshot){
grandgrandGrandchildSnapshot.forEach(function(grandgrandGrandchildSnapshot){
var doorKeyString = grandgrandGrandchildSnapshot.key;
var doorsOnRooms = admin.database().ref('/doors/' + doorKeyString);
doorsOnRooms.remove();
let clipOwners = admin.database().ref('/clipOwners/' + doorKeyString);
clipOwners.remove();
})
roomsOnDepts.remove();
})
})
deptsOnNode.remove(); // use the keys to delete the depts on depts main Node
})
});
admin.database().ref('/buildings/' + eachBuilding).set({});
admin.database().ref('/buildingsUserUid/' + userId + '/' + eachBuilding).remove();
});
}
}
return 0;
});

Node / MySQL - code: 'ER_PARSE_ERROR', when trying to insert ~800 records

I am working on small idea to collect errors from pages and to store them in DB and then use graph API to display information visually.
There is 8 sites and on each of them there is 100 entries - so 800 transactions per time.
I loop through each site and then sub-loop through table of errors and collect them.
I got it working if I make insert query on each of those sub-loops for all 800 entries but I am getting some sort of memory leak from so many transactions and after few minutes - Node breaks due to memory exceeding.
So I tried queuing all 800 entries into Array of Arrays and then performing multi-insert at the end of every iteration but I am getting ER_PARSE_ERROR.
var tabletojson = require('tabletojson');
var mysql = require("mysql");
var striptag = require("striptags");
var fs = require("fs");
var path = require('path');
var startCollector;
var iterations = 0;
var insertions = 0;
var duplicated = 0;
var datas = [];
var clients = ["ClientA", "ClientB", "ClientC", "ClientD", "ClientE", "ClientF", "ClientG", "ClientH"];
var appDir = path.dirname(require.main.filename);
var errorList = ["err1", "err2", "err3", "err4", "err5", "err6"];
var con = mysql.createPool({
host: "localhost",
user: "User",
password: "Password",
database: "errors"
});
function CollectErrors() {
startCollector = new Date();
for(var a = 0; a < clients.length; a++) {
(function(a) {
tabletojson.convertUrl("http://example.com" + clients[a] + "/page.php?limit=100", { stripHtmlFromCells: false }, function(response) {
var rs = response[0];
for(var l = rs.length-1; l > -1; l--) {
var newDate = formatDate(striptag(rs[l]["Date"]), striptag(rs[l]["Time"]));
var user = getUser(striptag(rs[l]["User"]));
var msg = striptag(rs[l]["Error"]);
var splitError = rs[l]["Error"].split("<a href=\"");
var link = getUrl(splitError[1]);
var id = getId(link);
var type = getType(striptag(splitError[0]));
var temp = [newDate, link, type, user, clients[a], id, msg];
datas.push(temp);
}
});
})(a);
}
con.getConnection(function(err, connection) {
connection.query("INSERT IGNORE INTO entries (time, url, type, author, client, uid, message) VALUES ?", [datas], function(err, rows) {
console.log(err);
});
connection.release();
datas = [];
});
setTimeout(CollectErrors, 10000);
}
function formatDate(date, time) {
var newdate = date.split("/").reverse().join("-");
var newtime = time+":00";
return newdate + " " + newtime;
}
function getUrl(uri) {
return "http://example.com/"+uri.split("\">Details")[0];
}
function getId(url) {
return decodeURIComponent((new RegExp('[?|&]' + "id" + '=' + '([^&;]+?)(&|#|;|$)').exec(url) || [null, ''])[1].replace(/\+/g, '%20')) || null;
}
function getType(error) {
for(var a = 0; a < errorList.length; a++) {
if(error.indexOf(errorList[a]) !== -1) {
return errorList[a];
}
}
return "Other";
}
function getUser(user) {
if(user == "" || user == " " || user == null) {
return "System";
}
return user;
}
CollectErrors();
I've tried mysql.createConnection too but that also gave me same issue.
I've been stuck for past 12 hours and I can't see what's wrong, I've even tried populating Datas table with just strings but got same error.

I've changed your code to use ES6 and correct modules features.
Useful links: correct pooling with mysql, correct insert query, async/await, IIFE, enhanced object
const tabletojson = require('tabletojson'),
mysql = require("mysql"),
striptag = require("striptags"),
fs = require("fs"),
path = require('path');
const startCollector,
iterations = 0,
insertions = 0,
duplicated = 0;
let datas = [];
const clients = ["ClientA", "ClientB", "ClientC", "ClientD", "ClientE", "ClientF", "ClientG", "ClientH"];
const appDir = path.dirname(require.main.filename);
const errorList = ["err1", "err2", "err3", "err4", "err5", "err6"];
const con = mysql.createPool({
host: "localhost",
user: "User",
password: "Password",
database: "errors"
});
// We'll use async/await from ES6
const collectErrors = async() => {
// Up to here I've only changed syntax to ES6
let startCollector = new Date();
// We'll try to iterate through each client. And we use here for..of syntax to allow us using await
for (let client of clients) {
// Please, check that client value return correct data. If not, change for..of to your for..each and client variable to clients[a]
const tbj = await tabletojson.convertUrl("http://example.com" + client + "/page.php?limit=100", {
stripHtmlFromCells: false
});
const result = tgj[0];
for (rs of result) {
// I can't check this part, but I hope your example was with correct values.
let newDate = formatDate(striptag(rs[l]["Date"]), striptag(rs[l]["Time"]));
let user = getUser(striptag(rs[l]["User"]));
let link = getUrl(splitError[1]);
let msg = striptag(rs[l]["Error"]);
let id = getId(link);
let splitError = rs[l]["Error"].split("<a href=\"");
let getType = getType(striptag(splitError[0]));
// ES6 enhanced object syntax
datas.push({
newDate,
user,
msg,
id,
splitError,
link,
getType,
temp: [newDate, link, type, user, client, id, msg]
});
}
}
// OK, here we have fulfilled datas array. And we want to save it.
con.getConnection((err, connection) => {
// Please, notice, here I've changed your insert query to prepared statement.
connection.query("INSERT IGNORE INTO entries SET ?", datas, (err, rows) => {
console.log(err);
connection.release();
datas = [];
});
});
// I don't see why do you need timeout here, so I've left it commented.
// setTimeout(CollectErrors, 10000);
};
// Here your other methods go....
// And to call your async function we'll use IIFE
(async() => {
await collectErrors();
})();
Probably there may be errors with mysql insert, but that's not for sure. If occurred, please write in comments and I'll help you with that.

Twitter API with node.js, get a list of followers

Hi I'm currently working on a Twitter bot with the Twitter API and Node.JS. I want the bot to return all of my followers and some of their features in an javascript object. Something like :
{['id', 'screen_name', 'name', 'screen_name', 'followers_count',
'friends_count']}
RN my code is :
var Twitter = new TwitterPackage(config);
var options =
{
screen_name: 'mowsolicious',
};
Twitter.get('followers/ids', options, function (err, data) { // returns a list of ids
var nbFollowers = data.ids.length
var id = []
console.log(nbFollowers) // how many followers I have
for (i=0 ; i <= nbFollowers ; i++) {
ids = data.ids
var id = ids[i]
Twitter.get('users/show/' + id, function(err, data) {
console.log(id + " - " + data.name + " - " + data.screen_name + " - " + data.followers_count + " - " + data.friends_count)
})
}
})
I'm pretty sure something is terribly wrong with my method (more precisely when I put the Twitter.get thing in the loop) and it returns a bunch of undefined in the console.
I tried to work with the API doc but I'm experiencing some troubles understanding it. If someone could help that would be great.
Thank you

Most likely, you get undefined because the user is not found :
[ { code: 50, message: 'User not found.' } ]
Checking err variable would take care of that. But looking at GET followers/id documentation, you should use GET users/lookup to efficiently request mutliple user objects (up to 100 user per request with user id delimited by comma)
Also, I assume you'd like a unique callback to be called when all requests are completed, using Promises will take care of that :
var res_array = [];
function getUserInfo(id_list) {
return Twitter.get('users/lookup', {
"user_id": id_list
}).then(function(data) {
res_array.push(data);
})
.catch(function(error) {
throw error;
})
}
Twitter.get('followers/ids', function(err, data) {
if (err) {
console.log(err);
return;
}
console.log("total followers : " + data.ids.length);
var requestNum = Math.floor(data.ids.length / 100);
var remainder = data.ids.length % 100;
var promises_arr = [];
for (var i = 0; i < requestNum; i++) {
promises_arr.push(getUserInfo(data.ids.slice(i * 100, i * 100 + 100).join(",")));
}
if (remainder != 0) {
promises_arr.push(getUserInfo(data.ids.slice(requestNum * 100, requestNum * 100 + 100).join(",")));
}
Promise.all(promises_arr)
.then(function() {
for (var i in res_array) {
for (var j in res_array[i]) {
var user = res_array[i][j];
console.log(user.id + " - " +
user.name + " - " +
user.screen_name + " - " +
user.followers_count + " - " +
user.friends_count)
}
}
})
.catch(console.error);
})

List of followers can be retrieved with superface sdk , try it based on the example below
npm install #superfaceai/one-sdk
npx #superfaceai/cli install social-media/followers
const { SuperfaceClient } = require('#superfaceai/one-sdk');
const sdk = new SuperfaceClient();
async function run() {
// Load the installed profile
const profile = await sdk.getProfile('social-media/followers');
// Use the profile
const result = await profile
.getUseCase('GetFollowers')
.perform({
profileId: '429238130'
});
return result.unwrap();
}
run();

This works pretty fine
import fetch from 'node-fetch'
async function getFollowers(username) {
const response = await fetch(`https://api.twitter.com/1.1/followers/list.json?screen_name=${username}`, {
headers: {
Authorization: `Bearer ${<yourbearertoken>}`
}
});
// Parse the response as JSON
const data = await response.json();
return data.users;
}
const followers = await getFollowers("<username>");

Trouble returning object through some Callbacks

I am able to get data back but I seem to be failing at getting the result back up through some methods above this:
car.js
'use strict';
var Q = require('q');
var pg = require('co-pg')(require('pg'));
var config = require('../../models/database-config');
var car = module.exports = {};
car.find = Q.async(function *(id)
{
var query = 'SELECT id, title, description FROM cars WHERE id = ' + id;
var connectionResults = yield pg.connectPromise(config.connection);
var client = connectionResults[0];
var done = connectionResults[1];
var result = yield client.queryPromise(query);
done();
console.log("value: " + result.rows[0].id);
return result.rows;
});
this returns a valid value for my console.log so I know I'm getting data back.
But now when I try to pass that back up the stack, here I seem to be losing it after this method:
database.js
module.exports = {
models: {
car: _carModel
},
find: Q.async(_find)
};
function _find(carId)
{
_carModel.find(carId)
.then(function(result){
console.log('result[0].id: ' + result[0].id);
return result;
})
.catch(function(error){
console.log("promise error: " + error);
})
.done();
};
So this also works, I get a valid value for console.log('result[0].id: ' + result[0].id);
But now when try to call this function, I lose the result:
gateway.js
var car = database.find(carId);
console.log("car: " + car.id);
...
here I get a'Cannot read property 'id' of undefined]'
UPDATE #2
So I am trying to propagate now the promise up, but still get undefined for the line console.log("returned car data: " + data); 'data' is undefined.
gateway.js
module.exports = {
data: function(someData){
_data = someData;
},
find: function(text, result){
if(!text){
results(null);
};
var endpoint = _endpoint.replace(/_text/g, text);
_client.query(endpoint, function(results){
var cars = [];
var car;
for (var i = 0; i < results.docs.length; i++){
var carId = results.docs[i].id;
car = database.find(carId)
.then(function(data){
console.log("returned car data: " + data);
})
.done();
cars.push(car);
}
result(cars);
});
}
database.js
'use strict';
var Q = require('q');
var _obituaryModel = require('../../models/postgreSQL/obituary');
module.exports = {
models: {
obituary: _carModel
},
find: Q.async(_find)
};
function _find(carId)
{
_carModel.find(carId)
.then(function(result){
console.log('result[0].id: ' + result[0].id);
return result;
})
.catch(function(error){
console.log("promise error: " + error);
})
.done();
};
carModel.js
'use strict';
var Q = require('q');
var pg = require('co-pg')(require('pg'));
var config = require('../../models/database-config');
var car = module.exports = {};
car.find = Q.async(function *(id)
{
var query = 'SELECT id, title, description FROM cars WHERE id = ' + id;
var connectionResults = yield pg.connectPromise(config.connection);
var client = connectionResults[0];
var done = connectionResults[1];
var result = yield client.queryPromise(query);
done();
console.log("value: " + result.rows[0].id);
return result.rows;
});

You have a promise object at your disposal, use it.
In database.js:
return _obituaryModel.find(carId)
and gateway.js
var car = database.find(carId);
car.then(function (data) {
console.log(data);
});

You can't return data from an asynchronous function.
See here: How to return value from an asynchronous callback function?

You are printing result[0].id on the console but reading result.id on the actual call:
Try
var car = database.find(carId);
console.log("car: " + car[0].id);
...