We Keep Coding

callback is not a function in mongoDB nodejs

I am trying to validate if an email or username already exist in my mongoDB users collection in nodejs in my User Model.
But whenever i try to do that i get this error callback is not a function, when it finds that a username already exists in the next process that i am pushing to my processes array. meaning that the first process gets executed normally and there are no errors, but if an email doesnt already exists it goes to the next process and the error happens.
What am i doing wrong? How can I validate more than one field and return a response for that specific error?
here is my code :
register(params) {
return new Promise((resolve, reject) => {
var processes = [];
//check if the email is already registered
let user = new models.User()
processes.push((callback) => {
user.emailExists(params.email,null).then((response) => {
if(response && response.length) {
callback({errorCode: 403, msg: 'User Already Exists.'});
} else {
db.collection("Users").insertOne(options, function (error, response) {
if (error) {
callback(error)
} else {
console.log(response.ops[0])
if(response.ops[0]) {
callback(null, {result: response.ops[0]});
} else {
callback({errorCode: 403, msg: 'User sign-up failed.'})
}
}
})
}
}, (error) => {
reject(error);
});
});
processes.push((callback) => {
user.usernameExists(params.username).then((response) => {
if(response && response.length) {
callback({errorCode: 403, msg: 'username Already Exists.'});
} else {
db.collection("Users").insertOne(options, function (error, response) {
if (error) {
callback(error) //this is line 91
} else {
console.log(response.ops[0])
if(response.ops[0]) {
callback(null, {result: response.ops[0]});
} else {
callback({errorCode: 403, msg: 'User sign-up failed.'})
}
}
})
}
}, (error) => {
reject(error);
});
});
async.waterfall(processes, function (error, data) {
if (error) {
reject(error);
} else {
resolve(data);
}
})
})
}
emailExists(email, password) {
// check that required params are set
if (!email) {
return new Promise(function (resolve, reject) {
console.log("Parameters not set")
reject({errorCode: 403, msg: 'invalid.'});
});
}
return new Promise(function (resolve, reject) {
db.collection("Users").find({
"email": email.toLowerCase()
}).limit(1).toArray(function (error, response) {
if (error) {
reject(error);
} else {
if (response && response.length) {
if (!password) {
resolve(response);
} else {
bcrypt.compare(password, response[0].password, function (err, success) {
if (err) {
reject(err);
} else {
if (success) {
resolve(true);
} else {
resolve(false);
}
}
});
}
} else {
resolve(false);
}
}
});
});
}
usernameExists(params) {
if (!params || params.username) {
return new Promise(function (resolve, reject) {
console.log("params not set");
reject({errorCode: 403, msg: 'Invalid.'});
});
}
return new Promise(function (resolve, reject) {
db.collection("Users").find({
"username": params.username
}).limit(1).toArray(function (error, response) {
if (error) {
reject(error);
} else {
if (response && response.length) {
resolve(true)
} else {
resolve(false);
}
}
});
})
}
error log:

There are so many things wrong in the code that it's hard to point to something specific tbh.
Please read how to use waterfall:
An array of async functions to run. Each function should complete with any number of result values. The result values will be passed as arguments, in order, to the next task.
It means when you call
callback(null, {result: response.ops[0]});
the next process is being called with 2 arguments:
process({result: response.ops[0]}, callback)
you define your processes with only 1 argument, the rest are ignored. The first one is a json, so you get the error that it is not a function when you try to invoke it.
Now, how to fix it. Don't use waterfall. You don't need it. Bare async/await is more than enough and will make code much more readable.

AWS Lambda call 3 async functions by 1 Lambda call

Ok, i'm done. Please someone help me :(
I don't know how js and lambda works
What i have to do:
Send GET request and get response.
Write data from response to DynamoDb
I can do it 1by1 but can't do everything by 1 lambda call.
My code:
const https = require('https');
const crypto = require("crypto");
const AWS = require('aws-sdk');
const DynamoDb = new AWS.DynamoDB({region: 'eu-central-1'});
exports.handler = async (event) => {
let response;
console.log("Start");
let steamTicket;
let steamId;
if(event.body){
const body = JSON.parse(event.body);
if(body.steamticket && body.steamid){
steamTicket = body.steamticket;
steamId = body.steamid;
}
else{
response = {
statusCode: 400,
body: JSON.stringify({
authenticated: false,
reason: 'cant find steamid or steamticket in your request'
})
};
return response;
}
}
else{
response = {
statusCode: 400,
body: JSON.stringify({
authenticated: false,
reason: 'cant find request body'
})
};
return response;
}
await httprequest(steamTicket).then((data) =>{
if(data.response && data.response.params){
if(data.response.params.result == 'OK' && data.response.params.steamid == steamId){
console.log(JSON.stringify(data));
const sessionId = crypto.randomBytes(16).toString("hex");
console.log('Generated session id: ' + sessionId);
PutToDB(sessionId, steamId);
}
else{
response = {
statusCode: 400,
body: JSON.stringify({
authenticated: false,
reason: 'steam response is not OK or session != steamId'
})
};
return response;
}
}
else{
response = {
statusCode: 400,
body: JSON.stringify({
authenticated: false,
reason: 'invalid response from steam: ' + JSON.stringify(data)
})
};
return response;
}
});
};
async function PutToDB(sessionId, steamId){
var WriteParams = {
RequestItems:{
SteamSessions: []
}
};
WriteParams.RequestItems.SteamSessions.push({
PutRequest:{
Item: {
SteamId: {S: steamId},
SessionId: {S: sessionId},
ttl: {N: (Math.floor(Date.now() / 1000) + 600).toString()}
}
}
});
console.log('SessionIdToWrite: ' + sessionId);
return new Promise((resolve, reject) =>{
DynamoDb.batchWriteItem(WriteParams, function(err, data){
if(err){
console.log("Error", err);
}
else{
console.log("Success write", JSON.stringify(data));
}
})
})
}
async function httprequest(steamTicket) {
return new Promise((resolve, reject) => {
const options = {
host: 'partner.steam-api.com',
path: '/ISteamUserAuth/AuthenticateUserTicket/v1/?key=somekey&appid=someid&ticket=' + steamTicket,
port: 443,
method: 'GET'
};
const req = https.request(options, (res) => {
if (res.statusCode < 200 || res.statusCode >= 300) {
return reject(new Error('statusCode=' + res.statusCode));
}
var body = [];
res.on('data', function(chunk) {
body.push(chunk);
});
res.on('end', function() {
try {
body = JSON.parse(Buffer.concat(body).toString());
} catch(e) {
reject(e);
}
resolve(body);
});
});
req.on('error', (e) => {
reject(e.message);
});
// send the request
req.end();
});
}
I lost way already, i'm not even sure it should work like that.
And most confusing thing! This b give me this test results:
Run 1:
2021-03-05T13:28:47.741Z INFO Start
2021-03-05T13:28:48.612Z INFO {"response":{"params":{"result":"OK","steamid":"mysteamid","ownersteamid":"mysteamid","vacbanned":false,"publisherbanned":false}}}
2021-03-05T13:28:48.650Z INFO Generated session id: 6a5633a5f862d8663d0fe546a9c89feb
2021-03-05T13:28:48.650Z INFO SessionIdToWrite: 6a5633a5f862d8663d0fe546a9c89feb
DynamoDb is empty, here we can't see log from DynamoDb.batchWriteItem result.
Run 2:
2021-03-05T13:29:53.308Z INFO Start
2021-03-05T13:29:53.674Z INFO Success write {"UnprocessedItems":{}}
2021-03-05T13:29:54.048Z INFO {"response":{"params":{"result":"OK","steamid":"mysteamid","ownersteamid":"mysteamid","vacbanned":false,"publisherbanned":false}}}
2021-03-05T13:29:54.048Z INFO Generated session id: 05c62de782202fc100cea9d47e38242c
2021-03-05T13:29:54.048Z INFO SessionIdToWrite: 05c62de782202fc100cea9d47e38242c
And after second run i can see in DynamoDb sessionId from FIRST RUN (6a5633a5f862d8663d0fe546a9c89feb)
If i run it again, there will be id from 2nd run
I think it continues to run previous tasks on new run? Or what? I'm lost
Thank you for any help with it

You need to call reject / resolve in the DynamoDb.batchWriteItem call.
return new Promise((resolve, reject) =>{
DynamoDb.batchWriteItem(WriteParams, function(err, data){
if(err){
console.log("Error", err);
reject(err);
}
else{
console.log("Success write", JSON.stringify(data));
resolve();
}
})
})

Recursive loop with promises crawler NodeJS

I'm trying to use Recursive Loop and Promises to Scrape a website.
But it fails.. It make the request only for the first page and at the second the program stops giving to me unhandled promise rejection warning
I have this three JS files:
scrapeAll.js (is the recursive loop that calls scrapePage.js)
scrapePage.js
scrapeComponents.js
scrapeAll.js:
var indexPage = 0;
scrapePage(indexPage).then((json)=>{
console.log(JSON.stringify(json, null, 4));
if(indexPage === Number.MAX_SAFE_INTEGER){
console.log("MAX SAFE INTEGER");
return;
}
save(json);
indexpage++;
scrapePage(indexPage);
}).catch((data)=>{
console.log(data);
if(indexPage === Number.MAX_SAFE_INTEGER){
console.log("MAX SAFE INTEGER");
return;
}
indexPage++;
scrapePage(indexPage);
});
ScrapePage.JS
let makeRequestCounter = 0;
function scrapePage(number) {
return new Promise((resolve, reject) => {
let url = URL + number;
let options = {
url: url,
headers: {
Host: SITE,
Connection: "keep-alive",
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Language": "it-IT,it;q=0.9,en-US;q=0.8,en;q=0.7",
"Cache-Control": "max-age=0",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36",
"Cookie": restoreCookieToString()
}
};
makeRequest(options).then((jsonData) => {
resolve(jsonData);
}).catch((error) => {
//REQUEST_LIMIT_EXCEEDED
if (error === CONSTANTS.REQUEST_LIMIT_EXCEEDED) {
reject(CONSTANTS.REQUEST_LIMIT_EXCEEDED);
}
//ALREADY_EXIST
else if (error === CONSTANTS.ALREADY_EXIST) {
reject(CONSTANTS.ALREADY_EXIST);
}
else if (error === 404) {
reject("no data found at this page");
}
//error can beeconnrefused or econnreset
else if (error.code !== undefined) {
//econnrefused
if (error.code === CONSTANTS.ECONNREFUSED) {
reject("WRONG_URL", url);
}
//econnreset
else if (error.code === CONSTANTS.ECONNRESET) {
console.log("\neconnreset error\n");
makeRequest(options);
}
}
}
);
});
}
function makeRequest(options) {
return new Promise((resolve, reject) => {
let json = {
category: [],
imgs: [],
title: "",
description: "",
url: ""
};
if (makeRequestCounter === CONSTANTS.REQUEST_LIMIT) {
reject(CONSTANTS.REQUEST_LIMIT_EXCEEDED);
}
makeRequestCounter++;
console.log("request to: ", options.url);
request(options, function (error, response, html) {
if (error) {
//error: possible econnreset econnrefused
reject(error);
} else {
if (response.statusCode === 200) {
cookieSave(response.headers);
//---------- check if in db the url is already saved -------------//
check(response.request.uri.href, (err) => {
if (!err) {
reject(CONSTANTS.ALREADY_EXIST);
}
});
//----------finish checking, is new -------------------//
//GETTING TITLE
title(html, json_recipe).then((json) => {
//GETTING category
category(html, json).then((json) => {
//GETTING images
imgs(html, json).then((json) => {
description(html, json).then((json) => {
json.url = response.request.uri.href;
resolve(json);
//description error
}).catch((error) => {
console.log(error);
});
//images error
}).catch((error) => {
console.log(error);
});
//category error
}).catch((error) => {
console.log(error);
});
//title error
}
).catch((error) => {
console.log(error);
});
}
//no data in this page
if (response.statusCode === 404) {
reject(response.statusCode);
}
}
});
});
}
scrapeComponents.js
...
function description(html, json) {
return new Promise((resolve, reject) => {
const $ = cheerio.load(html);
let description = $('.submitter__description').text().trim();
json.description = JSON.parse(description);
resolve(json);
});
}
...
error:
UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 1): no data found at this page
The program make the first request and return correctly at the scrapeAll.js that correctly the scrapePage(indexPage = 1).
The second time my program do exactly same as first time but when is time to return to the scrapeAll.js ( reject("no data found at this page"); in ScrapePage.js ) the program ends with the error.
Both the pages are without data but program fails also with good pages saving only the first.
I think that I made a big mistake with promises.
Thank you very much guys.

Your call for scrapPage function is running only once and you are not calling it iteratively. You might have to call it in iteration using a function. Update your scrapeAll.js:
function callScrapPage() {
var indexPage = 0;
while (indexPage < Number.MAX_SAFE_INTEGER) {
scrapePage(indexPage).then((json) => {
console.log(JSON.stringify(json, null, 4));
save(json);
indexpage++;
}
}
}

The problem is that one or more of your calls to scrapePage(indexPage) in scrapeAll.js are failing. You cannot recursively call a promise the way you might with other code, so you need a .then and .catch on the additional calls as well. Adding a .catch to the other calls will enable you to see the true source of failure.
scrapePage(indexPage)
.then((json)=>{
console.log(JSON.stringify(json, null, 4));
if(indexPage === Number.MAX_SAFE_INTEGER){
console.log("MAX SAFE INTEGER");
return;
}
save(json);
indexpage++;
scrapePage(indexPage).catch(e => console.log(e));
})
.catch((data)=>{
console.log(data);
if(indexPage === Number.MAX_SAFE_INTEGER){
console.log("MAX SAFE INTEGER");
return;
}
indexPage++;
scrapePage(indexPage).catch(e => console.log(e));
});

JS: TypeError on forEach loop

I'm grabbing the signature element from a jwt then calling the post('/audit') and using the signature in the forEach loop. However, the forEach loop throws an error :
TypeError: toAudit.forEach is not a function
What is causing the error? How do I fix it?
(server.js)
//retreives signature based on uuid without verifying asset. used for getting signatures for logs.
function getSignature(uuid){
console.log('start getSignature');
return new Promise(function (resolve, reject){
request.post({
url: `http://${ksiIP}:${ksiPort}/KsiProxy/rest/util/getSignature`,
headers: {
'Content-Type': 'application/json',
'Authorization' : 'Basic Og=='
}, json: {'uuid': uuid}
}, function(error, response, body){
if(error || (response.statusCode != 200)){
return reject((error ? error : body));
}
if(body.resultcode != 0){
console.log("Error: " + requestResult.info);
console.log("Asset could not be verified.");
return reject(new Error(body.info));
}
//add identification as well
resolve(body.signature);
}).auth('','',true);
});
}
dcServer.post('/audit', function(req,res){
console.log('start /audit');
if(req.jwtPayload.description != "admin"){
respondWithJSON(401, 401, "Not authorised for this functionality.", res);
return;
}
let toAudit = req.jwtPayload.signature
console.log('jwtPayload:', req.jwtPayload);
console.log("ids: ", toAudit);
let failed = 0;//counter to increase if signatures couldnt be retreived.
toAudit.forEach( (element, index, array) => { //**error point HERE**
getSignature(element.signature)
.then((error, result) => {
(result) => { array[index].signature = result };
(error) => {
failed++;
array[index].signature = "unavailable";
console.log(error.message);
}
});
});

How to solve promise issue?

I am new to Promise concepts and trying to wrap my head around but now I`` am confused here
const request = require("request");
const cheerio = require("cheerio");
const XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
var url = require("url");
module.exports = {
resturant: resturant,
};
var resturanttables = [];
function resturant(url, day) {
return new Promise(function(resolve, reject) {
request(url, function(error, response, html) {
if (error) {
return reject(error);
} else if (response.statusCode !== 200) {
return reject("Something is wrong! CINEMA")
}
httplogin("zeke", "coys", url, day);
console.log(resturanttables, "i am here");
resolve(resturanttables);
});
});
}
function httpafterlogin(url, cookie, day) {
request.get({
headers: {
'content-type': 'text/html',
'Cookie': cookie
},
url: url,
},
function(error, response, body) {
console.log(day)
var $ = cheerio.load(body);
if (day === "Friday") {
$(".WordSection2 p span ").each(function(li) {
// console.log(day, $(this).text())
resturanttables.push($(this).text());
console.log(resturanttables, "nside");
});
} else if (day === "Saturday") {
$(".WordSection4 p span").each(function(li) {
resturanttables.push($(this).text())
});
} else {
$(".WordSection6 p span").each(function(li) {
resturanttables.push($(this).text())
});
}
});
}
function httplogin(username, password, urls, day) {
request.post({
headers: {
'content-type': 'application/x-www-form-urlencoded'
},
url: urls,
form: {
"username": username,
"password": password
}
}, function(error, response, body) {
var cookie = response.caseless.dict['set-cookie'][0];
var location = response;
console.log(response.statusCode);
cookie = cookie.substring(0, cookie.indexOf(';'));
// httpafterlogin('http://vhost3.lnu.se:20080/dinner/'+response.headers.location, cookie);
var newurls = url.resolve(urls, response.headers.location)
httpafterlogin(newurls, cookie, day);
// console.log(response.headers, "jdjdjjdjdjjdjdjdjjdjjdjdj")
});
}
and then I call the function
loadPage.resturant("http://vhost3.lnu.se:20080/dinner/login", "Friday").then(function(data) {
console.log(data, "did it work now ")
})
the problem is that it returns the empty array. But when i tried to check and console.log in the afterlogin function and i could see that the array was actually filled, but that code runs after the promise has been resolved.
IN SHORT: how can I bound the resolve in restaurant promise not to send the data until the login function is completed?
in other words how can i get the filled array with information from afterlogin funtion?

rewrite httplogin and httpafterlogin to return promises:
function httpafterlogin (url, cookie, day) {
return new Promise(function (resolve, reject) {
request.get({
headers: {
'content-type': 'text/html',
'Cookie': cookie
},
url: url
}, function (error, response, body) {
if (error) {
reject(error);
} else {
resolve(body);
}
});
}).then(function (body) {
console.log(day);
var $ = cheerio.load(body);
if (day === "Friday") {
$(".WordSection2 p span").each(function (li) {
// console.log(day, $(this).text());
resturanttables.push($(this).text());
console.log(resturanttables, "nside");
});
} else if (day === "Saturday") {
$(".WordSection4 p span").each(function (li) {
resturanttables.push($(this).text());
});
} else {
$(".WordSection6 p span").each(function(li) {
resturanttables.push($(this).text());
});
}
});
}
function httplogin(username, password, urls, day) {
return new Promise(function (resolve, reject) {
request.post({
headers: {
'content-type': 'application/x-www-form-urlencoded'
},
url: urls,
form: {
"username": username,
"password": password
}
}, function(error, response, body) {
if (error) {
reject(error);
} else {
resolve(response);
}
});
}).then(function (response) {
var cookie = response.caseless.dict['set-cookie'][0];
var location = response;
console.log(response.statusCode);
cookie = cookie.substring(0, cookie.indexOf(';'));
var newurls = url.resolve(urls, response.headers.location)
return httpafterlogin(newurls, cookie, day);
});
}
then use .then like rsp suggested:
function resturant(url, day) {
return new Promise(function(resolve, reject) {
request(url, function(error, response, html) {
if (error) {
return reject(error);
} else {
resolve(response);
}
})
}).then(function (response) {
if (response.statusCode !== 200) {
throw new Error("Something is wrong! CINEMA");
}
return httplogin("zeke", "coys", url, day)
}).then(function () {
console.log(resturanttables, "i am here");
return resturanttables;
});
}
this way, the block containing resolve(restautanttables) will not get called until httplogin completes

Use promises throughout your code - you can simplify your code using the request-promise package in place of the request package. All requests become promises and the code is easier to read and maintain.
const rp = require("request-promise");
const cheerio = require("cheerio");
const url = require("url");
function resturant(url, day) {
rp(url)
.then(function(){
// URL returned a 200 response
// so attempt to perform login
httplogin("zeke", "coys", url, day)
.then(function (data) {
// Promise is resolved here
return data;
});
})
.catch(function(error){
// just throwing the error
throw error;
});
}
function httplogin(username, password, urls, day) {
var options = {
headers: {
"content-type": "application/x-www-form-urlencoded"
},
uri: urls,
form: {
username: username,
password: password
},
method: "POST",
resolveWithFullResponse: true
};
rp(options)
.then(function (response) {
// POST succeeded
// grab the cookie
var cookie = response.caseless.dict['set-cookie'][0]
.substring(0, cookie.indexOf(';'));
// get new url string
var newurls = url.resolve(urls, response.headers.location);
httpafterlogin(newurls, cookie, day)
.then(function (tables) {
return tables;
})
.catch(function (error) {
// just throwing the error
throw error;
});
})
.catch(function (error) {
// Login failure
// just throwing the error
throw error;
});
}
function httpafterlogin(url, cookie, day) {
var options = {
headers: {
"content-type": "text/html",
"Cookie": cookie
},
uri: url,
transform: function (body) {
return cheerio.load(body);
}
};
rp(options)
.then(function ($) {
// body has been transformed and
// can now be processed with jQuery
// initialise the tables array
var tables = [];
// DRY code
// set default selector
var selector = ".WordSection6 p span";
// change the selector for Friday/Saturday
if (day === "Friday") {
selector = ".WordSection2 p span ";
} else if (day === "Saturday") {
selector = ".WordSection4 p span";
}
// process the selected section
$( selector ).each(function(li) {
tables.push($(this).text())
});
// crawling complete
return tables;
})
.catch(function (error) {
// Crawling failure
// just throwing the error
throw error;
});
}

If you don't want the promise to get resolved before the login is completed then you will either have to make your httplogin function take a callback and run it like this:
httplogin("zeke", "coys", url, day, function (err) {
if (err) {
reject(err);
} else {
resolve(resturanttables);
}
});
or make it return a promise and run it for example like this:
httplogin("zeke", "coys", url, day).then(function () {
resolve(resturanttables);
}).catch(function (err) {
reject(err);
});
There are more ways to do it with promises but this is the simplest way.
Either way you have to make your httplogin function signal its completion by either calling the callback that it takes as an argument or resolving the promise that it returns.