I am using socket.io to communicate the swift client of my app with the server. Essentially, the client joins a socket connection upon opening the app and a job is instantly added to a Redis queue (it's a job that takes anywhere from a few seconds to like 15ish seconds). There's a response from the server to the client of the job id. While this job is processing, SOMETIMES the client will disconnect. There doesn't seem to be a rhyme or reason behind this, as the time of disconnection is totally inconsistent and it's also not like the disconnection is happening at a specific point in the function. I thought maybe I was manually disconnecting from the client side so I set up socket emissions right before each disconnect on the client side (when these emissions were emitted to the server, the server prints something that tells me where the disconnect came from). This showed me that the disconnect is automatic, because the emission is never received by the client before ending the socket connection. This is running on Heroku. Here's my code:
//queue initialization
const queue = new Queue('queue', process.env.REDIS_URL)
//client pings this endpoint to get the job id in the queue
app.post('/process', async function(request, response) {
let job = await queue({request: request.body});
console.log("Logging job as " + job.id)
response.json({ id: job.id });
});
queue.process(10, async (job) => { //10 is the max workers per job
console.log("Started processing")
const client = await pool.connect()
let item = job.data.request
let title = item.title
let subtitle = item.subtitle
let id = item.id
io.to(id).emit("Processing1", ""); //added emissions like these because I thought maybe the socket was timing out, but this didn't help
console.log("Processing1");
try {
await client.query('BEGIN')
let geoData = await //promise of geocoding endpoint api function
let lengthOfGeoData = geoData.context.length
io.to(id).emit("Processing2", "");
console.log("Processing2");
var municipality = ""
var area = ""
var locality = ""
var place = ""
var district = ""
var region = ""
var country = ""
//for loop to go through geoData and set the above values
if (municipality != "") {
console.log("Signing in from " + municipality + ", " + area);
} else {
console.log("Signing in from " + area)
}
await scrape(municipality, area, id);
await client.query('COMMIT')
} catch(err) {
await client.query('ROLLBACK')
console.log(err)
}
try {
await client.query('BEGIN')
const array = await //a function that queries a Postgres db for some rows, makes json objects out of them, and pushes to the 'array' variable
var array2 = []
for (a of array) {
let difference = getDifference(title, subtitle, a.title, a.subtitle) //math function
if (difference <= 10) {
array.push(a)
}
}
io.to(id).emit("Processing9", "");
console.log("Processing9");
await client.query('COMMIT')
} catch(err) {
await client.query('ROLLBACK')
console.log("ERROR: Failed arrayHelperFunction")
console.log(err)
} finally {
client.release()
console.log("About to emit this ish to " + id) //should emit to socket here ideally to notify that the processing is done and results can be polled
io.to(id).emit("finishedLoading", "")
return array2;
}
});
//when the client polls the queue after it's received the 'done' notifier from the server
app.post('/poll', async function(request, response) {
console.log("Polling")
let id = request.body.id
const results = await queue(id);
for (r of results.returnvalue) {
console.log("Sending " + r.title);
}
response.send(results.returnvalue)
});
//scrape
async function scrape(municipality, area, id) {
const client = await pool.connect();
try {
await client.query('BEGIN')
var location = ""
if (municipality != "") {
location = municipality + ", " + area
} else {
location = area
}
let inDatabase = await client.query('SQL statement AS it_does_exist', [params]);
io.to(id).emit("Processing3", "");
console.log("Processing3");
if (inDatabase.rows[0].it_does_exist == false) {
let query = "book clubs near " + location
var terminationTime = new Date()
terminationTime.setHours(terminationTime.getHours() + 4);
let date = ("0" + terminationTime.getDate()).slice(-2);
let month = ("0" + (terminationTime.getMonth() + 1)).slice(-2);
let year = terminationTime.getFullYear();
let hours = terminationTime.getHours();
let minutes = terminationTime.getMinutes();
let seconds = terminationTime.getSeconds();
let timestamp = year + "-" + month + "-" + date + " " + hours + ":" + minutes + ":" + seconds
try {
await client.query(`SQL statement`, [params]);
} catch(err) {
console.log("FAILURE: scrape() at 1.")
console.log(err)
}
var queryLocation = "New York,New York,United States" //default search origination is here
var queryGLCode = "US"
io.to(id).emit("Processing4", "");
console.log("Processing4");
try {
await fetch('https://serpapi.com/locations.json?q='+municipality+'&limit=10', { method : "GET" })
.then(res => res.json())
.then((json) => {
for (let index = 0; index < 10; index++) {
let locationAPIName = json[index].canonical_name
let locationAPICode = json[index].country_code
let resultLatitude = json[index].gps[1];
let resultLongitude = json[index].gps[0];
}
});
} catch(err) {
console.log("FAILURE: scrape() at 2.")
console.log(err)
}
io.to(id).emit("Processing5", "");
console.log("Processing5");
try {
await Promise.all([
searchEvents({engine: "google_events", q: query, location: queryLocation, hl: "en", gl: queryGLCode}).then(data => async function(){
try {
await client.query('BEGIN');
let results = data.events_results
if (results != null) {
console.log("first HAD results")
for (result of results) {
var fixedAddress = result.address[0]
let address = fixedAddress + ", " + result.address[1]
let title = result.title + address
var description = result.description
let geoData = await geocode(address); //mapbox geocode the address
let latitude = Number(geoData.center[0]);
let longitude = Number(geoData.center[1]);
await client.query(`SQL statement`, [params]);
}
io.to(id).emit("Processing6", "");
console.log("Processing6");
} else {
console.log("first DID NOT have results")
}
console.log("FIRST BLOCK")
await client.query('COMMIT');
} catch(err) {
console.log("Results[0] not found.")
console.log(err)
await client.query('ROLLBACK');
}
}()),
searchEvents({engine: "google_events", q: query, location: queryLocation, hl: "en", gl: queryGLCode, start: "10"}).then(data => async function(){
// same as the one above, just with an offset
}()),
searchEvents({engine: "google_events", q: query, location: queryLocation, hl: "en", gl: queryGLCode, start: "20"}).then(data => async function(){
// same as the one above, but with a different offset
}())
])
} catch(err) {
console.log("FAILURE: scrape() at 3.")
console.log(err)
}
} else {
console.log("Location already in the database.")
}
await client.query('COMMIT')
} catch(err) {
await client.query('ROLLBACK')
console.log(err)
} finally {
client.release()
return "Resolved";
}
}
//Client establish socket connection
func establishConnection(_ completion: (() -> Void)? = nil) {
let socketUrlString: String = appState.server
self.manager = SocketManager(socketURL: URL(string: socketUrlString)!, config: [.log(false), .reconnects(true), .extraHeaders(["header": "customheader"])])
self.socket = manager?.defaultSocket
self.socket?.connect()
self.socket?.once(clientEvent: .connect, callback: { (data, emitter) in
if completion != nil{
completion!()
}
})
//other socket functions
}
//Client initial post request
func process() {
let server = "serverstring" + "process"
let title = "title"
let subtitle = "subtitle"
let package = BookPackage(title: title, subtitle: subtitle, id: mySocketID) //this is after the initial connection
print("package is \(package)")
guard let url = URL(string: server) else { return }
var urlRequest = URLRequest(url: url)
urlRequest.addValue("application/json", forHTTPHeaderField: "Content-Type")
urlRequest.addValue("application/json", forHTTPHeaderField: "Accept")
urlRequest.httpMethod = "POST"
guard let data = try? JSONEncoder().encode(package) else { return }
urlRequest.httpBody = data
let task = URLSession.shared.dataTask(with: urlRequest) {
(data, response, error) in
if let error = error {
print(error)
return
}
guard let data = data else { return }
guard let dataString = String(data: data, encoding: String.Encoding.utf8) else { return }
let jsonData = Data(dataString.utf8)
var decodedJob: Job? = nil
do {
decodedJob = try JSONDecoder().decode(Job.self, from: jsonData) //Job is just a struct in the same form as the json object sent back from the server
} catch {
print(error.localizedDescription)
}
DispatchQueue.main.async {
self.appState.pendingJob = decodedJob
}
}
// start the task
task.resume()
}
The only consistent part of this bug is the logs right before the user disconnects (side note: 'reason of disconnect' and 'DISCONNECTED USER' fire on the socket.on('disconnect') event:
https://i.stack.imgur.com/7fjuU.png
https://i.stack.imgur.com/z5bmL.png
https://i.stack.imgur.com/aHNt3.png
https://i.stack.imgur.com/64WYI.png
You should be blocking the event loop with await. There is a heartbeat that the client sends every once in a while (which is defined with pingTimeout).
Since no ping is received by the server, it is disconnected.
You should isolate this process. Either find a way to use it with a worker/background process or async, additionally increasing pingTimeout on serverside might help you.
The solution to your problem is to modify the pingTimeout when initiating the server.
From Socket.io:
The server sends a ping, and if the client does not answer with a pong within pingTimeout ms, the server considers that the connection is closed.
Similarly, if the client does not receive a ping from the server
within pingInterval + pingTimeout ms, the client also considers that
the connection is closed.
const io = new Server(httpServer, {
pingTimeout: 30000
});
You can change the transport from the default to:
const io = new Server(httpServer, {
transports: ['polling', 'websocket'],
});
This might resolve the issue, else you can also try canging the upgradeTimeout and pingTimeout
Related
Hi as i am new to NodeJS as a follow-up to my previous question (Having problems getting new Array Value after looping in NodeJS) i have managed to get my code mostly working for parsing a list of email and querying them to the database, unable to get the return value in my promise when i get a response back from NodeJS API.
I am using MySQL 2.18.1 for my database and 4.17.1
Any idea i can solve it? Been trying it for a few hours.
Logs:
IM OUT HERE
Promise { { recipients: [] } }
{ recipients: [] }
retrieve_for_notification.js
async function processEmails(emails) {
var retrieveValues = {
recipients: []
};
// emails consist of those who were mentioned/notified, check condition whether they are eligible to be placed in receipients list
for (var i = 0; i < emails.length; i++) {
console.log(emails[i]);
// 1 - check for suspended
// 2 - check whether teacher and student pair is registered
var sql1 = 'SELECT COUNT(*) as count_value FROM school.schoolinformation WHERE email = ? AND user_status = ?';
var sql2 = 'SELECT COUNT(*) as count_value2 FROM school.registration_relationship WHERE teacher_email = ? AND student_email = ?';
var sql3 = 'SELECT COUNT(*) as count_value3 FROM school.schoolinformation WHERE email = ? AND user_type = ?';
var sqlvalues1 = [emails[i], 1];
var sqlvalues2 = [teacher, emails[i]];
var sqlvalues3 = [emails[i], 0];
// check for suspended
con.pool.query(sql1, sqlvalues1, async function (err1, result1) {
if (err1) throw err1;
var res1 = await getResult(sql1, sqlvalues1)
// console.log("(1) res value is %s", res1[0].count_value);
if (res1 > 0) return; // if result found skip to next email
// check whether teacher and student pair is registered
con.pool.query(sql2, sqlvalues2, async function (err2, result2) {
if (err2) throw err2;
var res2 = await getResult(sql2, sqlvalues2)
// teacher and student pair is not registered
if (res2 == 0) {
// check whether student mentioned is a valid student
con.pool.query(sql3, sqlvalues3, async function (err3, result3) {
if (err3) throw err3;
var res3 = await getResult(sql3, sqlvalues3)
// student is valid
if (res3 == 0) {
retrieveValues.recipients.push(sqlvalues3[0]);
}
});
}
else {
retrieveValues.recipients.push(sqlvalues2[0]);
}
});
});
};
return recipientsList;
}
var recipientsList = processEmails(emails);
console.log("IM OUT HERE");
console.log(recipientsList);
// Resolve promise and response send, not using helper for this
var p2 = Promise.resolve(recipientsList);
p2.then(function(v) {
console.log(v);
response.write(JSON.stringify(v, null, 3));
response.send.bind(response);
response.end();
}, function(e) {
console.error(e); // TypeError: Throwing
});
function getResult(sql, sqlvalues) {
// console.log("getResult SQL Query: %s", sql);
return new Promise(function (resolve, reject) {
con.pool.query(sql, sqlvalues, function (err, result) {
if (err) {
reject(err)
} else {
resolve(result)
}
})
})
}
May by use - "response.statusCode = responseCode" for old version express
I have multiple API calls that I'm using to delete user data for automated tests.
First I get an access token
async function getAccessToken(email, pwd) {
try {
const form = {email: email, password: pwd};
let config = {
headers: {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
}
};
const accessToken = await axios.post(`${process.env.API_URL}/test/token`, qs.stringify(form), config);
console.log(accessToken.data.data.accessToken);
return accessToken.data.data.accessToken
}
catch(e) {
console.error(``+email+` produced the Error = ` + e);
return 0;
}
}
I then use that token to delete the data needed
async function TMDeleteAllGoals (emailAddress) {
try {
var accessToken = await setup.getAccessToken(emailAddress, 'Test4321');
var userId = await user.getUserId(emailAddress);
var goalids = await getAllTMGoalID(emailAddress);
console.log(`Deleting all goals for ` + emailAddress + ``);
for (const goalid of goalids) {
const response = await axios.delete(`${process.env.TM_API_URL}/test/users/`+userId+`/goals/`+goalids+``, {'headers': {Authorization: 'Bearer ' + accessToken + ''}});
}
}
catch(e) {
console.error(``+emailAddress+` produced the Error = ` + e);
return 0;
}
}
I then need to run this for 1000 different users, so I execute using a loop
var i;
for (i = 1; i < 1001; i++) {
//console.log(i);
deleteGoals.TMDeleteAllGoals(`loadtest${i}#test.com`);
}
The problem I am having is that here appears to be a max for concurrent calls to get my access token (not sure what the limit is yet). So if I hit it with too many users, it will start to return null instead of the token.
Is there a way I can put a timeout in between each call in the loop, to try and slow it down. I have tried this, but it doesn't execute anything.
const timeout = ms => new Promise(res => setTimeout(res, ms));
async function deleteTheGoals () {
for (var i = 1; i < 1001; i++) {
//console.log(i);
await timeout(500);
const deleteGoals = await deleteGoals.TMDeleteAllGoals(`loadtest${i}#test.com`);
}
}
You may have a max request config in your server, so even if you do a timeout it doesn't guarantee that the request will end before 500 milliseconds.
Why not send them one after another ?
async function arraysWithAllIndexes(array) {
for(const i of array) {
await deleteGoals.TMDeleteAllGoals(`loadtest${i}#test.com`);
}
};
I have to make an API request, which then I then take an ID out of the data I get back and put that into another API request. This was not my API design pattern but it is what it is.
My issue is, when it comes to sending a JSON response, the response is being sent before all of my requests are finished. Now I have tried with counters to check that all my requests have been made before sending the response but it does not work. I will show you where the issue is:
So I make in total 26 API requests (crazy right).
I have gone through debugging and seen that my counter correctly increments 26 times.
This is where I check the counter:
if ((counter === counterSum)) {
res.send(projects);
}
The issue is, this code is running before all my counters can 'count'.
For example, at this point if I log the counter, it reaches to about 22 instead of 26. Often I see the counter log the same number 5 times or so because it has logged before my counter has had the chance to increment.
I have been going on this hours now so if anybody has an idea within the realm of Node.js, that would be amazing.
I will add some very simplified code showing all the request stages I have to go through and where I do the final log check:
let departmentProcesses = JSON.parse('[' + processes + ']');
Object.keys(departmentProcesses).forEach(function (key) {
console.log('The process' + departmentProcesses[key]);
let val = departmentProcesses[key];
let id = val.id;
let proccessURL = //url with + id
// Get all instances by project id
request({url: processURL},
function (error, response, body) {
//Convert the XML into JSON
parseString(response.body, function (err, result) {
departmentData = JSON.stringify(result);
});
let instanceData = (JSON.parse(departmentData)).feed.entry;
// Pushes to an array so I can sum up all the iterations
counters.push(instanceData.length);
Object.keys(instanceData).forEach(function (key) {
let val = instanceData[key];
let processId = val.id[0];
let nextURL = //url with + id";
request({url: nextURL},
function (error, response, body) {
//Convert the XML into JSON
parseString(response.body, function (err, result) {
departmentData = JSON.stringify(result);
});
let instanceData = (JSON.parse(departmentData)).feed.entry;
let val = instanceData[0].id[0];
let todoLink = //url with + id";
request({url: todoLink},
function (error, response, body, callback) {
//Convert the XML into JSON
parseString(response.body, function (err, result) {
departmentData = JSON.stringify(result);
});
let instanceEntry = (JSON.parse(departmentData)).feed.entry;
if (typeof instanceEntry !== 'undefined' && instanceEntry) {
let finalLink = //url with + id";
request({url: finalLink},
function (error, response, body) {
//Convert the XML into JSON
parseString(response.body, function (err, result) {
departmentData = JSON.stringify(result);
});
let instanceEntry = (JSON.parse(departmentData)).feed.entry;
if (typeof instanceEntry !== 'undefined' && instanceEntry) {
upNext = {
// some data
};
let data = {
// some data
}
processInstances.push(data);
counter++;
}
});
}
else {
upNext = 'null';
let data = {
// some data
}
counter++;
}
console.log(counter);
let counterSum = counters.reduce(function (a, b) {
return a + b;
}, 0);
if ((counter === counterSum)) {
res.send(projects);
}
});
});
});
});
let data = {
processName: processName,
id: id,
instances: processInstances
};
projects.push(data);
});
I have a datastructure in the form of object structure . [![enter image description here][1]][1]
{
Invite1
{
Amount: 10,
PhoneNumber:9876543210,
Status:"Pending"
}
Invite2
{
Amount: 20,
PhoneNumber:1234566789,
Status:"Pending"
}
}
I have a condition when whose Invite(1,2,3) PhoneNumber matches with other document that invitee need to update the field as Status = true
When I try to update a field as Status = true It is updating at the end of the document.
Mycode need to update
var dbref = db1.collection('deyaPayUsers').doc(sendauthid).collection('Split').doc(sendauthid).collection('SentInvitations').doc(senderautoid);
var msg1 = receiverph + "" + status + " to pay $" + document.Amount;
var fulldoc = dbref.get()
.then(doc => {
if (!doc.exists) {
console.log('No such document');
} else {
console.log('Document data :', doc.data());
d1 = doc.data();
console.log("d1 is" + d1);
for (var k in d1) {
var p = d1[k].PhoneNumber;
console.log("ivitees phone" + p);
if (receiverph == p) // Here the condition is true of the invite phoneNumber then need to update
{
console.log("p" + PhoneNumber);
console.log("the phonenumber matches");
var updated = dbref.update({"Status":status});// Here It is updating
at the endof the document
other Method to update
d1.Status = status; // In d1 I have the document data
var setdata = dbref.set(d1);
Please if their is any approach help with me.
Thanks
If I understand correctly that you would like to update each InviteXX item in the document, here is a code that will work (I've kept the main part of your code):
var dbref = db1.collection('deyaPayUsers').doc(sendauthid).collection('Split').doc(sendauthid).collection('SentInvitations').doc(senderautoid);
var msg1 = receiverph +"" + status +" to pay $"+document.Amount;
const fulldoc = dbref.get()
.then(doc => {
if (doc.exists) {
//console.log("Document data:", doc.data());
const inviteUpdate = {}; //An object that we will update by looping over the InviteXX objects of the document
const d1 = doc.data();
//console.log("d1 is" + d1);
for (let key in d1) {
if (d1.hasOwnProperty(key)) {
const p = d1[key].PhoneNumber;
//console.log(key);
//console.log("invitees phone " + p);
if (receiverph === p) // Here the condition is true of the invite phoneNumber then need to update
{
inviteUpdate[key + '.status'] = true;
//The key point is here: we define the object field with a mix of dot notation and []
}
}
}
return dbref.update(inviteUpdate);
} else {
// doc.data() will be undefined in this case
console.log("No such document!");
throw "No such document"
}
})
.catch(function (error) {
console.log("Error:", error);
});
Note that:
You should return a promise in your Cloud Function. I don't have the full code of your function but probably you will return fulldoc. (You may have a look at this video https://www.youtube.com/watch?v=652XeeKNHSk&t=2s)
I don't know how do you get/initialize receiverph
You may use const or let instead of var in a Cloud Function (JavaScript ES6+)
Heyo,
I have a following function
async function fnIsOnScreenOnce(img, desc,iCounter,client,repeatDelay=0) {
await timeout(repeatDelay);
let screenshot= await client.screenshot()
let buf = new Buffer(screenshot.value, 'base64');
let img1 = cv.imdecode(buf)
let result = img1.matchTemplate(img, 5).minMaxLoc();
result.screenshot=img1;
if (result.maxVal <= 0.65) {
// Fail
const msg = "Can't see object yet";
throw new Error(result);
}
// All good
console.log("result:"+result)
logger.info("Found image on screen: "+desc);
return result;
}
Call of the function
function fnIsOnScreen(img,client, repeats = 5, desc, wait = 2000,repeatDelay) {
logger.info("Looking for image on screen:" +desc +" with " + repeats + " repeats ");
let iCounter = 0;
let init = ()=> timeout(wait).then((asd)=>{
const attempt = () => fnIsOnScreenOnce(img, desc, iCounter,client,repeatDelay).then((data=>{
let imagepath=fnMarkOnImage(data.screenshot,img,data,outputDir)
let description={};
description.action="Is image on screen ?";
description.desc=desc;
description.repeats=repeats;
description.wait=wait;
description.img=imagepath;
description.message="is this correct element ? if is then it was found correctly";
fnPushToOutputArray(description)
return data;
})).catch(err => {
console.log(JSON.stringify(err));
console.log(err);
console.log(err.result);
iCounter++;
if (iCounter === repeats) {
// Failed, out of retries
logger.info("Object not found : " + desc);
return Promise.reject("Object not found : " + desc);
}
// Retry after waiting
return attempt();
});
return attempt();
})
return init();
}
result object contains some date.
On error result contains {} object with no values in it. I would need to get all the values. So how can i pass result object through throw new error to retrieve it in catch ?
One way to return extra data with error is to extend Error class and add them your self
class MyError extends Error {
constructor(message, errorExtraParams) {
super(message);
this._errorExtraParams = errorExtraParams;
}
get errorExtraParams() {
return this._errorExtraParams;
}
}
throw new MyError("Error!!!", {})
//or
let mError = new MyError("Error!!!", {})
console.log(mError.errorExtraParams)
But I suggest you don't use throw Error, because I don't like to throw Errors for insignificant reasons. What I mean is that in your case there is no reason to throw error cause there is no error and no reason to create an error just to tell you code "Hey I didnt find the image" instead just return false.
async function fnIsOnScreenOnce(img, desc, iCounter, client, repeatDelay = 0) {
await timeout(repeatDelay);
let screenshot = await client.screenshot()
let buf = new Buffer(screenshot.value, 'base64');
let img1 = cv.imdecode(buf)
let result = img1.matchTemplate(img, 5).minMaxLoc();
result.screenshot = img1;
if (result.maxVal <= 0.65) {
const msg = "Can't see object yet";
return false;
}
// All good
console.log("result:" + result)
logger.info("Found image on screen: " + desc);
return result;
}
function fnIsOnScreen(img, client, repeats = 5, desc, wait = 2000, repeatDelay) {
logger.info("Looking for image on screen:" + desc + " with " + repeats + " repeats ");
let iCounter = 0;
let init = () => timeout(wait).then((asd) => {
let found = false;
do {
let found = await fnIsOnScreenOnce(img, desc, iCounter, client, repeatDelay)
} while (found !== false && iCounter++ < 10)
let imagepath = fnMarkOnImage(found.screenshot, img, found, outputDir)
let description = {};
description.action = "Is image on screen ?";
description.desc = desc;
description.repeats = repeats;
description.wait = wait;
description.img = imagepath;
description.message = "is this correct element ? if is then it was found correctly";
fnPushToOutputArray(description)
return found;
})
return init();
}
You should pass a String to the Error Object, so if you want to exchange an object you could use JSON.stringify() like this:
try {
throw new Error(JSON.stringify({result:"Hello, World"}));
}
catch(error) {
console.log(JSON.parse(error.message))
}
As you can see, this is how you would send data from a try to a catch through throwing errors. You can ofc make the second part in the catch way shorter:
error = JSON.parse(error.message);
You can try an approach like this
try {
const err = new Error("My Error Occurred");
err.extra ='Extra details';
throw err;
}
catch (error) {
console.log(error.extra)
}
As Error itself is an Object ,We can make use of this to pass extra variables of our choice