We Keep Coding

Nodemailer doesn't seem to be freeing up TCP Ports after a mail has been sent

As the question says, I'm having an issue with our bulk mailing server.
So for abit of background, This emailing application of ours runs on a CRON job, sending out mails every hour.
Every 12 hours, We send out our biggest bulks, there are around 8 different groups and each group can have anywhere between 2000 - 4000 mails in each group that needs to be sent.
This application is built in Nodejs and uses Nodemailer to handle the sending of the mails.
What I've noticed for awhile now is that every so often the server would essentially "freeze" up and the memory usage of the app would slowly climb.
Windows Event log showed this error after the last bulk emailing was meant to run, "A request to allocate an ephemeral port number from the global TCP port space has failed due to all such ports being in use."
I've gone through the documentation on the Nodemailer website and applied the following changes
Using a pooled SMTP - This is still currently in the application
Set maxMessages to infinity - I've removed this as it didn't seem to help
Set maxConnections to 20 - Also removed because it made no difference
Using a dedicated queue manager - This was my final attempt, I went with RabbitMQ and have applied their Publish/Subscribe model.
RabbitMQ has improved the performance alot but it still hasn't resolved the issue.
Publish Function
const PublishMails = (mailObj) => {
return new Promise((resolve, reject) => {
var publishMailResult = {};
if (mailObj.length > 0) {
var connection = global.RabbitMQConnection;
connection.createChannel((err, channel) => {
if (err) {
publishMailResult.Result = false;
publishMailResult.Message = err.stack;
resolve(publishMailResult);
//return process.exit(1);
}
channel.assertQueue(config.RabbitMQ.Queue_EmailQueue, {
durable: true
}, err => {
if (err) {
publishMailResult.Result = false;
publishMailResult.Message = err.stack;
resolve(publishMailResult);
//return process.exit(1);
}
var mailData = {}
for (var x = 0; x < mailObj.length; x++) {
mailData.from = 'XXX#XXX.com';
mailData.to = mailObj[x].Email;
mailData.firstName = mailObj[x].FirstName;
mailData.login = mailObj[x].Login;
mailData.email = mailObj[x].Email;
mailData.mailID = mailObj[x].MailID;
mailData.sendID = mailObj[x].SendID;
mailData.subject = "Email Message";
mailData.template = 'EmailTempLate';
channel.sendToQueue(config.RabbitMQ.Queue_EmailQueue,
Buffer.from(JSON.stringify(mailData)), {
persistent: true,
contentType: 'application/json'
});
if (x === mailObj.length - 1) {
channel.close();
publishMailResult.Result = true;
publishMailResult.Message = "All mails successfully published.";
resolve(publishMailResult);
}
}
});
})
} else {
publishMailResult.Result = false;
publishMailResult.Message = "No mails were received - Mails Not Published.";
resolve(publishMailResult);
}
});
}
Subscribe function
const SubscribeMails = (mailObj) => {
return new Promise((resolve, reject) => {
if (mailObj.PublishMailResult.Result == true) {
var options = {
viewEngine: {
extname: '.html',
layoutsDir: 'views/email/',
defaultLayout: 'Email_Template'
},
viewPath: 'views/email',
extName: '.html'
};
var transporter = nodemailer.createTransport(smtpTransport({
host: 'XXX.XXX.XXX.XX',
port: 25,
pool: true
}));
transporter.use('stream', require('nodemailer-dkim').signer({
domainName: 'XXX.com',
keySelector: 'main',
privateKey: 'XXXX'
}));
transporter.use('compile', hbs(options));
var connection = global.RabbitMQConnection;
connection.createChannel((err, channel) => {
if (err) {
console.error(err.stack);
return process.exit(1);
}
channel.assertQueue(config.RabbitMQ.Queue_EmailQueue, {
durable: true
}, err => {
if (err) {
console.error(err.stack);
return process.exit(1);
}
channel.prefetch(1);
channel.consume(config.RabbitMQ.Queue_EmailQueue, data => {
if (data === null) {
return;
}
let mail = JSON.parse(data.content.toString());
transporter.sendMail({
from: mail.from,
to: mail.to,
subject: mail.subject,
template: mail.template,
context: {
FirstName: mail.firstName,
Email: mail.email,
MailID: mail.mailID,
SendID: mail.sendID,
}
}, (err, info) => {
if (err) {
console.error(err.stack);
return channel.nack(data);
}
channel.ack(data);
channel.checkQueue(config.RabbitMQ.Queue_EmailQueue, function (checkErr, queueData) {
if (queueData != null) {
if (queueData.messageCount == 0) {
channel.close();
transporter.close(); // Added in to test if this free's up TCP ports - Didn't help
}
}
});
});
});
resolve(true);
});
});
}
});
}
It really feels like I'm meant to be somehow closing these TCP connections manually but I haven't seen anything written about this in the documentation or haven't seen it mentioned on any example code I've seen.
I'm adding in the Cron job that starts this process to perhaps help debug this issue.
var cronPCSec = '0';
var cronPCMin = '58';
var cronPCHour = '*';
var cronPCDay = '*';
var cronPCMonth = '*';
var cronPCDayOfWeek = '0-6';
var cronPCfulltimes = "" + cronPCSec + " " + cronPCMin + " " + cronPCHour + " " + cronPCDay + " " + cronPCMonth + " " + cronPCDayOfWeek + "";
var MailerCronJob = new CronJob({
cronTime: cronPCfulltimes,
onTick: function () {
let objCronJob = {};
modelPC.GetMails().then(function (mail) {
return mail;
}).then(PublishMails).then(function (PublishResult) {
objCronJob.PublishMailResult = PublishResult;
return objCronJob;
}).then(SubscribeMails).then(function (result) {
console.log("Completed Successfully");
}).catch(err => {
console.log("Failed");
console.log(err)
});
},
start: false
});
MailerCronJob.start();
Thanks

Invalid state Error on websockets when sending message

I'm working on app which send message via websockets (managed by django channels) and in return it receives json from django db as a message and renders frontend based on that json.
I have Invalid State Error when I try to send message by websocket, why? Messages send are usually Json. I works properly all the time but commented part doesn't and I don't know why please explain me.
function main() {
configGame();
}
function configGame() {
const socket = "ws://" + window.location.host + window.location.pathname;
const websocket = new WebSocket(socket);
const playerName = document.querySelector(".playerName_header").textContent;
function asignEvents() {
const ready_btn = document.querySelector(".--ready_btn");
const start_btn = document.querySelector(".--start_btn");
ready_btn.addEventListener("click", () => {
let mess = JSON.stringify({
player: playerName,
action: "ready",
});
sendMess(mess);
});
start_btn.addEventListener("click", () => {
let mess = JSON.stringify({
player: playerName,
action: "start",
});
sendMess(mess);
});
}
function openWebsocket() {
console.log("Establishing Websocket Connection...");
websocket.onopen = () => {
console.log("Websocket Connection Established!");
};
}
function setWebsocket() {
websocket.onmessage = (mess) => {
console.log(`Message: ${mess.data}`);
dataJson = JSON.parse(mess.data);
dataJson = JSON.parse(dataJson.message);
//Player Ready (jeszcze z max_players zrobic kontrolke)
if (dataJson.action === "player_ready") {
const playersReadyText = document.querySelector(".players_ready_text");
playersReadyText.textContent = `Players ready: ${dataJson.players_ready}`;
}
};
websocket.onclose = () => {
console.log("Websocket Connection Terminated!");
};
}
/*
function checkState() {
let mess = JSON.stringify({
player: playerName,
action: "game state",
});
sendMess(mess);
}
*/
function sendMess(messText) {
websocket.send(messText);
}
openWebsocket();
checkState(); //This one doesn't work
asignEvents();
setWebsocket();
}
// Asigning Event Listneres to DOM ELEMENTS
function asignEvents() {
const ready_btn = document.querySelector(".--ready_btn");
const start_btn = document.querySelector(".--start_btn");
ready_btn.addEventListener("click", () => {
console.log("Ready");
});
start_btn.addEventListener("click", () => {
console.log("Start");
});
}
main();
Error:
Console (Safari) returns InvalidState error and points to
method checkState and sendMess.
InvalidStateError: The object is in an invalid state.

Is the websocket connected?
sendMess(messText) {
if (websocket.readyState === WebSocket.OPEN) {
websocket.send(messText);
} else {
console.warn("websocket is not connected");
}
}

Connection resets after 60 seconds in node.js upload application

I've written an application in node.js consisting of a server and a client for storing/uploading files.
For reproduction purposes, here's a proof of concept using a null write stream in the server and a random read stream in the client.
Using node.js 12.19.0 on Ubuntu 18.04. The client depends on node-fetch v2.6.1.
The issue I have is after 60 seconds the connection is reset and haven't found a way to make this work.
Any ideas are appreciated.
Thank you.
testServer.js
// -- DevNull Start --
var util = require('util')
, stream = require('stream')
, Writable = stream.Writable
, setImmediate = setImmediate || function (fn) { setTimeout(fn, 0) }
;
util.inherits(DevNull, Writable);
function DevNull (opts) {
if (!(this instanceof DevNull)) return new DevNull(opts);
opts = opts || {};
Writable.call(this, opts);
}
DevNull.prototype._write = function (chunk, encoding, cb) {
setImmediate(cb);
}
// -- DevNull End --
const http = require('http');
const server = http.createServer();
server.on('request', async (req, res) => {
try {
req.socket.on('end', function() {
console.log('SOCKET END: other end of the socket sends a FIN packet');
});
req.socket.on('timeout', function() {
console.log('SOCKET TIMEOUT');
});
req.socket.on('error', function(error) {
console.log('SOCKET ERROR: ' + JSON.stringify(error));
});
req.socket.on('close', function(had_error) {
console.log('SOCKET CLOSED. IT WAS ERROR: ' + had_error);
});
const writeStream = DevNull();
const promise = new Promise((resolve, reject) => {
req.on('end', resolve);
req.on('error', reject);
});
req.pipe(writeStream);
await promise;
res.writeHead(200);
res.end('OK');
} catch (err) {
res.writeHead(500);
res.end(err.message);
}
});
server.listen(8081)
.on('listening', () => { console.log('Listening on port', server.address().port); });
testClient.js
// -- RandomStream Start --
var crypto = require('crypto');
var stream = require('stream');
var util = require('util');
var Readable = stream.Readable;
function RandomStream(length, options) {
// allow calling with or without new
if (!(this instanceof RandomStream)) {
return new RandomStream(length, options);
}
// init Readable
Readable.call(this, options);
// save the length to generate
this.lenToGenerate = length;
}
util.inherits(RandomStream, Readable);
RandomStream.prototype._read = function (size) {
if (!size) size = 1024; // default size
var ready = true;
while (ready) { // only cont while push returns true
if (size > this.lenToGenerate) { // only this left
size = this.lenToGenerate;
}
if (size) {
ready = this.push(crypto.randomBytes(size));
this.lenToGenerate -= size;
}
// when done, push null and exit loop
if (!this.lenToGenerate) {
this.push(null);
ready = false;
}
}
};
// -- RandomStream End --
const fetch = require('node-fetch');
const runSuccess = async () => { // Runs in ~35 seconds
const t = Date.now();
try {
const resp = await fetch('http://localhost:8081/test', {
method: 'PUT',
body: new RandomStream(256e6) // new RandomStream(1024e6)
});
const data = await resp.text();
console.log(Date.now() - t, data);
} catch (err) {
console.warn(Date.now() - t, err);
}
};
const runFail = async () => { // Fails after 60 seconds
const t = Date.now();
try {
const resp = await fetch('http://localhost:8081/test', {
method: 'PUT',
body: new RandomStream(1024e6)
});
const data = await resp.text();
console.log(Date.now() - t, data);
} catch (err) {
console.warn(Date.now() - t, err);
}
};
// runSuccess().then(() => process.exit(0));
runFail().then(() => process.exit(0));

I tried (unsuccessfully) to reproduce what you are seeing based on your code example. Neither the success call is completing in ~35 seconds nor is the error being thrown in 60 seconds.
However, that being said, I think what is happening here is that your client is terminating the request.
You can increase the timeout by adding a httpAgent to the fetch PUT call. You can then set a timeout in the httpAgent.
const http = require('http');
...
const runFail = async () => { // Fails after 60 seconds
const t = Date.now();
try {
const resp = await fetch('http://localhost:8081/test', {
method: 'PUT',
body: new RandomStream(1024e6),
agent: new http.Agent({ keepAlive: true, timeout: 300000 })
});
const data = await resp.text();
console.log(Date.now() - t, data);
} catch (err) {
console.warn(Date.now() - t, err);
}
};
See the fetch docs for adding a custom http(s) agent here
See options for creating http(s) agent here

This turned out to be a bug in node.js
Discussion here: https://github.com/nodejs/node/issues/35661

How to pull out handler using module exports?

I am building a node application, and trying to neatly organize my code. I wrote a serial module that imports the serial libs and handles the connection. My intention was to write a basic module and then reuse it over and over again in different projects as needed. The only part that changes per use is how the incoming serial data is handled. For this reason I would like to pull out following handler and redefine it as per the project needs. How can I use module exports to redefine only this section of the file?
I have tried added myParser to exports, but that gives me a null and I would be out of scope.
Handler to redefine/change/overload for each new project
myParser.on('data', (data) => {
console.log(data)
//DO SOMETHING WITH DATA
});
Example usage: main.js
const serial = require('./serial');
const dataParser = require('./dataParser');
const serial = require('./serial');
//call connect with CL args
serial.connect(process.argv[2], Number(process.argv[3]))
serial.myParser.on('data',(data) => {
//Do something unique with data
if (dataParser.parse(data) == 0)
serial.send('Error');
});
Full JS Module below serial.js
const SerialPort = require('serialport');
const ReadLine = require('#serialport/parser-readline');
const _d = String.fromCharCode(13); //char EOL
let myPort = null;
let myParser = null;
function connect(port, baud) {
let portName = port || `COM1`;
let baudRate = baud || 115200;
myPort = new SerialPort(portName, {baudRate: baudRate})
myParser = myPort.pipe(new ReadLine({ delimiter: '\n'}))
//Handlers
myPort.on('open', () => {
console.log(`port ${portName} open`)
});
myParser.on('data', (data) => {
console.log(data)
});
myPort.on('close', () => {
console.log(`port ${portName} closed`)
});
myPort.on('error', (err) => {
console.error('port error: ' + err)
});
}
function getPorts() {
let portlist = [];
SerialPort.list((err, ports) => {
ports.forEach(port => {
portlist.push(port.comName)
});
})
return portlist;
}
function send(data) {
myPort.write(JSON.stringify(data) + _d, function (err) {
if (err) {
return console.log('Error on write: ', err.message);
}
console.log(`${data} sent`);
});
}
function close() {
myPort.close();
}
module.exports = {
connect, getPorts, send, close
}

The problem is that a module is used where a class or a factory would be appropriate. myParser cannot exist without connect being called, so it doesn't make sense to make it available as module property, it would be unavailable by default, and multiple connect calls would override it.
It can be a factory:
module.exports = function connect(port, baud) {
let portName = port || `COM1`;
let baudRate = baud || 115200;
let myPort = new SerialPort(portName, {baudRate: baudRate})
let myParser = myPort.pipe(new ReadLine({ delimiter: '\n'}))
//Handlers
myPort.on('open', () => {
console.log(`port ${portName} open`)
});
myParser.on('data', (data) => {
console.log(data)
});
myPort.on('close', () => {
console.log(`port ${portName} closed`)
});
myPort.on('error', (err) => {
console.error('port error: ' + err)
});
function getPorts() {
let portlist = [];
SerialPort.list((err, ports) => {
ports.forEach(port => {
portlist.push(port.comName)
});
})
return portlist;
}
function send(data) {
myPort.write(JSON.stringify(data) + _d, function (err) {
if (err) {
return console.log('Error on write: ', err.message);
}
console.log(`${data} sent`);
});
}
function close() {
myPort.close();
}
return {
myParser, getPorts, send, close
};
}
So it could be used like:
const serial = require('./serial');
const connection = serial(...);
connection.myParser.on('data',(data) => {
//Do something unique with data
if (dataParser.parse(data) == 0)
connection.send('Error');
});

Uncaught InvalidStateError: Failed to execute 'send' on 'WebSocket': Still in CONNECTING state

When my page loads, I try to send a message to the server to initiate a connection, but it's not working. This script block is near the top of my file:
var connection = new WrapperWS();
connection.ident();
// var autoIdent = window.addEventListener('load', connection.ident(), false);
Most of the time, I see the error in the title:
Uncaught InvalidStateError: Failed to execute 'send' on 'WebSocket': Still in CONNECTING state
So I tried to catch the exception, as you can see below, but now it seems InvalidStateError is not defined and that produces a ReferenceError.
Here's the wrapper object for my websocket connection:
// Define WrapperWS
function WrapperWS() {
if ("WebSocket" in window) {
var ws = new WebSocket("ws://server:8000/");
var self = this;
ws.onopen = function () {
console.log("Opening a connection...");
window.identified = false;
};
ws.onclose = function (evt) {
console.log("I'm sorry. Bye!");
};
ws.onmessage = function (evt) {
// handle messages here
};
ws.onerror = function (evt) {
console.log("ERR: " + evt.data);
};
this.write = function () {
if (!window.identified) {
connection.ident();
console.debug("Wasn't identified earlier. It is now.");
}
ws.send(theText.value);
};
this.ident = function () {
var session = "Test";
try {
ws.send(session);
} catch (error) {
if (error instanceof InvalidStateError) {
// possibly still 'CONNECTING'
if (ws.readyState !== 1) {
var waitSend = setInterval(ws.send(session), 1000);
}
}
}
window.identified = true;
theText.value = "Hello!";
say.click();
theText.disabled = false;
};
};
}
I am testing using Chromium on Ubuntu.

You could send messages via a proxy function that waits for the readyState to be 1.
this.send = function (message, callback) {
this.waitForConnection(function () {
ws.send(message);
if (typeof callback !== 'undefined') {
callback();
}
}, 1000);
};
this.waitForConnection = function (callback, interval) {
if (ws.readyState === 1) {
callback();
} else {
var that = this;
// optional: implement backoff for interval here
setTimeout(function () {
that.waitForConnection(callback, interval);
}, interval);
}
};
Then use this.send in place of ws.send, and put the code that should be run afterwards in a callback:
this.ident = function () {
var session = "Test";
this.send(session, function () {
window.identified = true;
theText.value = "Hello!";
say.click();
theText.disabled = false;
});
};
For something more streamlined you could look into promises.

This error is raised because you are sending your message before the WebSocket connection is established.
You can solve it by doing this simply:
conn.onopen = () => conn.send("Message");
This onopen function waits for your WebSocket connection to establish before sending your message.

if you use one websocket client object and connect from random app places then object can be in connecting mode (concurent access).
if you want to exchange through only one websoket then
create class with promise and keep it in property
class Ws {
get newClientPromise() {
return new Promise((resolve, reject) => {
let wsClient = new WebSocket("ws://demos.kaazing.com/echo");
console.log(wsClient)
wsClient.onopen = () => {
console.log("connected");
resolve(wsClient);
};
wsClient.onerror = error => reject(error);
})
}
get clientPromise() {
if (!this.promise) {
this.promise = this.newClientPromise
}
return this.promise;
}
}
create singleton
window.wsSingleton = new Ws()
use clientPromise property in any place of app
window.wsSingleton.clientPromise
.then( wsClient =>{wsClient.send('data'); console.log('sended')})
.catch( error => alert(error) )
http://jsfiddle.net/adqu7q58/11/

Method 1: Check connection
You can resolve a promise when socket is connected:
async function send(data) {
await checkConnection();
ws.send(data);
}
Implementation
This trick is implemented using an array of resolvers.
let ws = new WebSocket(url);
let connection_resolvers = [];
let checkConnection = () => {
return new Promise((resolve, reject) => {
if (ws.readyState === WebSocket.OPEN) {
resolve();
}
else {
connection_resolvers.push({resolve, reject});
}
});
}
ws.addEventListener('open', () => {
connection_resolvers.forEach(r => r.resolve())
});
Method 2: Wait for connection
You can resolve a promise when socket is not connected:
const MAX_RETRIES = 4;
async function send(data, retries = 0) {
try {
ws.send(data);
}
catch (error) {
if (retries < MAX_RETRIES error.name === "InvalidStateError") {
await waitForConnection();
send(data, retries + 1);
}
else {
throw error;
}
}
}
Implementation
This trick is implemented using an array of resolvers.
let ws = new WebSocket(url);
let connection_resolvers = [];
let waitForConnection = () => {
return new Promise((resolve, reject) => {
connection_resolvers.push({resolve, reject});
});
}
ws.addEventListener('open', () => {
connection_resolvers.forEach(r => r.resolve())
});
My opinion is that the second method has a little bit good performance!

It is possible to use functions and readyState with setTimeout.
function openSocket()
{
webSocket = new WebSocket("");
}
function sendData()
{
if(webSocket.readyState)
{
webSocket.send(JSON.stringify(
{
"event" : "",
"message" : ""
}));
}
else
{
setTimeout(sendData, 1000);
}
}
function eventHandler()
{
webSocket.onmessage = function(e)
{
data = JSON.parse(e.data);
event = data.event;
switch (event)
{...}
}
}
openSocket();
sendData();
enentHandler();