We Keep Coding

Nodemailer doesn't seem to be freeing up TCP Ports after a mail has been sent

As the question says, I'm having an issue with our bulk mailing server.
So for abit of background, This emailing application of ours runs on a CRON job, sending out mails every hour.
Every 12 hours, We send out our biggest bulks, there are around 8 different groups and each group can have anywhere between 2000 - 4000 mails in each group that needs to be sent.
This application is built in Nodejs and uses Nodemailer to handle the sending of the mails.
What I've noticed for awhile now is that every so often the server would essentially "freeze" up and the memory usage of the app would slowly climb.
Windows Event log showed this error after the last bulk emailing was meant to run, "A request to allocate an ephemeral port number from the global TCP port space has failed due to all such ports being in use."
I've gone through the documentation on the Nodemailer website and applied the following changes
Using a pooled SMTP - This is still currently in the application
Set maxMessages to infinity - I've removed this as it didn't seem to help
Set maxConnections to 20 - Also removed because it made no difference
Using a dedicated queue manager - This was my final attempt, I went with RabbitMQ and have applied their Publish/Subscribe model.
RabbitMQ has improved the performance alot but it still hasn't resolved the issue.
Publish Function
const PublishMails = (mailObj) => {
return new Promise((resolve, reject) => {
var publishMailResult = {};
if (mailObj.length > 0) {
var connection = global.RabbitMQConnection;
connection.createChannel((err, channel) => {
if (err) {
publishMailResult.Result = false;
publishMailResult.Message = err.stack;
resolve(publishMailResult);
//return process.exit(1);
}
channel.assertQueue(config.RabbitMQ.Queue_EmailQueue, {
durable: true
}, err => {
if (err) {
publishMailResult.Result = false;
publishMailResult.Message = err.stack;
resolve(publishMailResult);
//return process.exit(1);
}
var mailData = {}
for (var x = 0; x < mailObj.length; x++) {
mailData.from = 'XXX#XXX.com';
mailData.to = mailObj[x].Email;
mailData.firstName = mailObj[x].FirstName;
mailData.login = mailObj[x].Login;
mailData.email = mailObj[x].Email;
mailData.mailID = mailObj[x].MailID;
mailData.sendID = mailObj[x].SendID;
mailData.subject = "Email Message";
mailData.template = 'EmailTempLate';
channel.sendToQueue(config.RabbitMQ.Queue_EmailQueue,
Buffer.from(JSON.stringify(mailData)), {
persistent: true,
contentType: 'application/json'
});
if (x === mailObj.length - 1) {
channel.close();
publishMailResult.Result = true;
publishMailResult.Message = "All mails successfully published.";
resolve(publishMailResult);
}
}
});
})
} else {
publishMailResult.Result = false;
publishMailResult.Message = "No mails were received - Mails Not Published.";
resolve(publishMailResult);
}
});
}
Subscribe function
const SubscribeMails = (mailObj) => {
return new Promise((resolve, reject) => {
if (mailObj.PublishMailResult.Result == true) {
var options = {
viewEngine: {
extname: '.html',
layoutsDir: 'views/email/',
defaultLayout: 'Email_Template'
},
viewPath: 'views/email',
extName: '.html'
};
var transporter = nodemailer.createTransport(smtpTransport({
host: 'XXX.XXX.XXX.XX',
port: 25,
pool: true
}));
transporter.use('stream', require('nodemailer-dkim').signer({
domainName: 'XXX.com',
keySelector: 'main',
privateKey: 'XXXX'
}));
transporter.use('compile', hbs(options));
var connection = global.RabbitMQConnection;
connection.createChannel((err, channel) => {
if (err) {
console.error(err.stack);
return process.exit(1);
}
channel.assertQueue(config.RabbitMQ.Queue_EmailQueue, {
durable: true
}, err => {
if (err) {
console.error(err.stack);
return process.exit(1);
}
channel.prefetch(1);
channel.consume(config.RabbitMQ.Queue_EmailQueue, data => {
if (data === null) {
return;
}
let mail = JSON.parse(data.content.toString());
transporter.sendMail({
from: mail.from,
to: mail.to,
subject: mail.subject,
template: mail.template,
context: {
FirstName: mail.firstName,
Email: mail.email,
MailID: mail.mailID,
SendID: mail.sendID,
}
}, (err, info) => {
if (err) {
console.error(err.stack);
return channel.nack(data);
}
channel.ack(data);
channel.checkQueue(config.RabbitMQ.Queue_EmailQueue, function (checkErr, queueData) {
if (queueData != null) {
if (queueData.messageCount == 0) {
channel.close();
transporter.close(); // Added in to test if this free's up TCP ports - Didn't help
}
}
});
});
});
resolve(true);
});
});
}
});
}
It really feels like I'm meant to be somehow closing these TCP connections manually but I haven't seen anything written about this in the documentation or haven't seen it mentioned on any example code I've seen.
I'm adding in the Cron job that starts this process to perhaps help debug this issue.
var cronPCSec = '0';
var cronPCMin = '58';
var cronPCHour = '*';
var cronPCDay = '*';
var cronPCMonth = '*';
var cronPCDayOfWeek = '0-6';
var cronPCfulltimes = "" + cronPCSec + " " + cronPCMin + " " + cronPCHour + " " + cronPCDay + " " + cronPCMonth + " " + cronPCDayOfWeek + "";
var MailerCronJob = new CronJob({
cronTime: cronPCfulltimes,
onTick: function () {
let objCronJob = {};
modelPC.GetMails().then(function (mail) {
return mail;
}).then(PublishMails).then(function (PublishResult) {
objCronJob.PublishMailResult = PublishResult;
return objCronJob;
}).then(SubscribeMails).then(function (result) {
console.log("Completed Successfully");
}).catch(err => {
console.log("Failed");
console.log(err)
});
},
start: false
});
MailerCronJob.start();
Thanks

How to seperate websocket/express server into different files?

I am having trouble wrapping my head around the best way to execute some code below from a separate file. I have it running well in my my app.js but obviously, I do not want to keep a bunch of code in only app.js and want to start moving things into other files but am at a loss on how to do so. My app.js more or less looks like this and is extremely bloated:
//web server
const express = require('express');
const app = express();
const port = 3000;
//raspberryPI
const { ChatClient } = require("dank-twitch-irc");
const { yay } = require('./JavaScript/yay');
const { readPin } = require('./JavaScript/readPin');
let ejs = require('ejs');
//listen on port 3000
app.listen(port, () => console.info('Listening on port', { port }));
app.use(express.static('./'))
app.set('view engine', 'ejs');
app.get('/', (req, res) => {
res.render('index');
})
//Setup raspberry pi stepper motor hat
let spec = {
steppers: [{ W1: 'M1', W2: 'M2' }, { W1: 'M3', W2: 'M4'}],
//steppers: [{ W1: 'M3', W2: 'M4' }],
};
const motorHat = require('motor-hat')(spec);
motorHat.init();
motorHat.steppers[0].setSteps(2048);
motorHat.steppers[0].setSpeed({ rpm: 5 });
let client = new ChatClient({
connection: {
type: "websocket",
secure: true,
}
});
//connected?
client.on("ready", () => console.log("Successfully connected to chat"));
client.on("close", (error) => {
if (error != null) {
console.error("Client closed due to error", error);
}
});
const keywordsList = [];
client.on("PRIVMSG", (msg, error) => {
console.log(`[#${msg.channelName}] ${msg.displayName}: ${msg.messageText}`);
const keywordFinder = /\b(^right)|(^left)|(^fire)\b/iy;
//empty array fixes null value
const keywords = msg.messageText.match(keywordFinder) || [];
if (msg.messageText === keywords[0]){
if (keywordsList.length>10){
keywordsList.shift();
}
keywordsList.push(`${msg.displayName}: ${msg.messageText}`);
console.log(keywordsList);
}
// keywordsList.forEach(keywords)
if (msg.messageText === "right") {
motorHat.steppers[0].stepSync('fwd', 12);
}
if (msg.messageText === "left") {
motorHat.steppers[0].stepSync('back', 12);
}
if (msg.messageText === "yay") {
yay();
}
if (msg.messageText === "pin") {
readPin();
}
if (error != null) {
console.error("Client closed due to error", error);
}
});
//connect to specific Twitch chat
client.connect();
client.join("yuhn");
and the code I am having trouble removing is this (sorry if redundant):
const keywordsList = [];
client.on("PRIVMSG", (msg, error) => {
console.log(`[#${msg.channelName}] ${msg.displayName}: ${msg.messageText}`);t
const keywordFinder = /\b(^right)|(^left)|(^fire)\b/iy;
//empty array fixes null value
const keywords = msg.messageText.match(keywordFinder) || [];
if (msg.messageText === keywords[0]){
if (keywordsList.length>10){
keywordsList.shift();
}
keywordsList.push(`${msg.displayName}: ${msg.messageText}`);
console.log(keywordsList);
}
// keywordsList.forEach(keywords)
if (msg.messageText === "right") {
motorHat.steppers[0].stepSync('fwd', 12);
}
if (msg.messageText === "left") {
motorHat.steppers[0].stepSync('back', 12);
}
if (msg.messageText === "yay") {
yay();
}
if (msg.messageText === "pin") {
readPin();
}
if (error != null) {
console.error("Client closed due to error", error);
}
});
What I thought I could do is just put the code in a separate file, make sure all my modules are connected (export client and import it into the separate files along with motorhat, etc) but it does not run. Any and all help is appreciated. I have attempted to read about this for a few hours now and I keep getting in depth guides to routing/express file structure...which is fine but I need to know if that's the direction to go in and if so, the first steps.

With my current understanding of your code, I would start by creating a file that contains the callback of your websocket event:
ws_callbacks.js
const motorHat = require('./main.js').motorHat
const keywordsList = [];
exports.PRIVMSG_callback = (msg, error) => {
console.log(`[#${msg.channelName}] ${msg.displayName}: ${msg.messageText}`);t
const keywordFinder = /\b(^right)|(^left)|(^fire)\b/iy;
//empty array fixes null value
const keywords = msg.messageText.match(keywordFinder) || [];
// Rest of your code
}
You can then access the callback by requiring ws_callbacks.js in your main file as so:
app.js
// ...
motorHat.init();
motorHat.steppers[0].setSteps(2048);
motorHat.steppers[0].setSpeed({ rpm: 5 });
exports.motorHat = motorHat
// ...
client.on("PRIVMSG", require('./ws_callbacks.js').PRIVMSG_callback )
// ...
Here, The callback accesses motorHat, which is why it is being exported within the app.js file.
Note that you can also export the other websocket callbacks (ready and close) in similar fashion.
If you prefer to keep all websocket-related code in a separate file, this is how I would write it:
chat.js
const { ChatClient } = require("dank-twitch-irc");
let client = new ChatClient({
connection: {
type: "websocket",
secure: true,
}
});
//connected?
client.on("ready", () => console.log("Successfully connected to chat"));
client.on("close", (error) => {
if (error != null) {
console.error("Client closed due to error", error);
}
});
client.on("PRIVMSG", require('./ws_callbacks.js').PRIVMSG_callback )
// Exporting client in case you want to use it somewhere else
exports.client = client
And then import it in app.js.
However, you have mentioned experiencing problems with importing and exporting client so I am unsure if this will work

Invalid state Error on websockets when sending message

I'm working on app which send message via websockets (managed by django channels) and in return it receives json from django db as a message and renders frontend based on that json.
I have Invalid State Error when I try to send message by websocket, why? Messages send are usually Json. I works properly all the time but commented part doesn't and I don't know why please explain me.
function main() {
configGame();
}
function configGame() {
const socket = "ws://" + window.location.host + window.location.pathname;
const websocket = new WebSocket(socket);
const playerName = document.querySelector(".playerName_header").textContent;
function asignEvents() {
const ready_btn = document.querySelector(".--ready_btn");
const start_btn = document.querySelector(".--start_btn");
ready_btn.addEventListener("click", () => {
let mess = JSON.stringify({
player: playerName,
action: "ready",
});
sendMess(mess);
});
start_btn.addEventListener("click", () => {
let mess = JSON.stringify({
player: playerName,
action: "start",
});
sendMess(mess);
});
}
function openWebsocket() {
console.log("Establishing Websocket Connection...");
websocket.onopen = () => {
console.log("Websocket Connection Established!");
};
}
function setWebsocket() {
websocket.onmessage = (mess) => {
console.log(`Message: ${mess.data}`);
dataJson = JSON.parse(mess.data);
dataJson = JSON.parse(dataJson.message);
//Player Ready (jeszcze z max_players zrobic kontrolke)
if (dataJson.action === "player_ready") {
const playersReadyText = document.querySelector(".players_ready_text");
playersReadyText.textContent = `Players ready: ${dataJson.players_ready}`;
}
};
websocket.onclose = () => {
console.log("Websocket Connection Terminated!");
};
}
/*
function checkState() {
let mess = JSON.stringify({
player: playerName,
action: "game state",
});
sendMess(mess);
}
*/
function sendMess(messText) {
websocket.send(messText);
}
openWebsocket();
checkState(); //This one doesn't work
asignEvents();
setWebsocket();
}
// Asigning Event Listneres to DOM ELEMENTS
function asignEvents() {
const ready_btn = document.querySelector(".--ready_btn");
const start_btn = document.querySelector(".--start_btn");
ready_btn.addEventListener("click", () => {
console.log("Ready");
});
start_btn.addEventListener("click", () => {
console.log("Start");
});
}
main();
Error:
Console (Safari) returns InvalidState error and points to
method checkState and sendMess.
InvalidStateError: The object is in an invalid state.

Is the websocket connected?
sendMess(messText) {
if (websocket.readyState === WebSocket.OPEN) {
websocket.send(messText);
} else {
console.warn("websocket is not connected");
}
}

Node.js - Returning response from https.request() showing weird symbols

I am making an experimental web proxy in Node.js. A lot of websites work like normal even Discord.com. The problem is on this website.
For some odd reason, it's showing a lot of weird symbols instead of the website. But if I go on any image on that site the images work fine. I would love the help. I would also prefer to use regular HTTPS / HTTP modules.
By the way, the commented out parts are for a WebSocket proxy so discard that.
const express = require('express'),
app = express(),
WebSocket = require('ws'),
fs = require('fs'),
https = require('https'),
http = require('http');
btoa = (str) => {
str = new Buffer.from(str).toString('base64');
return str;
};
atob = (str) => {
str = new Buffer.from(str, 'base64').toString('utf-8');
return str;
};
const server_options = {
key: fs.readFileSync('./ssl/default.key'),
cert: fs.readFileSync('./ssl/default.crt')
}
const server = https.createServer(server_options, app);
//server.on('upgrade', function upgrade(req, socket, head) {
// const wss = new WebSocket.Server({
// server: server
// });
// wss.on('connection', function connection(ws) {
// console.log(req.url.slice(1))
// const client = new WebSocket(req.url.slice(1));
//ws.on('message', function incoming(message) {
// client.send(message)
// });
// client.on('message', function incoming(data) {
// ws.send(data)
// })
// ws.on('error', function incoming(data) {
// client.send(data)
// });
// client.on('error', function incoming(data) {
// ws.send(data)
//});
// });
//});
app.use('/', async(req, res, next) => {
var proxy = {};
proxy.url = `https://www.startpage.com${req.url}`;
proxy.url = {
href: proxy.url,
hostname : proxy.url.split('/').splice(2).splice(0, 1).join('/'),
origin : proxy.url.split('/').splice(0, 3).join('/'),
encoded_origin : btoa(proxy.url.split('/').splice(0, 3).join('/')),
path : '/' + proxy.url.split('/').splice(3).join('/'),
protocol : proxy.url.split('\:').splice(0, 1).join(''),
}
proxy.requestHeaders = req.headers;
proxy.requestHeaders.host = proxy.url.hostname;
delete proxy.requestHeaders['accept-encoding']
proxy.requestHeaders['referer'] = proxy.url.href;
proxy.requestHeaders['origin'] = proxy.url.origin;
proxy.options = {
method: req.method,
headers: proxy.requestHeaders,
rejectUnauthorized: false
}
if (proxy.url.protocol == 'https') { proxy.protocol = https; }
else {proxy.protocol = http};
proxy.sendRequest = proxy.protocol.request(proxy.url.href, proxy.options, server_res => {
var body = [], redirect = false, redirect_value = '';
server_res.on('data', (chunk) => {
body.push(Buffer.from(chunk, 'binary'))
});
var ct = 'text/plain';
Object.entries(server_res.headers).forEach(([header_name, header_value]) => {
if (header_name.startsWith('content-encoding') || header_name.startsWith('x-') || header_name.startsWith('cf-') || header_name.startsWith('strict-transport-security') || header_name.startsWith('content-security-policy')) {
delete server_res.headers[header_name];
}
if (header_name.startsWith('location') || header_name.startsWith('Location')) {
redirect = true; redirect_value = header_value;
delete server_res.headers[header_name];
}
if (header_name == 'content-type') ct = header_value;
});
if (ct == null || typeof ct == 'undefined') ct = 'text/html';
if (redirect == true) { return res.redirect(307, '/' + redirect_value); };
server_res.on('end', () => {
body = Buffer.concat(body)
res.contentType(ct)
res.set(server_res.headers);
res.status(server_res.statusCode)
res.send(body)
});
});
proxy.sendRequest.on('error', err => {
res.send(err.toString())
});
if (req.method == 'POST') {
req.raw_body = '';
req.str_body = '';
req.on('data', chunk => {
req.raw_body += chunk.toString(); // convert Buffer to string
});
req.on('end', () => {
req.str_body = req.raw_body;
proxy.sendRequest.write(req.str_body);
proxy.sendRequest.end();
});
} else proxy.sendRequest.end();
});
server.listen('9000')

Web Push Notification for Chrome [closed]

Closed. This question needs details or clarity. It is not currently accepting answers.
Want to improve this question? Add details and clarify the problem by editing this post.
Closed 6 years ago.
Improve this question
How to create Web Chrome Notifications ?
I have been searching for how to create a web push chrome notification using payload and after a decent R&D, i would like to share my code with you all.

Client Side Code:
main.js
function base64UrlToUint8Array(base64UrlData) {
const padding = '='.repeat((4 - base64UrlData.length % 4) % 4);
const base64 = (base64UrlData + padding)
.replace(/\-/g, '+')
.replace(/_/g, '/');
const rawData = atob(base64);
const buffer = new Uint8Array(rawData.length);
for (let i = 0; i < rawData.length; ++i) {
buffer[i] = rawData.charCodeAt(i);
}
return buffer;
}
(function() {
if (!('serviceWorker' in navigator)) {
return;
}
return navigator.serviceWorker.register('sw.js')
.then(function(registration) {
console.log('service worker registered');
return navigator.serviceWorker.ready;
})
.then(function(reg) {
var channel = new MessageChannel();
channel.port1.onmessage = function(e) {
window.document.title = e.data;
}
reg.active.postMessage('setup', [channel.port2]);
var subscribeOptions = { userVisibleOnly: true };
// Figure out the vapid key
var searchParam = window.location.search;
vapidRegex = searchParam.match(/vapid=(.[^&]*)/);
if (vapidRegex) {
// Convert the base 64 encoded string
subscribeOptions.applicationServerKey = base64UrlToUint8Array(vapidRegex[1]);
}
console.log(subscribeOptions);
return reg.pushManager.subscribe(subscribeOptions);
})
.then(function(subscription) {
console.log(JSON.stringify(subscription));
window.subscribeSuccess = true;
window.testSubscription = JSON.stringify(subscription);
})
.catch(function(err) {
window.subscribeSuccess = false;
window.subscribeError = err;
});
})();
sw.js
'use strict';
let port;
let pushMessage;
console.log('Started', self);
self.addEventListener('install', function (event) {
self.skipWaiting();
console.log('Installed', event);
});
self.addEventListener('activate', function (event) {
console.log('Activated', event);
});
self.addEventListener('push', function (event) {
if (event.data) {
console.log(event.data);
}
var payload = event.data ? event.data.text() : 'no payload';
event.waitUntil(
self.registration.showNotification('Web Push Notification ', {
body: payload,
icon: 'images/abc.png',
}));
});
self.addEventListener('notificationclick', function(event) {
console.log('Notification click: tag', event.notification.tag);
event.notification.close();
var url = 'http://www.google.com/';
event.waitUntil(
clients.matchAll({
type: 'window'
})
.then(function(windowClients) {
console.log('WindowClients', windowClients);
for (var i = 0; i < windowClients.length; i++) {
var client = windowClients[i];
console.log('WindowClient', client);
if (client.url === url && 'focus' in client) {
return client.focus();
}
}
if (clients.openWindow) {
return clients.openWindow(url);
}
})
);
});
.................................................................................
SERVER SIDE CODE:
routers/notifications.js
var express = require('express');
var router = express.Router();
var webPush = require('web-push');
var gcmUrl = "https://android.googleapis.com/gcm/send/";
//this key is coming from gcm project
var GCM_API_KEY = "AIzaSyAI2OM3My5E8uulEGtQn31zfydBlCjhezZZWlpE";
router.get('/', function (req, res, next) {
res.send('Hello');
})
webPush.setGCMAPIKey(GCM_API_KEY);
router.post('/', function (req, res, next) {
var payload = req.body;
console.log(payload);
gcmUrl = "https://android.googleapis.com/gcm/send/"+req.body.registrationId;
console.log('sending request to GCM');
webPush.sendNotification(gcmUrl, {
RegistrationID: req.body.registrationId,
TTL: req.body.ttl,
payload: req.body.payload,
userPublicKey: req.body.key,
userAuth: req.body.authSecret,
})
.then(function (gcmResponse) {
console.log('gcm: ', gcmResponse);
res.sendStatus(201);
}).catch(function (gcmError) {
console.error("error frm gcm");
console.log(gcmError);
});
});
module.exports = router;
.................................................................................
Sending Request:
I used postman to send notification.
{
"ttl" : "5",
"payload" :"Your Request #123 is accepted" ,
"key" : "BBqMWJxSWFFcnvpowevnskdhdX0im4nADVSj9F_53xhxahcz-dnnR8wZv44o=",
"authSecret" : "elOOVcwciiaaavkYiDA==",
"registrationId": "doPVnhn3Ymc:APA91bFx_2tyfKs2xbpaocnakdrzdjS0ED9okiNEz-jECb3lC43kTqfltBZ54prNgtH3P_mBaDs5JOEQihhZld-E-vggxaUVUhyphe-oSoCE"
}

Check this guide, is not hard at all
https://developer.mozilla.org/en-US/docs/Web/API/Notifications_API/Using_the_Notifications_API

Here's the guide with examples for Google Chrome: https://developers.google.com/web/updates/2015/03/push-notifications-on-the-open-web