Struggling to find memory leak. NodeJS, Express, MongoDB - javascript

so I have a NodeJS with Express configured for API calls to get data from my backend Database.
Everything works great except there is a memory leak that I can't seem to solve no matter what I do. I have narrowed it down to a MongoDB fetch call, that grabs data from MongoDB.
Since the Data being sent is the same, and to avoid multiple requests to MongoDB, I created a top level variable that fetches that data so on a request it sends that, vs doing a fetch constantly as it would be 1000's of requests every minute.
I have also set the --max_old_space_size variable to 4096 and higher at 8192 and will eventually crash also.
Below is the fetch code.
//router get apis.js
import { Model } from '../../dbmodels/models.js';
let data = null;
// Bot interval
setInterval(async () => {
try {
data = await tokenModel.find({
$or: [
{ "currentRanks.minuteTokenRank": {$lt: 51} },
{ "currentRanks.fiveMinuteTokenRank": {$lt: 51} },
{ "currentRanks.fifteenMinuteTokenRank": {$lt: 51} },
{ "currentRanks.thirtyMinuteTokenRank": {$lt: 51} },
{ "currentRanks.hourlyTokenRank": {$lt: 51} },
{ "currentRanks.dailyTokenRank": {$lt: 51} },
{ "currentRanks.weeklyTokenRank": {$lt: 51} }
]
}).lean();
} catch (error) {
console.error(error);
return;
}
}, 45000);
export async function main(req, res) {
let dataRes = data;
try {
res.status(200).json(dataRes);
dataRes = null;
} catch {(err) => {
res.status(500).json({ message: err.message })
console.log('err', err.message)
}
}
//console.log('Get Top Data')
}
//main server.js file
import dotenv from "dotenv"
dotenv.config()
import express from 'express';
const app = express();
import { createServer } from 'https';
import { createServer as _createServer } from 'http';
import { readFileSync } from 'fs';
import compression from "compression";
import pkg from 'mongoose';
const { connect, connection } = pkg;
import cors from 'cors';
import auth from './utils/auth.js'
connect(process.env.DB_URL);
let mongoClient = connection;
mongoClient.on('error', (error) => console.error(error));
mongoClient.once('open', () => console.log(`Cncted to DB ${mongoClient.name}`));
app.use(compression());
app.use(cors({ origin: ['http://localhost:3000']}));
// Apis route
import apis from './apis/apis.js';
app.use('/api', auth, apis);
//listen both http & https
const httpServer = _createServer(app);
const httpsServer = createServer({
key: readFileSync('redacted.pem'),
cert: readFileSync('redacted.pem'),
}, app);
httpServer.listen(3000, () => console.log('Server Started port 3000'));
httpsServer.listen(3001, () => {
console.log('HTTPS Server running port 3001')
})

So looks like I was able to find the leak. It wasn't with any of the API's I posted. But a hidden one that I use or Web3. There is a known bug on the web3 package leaving connections open.
This is tied to the data I am retrieving above which is why it seemed to me like it was this API, but further troubleshooting found the real issue.
Here is the issue in case anyone uses web3 packages and runs into similar.
https://github.com/web3/web3.js/issues/3042#issuecomment-663622882

Related

mongoose fails to connect on first attempt

I'm using nodejs, express, and mongoose (6.9.0) on my app. It's deployed on Vercel, the first time I try to call to the api from my frontend app, the api shows me the following error on console
MongoDB disconnectedMongoose default connection has occured: MongooseServerSelectionError: Could not connect to any servers in your MongoDB Atlas cluster. One common reason is that you're trying to access the database from an IP that isn't whitelisted. Make sure your current IP address is on your Atlas cluster's IP whitelist: https://docs.atlas.mongodb.com/security-whitelist/
My api is wishlisted on MongoDB, and this only happens on the first call to the api, the next calls works perfectly. (this only happens in production)
This is my connect function
const { MONGO_DB_URI_TEST } = process.env;
const connectionString = MONGO_DB_URI_TEST;
const mongooseOptions = {
useUnifiedTopology: true,
useNewUrlParser: true,
};
if (!connectionString) {
console.error("Failed to import .env");
}
const connectMongo = () => {
mongoose.connect(connectionString, mongooseOptions);
mongoose.connection.on("connected", () => {
console.log("MongoDB is connected");
});
mongoose.connection.on("error", (error) => {
console.log(`Mongoose default connection has occured: ${error}`);
process.exit();
});
mongoose.connection.on("disconnected", () => {
console.log("MongoDB disconnected");
});
process.on("uncaughtException", () => {
mongoose.disconnect();
});
const closeConnection = function () {
mongoose.connection.close(() => {
console.log("MongoDB disconnected due to app termination");
process.exit(0);
});
};
process.on("SIGINT", closeConnection).on("SIGTERM", closeConnection);
};
export { connectMongo };
app.js (it has many middlewares irrelevant here)
const app = express();
connectMongo();
app.use("/", router);
export { app };
index.js
import { app } from "./src/app.js";
const PORT = process.env.PORT || 4000;
const server = app.listen(PORT, () => {
console.log("Server listening on port", PORT);
});
export default server;
How can I solve this? Thanks in advance.

failed to fetch (axios, nodejs, react)

My fetch takes too long until it fails
I tried chrome, edge and postman
other fetch requests from pixabay api are working great
I compared the code to other projects I've made and found nothing
I also added a proxy to "package.json" as suggested on some places and it didnt work
posting below parts of my code:
controller:
import axios from 'axios'
export const getAll = async () =>{
const response = await axios.get('https://pixabay.com/api/?key=25540812-faf2b76d586c1787d2dd02736')
.then(resp=>resp)
.catch(err=>console.log(err));
return response;
}
router:
import express from "express";
import * as homeController from '../controllers/home.controller.js'
const homeRouter = express.Router();
homeRouter.get('/all', homeController.getAll)
export default homeRouter
indexjs:
import express from "express"
import dotenv from "dotenv"
import homeRouter from './routers/home.router.js'
dotenv.config();
const PORT = 3000 //process.env.PORT
console.log(PORT);
const app = express();
app.use(express.json());
app.use(homeRouter)
app.listen(PORT, ()=>{console.log(`server is connected on port ${PORT}`)})
fetch:
const getAll = async()=>{
try {
const response = await fetch (`http://localhost:3000/all`)
console.log("hu");
if (!response.ok) {
throw new Error();
}
else{
console.log("ok");
}
const responseObj = await response.json();
console.log(responseObj);
}
catch (error) {
console.log(error);
}
}
useEffect(()=>{
getAll();
},[])
Posting the answer by #Jaromanda X for everyone to see:
"see this app.get('/', (req, res) => { ... where's you req and res ??? nowhere, that's where - hint: export const getAll = async (req, res) =>{"
Apparently EVERY controller made with express needs to send a response back (in the form of res.send)
Should be obvious but somehow I missed it
Thanks everyone!

How can I stop getting the 'Can't resolve async_hooks' error when using npm start?

I've been following a tutorial on how to connect to a mongoDB collection. The tutorial works fine but I'm trying the same code in a project I'm working on and have been getting the following error constantly:
./node_modules/raw-body/index.js
Module not found: Can't resolve 'async_hooks' in '*:\*\*\Desktop\Projects\testing-area\node_modules\raw-body'
I've tried:
-deleting node_modules and then running npm install
-running npm update to bring all dependencies to the latest version
-updating npm itself
I've read that async_hooks is used for backend work and if you try to use it in the frontend, it can cause this issue. Problem is, I don't really know a way around it.
Here's the code I'm trying to use to connect to the mongodb collection:
//give functions of mongo db to MongoClient
const { MongoClient } = require('mongodb')
let dbConnection
const bark = (input) => {
console.log(input)
}
module.exports = {
connectToDb: (cb) => {
MongoClient.connect("mongodb://localhost:27017/Treasures")
.then((client) => {
dbConnection = client.db()
return cb()
})
.catch(err => {
bark("----------")
bark("Pants shat when trying to connect to DB:\n")
bark(err)
return cb(err)
bark("----------")
})
},
getDb: () => dbConnection
}
And then in a component I have this, to try and get the data from the collection:
// Imports/Requires for mongoDb connection
const { ObjectID } = require("bson");
const express = require("express");
const { connectToDb, getDb } = require("../../db")
// COMPONENT STARTS HERE:
export const TreasureGen = () => {
//init app and middleware
const app = express();
//db connection
let db
connectToDb((err) => {
if(!err)
{
app.listen(3000, () => {
bark("App listening on port 3000")
})
db = getDb()
}
})

Socket.io on Heroku does not run as expected

I have a MERN application. I want to run this on Heroku. That also works so far. But I can't get socket.io to run on Heroku. My server listens on port 5555. Below I have listed all possible scenarios that I have already tried without success. What else can I do ?
I specify "ws://localhost:5555" or "http://localhost:5555" whatever it is, it works with a local address.
Thank you very much!
index.js Server
import express from "express";
import http from "http";
import { Server } from "socket.io";
const app = express();
const server = http.createServer(app);
const socketio = new Server(server, { cors: { origin: "*" } });
socketio.on("connect", (socket) => {
socket.on("addUser", (userId) => {
addUser(userId, socket.id);
console.log(users);
});
...
});
app.listen(process.env.PORT || 5050, () => {
verbindugZuMongoDb();
});
server.listen(5555);
App.js
import { io } from "socket.io-client";
useEffect(() => {
setSocket(io("wss://example.herokuapp.com:5555/")); // Also not working
setSocket(io("https://example.herokuapp.com:5555/")); // Also not working
setSocket(io()); // Also not working
setSocket(io(":5555")); // Also not working
setSocket(io("https://example.herokuapp.com/")); // Also not working
}, []);

Make Shopify API Post Request using Next.js and Koa Router (405 Method not allowed)

I'm not sure if this is a Shopify related issue or a general coding issue, but when I send a request to the API for Next.js or how Shopify has exemplified, I get a 405 error. I am using Koa Router (that was what Shopify has prebuilt with their Auth boilerplate setup) and MongoDB.
server.js
import addTest from "../helpers/add";
...
router.post("/api/add", async (ctx) => {
try {
/* Get the shop from the authorization header to prevent users from spoofing the data */
shopDomain = await getShopUrlFromSession(ctx.req, ctx.res);
if (shopDomain) {
addTest(ctx.req, ctx.res);
}
res.status(201).send(response[0]);
} catch (error) {
res.status(500).send(error.message);
}
});
add.js
import connectMongo from "../../../utils/connectMongo";
import { Shopify } from "#shopify/shopify-api";
import Test from "../../../models/testModel";
export default async function addTest(req, res) {
try {
console.log("CONNECTING TO MONGO");
await connectMongo();
console.log("CONNECTED TO MONGO");
const session = await Shopify.Utils.loadCurrentSession(req, res, true);
const client = new Shopify.Clients.Graphql(
session.shop,
session.accessToken
);
const shopId = `https://${session.shop}`;
console.log("CREATING DOCUMENT");
const test = await Test.create(req.body);
console.log("CREATED DOCUMENT");
res.json({ test });
} catch (error) {
console.log(error);
res.json({ error });
}
}

Categories