We Keep Coding

Node.js file blocks terminal after correct execution

I'm new to using Node and fetch and of course I'm having some problems with my code. Essentially I'm trying to implement a project where I get some json data through an API request and store it into a mysql database. These data is contained in multiple pages and therefore I used a simple for cycle for multiple fetching. I do this 2 times as I have to get data from 2 different object lists. For storing the data I first established a mysql connection and later I execute the sql query inside another for iterating the single object data.
It performes correctly both extraction of json data and storage in mysql database but once I execute node index.js on the terminal, the process keeps on running and the terminal gets suspended until I force the process to terminate.
I used why-is-node-running and found out this:
Here's the code of index.js:
import mysql from 'mysql';
import fetch from 'node-fetch';
import log from 'why-is-node-running';
const URL0 = "https://atlas.ripe.net/api/v2/probes/?status=1";
const sql = "INSERT INTO probes (id, country, longitude, latitude) VALUES (?,?,?,?)";
const sql1 = "INSERT INTO anchors (id, country, longitude, latitude) VALUES (?,?,?,?)";
const PG_SIZE = 100;
let num_pages_probes=120;
let i=0, j=1, k=1, a=0;
const con = mysql.createConnection({
host:'localhost',
user:'root',
password:'',
database:'probes&anchors'
});
con.connect((err)=>{
if(err){
console.log("Connection not proper");
}else{
console.log("connected");
}
});
/*
fetch(URL0)
.then((response) => {
if (!response.ok) {
throw new Error("HTTP error! status: "
+ response.status);
} else {
return response.json();
}
})
.then((data) => {
num_pages_probes = Math.ceil(data.count/PG_SIZE);
console.log(num_pages_probes);
});
*/
for (j; j<=2; j++){
console.log("j="+j);
let URLi = "https://atlas.ripe.net/api/v2/probes/?page="+j+"&status=1";
fetch(URLi)
.then((response) => {
if (!response.ok) {
throw new Error("HTTP error! status: "
+ response.status);
} else {
return response.json();
}
})
.then((data) => {
for (let probe of data.results){
i++;
let id0 = probe.id;
let country = probe.country_code;
let longitude = probe.geometry.coordinates[0];
let latitude = probe.geometry.coordinates[1];
con.query(sql, [id0, country, longitude, latitude], function (err, result) {
if (err) throw err;
console.log("1 record inserted");
});
console.log("id0: "+id0+"\t"+"cc: "+country+"\t"+"long: "+longitude+"\t"+"lati: "+latitude);
console.log(i);
}
// con.end();
});
}
for (k; k<=2; k++){
console.log("k="+k);
let URLi = "https://atlas.ripe.net/api/v2/anchors/?page="+k;
fetch(URLi)
.then((response) => {
if (!response.ok) {
throw new Error("HTTP error! status: "
+ response.status);
} else {
return response.json();
}
})
.then((data) => {
for (let anchor of data.results){
a++;
let id0 = anchor.id;
let country = anchor.country;
let longitude = anchor.geometry.coordinates[0];
let latitude = anchor.geometry.coordinates[1];
con.query(sql1, [id0, country, longitude, latitude], function (err, result) {
if (err) throw err;
console.log("1 record inserted");
});
console.log("id0: "+id0+"\t"+"cc: "+country+"\t"+"long: "+longitude+"\t"+"lati: "+latitude);
console.log(a);
}
});
}
setTimeout(function () {
log() // logs out active handles that are keeping node running
}, 100)
Can someone help me out please? I don't know where to put my hands on.
PS. I purposely limited the cycle to 2 but it would actually be like 120.

You are not closing your mysql connection which keep your proccess up.
You probably want to close your connection when all your fetch/inserts are done, the tricks here is to ensure you've completed all your inserts before closing your connection.
You can have a look at async/await syntax, it will help you ensure you are closing only when you've done your inserts.
A very simplified version would look like:
const fn = async () => {
const con = mysql.createConnection({ ... });
for (...) {
const res = await fetch({ ... });
const data = await res.json();
await con.query({ ... });
  }
await con.close();
}
fn();
NOTE: The mysql lib seems to only work with callback, so you will probably have to promisify the methods you need (see utils.promisify)

filterByFormula Result to Create New Record

I’m using Twilio Functions (node.js) to load to AirTable all incoming texts. I’m utilizing 2 tables -
“Receive Text” with columns "Inbound Text Date", "Inbound Text Content" and "UID".
“Users” with columns "UID" and Phone Number ("Your Number").
When a text comes in, all I have is the phone number. so I’m trying to use FilterByFormula with the phone number to retrieve the user’s UID from the “Users” table.
Here’s the code I’m running -
exports.handler = function(context, event, callback) {
var Airtable = require('airtable');
var base = new Airtable({apiKey: context.AIRTABLE_API_KEY}).base('app######');
var phone = event.From;
var format_phone = // format to +1 (###) ###-####
base('Users').select({
filterByFormula: `Your Number = "${format_phone}"`,
view: "Grid view"
}).eachPage(function page(records, fetchNextPage) {
records.forEach(function(record) {
console.log(record.id);
});
fetchNextPage();
}, function done(err) {
if (err) { console.error(err); return; }
});
inputbase = 'Receive Text';
message = {
'UID' : ['rec#######'],
'Inbound Text Date' : Date.now(),
'Inbound Text Content' : event.Body,
};
base(inputbase).create(message, function(err, record) {
if (err) { console.error(err); return; }
console.log(record.getId());
callback(null, message);
});
};
Right now for testing, the UID is hardcoded and it's working fine. I'm unsure how do I get the UID from the filterByFormula into a variable or just straight into the message variable.
Both the UID and Phone Number are unique values in “Users”, so I don't expect to receive more than 1 record back.
Thanks.

try this, let me know how it goes.
const Airtable = require('airtable');
exports.handler = function(context, event, callback) {
const base = new Airtable({apiKey: context.AIRTABLE_API_KEY}).base(context.AIRTABLE_DATABASE);
const phone = event.From;
const body = event.Body;
function createRecord(UID) {
const today = new Date();
const date = `${(today.getMonth()+1)}/${today.getDate()}/${today.getFullYear()}`;
const inputbase = 'Receive Text';
const message = {
'UID': UID,
'Inbound Text Date': date,
'Inbound Text Content': body,
};
base(inputbase).create(message, function(err, record) {
if (err) {
console.error(err);
callback("error");
}
console.log(record.getId());
callback(null, message);
});
}
base('Users').select({
filterByFormula: `{Your Number} = "${phone}"`,
view: "Grid view"
}).firstPage().then(records => {
if (!records[0]) {
console.log("**NO MATCH**");
createRecord("**NO MATCH**");
} else {
console.log("**MATCH**");
createRecord(records[0].fields.UID);
}
})
.catch(err => {
console.log(err);
callback("error");
});
};

Async Qestiong Node.js

I am new to Node.js, and I was so used to async language and not familiar with synchronized processing style.
I am working on a basic check out function, that check out items in a users cart and get the total. The sql works fine but I have trouble to get the total. Please see my code below
exports.checkout = (req, res) =>{
const schema = {
user_id: joi.number().required(),
}
const results = joi.validate(req.body,schema)
if (results.error){
//or someother redirect
res.status(400).send(results.error)
return;
}
let data = {
user_id: req.body.user_id
};
let order_id ;
let total = 0 ;
//first create order entry for the products and movies in the shopping cart
query = "INSERT INTO Order_table (user_id,order_time) VALUES (?, NOW());";
db.query(query,[data.user_id], async(err, resp) => {
if(err){
res.status(500).send(err);
console.log(`Check out Order [SQL ERROR]: ${err}`)
}
else{
console.log('Created order');
order_id=await resp.insertId;
console.log(order_id)
await console.log(await get_total(data.user_id,await order_id,res));
//await res.send(Promise.all(total));
//res.status(200).send(await total);
}
})
}
async function check_out_product(user_id,order_id,res){
var total = 0;
//Find all the products with details that is in the shopping cart belong to the given user_id.
query = "Select product_id,quantity,unit_price,product_tax_code from Shoping_Cart_Product join product on Shoping_Cart_Product.product_id = product.id where user_id =?";
await db.query(query,[user_id], async (err, resp) => {
if(err){
res.status(500).send(err);
console.log(`Check out Product [SQL ERROR]: ${err}`)
}
else{
console.log(`Check out product`)
if(resp){
// For each product insert them into the Order Detail table
await resp.forEach(async function(value){
//console.log(value['product_id']);
query = "INSERT INTO Order_Detail_Products (id, product_id,order_id,quantity) VALUES ((select id from (select * from Order_Detail_Products) AS temp where temp.order_id = ? and product_id=? ),?,?,?) ON DUPLICATE KEY UPDATE quantity = ?"
await db.query(query,[order_id,value['product_id'],value['product_id'],order_id,value['quantity'],value['quantity']], async (err, resp) => {
if(err){
res.status(500).send(err);
console.log(`Product order Insert [SQL ERROR]: ${err}`)
}
else{
console.log(`Product Order is inserted`)
total = await total + (value['quantity'] * value['unit_price'])
await console.log('current total is',total)
}
})
});
//after the insertion is done, delete the original entry from the shopping cart product table.
query = "Delete from Shoping_Cart_Product where user_id=?"
await db.query(query,[user_id], (err, resp) => {
if(err){
res.status(500).send(err);
console.log(`Shopping Cart Product delete [SQL ERROR]: ${err}`)
}
else{
console.log(`Shopping Cart Product is emptied`)
}
})
}
//res.status(200).send(resp[0]);
}
})
return Promise.resolve(total);
}
async function check_out_movie(user_id,order_id,res){
var total = 0;
//Find all the movies with details that is in the shopping cart belong to the given user_id.
query = "Select movie_id,method,price_rent,price_buy from Shoping_Cart_Movie join movie on Shoping_Cart_Movie.movie_id = movie.id where user_id =?";
await db.query(query,[user_id],async (err, resp) => {
if(err){
res.status(500).send(err);
console.log(`Check out Movie [SQL ERROR]: ${err}`)
}
else{
await console.log(`Check out Movie`)
if(resp){
// For each product insert them into the Order Detail table
await resp.forEach(async function(value){
var isrent;
if (value['method']=="rent"){
isrent = 0;
}else{
isrent =1;
}
query = "INSERT INTO Order_Detail_Movie (id, movie_id,order_id,isrent,duration) VALUES ((select id from (select * from Order_Detail_Movie) AS temp where temp.order_id = ? and movie_id=? ),?,?,?,DATE_ADD(NOW(), INTERVAL 7 DAY)) ON DUPLICATE KEY UPDATE isrent = ?"
await db.query(query,[order_id,value['movie_id'],value['movie_id'],order_id,isrent,isrent], async (err, resp) => {
if(err){
res.status(500).send(err);
console.log(`Movie order Insert [SQL ERROR]: ${err}`)
}
else{
console.log(`Movie Order is inserted`)
if(isrent == 1){
total = await total + value['price_rent']
await console.log('current total is',total)
}else{
total = await total + value['price_buy']
await console.log('current total is',total)
}
}
})
});
//after the insertion is done, delete the original entry from the shopping cart movie table.
query = "Delete from Shoping_Cart_Movie where user_id=?"
await db.query(query,[user_id], async (err, resp) => {
if(err){
res.status(500).send(err);
await console.log(`Shopping Cart Movie delete [SQL ERROR]: ${err}`)
}
else{
await console.log(`Shopping Cart Movie is emptied`)
}
})
}
}
})
return Promise.resolve(total);
}
async function get_total(user_id,order_id,res){
total = check_out_movie(user_id,order_id,res);
total += check_out_product(user_id,order_id,res,total);
return await total;
}
The output that I got is following:
Created order
77
[object Promise][object Promise]
Check out Movie
Check out product
Shopping Cart Movie is emptied
Product Order is inserted
current total is 3.99
Shopping Cart Product is emptied
I am wondering why is the promise would be logged before the functions are processed, despite that I set await for this function call.
and how do I get numeric out put from this console.log()

Append currently logged in user to JSON file

I am trying to find a way to get the currently logged in user and than append them to a JSON file. Below is my code to first read the dir, then get the most recent file, return it and then append the current user that is logged in.
I can append a string to the file but when trying to perform req.user it states
Cannot read property 'user' of undefined
What would I need to include in this file so that it knows what user is?
let fs = require("fs"),
express = require("express"),
_ = require("underscore"),
User = require("./models/user"),
path = require("path");
let getFileAddUser = () => {
let filePath = '../automation_projects/wss-automation-u/results/temp/';
fs.readdir(filePath, (err, files) => {
if (err) { throw err; }
let file = getMostRecentFile(files, filePath);
console.log(file);
fs.readFile(filePath + file, 'utf8', (err, data) => {
let json = JSON.parse(data);
if(err){
console.error(err);
return;
} else {
//Un-comment to write to most recent file.
//==================================================
//This should find the currently logged in user and append them to the most recent file found.
json.currentuser = req.user;
fs.writeFile(filePath + file, JSON.stringify(json), (error) => {
if(error){
console.error(error);
return;
} else {
console.log(json);
}
});
//==================================================
console.log(data);
}
});
});
};
//Get the most recent file from the results folder.
function getMostRecentFile(files, path) {
let out = [];
files.forEach(function(file) {
let stats = fs.statSync(path + "/" +file);
if(stats.isFile()) {
out.push({"file":file, "mtime": stats.mtime.getTime()});
}
});
out.sort(function(a,b) {
return b.mtime - a.mtime;
})
return (out.length>0) ? out[0].file : "";
}
module.exports = getFileAddUser;

Thanks to a knowledgeable co-worker and some further research we were able to get this working. I'd like to share the code we came up with to append the currently logged in user to our results file. You will also notice we got some help using the Ramada.js library.
let fs = require("fs"),
express = require("express"),
_ = require("underscore"),
User = require("./models/user"),
r = require("ramda"),
path = require("path");
//This will be our function to get the most recent file from our dir and
//return it to us. We than user this function below.
function getMostRecentFile(files, path) {
let out = [];
let f = r.tail(files);
console.log(files);
f.forEach(function(file) {
let stats = fs.statSync(path + "/" +file);
if(stats.isFile()) {
out.push({"file":file, "mtime": stats.mtime.getTime()});
}
});
out.sort(function(a,b) {
return b.mtime - a.mtime;
})
return (out.length>0) ? out[0].file : "";
}
//Passing in 'u' as a argument which can than be used in a route and pass in
//anything that we want it to be. In our case it was the currently logged
//in user.
let getUser = (u) => {
let user = u;
let filePath = '../automation_projects/wss-automation-u/results/temp/';
//Comment above and uncomment below for testing locally.
// let filePath = "./temp/";
let file = "";
//Below we read our dir then get the most recent file using the
//getMostRecentfile function above.
read_directory(filePath).then( files => {
file = getMostRecentFile(files, filePath)
console.log(file);
return(read_file(filePath + file))
}).then( x => {
// Here we parse through our data with x representing the data that we
//returned above.
let json = JSON.parse(x);
return new Promise(function(resolve, reject) {
json.currentuser = u;
//And finally we write to the end of the latest file.
fs.writeFile(filePath + file, JSON.stringify(json), (error) => {
if(error) reject(error);
else resolve(json);
// console.log(json);
});
});
});
}
let read_directory = (path) => {
return new Promise((resolve, reject) => {
fs.readdir(path, (err, items) => {
if (err){
return reject(err)
}
return resolve([path, ...items])
})
})
}
let read_file = (path) => {
return new Promise((resolve, reject) => {
fs.readFile(path, "utf8", (err, items) => {
if (err){
return reject(err)
}
return resolve(items)
})
})
}
module.exports = getUser;
Than below is an example route with how to use the getUser module. You will want to require it like you do everything else with node.js and dependencies. Hope this helps someone in the future.
let getUser = require("getuser");
//Make a route to use the getUser module and pass in our argument value.
app.get("/", (req, res) => {
//With in the get user function pass in whatever you want to equal 'u' from the getuser module.
getUser(req.user.username);
res.render("index", { username: req.user });
});

node-oracledb bind variable of null

Using the node-oracledb package with node.js, I'm trying to delete a row in my oracle database that has a null value in it. I need to be able to pass a null value in as a bind variable but it is throwing an error:
var query = "delete from table where event IS :event";
var bind_vars = [null];
connection.execute(query, bind_vars, { autoCommit: true }, function(error, results) {});
The error returned is
{ Error: ORA-00908: missing NULL keyword errorNum: 908, offset: 46 }

You don't need a bind variable for that...
Given the following table:
create table t (
c number
);
insert into t values (1);
insert into t values (null);
insert into t values (3);
insert into t values (null);
insert into t values (5);
commit;
This should work:
const oracledb = require('oracledb');
const config = require('./dbConfig.js');
async function runTest() {
let conn;
try {
conn = await oracledb.getConnection(config);
let result;
let value = 1;
if (value === null) {
result = await conn.execute('delete from t where c is null');
} else { // other values should have a bind
result = await conn.execute('delete from t where c = :value', [value]);
}
// Note that the work was not committed.
console.log('Rows deleted:', result.rowsAffected);
} catch (err) {
console.error(err);
} finally {
if (conn) {
try {
await conn.close();
} catch (err) {
console.error(err);
}
}
}
}
runTest();

what about change the query
var query = "delete from table where nvl(event, '__') = nvl(:event, '__')";
var bind_vars = [null];
connection.execute(query, bind_vars, { autoCommit: true }, function(error, results) {});
it change null column and value to another character
change (__) with your own character