We Keep Coding

Node.js - Cannot append global variable when using fs

Im trying to read multiple xml files and parse data from them and i managed to do that but now new problem appeared.
allData variable is never changed, no matter what i do. What am i supposed to do here?
I dont know what to do or what to try, this is my first time working with files and im honestly surprised ive managed to come this far.
var parseString = require('xml2js').parseString;
var fs = require('fs')
var allData = {
store: []
}
function readFiles(__dirname, onFileContent, onError) {
fs.readdir(__dirname + '\\parse\\', function (err, filenames) {
if (err) {
return;
}
filenames.forEach(function (filename) {
console.log(filename)
fs.readFile(__dirname + '\\parse\\' + filename, 'utf-8', function (err, content) {
if (err) {
console.log(err)
return;
}
parseString(content, function (err, result) {
let tempObj = {}
let data = result.storeD[0]
if (data.name) {
tempObj['name'] = data.name[0];
}
if (data.price) {
tempObj['price'] = data.price[0];
}
//more of the same type of code
console.log(tempObj)
//output: { name: 'Data1', price: '1000' }
allData.store.push(tempObj)
})
})
})
});
console.log("All data: ",allData)
//Outputs once at the begining
//output: All data: { store: [] }
}
readFiles(__dirname)
SOLVED
adjusted code to use.readFileSync()(removed callback function) and now it works.
var parseString = require('xml2js').parseString;
var fs = require('fs')
var allData = {
store: []
}
function readFiles(__dirname, onFileContent, onError) {
fs.readdir(__dirname + '\\parse\\', function (err, filenames) {
if (err) {
return;
}
filenames.forEach(function (filename) {
console.log(filename)
let file = fs.readFileSync(__dirname + '\\parse\\' + filename, 'utf-8')
parseString(file, function (err, result) {
let tempObj = {}
let data = result.storeD[0]
if (data.name) {
tempObj['name'] = data.name[0];
}
if (data.price) {
tempObj['price'] = data.price[0];
}
//more of the same type of code
console.log(tempObj)
//output: { name: 'Data1', price: '1000' }
allData.store.push(tempObj)
})
})
console.log("All data: ",allData)
});
//Outputs once at the begining
//output: All data: { store: [] }
}
readFiles(__dirname)

The .readdir() and .readFile() methods are async, so in fact the console.log() is executed before all of the readFile operations.
In order to access the allData variable after these operations are complete, you have to either make them sync using .readFileSync() instead or you need to promisify the .readFile() method and wait for all of the promises to resolve.

Nodejs async-waterfall callback function is not defined

I am trying to save the file and details of it in the database.
Waterfall function with two calls but the second function is not waiting for the first function to finish.
Even without using waterfall eachseries doesn't work as expected. It doesn't wait for the record to be created and hence uniqueness error occurs due to same id.
What am I doing wrong here and how can I fix it?
Thanks!
async.eachSeries(uploadedPhotos, async function (uploadedFile, callback) {
async.waterfall([
async function() {
var lastUser = await TblUserFiles.find({}).sort('id DESC').limit(1);
// fileID remains undefined if i remove async-await from the function
var fileID = lastUser[0].id;
fileID += 1;
cbb(null, fileID, uploadedFile);
},
async function(file_id, uploadedFile, cbb) {
sails.log("save file id is " + file_id);
sails.log("savee file id is " + uploadedFile['filename']);
var today = moment(new Date()).format('YYYY-MM-DD HH:mm:ss');
await TblUserFiles.findOrCreate({ customer_no: req.session.userId, file_type: 'profile_image' }, {
id: fileID,
customer_no: req.session.userId,
file_name: uploadedFile['filename'],
file_type: 'profile_image',
is_approved: 'No',
approved_by: 0,
approved_on: today,
approved_from: ip,
uploaded_date: today,
modified_date: today
}).exec(async (err, user, wasCreated) => {
if (err) { return res.serverError(err); }
if (wasCreated) {
// created a new user
sails.log("new file was uploaded...")
return cbb(err, "done");
// return res.send("sent");
}
else {
// found existing user
var user = await TblUserFiles.create({
id: fileID,
customer_no: req.session.userId,
file_name: uploadedFile["filename"],
file_type: "image",
is_approved: "No",
approved_by: 0,
approved_on: today,
approved_from: ip,
uploaded_date: today,
modified_date: today
}).intercept(err => {
// Return a modified error here (or a special exit signal)
// and .create() will throw that instead
err.message = "Uh oh: " + err.message;
return err;
}).fetch();
if (user) {
sails.log("found existing files..")
return cbb(err, "done");
}
sails.log("this should not be called");
}
});
}
], (err, success) => {
if (err) sails.log(err);
return callback(err, 'done')
});
}, function (err) {
// if any of the saves produced an error, err would equal that error
if (err) {
sails.log(err);
} else {
return res.json({
message: uploadedPhotos.length + ' file(s) uploaded successfully!',
files: uploadedPhotos
});
}
});

Updated answer:
I rewrite the code.
I forgot that async.js handles promise in different way. Basically, you don't use callback(). Use return instead. If an error occurs, use throw new Error(message).
See here for more information. Read topic: Using ES2017 async functions.
async.eachSeries(uploadedPhotos, async uploadedFile => {
var lastUser = await TblUserFiles.find({}).sort('id DESC').limit(1);
var fileID = lastUser[0].id;
fileID += 1;
sails.log("save file id is " + file_id);
sails.log("savee file id is " + uploadedFile['filename']);
var today = moment(new Date()).format('YYYY-MM-DD HH:mm:ss');
await TblUserFiles.findOrCreate(
{ customer_no: req.session.userId, file_type: 'profile_image' },
{
id: fileID,
customer_no: req.session.userId,
file_name: uploadedFile['filename'],
file_type: 'profile_image',
is_approved: 'No',
approved_by: 0,
approved_on: today,
approved_from: ip,
uploaded_date: today,
modified_date: today
}
).exec(async (err, user, wasCreated) => {
if (err) throw new Error(err);
if (wasCreated) {
sails.log("new file was uploaded...");
return;
} else {
await TblUserFiles.create({
id: fileID,
customer_no: req.session.userId,
file_name: uploadedFile["filename"],
file_type: "image",
is_approved: "No",
approved_by: 0,
approved_on: today,
approved_from: ip,
uploaded_date: today,
modified_date: today
})
.intercept(err => {
throw new Error("Uh oh: " + err.message);
}).fetch();
if (user) {
sails.log("found existing files..");
return;
} else {
sails.log("this should not be called");
return;
}
}
});
}, err => {
// Don't call res.serverError() or res.json() inside the async loop!
if (err) {
sails.log(err);
res.serverError(err);
}
else {
res.json({
message: uploadedPhotos.length + ' file(s) uploaded successfully!',
files: uploadedPhotos
});
}
});
I think the first problem here is that the you forget to give the first task a callback function(cbb in this case).
async.waterfall([
async function(cbb) {
var lastUser = await TblUserFiles.find({}).sort('id DESC').limit(1);
// fileID remains undefined if i remove async-await from the function
var fileID = lastUser[0].id;
fileID += 1;
cbb(null, fileID, uploadedFile);
},
async function(file_id, uploadedFile, cbb) {
...
Secondly, you shouldn't return a callback. Callback is a function not a promise. Just use them normally.
By the way, const async = require('async'); will NOT override async keyword. Compiler can tell the difference between them. This is proved by the following script example:
const async = require('async');
let runPromise = (name, timer, success = true) => {
console.log(`${name} starts.`);
return new Promise((resolve, reject) => {
if (success) {
setTimeout(function () {
resolve(`${name} finished after ${timer / 1000} seconds(resolved).`);
}, timer);
} else {
reject(`${name} failed(rejected).`);
}
});
};
async function asyncFunction() {
let txt = await runPromise('A', 1000);
console.log(txt);
}
asyncFunction();

array push results in empty array javascript

I am trying to store API results into an array.
The data is displayed in console, but on pushing the data into an array, the array is still empty.
Here's the code:
app.post('/fetchFavoriteTweets/', verifyToken, function(req, res) {
var favorites = [];
dbConn.then( function (database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne(
{ _id: ObjectId(req.userId) }, function(err, result) {
if(err) throw err;
if(!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({msg:'record not found'});
}
else {
result.favorite_tweets.forEach(function (tweet) {
T.get('statuses/show', {id: tweet.id}, function(err, data, response) {
if(!err){
favorites.push(data);
console.log(data); //this returns data
} else {
console.log(err);
}
});
});
console.log(favorites);
// res.status(200).json({msg:'success', data:favorites});
}
});
}).catch(function(e){console.log(e)})
});

It looks like you're defining the favorites array within the scope of the function callback. Try putting var favorites = []; above you app.post() call instead.
Also, keep in mind that it will only have a value after the callback is complete, so any synchronous code later down the line will only see the empty array value.

I've updated your code to get favorites by storing separately the promise and call it afterwards:
UPDATE
As you can see in the demo, i have 2x console.log at the bottom, the first one(C1) is contained in the promise favoritesPromise () and the second (C2) is after the promise.
Synchronous actions will never wait for asynchronus actions to take place, therefore in my example C2 will always be outputted before C1, even if console.log(1 ... ) is before console.log(2 ... ), they'll appear reversed in the console.
In the promise i added a setTimeout of 1ms to mock a request, it was all it took to achieve the current output. Another thing you can test is removing the setTimeout then output will change a bit, your promise becomes synchronus until it reaches resolve(favorites), that means favorites has all the data by now, but when resolve takes place, it becomes async, and in your console you will still see C2 first (but now with data) and C1 second.
In my earlier answer i tried to implement this reasoning to your code.
Keep it async folks!
var favorites = [];
var favoritesPromise = () => {
return new Promise((resolve, reject) => {
console.log('Retrieving data from the internet.');
// This timeout mocks your request to anything that is async or promie
setTimeout(() => {
console.log('Request done')
let resultFavorite_tweets = [{
id: 1,
name: 'a dog'
}, {
id: 2,
name: 'a cat'
}];
resultFavorite_tweets.forEach(item => {
favorites.push(item.name);
})
resolve(favorites);
// if you have an error use
// reject(err)
}, 1);
});
}
favoritesPromise().then(favList => {
console.log(1, 'this will always contain data from the internet, but will always be last', favList);
})
console.log(2, 'this will be empty (unless you remove setTimeout), but will always be first', favorites);
app.post('/fetchFavoriteTweets/', verifyToken, function(req, res) {
const favoritesPromise = () => {
return new Promise((resolve, reject) => {
var favorites = [];
dbConn.then(function(database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne({
_id: ObjectId(req.userId)
}, function(err, result) {
if (err) reject(err);
if (!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({
msg: 'record not found'
});
} else {
result.favorite_tweets.forEach(function(tweet) {
T.get('statuses/show', {
id: tweet.id
}, function(err, data, response) {
if (!err) {
favorites.push(data);
console.log(data); //this returns data
} else {
console.log(err);
reject(err);
}
});
resolve(data);
});
console.log(favorites);
// res.status(200).json({msg:'success', data:favorites});
}
});
}).catch(function(e) {
reject(e)
})
});
}
// Here you call the promise to retrieve "favorites"
favoritesPromise().then(favoritesList => {
console.log('your favorites array', favoritesList)
})
})

Try next code
app.post('/fetchFavoriteTweets/', verifyToken, function (req, res) {
var favorites = [];
dbConn.then(function (database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne(
{ _id: ObjectId(req.userId) }, function (err, result) {
if (err) throw err;
if (!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({ msg: 'record not found' });
}
else {
// Counter
let count = result.favorite_tweets.length;
result.favorite_tweets.forEach(function (tweet) {
T.get('statuses/show', { id: tweet.id }, function (err, data, response) {
// Decrease count
count -= 1;
if (!err) {
favorites.push(data);
// Check if count is zero
if (count === 0) {
console.log(favorites);
res.status(200).json({msg:'success', data:favorites});
}
} else {
console.log(err);
}
});
});
}
});
}).catch(function (e) { console.log(e) })
});

How to avoid a callback in promise

I am new to NodeJS and JavaScript. I am badly stuck in a problem:
I want to generate QR image of 'some text' and after generating it, I want to query my MySQL database and insert the image to database.
Problem is that QRCode.toDataURL of SOLDAIR module goes in running state and query is called before the QR image is returned from the .toDataUrl function.
Hence it results in error.
I tried everything, promises, nested promises, counters, if statements etc., but I am unable to find a solution.
My code:
router.post('/generateTicket', (req,res) => {
const query1 = `SELECT * FROM booking ORDER BY bookingID DESC LIMIT 1`;
const query2 = `INSERT INTO ticket (ticket_image,BookingID) SET ?`;
let bookingID;
let count;
let ticket_data = {};
Promise.using(mysql.getSqlConn(), conn => {
conn.query(query1).then(resultOfQuery1 => {
bookingID = resultOfQuery1[0].BookingID;
count = resultOfQuery1[0].PeopleCount;
console.log("ID = " + bookingID + " people count = "+count);
promiseToCreateQRcode().then(function (URLfromResolve) {
console.log("the url is " + URLfromResolve );
}).catch(function (fromReject) {
console.log("Rejected "+ fromReject);
}); // catch of promise to create QR
}).catch(err => {
res.json({ status: 500, message: 'Error Occured in query 1 ' + err });
}); // catch of query 1
});
});
var opts = {
errorCorrectionLevel: 'H',
type: 'image/png',
rendererOpts: {
quality: 0.3
}
};
let promiseToCreateQRcode = function () {
let QRImage;
return new Promise(function (resolve,reject) {
QRCode.toDataURL('text', function (err, url) {
if (err) throw err
console.log("\n"+url+"\n");
QRImage = url;
});
if (QRImage)
resolve(QRImage);
else
reject("error occured in url");
});
};
As u can see, the program jumps to if statement and the QR image is not generated yet, hence it goes in "reject":

Try this,
let promiseToCreateQRcode = function () {
return new Promise(function (resolve,reject) {
QRCode.toDataURL('text', function (err, url) {
if (err){
reject(err); // or some message
} else {
resolve(url);
}
});
});
};
This way promise will be resolved only when toDataURL returns QR image.

Have a look at How do I convert an existing callback API to promises?. You need to call resolve or reject in the asynchronous callback!
function promiseToCreateQRcode() {
return new Promise(function(resolve,reject) {
QRCode.toDataURL('text', function (err, url) {
if (err) {
reject(err);
} else {
console.log("\n"+url+"\n");
resolve(url);
}
});
});
}
Using this extra QRImage variable like you did cannot work.

Amazon S3 Node.js SDK deleteObjects

I am trying to delete several objects after copying them to a different folder.
My code is like:
var deleteParam = {
Bucket: 'frontpass-test',
Delete: {
Objects: [
{Key: '1.txt'},
{Key: '2.txt'},
{Key: '3.txt'}
]
}
};
s3.deleteObjects(deleteParam, function(err, data) {
if (err) console.log(err, err.stack);
else console.log('delete', data);
});
and the returned data is:
delete { Deleted: [ { Key: '1.txt' }, { Key: '3.txt' }, { Key: '2.txt' } ],
Errors: [] }
so I assume the deletion is completed. But the objects are still exist in the folder, is there something wrong with my code?
I also tried to delete objects using for loop and s3.deleteObject, but it only delete the last object in my list of files.
for (var i = 0; i < files.length; i++) {
var copyParams = {
Bucket: 'frontpass-test',
CopySource: 'frontpass-test/unsold/' + files[i].filename,
Key: 'sold/' + files[i].filename
};
var deleteParam = {
Bucket: 'frontpass-test',
Key: 'unsold/' + files[i].filename
};
s3.copyObject(copyParams, function(err, data) {
if (err) console.log(err, err.stack);
else {
s3.deleteObject(deleteParam, function(err, data) {
if (err) console.log(err, err.stack);
else console.log('delete', data);
});
}
});
}
Any idea on how to delete objects in my case? Thanks in advance.

Well the first example looks good. Do you have object versioning turned on in the bucket? That would keep a copy of a file even after you delete it.
The second example actually contains some bugs that would explain why only the last one gets deleted. Because Node.js is asynchronous, when you hit the copyObject function call, the loop iteration ends and goes to the next iteration, not waiting for the callback on copyObject to be called. You try to define the params variables for each iteration of the loop with the var keyword, but because Javascript has function level scope not block level scope, you aren't actually creating new variables on each iteration. You only have one instance of copyParmas and deleteParams. So you quickly run through the loop and deleteParams stays on the value it receives in the last iteration of the loop. Then eventually the callbacks to the copyObject calls start firing, and they all call deleteObject with deleteParams which by now is the last one. In order to make multiple asynchronous calls in a loop, I like to use the async library. Using it, you could do the following:
async.each(files, function iterator(file, callback) {
var copyParams = {
Bucket: 'frontpass-test',
CopySource: 'frontpass-test/unsold/' + file.filename,
Key: 'sold/' + file.filename
};
var deleteParam = {
Bucket: 'frontpass-test',
Key: 'unsold/' + file.filename
};
s3.copyObject(copyParams, function(err, data) {
if (err) callback(err);
else {
s3.deleteObject(deleteParam, function(err, data) {
if (err) callback(err)
else {
console.log('delete', data);
callback();
}
});
}
});
}, function allDone(err) {
//This gets called when all callbacks are called
if (err) console.log(err, err.stack);
});

Just had to implement folder rename on top of s3, I did it as follows: (promise api)
_getDataForItemRename(from, to) {
return s3.listObjectsV2({Bucket: services.conf.workspace, Prefix: from}).promise()
.then((data) => {
const toCopy = [];
const toRemove = [];
const s3Contents = Object.assign([], data.Contents);
// in case of a single dir (with no children)
if (s3Contents.length === 0) {
s3Contents.push({Key: from});
}
s3Contents.forEach((item) => {
const copyPromise = s3.copyObject({
Bucket: services.conf.workspace,
Key: to,
CopySource: services.conf.workspace + '/' + item.Key
}).promise();
const deletePromise = s3.deleteObjects({
Bucket: services.conf.workspace,
Delete: {Objects: [{Key: from}]}
}).promise();
toCopy.push(copyPromise);
toRemove.push(deletePromise);
});
return {copy: toCopy, remove: toRemove};
}).catch((err) => {
return Promise.reject(err);
});
}
return this._getDataForItemRename(_from, _to).then((files) => {
return Promise.all(files.copy).then(() => {
return Promise.all(files.remove).then(result => {
return result;
});
});
}).catch((err) => {
return Promise.reject(err);
});