We Keep Coding

Node Function Returning Empty Array

In the following node function, it is returning an empty array. Not sure why its doing that. Could this be a async await issue? Would appreciate any help. Thank you
const folderPath = '/public/home.html'
function getCircuitAndFuse(folderPath){
//List containing circuit name with its fuse
let temporaryList = [];
let finalCircuitAndFuseList = []
fs.readFile(__dirname + folderPath, (error, data)=>{
if(error){
console.log(`Unable to read file: ${error}`)
}else{
var $ = cheerio.load(data)
$('img').each(function(index, element){
let getClassAtr = element.attribs.class
temporaryList.push(getClassAtr.slice(0, getClassAtr.lastIndexOf(" ")))
})
finalCircuitAndFuseList = [...new Set(temporaryList)]
}
})
return finalCircuitAndFuseList;
}
let getInfo = getCircuitAndFuse(folderPath)
// Returning empty array
console.log(getInfo)
***Server code****
const server = http.createServer(function(req, res){
res.writeHead(200, {'Content-Type': 'text/plain'})
res.end()
}).listen(port, ()=>{
console.log(`Server listening on port ${port}. Press Ctrl-C to terminate...`)
})

getCircuitAndFuse must return Promise like this:
function getCircuitAndFuse(folderPath) {
return new Promise((resolve, reject) => {
//List containing circuit name with its fuse
let temporaryList = [];
fs.readFile(__dirname + folderPath, (error, data) => {
if (error) {
console.log(`Unable to read file: ${error}`);
} else {
var $ = cheerio.load(data);
$('img').each(function (index, element) {
let getClassAtr = element.attribs.class;
temporaryList.push(
getClassAtr.slice(0, getClassAtr.lastIndexOf(' '))
);
});
resolve([...new Set(temporaryList)]);
}
});
});
}
getCircuitAndFuse(folderPath).then((getInfo) => {
// do something with `getInfo`
});

Another alternative to Faruk's answer would be to just use fs.readFileSync instead of wrapping your function in a promise and requiring some of that extra ceremony. Using fs.readFileSync will ensure that your function doesn't return prematurely.
Here is your code rewritten with that in mind:
function getCircuitAndFuse(folderPath) {
try {
let temporaryList = [];
const data = fs.readFileSync(__dirname + folderPath);
const $ = cheerio.load(data);
$("img").each(function (index, element) {
let getClassAtr = element.attribs.class;
temporaryList.push(getClassAtr.slice(0, getClassAtr.lastIndexOf(" ")));
});
return [...new Set(temporaryList)];
} catch (error) {
console.log(error);
}
}

How to convert JavaScript functions that use multiple callbacks, to use promises?

I am trying to convert the way I use asynchronous functions from callbacks to promises.
I understand this basic conversion shown here, where callbacks are converted to the resolve and reject functions:
// CALLBACK
const getData = (id, callback) => {
setTimeout(() => {
if (!id) return callback('ERROR: id is missing')
return callback("The data for id " + id);
}, 1000)
};
getData(111, console.log);
getData(222, console.log);
getData(null, console.log);
// PROMISE
const getData2 = id => {
return new Promise((resolve, reject) => {
setTimeout(() => {
if (!id) reject('id is missing');
resolve("The data for id " + id);
}, 1000);
});
};
getData2(333).then(console.log).catch((message) => console.log("ERROR: " + message));
getData2(444).then(console.log).catch((message) => console.log("ERROR: " + message));
getData2(null).then(console.log).catch((message) => console.log("ERROR: " + message));
But I often use callbacks as in the following scenerio where you can have a process which takes a long time and sends out bits of data in numerous callbacks back to the calling code as it processes its information:
sleep = function (ms) {
var start = new Date().getTime();
let now = 0;
let difference = 0;
for (var i = 0; i < 1e17; i++) {
now = new Date().getTime();
difference = now - start;
if (difference > ms) {
break;
}
}
}
const goShopping = (list, cbItemReport, cbFinished, cbError) => {
let count = 0;
let numberFound = 0;
const randomError = Math.floor(Math.random() * 3);
if (randomError == 0) {
cbError('Something went wrong, trip aborted.');
} else {
list.forEach(item => {
const randomFound = Math.floor(Math.random() * 4);
if (randomFound > 0) {
cbItemReport(item, true, ++count);
numberFound++;
} else {
cbItemReport(item, false, ++count);
}
sleep(1000);
})
cbFinished(`Bought ${numberFound} things.`);
}
}
goShopping(['milk', 'eggs', 'sugar', 'bread'],
(item, found, count) => {
console.log(`Item #${count} "${item}" was ${found ? 'found' : 'not found'}.`);
},
(message) => {
console.log("Returned from shopping: " + message);
},
(error) => {
console.log("ERROR: " + error);
});
How would I convert this latter use of callbacks to promises? In this case, the three callbacks cbItemReport, cbFinished, cbError are too many to map to the two that Promise has, i.e. only resolve (cbFinished) and reject (cbError), or what am I missing here?

From what it sounds like, you are looking to implement something like the RxJs library, so why not just use it?
Check out RxJs here
For example a call could look like this then:
const sub = new Subject(); // Create a subject
sub.asObservable().subscribe( // Subscribe to that subject as obserable
({item, found, count}) => { // next
console.log(`Item #${count} "${item}" was ${found ? 'found' : 'not found'}.`);
},
error => { // error
console.log("ERROR: " + error);
},
message => { // complete
console.log("Returned from shopping: " + message);
}
);
const goShopping = (list) => {
let count = 0;
let numberFound = 0;
const randomError = Math.floor(Math.random() * 3);
if (randomError == 0) {
sub.error('Something went wrong, trip aborted.'); // push an error to the subject
} else {
list.forEach(item => {
const randomFound = Math.floor(Math.random() * 4);
if (randomFound > 0) {
sub.next({item: item, found: true, count: ++count}); // push a result to subject (single object)
numberFound++;
} else {
sub.next({item: item, found: true, count: ++count}); // same as above
}
sleep(1000);
})
sub.complete(`Bought ${numberFound} things.`); // push complete to subject. after that no next is allowed anymore
}
}
goShopping(['milk', 'eggs', 'sugar', 'bread'])

Call resolve from cbFinished, reject from cbError and write a cbItemReport that does whatever you want it to do to each item, just as in the existing code.

Sync a forEach with an event inside (Firebase & node 6.11.5)

I'm reading, bit per bit, all the PNG files inside a directory and I've to summarize some data in a json.
The problem is that, if I understand, the PNG reader send an async event "parsed" when finished. That cause that the function exits beforse json is populated...
I'm using node 6.11.5, so I cannot use sync/await.
var fs = require('fs'),
PNG = require('pngjs').PNG;
exports.movie = functions.https.onRequest((req, res) => {
console.log('********** START FUNCTION ************');
var movieFolder = 1;
if (req.query.id) movieFolder = '../movies/' + req.query.id + '/png/';
var exitJson = [];
fs.readdir(movieFolder, (err, files) => {
files.forEach((file) => {
fs.createReadStream(movieFolder + file)
.pipe(new PNG({
filterType: 1
}))
.on('parsed', function () {
console.log('Parsing: ' + movieFolder + file);
exitJson.push({
width: this.width,
height: this.height,
data: []
});
});
});
});
console.log('************* FINISHED *************');
res.status(200).json(exitJson);
});

You can use a simple itemsProcessed counter to detect if all of your callbacks were resolved.
var movieFolder = 1;
if (req.query.id) movieFolder = '../movies/' + req.query.id + '/png/';
var exitJson = [];
var itemsProcessed = 0;
fs.readdir(movieFolder, (err, files) => {
files.forEach((file) => {
fs.createReadStream(movieFolder + file)
.pipe(new PNG({
filterType: 1
}))
.on('parsed', function () {
console.log('Parsing: ' + movieFolder + file);
exitJson.push({
width: this.width,
height: this.height,
data: []
});
itemsProcessed++;
if (itemsProcessed === files.length) {
console.log('************* FINISHED *************');
res.status(200).json(exitJson);
}
});
});
});

You can load files one by one through recursion calls.
Don't forget to check errors.
exports.movie = functions.https.onRequest((req, res) => {
var movieFolder = 1;
if (req.query.id)
movieFolder = '../movies/' + req.query.id + '/png/';
var exitJson = [];
fs.readdir(movieFolder, function (err, files) {
var sendError = (err) => res.status(500).send(err.message);
if (err)
return sendError(err);
function loadFile (i) {
if (i == files.length)
return res.status(200).json(exitJson); // !!!DONE!!!
var file = files[i];
fs.createReadStream(movieFolder + file)
.pipe(new PNG({filterType: 1}))
.on('parsed', function () {
console.log('Parsing: ' + movieFolder + file);
exitJson.push({width: this.width, height: this.height, data: []});
loadFile (i + 1); // go to next file
})
.on('error', sendError);
}
loadFile(0); // start recursion
});
});

const exports={};const sizes={'foo.png':[100,200],'bar.png':[200,200],'baz.png':[300,200]};Promise.delay = (t) => new Promise(r => setTimeout(r, t)); const randomTime = (a = 500, b = 1500) => Math.floor(Math.random() * b) + a;
const require=src=>({'fs':{readdir:(d,c)=>{Promise.delay(randomTime()).then(() => c(null,['foo.png','bar.png','baz.png']))},createReadStream:(path)=>({pipe:(f)=>({on:(e,c)=>{const s=sizes[path.split('/').slice(-1)[0]];const a={width:s[0],height:s[1]};a.c=c;Promise.delay(randomTime()).then(() => a.c())}})})},'pngjs':{PNG:class PNG{constructor(a){}}},'firebase-functions':{https:{onRequest:(handler)=>{handler({query:({id:2})},{status:(s)=>({json:(a) => document.getElementById('res').innerHTML = `<pre><code>${JSON.stringify(a, null, 4)}</code></pre>`})})}}}})[src];
// ------------------- ignore the above
const fs = require('fs');
const PNG = require('pngjs').PNG;
const functions = require('firebase-functions');
/**
* Using a new Promise, we can perform multiple async tasks all contained
* within that one Promise which can be resolved or rejected. We read the
* folder directory for its files and pass it on to our Promised 'readFiles'.
*/
function readMovieFiles(folder) { console.log('readMovieFiles', folder)
return new Promise((res, rej) => {
fs.readdir(folder, (err, files) => {
readFiles(files, folder).then(res).catch(rej)
});
});
}
/**
* Given an array of file names within a folder, we can chain together the
* file promises using the reduce method. Starting at an initial value of
* Promise<[]>, each file in the array will be read sequentially.
*/
function readFiles(files, folder) { console.log('readFiles', folder, files)
return Promise.all(files.map(name => readFile(folder + name)));
}
/**
* We read a file and in the parsed callback, we call the res() and pass it
* the newly constructed array containing the newest file to be parsed.
*/
function readFile(path) { console.log('readFile', path)
return new Promise((res, rej) => {
fs.createReadStream(path)
.pipe(new PNG({ filterType: 1 }))
.on('parsed', function() {
console.log('parsedFile', path)
res({
data: [],
width: this.width,
height: this.height
});
});
});
}
exports.movie = functions.https.onRequest((req, res) => {
console.log('********** START FUNCTION ************');
if (!req.query.id) req.query.id = 1;
readMovieFiles(`../movies/${req.query.id}/png/`).then(exitJson => {
res.status(200).json(exitJson);
}).catch(error => {
res.status(500).json(error);
});
console.log('************* FINISHED *************');
});
<pre><code id="res"></code></pre>

How to pull out handler using module exports?

I am building a node application, and trying to neatly organize my code. I wrote a serial module that imports the serial libs and handles the connection. My intention was to write a basic module and then reuse it over and over again in different projects as needed. The only part that changes per use is how the incoming serial data is handled. For this reason I would like to pull out following handler and redefine it as per the project needs. How can I use module exports to redefine only this section of the file?
I have tried added myParser to exports, but that gives me a null and I would be out of scope.
Handler to redefine/change/overload for each new project
myParser.on('data', (data) => {
console.log(data)
//DO SOMETHING WITH DATA
});
Example usage: main.js
const serial = require('./serial');
const dataParser = require('./dataParser');
const serial = require('./serial');
//call connect with CL args
serial.connect(process.argv[2], Number(process.argv[3]))
serial.myParser.on('data',(data) => {
//Do something unique with data
if (dataParser.parse(data) == 0)
serial.send('Error');
});
Full JS Module below serial.js
const SerialPort = require('serialport');
const ReadLine = require('#serialport/parser-readline');
const _d = String.fromCharCode(13); //char EOL
let myPort = null;
let myParser = null;
function connect(port, baud) {
let portName = port || `COM1`;
let baudRate = baud || 115200;
myPort = new SerialPort(portName, {baudRate: baudRate})
myParser = myPort.pipe(new ReadLine({ delimiter: '\n'}))
//Handlers
myPort.on('open', () => {
console.log(`port ${portName} open`)
});
myParser.on('data', (data) => {
console.log(data)
});
myPort.on('close', () => {
console.log(`port ${portName} closed`)
});
myPort.on('error', (err) => {
console.error('port error: ' + err)
});
}
function getPorts() {
let portlist = [];
SerialPort.list((err, ports) => {
ports.forEach(port => {
portlist.push(port.comName)
});
})
return portlist;
}
function send(data) {
myPort.write(JSON.stringify(data) + _d, function (err) {
if (err) {
return console.log('Error on write: ', err.message);
}
console.log(`${data} sent`);
});
}
function close() {
myPort.close();
}
module.exports = {
connect, getPorts, send, close
}

The problem is that a module is used where a class or a factory would be appropriate. myParser cannot exist without connect being called, so it doesn't make sense to make it available as module property, it would be unavailable by default, and multiple connect calls would override it.
It can be a factory:
module.exports = function connect(port, baud) {
let portName = port || `COM1`;
let baudRate = baud || 115200;
let myPort = new SerialPort(portName, {baudRate: baudRate})
let myParser = myPort.pipe(new ReadLine({ delimiter: '\n'}))
//Handlers
myPort.on('open', () => {
console.log(`port ${portName} open`)
});
myParser.on('data', (data) => {
console.log(data)
});
myPort.on('close', () => {
console.log(`port ${portName} closed`)
});
myPort.on('error', (err) => {
console.error('port error: ' + err)
});
function getPorts() {
let portlist = [];
SerialPort.list((err, ports) => {
ports.forEach(port => {
portlist.push(port.comName)
});
})
return portlist;
}
function send(data) {
myPort.write(JSON.stringify(data) + _d, function (err) {
if (err) {
return console.log('Error on write: ', err.message);
}
console.log(`${data} sent`);
});
}
function close() {
myPort.close();
}
return {
myParser, getPorts, send, close
};
}
So it could be used like:
const serial = require('./serial');
const connection = serial(...);
connection.myParser.on('data',(data) => {
//Do something unique with data
if (dataParser.parse(data) == 0)
connection.send('Error');
});

Node.js - How to return after all asynchronous calls have finished

As shown bellow, I am pushing the object link_to_json returns into an array allShirts declared in html_to_json.
However, the console.dir on the third last line and the return value of html_to_json logs an array of undefined references. Which I presume is because console.dir and return is executed before link_to_json functions finished.
How do I ensure the return value of html_to_json is a filled up allShirts array?
//Go to individual links and scrape relevant info
const link_to_json = (link) => {
request(link, (err, res, body) => {
if (!error_handler(err, res, link)) {
const $ = cheerio.load(body);
const shirt_detail = $('.shirt-details').find('h1').text();
const Title = shirt_detail.substr(shirt_detail.indexOf(' ') + 1);
const Price = shirt_detail.substr(0, shirt_detail.indexOf(' '));
const ImageURL = $('.shirt-picture').find('img').attr('src');
const URL = link;
return new Shirt(Title, Price, ImageURL, URL);
} else return {};
});
}
//Crawl through all individual links listed in Root
const html_to_json = body => {
const allShirts = [];
const $ = cheerio.load(body);
$('.products').find('a').each((index, val) => {
allShirts.push(link_to_json(rootURL + $(val).attr('href')));
});
console.dir(allShirts); // <--- HERE
return allShirts;
}

There's a few ways to go after this, but I like the Async library for this sort of thing.
How I'd handle your problem is to actually get all the URLs first, so changing your body scrape to something like this instead:
const shirtLinks = [];
$('.products').find('a').each((index, val) => {
shirtLinks.push(rootURL + $(val).attr('href'));
});
You need your conversion function to be asynchronous as well:
const linkToJSON = (link, cb) => {
request(link, (err, res, body) => {
if (!error_handler(err, res, link)) {
const $ = cheerio.load(body);
const shirt_detail = $('.shirt-details').find('h1').text();
const Title = shirt_detail.substr(shirt_detail.indexOf(' ') + 1);
const Price = shirt_detail.substr(0, shirt_detail.indexOf(' '));
const ImageURL = $('.shirt-picture').find('img').attr('src');
const URL = link;
return cb(null, new Shirt(Title, Price, ImageURL, URL));
}
return cb();
});
}
Then use async to map them across the async function that fetches the data:
async.map(shirtLinks, linkToJSON, (err, results) => {
console.dir(results);
});

This is how I would do it. I find it easier to debug this way.
let getShirtDetailsBody = (link) => {
return new Promise((resolve, reject) => {
request(link, (err, res, body) => {
if (err) {
reject(err)
} else {
resolve(body)
}
})
})
}
let getShirt = (body) => {
const $ = cheerio.load(body);
const shirt_detail = $('.shirt-details').find('h1').text();
const Title = shirt_detail.substr(shirt_detail.indexOf(' ') + 1)
const Price = shirt_detail.substr(0, shirt_detail.indexOf(' '))
const ImageURL = $('.shirt-picture').find('img').attr('src')
const URL = link
return new Shirt(Title, Price, ImageURL, URL)
}
let getAllProductsShirtsBody = (body) => {
const $ = cheerio.load(body)
return Promise.all($('.products').find('a').map((index, val) => {
return getShirtDetailsBody(`rootURL${$(val).attr('href')}`)
}))
}
getAllProductsShirtsBody(yourbody).then(allShirtsBody => {
const allShirts = allShirtsBody.map(shirtBody => { return getShirt(shirtBody) })
console.log(allShirts)
}).catch(err => { console.log(err) })