We Keep Coding

Can't get rid of a global variable (Need to make it non-global)

I'm working on a web app that shows the total duration of a playlist. Here we're dealing with the YouTube API. And I want to know how should I get rid of the global variable newPageToken. Also I still need to use it in the third function on this snippet.
let newPageToken = null;
// Next page for more results (Max 50 per page)
function getNextTokenURL() {
console.log(newPageToken);
return newPageToken
? `${playlistItemsURL}&playlistId=${extractedPlaylistIDId}&pageToken=${newPageToken}&key=${API_KEY}`
: `${playlistItemsURL}&playlistId=${extractedPlaylistIDId}&key=${API_KEY}`;
}
async function getVideoIdsForPageToken() {
try {
const { data } = await axios.get(getNextTokenURL());
const nextPageToken = data.nextPageToken;
const videoIds = data.items.map((video) => {
return video.contentDetails.videoId;
});
return { videoIds, nextPageToken };
} catch (e) {
if (e.response) {
const { code, message } = e.response.data.error;
throw new Error(`StatusCode ${code}. Reason: ${message}`);
console.log("Errow while fetching videos list.");
} else {
throw new Error(e.message);
}
}
}
// Navigates between the videos per page and adds them (Maximum 50)
async function getPlaylistData() {
try {
const { videoIds, nextPageToken } = await getVideoIdsForPageToken();
let pageToken = nextPageToken;
newPageToken = pageToken;
const returnedVideoIds = [];
returnedVideoIds.push(getDetailsForVideoIds(videoIds));
const videoGroups = await Promise.all(returnedVideoIds);
for (const group of videoGroups) {
for (const video of group) {
finalTotalDuration += returnedToSeconds(video.contentDetails.duration);
}
}
// console.log(videoIds);
if (nextPageToken) {
await getPlaylistData();
}
} catch (e) {
throw new Error(e.message);
console.log("Error while navigating between video pages.");
}
}```

Assumptions
finalTotalDuration is also a global variable declared somewhere (Not a good idea)
You call getPlaylistData for multiple playlist for multiple users
Solution
You need to ensure the getPlaylistData is standalone and returns the finalTotalDuration as a return value (not set a global one)
To make it standalone it has to be iterative in nature. It should be a recursive function which does the following
async function getPlaylistTotalDuration(newPageToken) {
// Step 1: Create the required query URL based on the newPageToken parameter
// Step 2: Start a local duration counter
// Step 3: Get the video details based on the URL created in Step 1
// Step 4: Get the durations in seconds and add it to the local duration counter created in Step 2
// Step 5: Check if the return of Step 3 has a nextPageToken, if so do a recursive call to self with the new token
// Step 6: Return the final value, which will propogate back in a recursive function
}
You can simply call the function like
let finalTotalDuration = getPlaylistTotalDuration(null); // or getPlaylistTotalDuration();
for example the below getPlaylistTotalDuration is a replacement to your getPlaylistData method
async function getVideoIdsForPageToken(url) {
try {
const { data } = await axios.get(url);
const nextPageToken = data.nextPageToken;
const videoIds = data.items.map((video) => {
return video.contentDetails.videoId;
});
return { videoIds, nextPageToken };
} catch (e) {
if (e.response) {
const { code, message } = e.response.data.error;
throw new Error(`StatusCode ${code}. Reason: ${message}`);
console.log("Errow while fetching videos list.");
} else {
throw new Error(e.message);
}
}
}
async function getPlaylistTotalDuration(newPageToken) {
try {
// Step 1: Create the required query URL based on the newPageToken parameter
let url = newPageToken
? `${playlistItemsURL}&playlistId=${extractedPlaylistIDId}&pageToken=${newPageToken}&key=${API_KEY}`
: `${playlistItemsURL}&playlistId=${extractedPlaylistIDId}&key=${API_KEY}`;
// Step 2: Start a local duration counter
let totalDuration = 0;
// Step 3: Get the video details based on the URL created in Step 1
const { videoIds, nextPageToken } = await getVideoIdsForPageToken(url);
const returnedVideoIds = [];
returnedVideoIds.push(getDetailsForVideoIds(videoIds));
const videoGroups = await Promise.all(returnedVideoIds);
for (const group of videoGroups) {
for (const video of group) {
// Step 4: Get the durations in seconds and add it to the local duration counter created in Step 2
totalDuration += returnedToSeconds(video.contentDetails.duration);
}
}
// Step 5: Check if the return of Step 3 has a nextPageToken, if so do a recursive call to self with the new token
if (nextPageToken) {
totalDuration += await getPlaylistTotalDuration(nextPageToken);
}
// Step 6: Return the final value, which will propogate back in a recursive function
return totalDuration;
} catch (e) {
throw new Error(e.message);
console.log("Error while navigating between video pages.");
}
}
Note: I have not actually ran the above code, but hopefully you get an idea of what needs to be done.

Return upon another file check

So I'm trying to run a few checks inside a file. Lets say inside checks.js I have
module.exports = async (content) => {
// Check no.1
if (content.id != 'SomeID') return;
// Check no.2
if (content.length > 20) return;
//..etc
}
And in my main file I am requiring this file. I want it to return in the original file depending on the outcome of checks.js So lets say the content id isn't the same as 'SomeID', I want it to return and not to continue the rest of the code in my main file. I did require('filePath')(content) But it doesn't actually return in the main file as it should by instructions in checks.js What am I doing wrong and what should I be doing. Thank you in advance!

checks.js is returning an AsyncFunction, you must await it.
checks.js:
module.exports = async (content) => {
// Check no.1
if (content.id != 'SomeID') return;
// Check no.2
if (content.length > 20) return;
//..etc
return true // maybe your not returning truthy?
}
index.js:
const checks = require('./checks');
(async () => {
console.log('typeof checks()', typeof checks);
console.log('instance of', checks.constructor.name);
//
let content = {
id: 'SomeID'
};
if (await checks(content)) {
console.log('1. passed');
} else {
console.log('1. failed');
}
//
content = {
id: 'WrongID'
};
if (await checks(content)) {
console.log('2. passed');
} else {
console.log('2. failed');
}
})();
Will output when run:
typeof checks() function
instance of AsyncFunction
1. passed
2. failed

See Understanding async/await on NodeJS for more details.

looping the callback function in node js

This is a piece of code which writes data to a ble device and reads data from it. data is written to the device in the form of a buffer. the value in 'mydata' (AAAD0000) is the command to be written in order to read the data.
function named chara3() consists of write and read function which is a callback function in which the command is passed read back.
My requirement is the 'mydata' value which i said earlier, the last two zeros is the memory address. i need to read the data in different memory addresses starting from zero to 59. That is AAAD0000 to AAAD0059. so of course i need to run a loop. If I'm reading the zeroth location, the code is quite fine and i got the output as well but when i tried to make it inside a loop, the code is all a mess. the read part is not executing.
can any one suggest a better way to read data from zeroth memory location to 59th memory location (AAAD0000 to AAAD0059)???
first command writes to it
then reads data
memory location incremented by 1
this should repeat up to 59
var mydata = 'AAAD0000';
function chara3() {
var buff2 = new Buffer(mydata, 'hex');
SensorCharacteristic.write(buff2, false, function(error) { //[0x002d]
console.log('Writing command SUCCESSFUL',mydata);
if (!error) {
SensorCharacteristic.read((error, data) => {
console.log("i just entered");
if (data.toString('hex') != '0000') {
console.log('Temperature History: ', data.toString('hex'));
enter();
}
else {
console.log('contains null value');
} //else
});
}
function enter()
{
mydata = (parseInt(mydata, 16) + 00000001).toString(16);
}
}); //.write
} //chara3
there's no error. But some part of the code is not executing.

You can use the recursion by wrapping your methods into a promise
async function chara3(mydata = 'AAAD0000') {
if (mydata === 'AAAD0059') {
return;
}
var buff2 = new Buffer(mydata, 'hex');
return new Promise((resolve) => {
SensorCharacteristic.write(buff2, false, function (error) { //[0x002d]
console.log('Writing command SUCCESSFUL', mydata);
if (!error) {
SensorCharacteristic.read(async (error, data) => {
console.log("i just entered");
if (data.toString('hex') != '0000') {
console.log('Temperature History: ', data.toString('hex'));
let next = await chara3(enter())
return resolve(next);
}
else {
console.log('contains null value');
return resolve();
} //else
});
}
}); //.write
});
} //chara3
function enter() {
return (parseInt(mydata, 16) + 00000001).toString(16);
}
Also if you can convert your methods SensorCharacteristic.write and SensorCharacteristic.read into promises you can simply map
function chara3(mydata) {
var buff2 = new Buffer(mydata, 'hex');
await SensorCharacteristic.write(buff2, false);
console.log('Writing command SUCCESSFUL', mydata);
let data = await SensorCharacteristic.read();
if (data.toString('hex') != '0000') {
console.log('Temperature History: ', data.toString('hex'));
enter();
} else {
console.log('contains null value');
}
};
let promiseArray = Array(60).fill().map((_, i) => (parseInt('AAAD0000', 16) + i).toString(16)).map(chara3);
Promise.all(promiseArray).then(() => console.log('done'));

Javascript: How to check if async operation is still pending / In progress?

I would like to know if it is somehow possible to check if an asynchronous operation in Javascript is still pending..
Because I am doing a database request on calling a specific URL... While the db call is still in progress, I want to stop any other incoming db-calls (which means, stop any further calls to that URL in case the db-request is still pending).
Is that somehow possible?
Because the database call takes up to minutes, and I don't want to launch another database-call while the first is still in progress.. The problem is, I somehow cannot figure out how to check if the call has started and is still in progress, because the response comes only after the .then() clause when the process has already finished.
this is my db-call function:
const getWriteIndex = async () => {
return Promise.all(someFunction1, someFunction2...).then(...) {
writeMessageObject = checkDocuments(...);
return Promise.resolve(writeMessageObject);
})).catch((err) => {
return Promise.reject(err);
});
}
This is my URL/Route Call function with express:
router.get("/v1/...", someMiddleware(), async function(req,res,next) {
if (read_cached() && initialised_read) {
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
} else {
try {
//HOW CAN I CHECK HERE IF THE DB-CALL IS ALREADY IN PROGRESS?
readmsg_obj.message = '';
getReadIndex().then((message) => {
initialised_read = true;
readmsg_obj = {...message};
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
}).catch((reject) => {
logger.error(`/../... : ${reject}`);
initialised_read = false;
res.status(500).send(reject);
});
} catch(err) {
logger.error(`/v1/... : ${err}`);
res.status(500).send(err);
};
}
});

hmm I found a workaround here:
https://ourcodeworld.com/articles/read/317/how-to-check-if-a-javascript-promise-has-been-fulfilled-rejected-or-resolved
so I wrote that function to check for promise stati, but I am still wondering if it's not somehow possible to query for static promise properties to get their actual state ;) (but weirdly, I didn't find any on the web).
const checkPendingRequest= (promise) => {
if (promise.isResolved) return promise;
// Set initial state
var isPending = true;
var isRejected = false;
var isFulfilled = false;
// Observe the promise, saving the fulfillment in a closure scope.
var result = promise.then(
function(v) {
isFulfilled = true;
isPending = false;
return v;
},
function(e) {
isRejected = true;
isPending = false;
throw e;
}
);
result.isFulfilled = function() { return isFulfilled; };
result.isPending = function() { return isPending; };
result.isRejected = function() { return isRejected; };
return result;
}
So I fixed my function for the request:
router.get("/v1/...", someMiddleware(), async function(req,res,next) {
if (read_cached() && initialised_read) {
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
} else {
try {
readmsg_obj.message = '';
if ((dbQueryPromiseRead != null) && dbQueryPromiseRead.isPending()) {
logger.info(`Database request for Index-read is still pending!`);
return;
}
dbQueryPromiseRead = checkPendingRequest(getReadIndex());
dbQueryPromiseRead.then((message) => {
initialised_read = true;
readmsg_obj = {...message};
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
}).catch((reject) => {
logger.error(`/../... : ${reject}`);
initialised_read = false;
res.status(500).send(reject);
});
} catch(err) {
logger.error(`/v1/... : ${err}`);
res.status(500).send(err);
};
}
});

You need to try add in node.js like global.dbCallState flag if operation is still running.
This global var one for all modules.
Do not change this object like global = new Object();, but you can use child field's.
https://nodejs.org/api/globals.html
You can change it in another module like global.dbCallState = false.
It not best solution, but it can help.
But i don't know, why you want only one connection. Its not good solution to block I/O in node.js

Unreliable behaviour in Node.js

I have a Node.js application that, upon initialisation, reads two tables from an SQL database and reconstructs their relationship in memory. They're used for synchronously looking up data that changes (very) infrequently.
Problem: Sometimes I can't access the data, even though the application reports successfully loading it.
Code:
constants.js
module.exports = {
ready: function () { return false; }
};
var log = sysLog('core', 'constants')
, Geo = require('../models/geo.js');
var _ready = false
, _countries = []
, _carriers = [];
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
var obj = country.toPlainObject()
, id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
_countries = countries;
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
_carriers = carriers;
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
var log = sysLog('core', 'utils')
, constants = require('./constants');
module.exports = {
getCountryByISO: function(iso) {
if (!iso) {
return;
}
if ('string' != typeof iso) {
throw new Error('getCountryByISO requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
switch (iso.length) {
case 2:
return _.findWhere(constants.countries(), { 'iso2' : iso.toUpperCase() });
case 3:
return _.findWhere(constants.countries(), { 'iso3' : iso.toUpperCase() });
default:
throw new Error('getCountryByISO requires a 2 or 3 letter ISO code');
}
},
getCarrierByCode: function(code) {
if (!code) {
return;
}
if ('string' != typeof code) {
throw new Error('getCarrierByCode requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'code' : code });
},
getCarrierByHandle: function(handle) {
if (!handle) {
return;
}
if ('string' != typeof handle) {
throw new Error('getCarrierByHandle requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'handle' : handle });
}
};
Use case
if (data.handle) {
carrier = utils.getCarrierByHandle(data.handle);
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}
What's going on: All errors are logged; as soon as I see an error (i.e. "Unknown carrier") in the logs, I check the SQL database to see if it should've been recognised. That has always been the case so far, so I check the debug log to see if data was loaded. I always see "Loaded X countries" and "Loaded Y carriers" with correct values and no sign of "Could not load constants" or any other kind of trouble.
This happens around 10% of the time I start the application and the problem persists (i.e. didn't seem to go away after 12+ hours) and seems to occur regardless of input, leading me to think that the data isn't referenced correctly.
Questions:
Is there something wrong in constants.js or am I doing something very obviously wrong? I've tried setting it up for cyclical loading (even though I am not aware of that happening in this case).
Why can't I (sometimes) access my data?
What can I do to figure out what's wrong?
Is there any way I can work around this? Is there anything else I could to achieve the desired behaviour? Hard-coding the data in constants.js is excluded.
Additional information:
constants.reload() is never actually called from outside of constants.js.
constants.js is required only in utils.js.
utils.js is required in app.js (application entry); all files required before it do not require it.
SQL access is done through an in-house library built on top of knex.js and bluebird; so far it's been very stable.
Versions:
Node.js v0.10.33
underscore 1.7.0
bluebird 2.3.11
knex 0.6.22

}).finally(function () {
_ready = true;
});
Code in a finally will always get called, regardless of if an error was thrown up the promise chain. Additionally, your reload().catch(/* ... */) clause will never be reached, because finally swallows the error.
Geo.Country.find() or Geo.Carrier.Descriptor.find() could throw an error, and _ready would still be set to true, and the problem of your countries and carriers not being set would persist.
This problem would not have occurred if you had designed your system without a ready call, as I described in my previous post. Hopefully this informs you that the issue here is really beyond finally swallowing a catch. The real issue is relying on side-effects; the modification of free variables results in brittle systems, especially when asynchrony is involved. I highly recommend against it.

Try this
var log = sysLog('core', 'constants');
var Geo = require('../models/geo.js');
var index;
var _countries;
var _carriers;
function reload() {
index = Object.create(null);
_countries = Geo.Country.find().map(function (country) {
var obj = country.toPlainObject();
var id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
});
_carriers = _countries.then(function(countries) {
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
});
});
return _carriers;
}
reload().done();
module.exports = {
reload : reload,
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};

constants.reload() is never actually called from outside of
constants.js.
That's your issue. constants.reload() reads from a database, which is an aysnchronous process. Node's require() is a synchronous process. At the time constants.js is required in utils.js and the module.exports value is returned, your database query is still running. And at whatever point in time that app.js reaches the point where it calls a method from the utils module, that query could still be running, resulting in the error.
You could say that requiring utils.js has the side-effect of requiring constants.js, which has the side-effect of executing a database query, which has the side-effect of concurrently modifying the free variables _countries and _carriers.
Initialize _countries and _carriers as unresolved promises. Have reload() resolve them. Make the utils.js api async.
promises.js:
// ...
var Promise = require('bluebird');
var countriesResolve
, carriersResolve;
var _ready = false
, _countries = new Promise(function (resolve) {
countriesResolve = resolve;
})
, _carriers = new Promise(function (resolve) {
carriersResolve = resolve;
});
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
// ...
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
countriesResolve(countries);
return Geo.Carrier.Descriptor.find().map(function (carrier) {
// ...
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
carriersResolve(carriers);
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
getCarrierByHandle: function(handle) {
// ...
return constants.carriers().then(function (carriers) {
return _.findWhere(carriers, { 'handle' : handle });
});
}
Use case:
utils.getCarrierByHandle(data.handle).then(function (carrier) {
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}).then(function () {
// ... next step in application logic
});
This design will also eliminate the need for a ready method.
Alternatively, you could call constants.reload() on initialization and hang all possibly-dependent operations until it completes. This approach would also obsolete the ready method.
What can I do to figure out what's wrong?
You could have analyzed your logs and observed that "Loaded X countries" and "Loaded Y carriers" were sometimes written after "Unknown carrier", helping you realize that the success of utils.getCarrierByHandle() was a race condition.