Scenario
I need a background upload queue that sends files to a server.
The queue should send the files in sequential order as they are pushed into the queue (FIFO).
My solution
var pendingFiles = [];
var filesOperation = null;
uploadNextAsync = function(file) {
var next;
if (!pendingFiles.length) {
return WinJS.Promise.as();
}
next = pendingFiles.shift();
fileslogger.debug("Uploading " + next);
return fileQuery.folder.getFileAsync(next).then(function(file) {
return Server.sendFileAsync(file).then(function() {
return filesOk += 1;
}, function(error) {
filesNok += 1;
return logger.error(error.message, error);
}).then(function() {
if (pendingFiles.length) {
return uploadNextAsync(inspection);
}
});
});
};
createTaskForFile = function(file) {
if (pendingFiles.length == 0) {
pendingFiles = [file.name]
filesOperation = uploadNextAsync(file);
} else {
pendingFiles.push(file.name);
return filesOperation.then(function() {
return uploadNextAsync(file);
});
}
};
Problem
It seems that sometimes if createTaskForFile is called very quickly in succession then 2 files end up being uploaded at the same time. So somewhere is a little glitch either in the createTastForFile function on how it uses the fileOperation.then construct or inside the uploadNextAsync does something wrong?
Your problem is that pendingFiles is always empty. In createTaskForFile, you would set it to an one-element array then, but immediately call uploadNextAsync() which shifts it out. I guess your script might work if you shifted the file after the file has been uploaded.
However, you actually don't need this array. You can just queue the action to filesOperation, which would be a promise representing the upload of all current files.
var filesOperation = WinJS.Promise.as();
function createTaskForFile(file) {
return filesOperation = filesOperation.then(function() {
return uploadNextAsync(file);
});
}
function uploadAsync(next) {
fileslogger.debug("Uploading " + next.name);
return fileQuery.folder.getFileAsync(next.name).then(function(file) {
return Server.sendFileAsync(file);
}).then(function() {
return filesOk += 1;
}, function(error) {
filesNok += 1;
return logger.error(error.message, error);
});
}
Related
In my code I deal with multiple JSON requests that need to be parsed in an order.
let allRadio_data, allHistory_data, iTunes_data;
$.when(parseRadioList())
.then(function(list) {
radioList_data = list;
return $.when(
parseEachRadioData(radioList_data),
ParseEachHistoryData(radioList_data)
);
})
.then(function() {
console.log(allRadio_data);
console.log(allHistory_data);
return $.when(_parseiTunes());
})
.then(function(iTunesInfo) {
iTunes_data = iTunesInfo;
return _cacheOptions();
})
function _cacheOptions() {
// FINAL function
}
/////////
function parseRadioList() {
return $.getJSON("https://api.myjson.com/bins/xiyvr");
}
function _parseiTunes() {
return $.getJSON("https://itunes.apple.com/search?term=jackson&limit=10&callback=?")
}
function parseEachRadioData(radioList) {
allRadio_data = [];
$.each(radioList, function(index, radio) {
$.when($.getJSON(radio.url + "/stats?sid=" + radio.stream_id + "&json=1&callback=?"))
.then(function(data) {
allRadio_data.push(data);
});
})
}
function ParseEachHistoryData(radioList) {
allHistory_data = [];
$.each(radioList, function(index, radio) {
$.when($.getJSON(radio.url + "/played?sid=" + radio.stream_id + "&type=json&callback=?"))
.then(function(data) {
allHistory_data.push(data);
});
})
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
Right now the code is running, but where I do console.log(allRadio_data); it is empty. However, if I do settimeout() to delay it for a second the data is completed. This means then() is not running on time.
This is the structure I am looking for:
JSON file 1 is parsed. parseRadioList()
JSON1 is an array of multiple entries of JSON URLS.
Run through the URLs within JSON1 array and do getJSON for each. parseEachRadioData(radioList_data) & ParseEachHistoryData(radioList_data)
Push data of each JSON in one general Array.
Once completed, parse JSON2 _parseiTunes()
Any Idea how to make this code running in the right structure.
Thanks in advance.
For a start, parseEachRadioData and ParseEachHistoryData don't return anything at all, let alone a Promise - so it's impossible to wait on them
Also, you're overusing $.when ... in fact you never need to use it, just use regular promises since jQuery $.getJSON etc return a usable Promise-like object
i.e. your code could be
let allRadio_data, allHistory_data, iTunes_data;
parseRadioList()
.then(function(list) {
radioList_data = list;
return Promise.all([
parseEachRadioData(radioList_data),
ParseEachHistoryData(radioList_data)
]);
})
.then(function(result) {
allRadio_data = result[0];
allHistory_data = result[1];
console.log(allRadio_data);
console.log(allHistory_data);
return _parseiTunes();
})
.then(function(iTunesInfo) {
iTunes_data = iTunesInfo;
return _cacheOptions();
})
function _cacheOptions() {
// FINAL function
}
/////////
function parseRadioList() {
return $.getJSON("https://api.myjson.com/bins/xiyvr");
}
function _parseiTunes() {
return $.getJSON("https://itunes.apple.com/search?term=jackson&limit=10&callback=?")
}
function parseEachRadioData(radioList) {
return Promise.all(radioList.map(radio => $.getJSON(radio.url + "/stats?sid=" + radio.stream_id + "&json=1&callback=?")));
}
function ParseEachHistoryData(radioList) {
return Promise.all(radioList.map(radio => $.getJSON(radio.url + "/played?sid=" + radio.stream_id + "&type=json&callback=?")));
}
from a first view
$.when(parseRadioList()) // your code will wait her
.then(function (list) {
radioList_data = list;
return $.when(
parseEachRadioData(radioList_data), // this two method are void they will complete
ParseEachHistoryData(radioList_data) // immediately without waithing for getJson...
);
})
I would like to know if it is somehow possible to check if an asynchronous operation in Javascript is still pending..
Because I am doing a database request on calling a specific URL... While the db call is still in progress, I want to stop any other incoming db-calls (which means, stop any further calls to that URL in case the db-request is still pending).
Is that somehow possible?
Because the database call takes up to minutes, and I don't want to launch another database-call while the first is still in progress.. The problem is, I somehow cannot figure out how to check if the call has started and is still in progress, because the response comes only after the .then() clause when the process has already finished.
this is my db-call function:
const getWriteIndex = async () => {
return Promise.all(someFunction1, someFunction2...).then(...) {
writeMessageObject = checkDocuments(...);
return Promise.resolve(writeMessageObject);
})).catch((err) => {
return Promise.reject(err);
});
}
This is my URL/Route Call function with express:
router.get("/v1/...", someMiddleware(), async function(req,res,next) {
if (read_cached() && initialised_read) {
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
} else {
try {
//HOW CAN I CHECK HERE IF THE DB-CALL IS ALREADY IN PROGRESS?
readmsg_obj.message = '';
getReadIndex().then((message) => {
initialised_read = true;
readmsg_obj = {...message};
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
}).catch((reject) => {
logger.error(`/../... : ${reject}`);
initialised_read = false;
res.status(500).send(reject);
});
} catch(err) {
logger.error(`/v1/... : ${err}`);
res.status(500).send(err);
};
}
});
hmm I found a workaround here:
https://ourcodeworld.com/articles/read/317/how-to-check-if-a-javascript-promise-has-been-fulfilled-rejected-or-resolved
so I wrote that function to check for promise stati, but I am still wondering if it's not somehow possible to query for static promise properties to get their actual state ;) (but weirdly, I didn't find any on the web).
const checkPendingRequest= (promise) => {
if (promise.isResolved) return promise;
// Set initial state
var isPending = true;
var isRejected = false;
var isFulfilled = false;
// Observe the promise, saving the fulfillment in a closure scope.
var result = promise.then(
function(v) {
isFulfilled = true;
isPending = false;
return v;
},
function(e) {
isRejected = true;
isPending = false;
throw e;
}
);
result.isFulfilled = function() { return isFulfilled; };
result.isPending = function() { return isPending; };
result.isRejected = function() { return isRejected; };
return result;
}
So I fixed my function for the request:
router.get("/v1/...", someMiddleware(), async function(req,res,next) {
if (read_cached() && initialised_read) {
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
} else {
try {
readmsg_obj.message = '';
if ((dbQueryPromiseRead != null) && dbQueryPromiseRead.isPending()) {
logger.info(`Database request for Index-read is still pending!`);
return;
}
dbQueryPromiseRead = checkPendingRequest(getReadIndex());
dbQueryPromiseRead.then((message) => {
initialised_read = true;
readmsg_obj = {...message};
res.setHeader('Content-Type', 'application/json');
res.json(readmsg_obj);
}).catch((reject) => {
logger.error(`/../... : ${reject}`);
initialised_read = false;
res.status(500).send(reject);
});
} catch(err) {
logger.error(`/v1/... : ${err}`);
res.status(500).send(err);
};
}
});
You need to try add in node.js like global.dbCallState flag if operation is still running.
This global var one for all modules.
Do not change this object like global = new Object();, but you can use child field's.
https://nodejs.org/api/globals.html
You can change it in another module like global.dbCallState = false.
It not best solution, but it can help.
But i don't know, why you want only one connection. Its not good solution to block I/O in node.js
I'm hitting an API which returns all details on kills in a game, the first endpoint returns an id to the kill event, then a second endpoint is hit to retrieve the killer and killed names.
Because of the way this API is set up I need to make a request to first get the event ID and then wait for all id's in the returned array to get a result and then process the entire kill array:
requestify.get(url).then(function (response) {
var events = [];
if (response.body && response.body.length > 0) {
data = JSON.parse(response.body);
if (data.hasOwnProperty('events')) {
events = data.events.map(function(event) {
return this.getDataForHeroKillId(event.id, function(killInfo) {
return { killer: killInfo.killer, killed: killInfo.killed, timestamp: event.time };
});
}.bind(this));
console.log('events is: ', events);
}
}
return Promise.all(events);
}.bind(this));
My getKillInformation function looks like this:
KillFeed.prototype.getKillInformation = function(id, cb) {
var data = null;
requestify.get(url).then(function (response) {
var event = {};
if (response.body && response.body.length > 0) {
data = JSON.parse(response.body);
if (data.hasOwnProperty('Killer')) {
event = { killer: data.Killer, killed: data.Killed};
}
}
cb(event);
});
};
In the second method I was hoping that I could callback the result of each child request and then once they had all been executed my new array would hold the data. But due to the event driven nature of JS I found that my code block continues to return an empty events array as this code is obviously non blocking (understandably as blocking the event queue whilst making a HTTP request is not ideal). How can I implement this?
One uses promises for this.
requestify.get(url).then(function (response) {
var events = [];
if (response.body && response.body.length > 0) {
var data = JSON.parse(response.body);
if (data.hasOwnProperty('events')) {
// Trying to process the kill information here
events = data.events.map(function(event) {
return this.getKillInformation(event.id).then(function(killInfo) {
return { killer: killInfo.killer, killed: killInfo.killed, timestamp: event['time1'] };
});
}.bind(this));
}
}
return Promise.all(events);
});
KillFeed.prototype.getKillInformation = function(id) {
var url = 'internal_url';
return requestify.get(url).then(function (response) {
if (response.body && response.body.length > 0) {
var data = JSON.parse(response.body);
if (data.hasOwnProperty('killer')) {
return { killer: data.Killer, killed: data.Killed };
}
}
});
};
You could use async and its waterfall method. Async is a NodeJS module, but it can be used in browser, too.
I have a Node.js application that, upon initialisation, reads two tables from an SQL database and reconstructs their relationship in memory. They're used for synchronously looking up data that changes (very) infrequently.
Problem: Sometimes I can't access the data, even though the application reports successfully loading it.
Code:
constants.js
module.exports = {
ready: function () { return false; }
};
var log = sysLog('core', 'constants')
, Geo = require('../models/geo.js');
var _ready = false
, _countries = []
, _carriers = [];
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
var obj = country.toPlainObject()
, id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
_countries = countries;
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
_carriers = carriers;
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
var log = sysLog('core', 'utils')
, constants = require('./constants');
module.exports = {
getCountryByISO: function(iso) {
if (!iso) {
return;
}
if ('string' != typeof iso) {
throw new Error('getCountryByISO requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
switch (iso.length) {
case 2:
return _.findWhere(constants.countries(), { 'iso2' : iso.toUpperCase() });
case 3:
return _.findWhere(constants.countries(), { 'iso3' : iso.toUpperCase() });
default:
throw new Error('getCountryByISO requires a 2 or 3 letter ISO code');
}
},
getCarrierByCode: function(code) {
if (!code) {
return;
}
if ('string' != typeof code) {
throw new Error('getCarrierByCode requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'code' : code });
},
getCarrierByHandle: function(handle) {
if (!handle) {
return;
}
if ('string' != typeof handle) {
throw new Error('getCarrierByHandle requires a string');
}
if (!constants.ready()) {
throw new UnavailableError('Try again in a few seconds');
}
return _.findWhere(constants.carriers(), { 'handle' : handle });
}
};
Use case
if (data.handle) {
carrier = utils.getCarrierByHandle(data.handle);
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}
What's going on: All errors are logged; as soon as I see an error (i.e. "Unknown carrier") in the logs, I check the SQL database to see if it should've been recognised. That has always been the case so far, so I check the debug log to see if data was loaded. I always see "Loaded X countries" and "Loaded Y carriers" with correct values and no sign of "Could not load constants" or any other kind of trouble.
This happens around 10% of the time I start the application and the problem persists (i.e. didn't seem to go away after 12+ hours) and seems to occur regardless of input, leading me to think that the data isn't referenced correctly.
Questions:
Is there something wrong in constants.js or am I doing something very obviously wrong? I've tried setting it up for cyclical loading (even though I am not aware of that happening in this case).
Why can't I (sometimes) access my data?
What can I do to figure out what's wrong?
Is there any way I can work around this? Is there anything else I could to achieve the desired behaviour? Hard-coding the data in constants.js is excluded.
Additional information:
constants.reload() is never actually called from outside of constants.js.
constants.js is required only in utils.js.
utils.js is required in app.js (application entry); all files required before it do not require it.
SQL access is done through an in-house library built on top of knex.js and bluebird; so far it's been very stable.
Versions:
Node.js v0.10.33
underscore 1.7.0
bluebird 2.3.11
knex 0.6.22
}).finally(function () {
_ready = true;
});
Code in a finally will always get called, regardless of if an error was thrown up the promise chain. Additionally, your reload().catch(/* ... */) clause will never be reached, because finally swallows the error.
Geo.Country.find() or Geo.Carrier.Descriptor.find() could throw an error, and _ready would still be set to true, and the problem of your countries and carriers not being set would persist.
This problem would not have occurred if you had designed your system without a ready call, as I described in my previous post. Hopefully this informs you that the issue here is really beyond finally swallowing a catch. The real issue is relying on side-effects; the modification of free variables results in brittle systems, especially when asynchrony is involved. I highly recommend against it.
Try this
var log = sysLog('core', 'constants');
var Geo = require('../models/geo.js');
var index;
var _countries;
var _carriers;
function reload() {
index = Object.create(null);
_countries = Geo.Country.find().map(function (country) {
var obj = country.toPlainObject();
var id = obj.id;
delete obj.id;
index[id] = obj;
return Object.freeze(obj);
});
_carriers = _countries.then(function(countries) {
return Geo.Carrier.Descriptor.find().map(function (carrier) {
var obj = carrier.toPlainObject();
if (obj.country) {
obj.country = index[obj.country];
}
return Object.freeze(obj);
});
});
return _carriers;
}
reload().done();
module.exports = {
reload : reload,
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
constants.reload() is never actually called from outside of
constants.js.
That's your issue. constants.reload() reads from a database, which is an aysnchronous process. Node's require() is a synchronous process. At the time constants.js is required in utils.js and the module.exports value is returned, your database query is still running. And at whatever point in time that app.js reaches the point where it calls a method from the utils module, that query could still be running, resulting in the error.
You could say that requiring utils.js has the side-effect of requiring constants.js, which has the side-effect of executing a database query, which has the side-effect of concurrently modifying the free variables _countries and _carriers.
Initialize _countries and _carriers as unresolved promises. Have reload() resolve them. Make the utils.js api async.
promises.js:
// ...
var Promise = require('bluebird');
var countriesResolve
, carriersResolve;
var _ready = false
, _countries = new Promise(function (resolve) {
countriesResolve = resolve;
})
, _carriers = new Promise(function (resolve) {
carriersResolve = resolve;
});
function reload() {
_ready = false;
var index = Object.create(null);
return Geo.Country.find().map(function (country) {
// ...
}).then(function (countries) {
log.debug('Loaded ' + countries.length + ' countries');
countriesResolve(countries);
return Geo.Carrier.Descriptor.find().map(function (carrier) {
// ...
}).then(function (carriers) {
log.debug('Loaded ' + carriers.length + ' carriers');
carriersResolve(carriers);
});
}).finally(function () {
_ready = true;
});
}
reload().catch(function (err) {
log.crit({ message: 'Could not load constants', reason: err });
process.exit(-42);
}).done();
module.exports = {
reload : reload,
ready : function () { return _ready; },
countries : function () { return _countries; },
carriers : function () { return _carriers; }
};
utils.js
getCarrierByHandle: function(handle) {
// ...
return constants.carriers().then(function (carriers) {
return _.findWhere(carriers, { 'handle' : handle });
});
}
Use case:
utils.getCarrierByHandle(data.handle).then(function (carrier) {
if (_.isEmpty(carrier)) {
throw new InternalError('Unknown carrier', { handle: data.handle });
}
}).then(function () {
// ... next step in application logic
});
This design will also eliminate the need for a ready method.
Alternatively, you could call constants.reload() on initialization and hang all possibly-dependent operations until it completes. This approach would also obsolete the ready method.
What can I do to figure out what's wrong?
You could have analyzed your logs and observed that "Loaded X countries" and "Loaded Y carriers" were sometimes written after "Unknown carrier", helping you realize that the success of utils.getCarrierByHandle() was a race condition.
I am facing following issue:
I am calling in foreach cycle following browse function. When the rb.wsc.browse(symbol) is called the program do some WebSocket request and when the message is returned the event is emmited. The problem is that I always get the same browseData even when I know that the event is emited with different one. I think that this is closure issue, but I don't know how to solve it.
function browse(rb, symbol, callback) {
var result = function(wsc, browseData) {
wsc.off('browse', result);
wsc.off('failed', result);
var err = null;
if (wsc.errno < 0) {
err = new Error("Browsing symbol " + symbol + " failed!");
err.status = wsc.errno;
} else {
saveBrowseData(rb, browseData);
}
callback(err, symbol);
};
// Register temporary listeners
rb.wsc.on('browse', result);
rb.wsc.on('failed', result);
// Browse symbol
rb.wsc.browse(symbol);
}
RexBrowser.prototype.refresh = function() {
var that = this;
var browseRequestNumber = 1;
var browseResult = function(err, symbol) {
browseRequestNumber--;
var item = that.getSymbol(symbol);
_.each(item.children, function(child) {
if (child.browse) {
browseRequestNumber++;
debug("Browsing: " + child.cstring);
browse(that,child.cstring, browseResult);
}
});
if (browseRequestNumber === 0) {
that.emit('refresh', that);
}
};
// Start recursive browsing
browse(this,'$', browseResult);
};-
You could try using a IIFE:
} else {
function(rbInner, browseDataInner){
saveBrowseData(rbInner, browseDataInner);
}(rb, browseData);
}
This makes sure the variables used by / in saveBrowseData have the values they have when the function is called.