I recently learned that azure mobile services limits their query search to 50 (or 1000 if you change the default), but my app needs to be able to access unlimited records. I created this service to return a list that contains all of the records. I made the skip count 1 because I wanted to make sure it worked on a small scale before doing a skip count like 50. After debugging this code it keeps returning two of the same records with one of them different. There are three completely different records in my database so I am confused why it is picking up one of them twice. I made sure that when the request was made that skip amount was either 0,1,2.
return new Promise(function(resolve, reject) {
var x = 0
var total = 1;
list = [];
console.log(list.length);
while(x <= 3){
x++;
var query = table.where(where).includeTotalCount().skip(skipAmount).take(1).read().done(function (results) {
total = results.totalCount;
if(results[0] != undefined)
{
for (var i = results.length - 1; i >= 0; i--) {
list.push(results[i]);
console.log(results[i]);
}
}
else
{
resolve(list);
}
}, function (err) {
reject(err);
});
skipAmount++;
}
});
}
}
You may need to use the .orderBy() or .orderByDescending() query methods to make sure that the sorted data are returned.
table
.where(where)
.orderBy('updatedAt')
.includeTotalCount()
.skip(skipAmount)
.take(1)
.read()
.then(success, failure);
Related
I have set a timeout for sending back an error. The problem is that if I need to clear the timeout with clearTimeOut() it does indeed kill it, I can see that as the value of errTimeout's _kill shows true in the debugger. But for some reason node still keeps running the script until the timeOutPeriod is over. I guess it won't really create issues in production because the calling function will receive the returned value. But it still kinda pisses me off that it keeps waiting instead of killing the script.
return new Promise((resolve,reject) => {
function checkResponse () {
//creates a timeout that returns an error if data not receieved after the specified time.
let errTimeout = setTimeout(reject, config.timeOutPeriod);
//the +1 is there because we originally reduced one, we need to use the physical number now.
if(layerKeycodes.length !== (rows+1) * (columns+1)){
//if the array is not complete, check again in 100 ms
setTimeout(checkResponse, 100);
} else {
//clear the error timeout
clearTimeout(errTimeout);
//send the layerKeycodes to the calling function
resolve(layerKeycodes);
}
}
It looks like this code is something you're trying to fit into getLayerKeycodes() from this other question to somehow know when all the data has been received from your keyboard hardware.
I'll illustrate how you can plug into that without using timers. Here's what you started with in that other question:
Your original function
const getLayerKeycodes = (keyboard, layer, rows, columns) => {
//array that stores all the keycodes according to their order
let layerKeycodes = [];
//rows and columns start the count at 0 in low level, so we need to decrease one from the actual number.
columns --;
rows --;
//loop that asks about all the keycodes in a layer
const dataReceived = (err, data) => {
if(err) {
return err;
}
// push the current keycode to the array
// The keycode is always returned as the fifth object.
layerKeycodes.push(data[5]);
console.log(layerKeycodes);
};
for (let r = 0 , c = 0;c <= columns; r ++){
//callback to fire once data is receieved back from the keyboard.
if(r > rows){
c++;
//r will turn to 0 once the continue fires and the loop executes again
r = -1;
continue;
}
//Start listening and call dataReceived when data is received
keyboard[0].read(dataReceived);
//Ask keyboard for information about keycode
// [always 0 (first byte is ignored),always 0x04 (get_keycode),layer requested,row being checked,column being checked]
keyboard[0].write([0x01,0x04,layer,r,c]);
}
console.log(layerKeycodes);
}
Manually created promise to resolve upon completion of all rows/columns
And, you can incorporate the completion detection code inside of the dataReceived() function without any timers and without reworking much of the rest of your logic like this:
const getLayerKeycodes = (keyboard, layer, rows, columns) => {
return new Promise((resolve, reject) => {
//array that stores all the keycodes according to their order
const layerKeycodes = [];
const totalCells = rows * columns;
let abort = false;
//rows and columns start the count at 0 in low level, so we need to decrease one from the actual number.
columns--;
rows--;
// function that gets with keyboard data
function dataReceived(err, data) => {
if (err) {
abort = true; // set flag to stop sending more requests
reject(err);
return;
}
// push the current keycode to the array
// The keycode is always returned as the fifth object.
layerKeycodes.push(data[5]);
// now see if we're done with all of them
if (layerKeycodes.length >= totalCells) {
resolve(layerKeycodes);
}
}
// loop that asks about all the keycodes in a layer
for (let r = 0, c = 0; c <= columns; r++) {
// stop sending more requests if we've already gotten an error
if (abort) {
break;
}
//callback to fire once data is receieved back from the keyboard.
if (r > rows) {
c++;
//r will turn to 0 once the continue fires and the loop executes again
r = -1;
continue;
}
//Start listening and call dataReceived when data is received
keyboard[0].read(dataReceived);
//Ask keyboard for information about keycode
// [always 0 (first byte is ignored),always 0x04 (get_keycode),layer requested,row being checked,column being checked]
keyboard[0].write([0x01, 0x04, layer, r, c]);
}
}
}
}
A simplified version by promisifying the read function
And, here's a bit simpler version that promisifies the read function so we can use await on it and then just use an async function and a dual nested for loop for simpler loop mechanics.
const util = require('util');
async function getLayerKeycodes(keyboard, layer, rows, columns) => {
// promisify the keyboard.read()
const readKeyboard = util.promisify(keyboard[0].read).bind(keyboard[0]);
//array that stores all the keycodes according to their order
const layerKeycodes = [];
// loop that asks about all the keycodes in a layer
for (let rowCntr = 0; rowCntr < rows; rowCntr++) {
for (let colCntr = 0; colCntr < columns; colCntr++) {
// Start listening and collect the promise
const readPromise = readKeyboard();
// Ask keyboard for information about keycode
// [always 0 (first byte is ignored),always 0x04 (get_keycode),layer requested,row being checked,column being checked]
keyboard[0].write([0x01, 0x04, layer, rowCntr, colCntr]);
// wait for data to come in
const data = await readPromise;
// push the current keycode to the array
// The keycode is always returned as the fifth object.
layerKeycodes.push(data[5]);
}
}
return layerCodes;
}
This also makes sure that we send a write, then wait for the data from that write to come back before we sent the next write which seems like a potentially safer way to handle the hardware. Your original code fires all the writes at once which might work, but it seems like the reads could come back in any order. This guarantees sequential order in the layerCodes array which seems safer (I'm not sure if that matters with this data or not).
Error handling in this last version is somewhat automatically handled by the async function and the promises. If the read returns an error, then the readPromise will automatically reject which will abort our loop and in turn reject the promise that the async function returned. So, we don't have to do the abort checking that the previous function with the manually created promise had.
Now, of course, I don't have the ability to run any of these to test them so it's possible there are some typos somewhere, but hopefully you can work through any of those and see the concept for how these work.
I have a for loop like so:
var currentLargest
for (var site in sites) {
// Each site is a website (www.mysite1.com, www.mysite2.com, etc.)
$.getJSON(site).then(function(data) {
if (data > currentLargest) {
currentLargest = data
}
});
}
// Here is where I want to use "currentLargest"
It checks multiple different websites to get the largest value. However, I can't extract the data I need after the for loop. I can only interact with currentLargest from within the promise, but that data is only relevant for ONE of the sites (it hasn't finished looping through all them until after the for loop).
How can I access the currentLargest value after the for loop containing the getJSON promises?
// Your code:
/*
var currentLargest
for (var site in sites) {
// Each site is a website (www.mysite1.com, www.mysite2.com, etc.)
$.getJSON(site).then(function(data) {
if (data > currentLargest) {
currentLargest = data
}
});
}
*/
// Updated:
const sitePromisesArr = sites.map((site) => $.getJSON(site));
$.when(sitePromisesArr)
.then((valArr, index) => {
// The "valArr" here will be an Array with the responses
const largest = valArr.reduce((current, next) => next > current ? next : current, 0);
// This will be the largest value.
console.log(largest);
// This will be the site that contains the largest value.
console.log(sitePromisesArr[index]);
});
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
You need to use a async loop and then execute a callback once all the async getJSON requests are finished. You could use http://caolan.github.io/async/docs.html#each to help with this.
I use Parse in iOS to run a cloud code method that gets an ID in it's request and receives a number in the response.
The purpose of the cloud code function is to take the request ID and add it to a field of 3 different users.
Here is the cloud code method in Javascript:
amount = 3;
// Use Parse.Cloud.define to define as many cloud functions as you want.
// For example:
Parse.Cloud.define("addToIDs", function(request, response) {
var value = request.params.itemId;
var query = new Parse.Query(Parse.User);
query.ascending("createdAt");
query.limit(100);
query.find({
success: function(results) {
var sent = 0;
for (var i = 0; i < results.length; i++) {
var idlst = results[i].get("idString");
if (idlst != null && idlst.indexOf(value) <= -1) {
idlst += value+"|";
results[i].set("idString", idlst);
results[i].save();
sent = sent+1;
}
if (sent >= amount) {
break;
}
}
response.success(sent);
},
error: function() {
response.error("Test failed");
}
});
});
When running this cloud code method I get a response of '3' meaning it called .save for 3 users. The problem is that when i go back to look in the Database viewer in the parse website it actually only updated a single user (Its always the same user). No matter how many times i run this code, it will only actually update the first user..
Anyone know why this is happening?
Both save and saveAll are asynchronous, so you should make sure the saving process is done.
Also note that, the user object can only be updated by the owner or request with masterkey.
The following code should work:
var amount = 3;
Parse.Cloud.define("addToIDs", function(request, response) {
var value = request.params.itemId;
var query = new Parse.Query(Parse.User);
query.ascending("createdAt");
query.limit(100);
return query.find()
.then(function(results) { // success
var toSave = [];
var promise = new Parse.Promise();
for (var i = 0; i < results.length; i++) {
var idlst = results[i].get("idString");
if (idlst != null && idlst.indexOf(value) <= -1) {
idlst += value+"|";
results[i].set("idString", idlst);
toSave.push(results[i]);
}
if (toSave.length >= amount) {
break;
}
}
// use saveAll to save multiple object without bursting multiple request
Parse.Object.saveAll(toSave, {
useMasterKey: true,
success: function(list) {
promise.resolve(list.length);
},
error: function() {
promise.reject();
}
});
return promise;
}).then(function(length) { // success
response.success(length);
}, function() { // error
response.error("Test failed");
});
});
The reason this is happening is two-fold:
save() is an asynchronous method, and
response.success() will immediately kill your running code as soon as it's called.
So what's happening is that inside your for loop you're running save() several times, but since it's asynchronous, they're simply thrown into the processing queue and your for loop continues on through. So it's quickly throwing all of your save()'s into the processing queue, and then it reaches your response.success() call but, by the time it's reached, only one of the save()'s has had a chance to successfully process.
I have only recently started developing for node.js, so forgive me if this is a stupid question - I come from Javaland, where objects still live happily sequentially and synchronous. ;)
I have a key generator object that issues keys for database inserts using a variant of the high-low algorithm. Here's my code:
function KeyGenerator() {
var nextKey;
var upperBound;
this.generateKey = function(table, done) {
if (nextKey > upperBound) {
require("../sync/key-series-request").requestKeys(function(err,nextKey,upperBound) {
if (err) { return done(err); }
this.nextKey = nextKey;
this.upperBound = upperBound;
done(nextKey++);
});
} else {
done(nextKey++);
}
}
}
Obviously, when I ask it for a key, I must ensure that it never, ever issues the same key twice. In Java, if I wanted to enable concurrent access, I would make make this synchronized.
In node.js, is there any similar concept, or is it unnecessary? I intend to ask the generator for a bunch of keys for a bulk insert using async.parallel. My expectation is that since node is single-threaded, I need not worry about the same key ever being issued more than once, can someone please confirm this is correct?
Obtaining a new series involves an asynchronous database operation, so if I do 20 simultaneous key requests, but the series has only two keys left, won't I end up with 18 requests for a new series? What can I do to avoid that?
UPDATE
This is the code for requestKeys:
exports.requestKeys = function (done) {
var db = require("../storage/db");
db.query("select next_key, upper_bound from key_generation where type='issue'", function(err,results) {
if (err) { done(err); } else {
if (results.length === 0) {
// Somehow we lost the "issue" row - this should never have happened
done (new Error("Could not find 'issue' row in key generation table"));
} else {
var nextKey = results[0].next_key;
var upperBound = results[0].upper_bound;
db.query("update key_generation set next_key=?, upper_bound=? where type='issue'",
[ nextKey + KEY_SERIES_WIDTH, upperBound + KEY_SERIES_WIDTH],
function (err,results) {
if (err) { done(err); } else {
done(null, nextKey, upperBound);
}
});
}
}
});
}
UPDATE 2
I should probably mention that consuming a key requires db access even if a new series doesn't have to be requested, because the consumed key will have to be marked as used in the database. The code doesn't reflect this because I ran into trouble before I got around to implementing that part.
UPDATE 3
I think I got it using event emitting:
function KeyGenerator() {
var nextKey;
var upperBound;
var emitter = new events.EventEmitter();
var requesting = true;
// Initialize the generator with the stored values
db.query("select * from key_generation where type='use'", function(err, results)
if (err) { throw err; }
if (results.length === 0) {
throw new Error("Could not get key generation parameters: Row is missing");
}
nextKey = results[0].next_key;
upperBound = results[0].upper_bound;
console.log("Setting requesting = false, emitting event");
requesting = false;
emitter.emit("KeysAvailable");
});
this.generateKey = function(table, done) {
console.log("generateKey, state is:\n nextKey: " + nextKey + "\n upperBound:" + upperBound + "\n requesting:" + requesting + " ");
if (nextKey > upperBound) {
if (!requesting) {
requesting = true;
console.log("Requesting new series");
require("../sync/key-series-request").requestSeries(function(err,newNextKey,newUpperBound) {
if (err) { return done(err); }
console.log("New series available:\n nextKey: " + newNextKey + "\n upperBound: " + newUpperBound);
nextKey = newNextKey;
upperBound = newUpperBound;
requesting = false;
emitter.emit("KeysAvailable");
done(null,nextKey++);
});
} else {
console.log("Key request is already underway, deferring");
var that = this;
emitter.once("KeysAvailable", function() { console.log("Executing deferred call"); that.generateKey(table,done); });
}
} else {
done(null,nextKey++);
}
}
}
I've peppered it with logging outputs, and it does do what I want it to.
As another answer mentions, you will potentially end up with results different from what you want. Taking things in order:
function KeyGenerator() {
// at first I was thinking you wanted these as 'class' properties
// and thus would want to proceed them with this. rather than as vars
// but I think you want them as 'private' members variables of the
// class instance. That's dandy, you'll just want to do things differently
// down below
var nextKey;
var upperBound;
this.generateKey = function (table, done) {
if (nextKey > upperBound) {
// truncated the require path below for readability.
// more importantly, renamed parameters to function
require("key-series-request").requestKeys(function(err,nKey,uBound) {
if (err) { return done(err); }
// note that thanks to the miracle of closures, you have access to
// the nextKey and upperBound variables from the enclosing scope
// but I needed to rename the parameters or else they would shadow/
// obscure the variables with the same name.
nextKey = nKey;
upperBound = uBound;
done(nextKey++);
});
} else {
done(nextKey++);
}
}
}
Regarding the .requestKeys function, you will need to somehow introduce some kind of synchronization. This isn't actually terrible in one way because with only one thread of execution, you don't need to sweat the challenge of setting your semaphore in a single operation, but it is challenging to deal with the multiple callers because you will want other callers to effectively (but not really) block waiting for the first call to requestKeys() which is going to the DB to return.
I need to think about this part a bit more. I had a basic solution in mind which involved setting a simple semaphore and queuing the callbacks, but when I was typing it up I realized I was actually introducing a more subtle potential synchronization bug when processing the queued callbacks.
UPDATE:
I was just finishing up one approach as you were writing about your EventEmitter approach, which seems reasonable. See this gist which illustrates the approach. I took. Just run it and you'll see the behavior. It has some console logging to see which calls are getting deferred for a new key block or which can be handled immediately. The primary moving part of the solution is (note that the keyManager provides the stubbed out implementation of your require('key-series-request'):
function KeyGenerator(km) {
this.nextKey = undefined;
this.upperBound = undefined;
this.imWorkingOnIt = false;
this.queuedCallbacks = [];
this.keyManager = km;
this.generateKey = function(table, done) {
if (this.imWorkingOnIt){
this.queuedCallbacks.push(done);
console.log('KG deferred call. Pending CBs: '+this.queuedCallbacks.length);
return;
};
var self=this;
if ((typeof(this.nextKey) ==='undefined') || (this.nextKey > this.upperBound) ){
// set a semaphore & add the callback to the queued callback list
this.imWorkingOnIt = true;
this.queuedCallbacks.push(done);
this.keyManager.requestKeys(function(err,nKey,uBound) {
if (err) { return done(err); }
self.nextKey = nKey;
self.upperBound = uBound;
var theCallbackList = self.queuedCallbacks;
self.queuedCallbacks = [];
self.imWorkingOnIt = false;
theCallbackList.forEach(function(f){
// rather than making the final callback directly,
// call KeyGenerator.generateKey() with the original
// callback
setImmediate(function(){self.generateKey(table,f);});
});
});
} else {
console.log('KG immediate call',self.nextKey);
var z= self.nextKey++;
setImmediate(function(){done(z);});
}
}
};
If your Node.js code to calculate the next key didn't need to execute an async operation then you wouldn't run into synchronization issues because there is only one JavaScript thread executing code. Access to the nextKey/upperBound variables will be done in sequence by only one thread (i.e. request 1 will access first, then request 2, then request 3 et cetera.) In the Java-world you will always need synchronization because multiple threads will be executing even if you didn't make a DB call.
However, in your Node.js code since you are making an async call to get the nextKey you could get strange results. There is still only one JavaScript thread executing your code, but it would be possible for request 1 to make the call to the DB, then Node.js might accept request 2 (while request 1 is getting data from the DB) and this second request will also make a request to the DB to get keys. Let's say that request 2 gets data from the DB quicker than request 1 and update nextKey/upperBound variables with values 100/150. Once request 1 gets its data (say values 50/100) then it will update nextKey/upperBound. This scenario wouldn't result in duplicate keys, but you might see gaps in your keys (for example, not all keys 100 to 150 will be used because request 1 eventually reset the values to 50/100)
This makes me think that you will need a way to sync access, but I am not exactly sure what will be the best way to achieve this.
geturls(data,function(urls){
var data = {
"data": [
{ "userProfile": userP },
{ "urls": urls }
]
};
res.send(data);
});
function getUrls(data,done){
links = new Array();
for (var i=0; i<data.length; i++){
user = data[i]
Url.find({where:{data.id}}).success(function(url){
links.push({
"url": ur.text,
"date": data.syncedTime
});
if (urls.length == data.length){
done(links);
}
});
}
}
My problem with my code is this:
I'm returning the response through a callback once data collected in my array equals the length of the parent array. This is obviously a very dangerous and not so elegant solution. As, suppose I get a .failure from Url database, then my urls.length won't be equal with data.length. So, I'm a bit confused how to go about this.
Any help?
It will be easy for you, if you use async.js.
I used mapSeries here. It takes 3 parameters.
collection/array
iterator, which will be called for each item in the passed collection/array with 2 arguments. 1. item in collection, 2. callback. After completing the job in iterator, You should call the callback in node style(err first, results follows).
Final callback, which will be called after all the items in the collection mapped.
function getUrls(data,done){
var async = require('async');
async.mapSeries(data, function(user, cb) {//If you want it to be async `async.map`
Url.find({where:{user.id}}).success(function(url){
cb(null, {
"url": url.text,
"date": user.syncedTime
});
});
}, function(err, results) {
//results is an array. Its the same as `links` in your old code.
done(results);
});
}
geturls(data,function(urls){
var data = {
"data": [
{ "userProfile": userP },
{ "urls": urls }
]
};
res.send(data);
});
Use recursion:
function getUrls(data,done) {
var links = new Array();
function doGetUrl(i) {
var user = data[i];
Url.find({where:{data.id}}).
success(function(url){
links.push({
"url": ur.text,
"date": data.syncedTime
});
if (links.length == data.length){
done(links);
} else {
doGetUrl(i + 1); // get next url
}
}).
failure(function(err) {
doGetUrl(i); // on error, try to get current url again
// other error handling code
});
}
doGetUrl(0);
}
I would probably make use of the complete callback, in jQuery terms. Have a counter that records how many records have been processed and update this in complete, as this executes on success or failure. Then when that counter is >= the length of the data array you can exit.
As an aside, I would always do a >= rather than an == for the comparison you are doing there, that way, if for any crazy reason, the count is upped more than it should you still exit.
If alll you want to do is avoif the problem of checking links.length to determine when you are done then I think its just a matter of adding a separate counter that gets incremented even if the urk database fails. If you do that you can continue using your current stype where the async requests are run in parallel.
var nreq = 0;
for (var i=0; i<data.length; i++){
doTheAsyncOperation(function(){
//Run this part in both the success and error cases
nreq = nreq + 1;
if(nreq >= data.length){ done(links) }
})
}
On the other hand, if you want to run one query after the other you will need to rewrite the for to use recursion. This time, you don't need to worry about keeping a separate counter since you know when the final request runs:
function loop(i){
if(i >= data.length){
done(links);
}else{
doTheAsyncOperation(function(){
loop(i+1);
})
}
}
loop(0);
Finally, its good to know how to code this sort of patterns yourself but in the long run I highly recommend using a control flow library to keep things cleaner.