Our firebase real-time database reaches 29k+, after that our app keeps on loading and loading and not showing any more data. i used this script to populate data return from firebase:
firebase.database().ref('/registrant/').orderByChild('id').on("value", function(snapshot) {
console.log(snapshot.val());
});
any help will be much appreciated thanks a lot.
Along with Franks answer, you could also do a staggered approach where you use a shallow query to get the top-level elements and load them asynchronously.
To my knowledge it is only available to the rest API, but it gives you the root nodes to iterate through directly
import request from 'request';
request({ url: "https://[YOUR-APP-ID].firebaseio.com/path/to/data.json?shallow=true" }, (error, response, body) => {
const shallowData = JSON.parse(body);
console.log(shallowData);
});
Source:
https://firebase.google.com/docs/database/rest/retrieve-data#shallow
https://firebase.google.com/docs/reference/rest/database#section-param-shallow
If you're loading 29K nodes, is it possible that you're running low on memory in where it is that you're running this code?
If so, consider loading less data by for example by add a limit... clause to your query:
firebase.database().ref('/registrant/').orderByChild('id').limitToFirst(100).on("value", function(snapshot) {
...
Alright got it to work this is derived from Frank and DIGI's answer kudos to them what i did is this:
let start = 20;
let end = 30;
$scope.execute = function() {
start = start + 10;
end = end + 10;
var settings = {
"url": "https://lcqrregistry-default-rtdb.firebaseio.com/registrant.json?orderBy=\"age\"&startAt=" + start + "&endAt=" + end + "&print=pretty",
"method": "GET",
"timeout": 0,
};
$.ajax(settings).done(function(response) {
console.log(start,end)
$timeout(function() {
$scope.$apply(function() {
let returnArr = [];
angular.forEach(response, function(value, key) {
returnArr.push(value);
});
$scope.registereds = returnArr;
console.log($scope.registereds)
});
})
});
}
var settings = {
"url": "https://my-url/registrant.json?orderBy=\"age\"&startAt=" + 18 + "&endAt=" + 30 + "&print=pretty",
"method": "GET",
"timeout": 0,
};
$.ajax(settings).done(function(response) {
$timeout(function() {
$scope.$apply(function() {
let returnArr = [];
angular.forEach(response, function(value, key) {
returnArr.push(value);
});
$scope.registereds = returnArr;
console.log($scope.registereds)
});
})
});
I filter by age and just add the next button to populate the filtered data
Related
I'm new to learning Node.js, so I'm still getting used to asynchronous programming and callbacks. I'm trying to insert a record into a MS SQL Server database and return the new row's ID to my view.
The mssql query is working correctly when printed to console.log. My problem is not knowing how to properly return the data.
Here is my mssql query - in addJob.js:
var config = require('../../db/config');
async function addJob(title) {
var sql = require('mssql');
const pool = new sql.ConnectionPool(config);
var conn = pool;
let sqlResult = '';
let jobID = '';
conn.connect().then(function () {
var req = new sql.Request(conn);
req.query(`INSERT INTO Jobs (Title, ActiveJD) VALUES ('${title}', 0) ; SELECT ##IDENTITY AS JobID`).then(function (result) {
jobID = result['recordset'][0]['JobID'];
conn.close();
//This prints the correct value
console.log('jobID: ' + jobID);
}).catch(function (err) {
console.log('Unable to add job: ' + err);
conn.close();
});
}).catch(function (err) {
console.log('Unable to connect to SQL: ' + err);
});
// This prints a blank
console.log('jobID second test: ' + jobID)
return jobID;
}
module.exports = addJob;
This is my front end where a modal box is taking in a string and passing it to the above query. I want it to then receive the query's returned value and redirect to another page.
// ADD NEW JOB
$("#navButton_new").on(ace.click_event, function() {
bootbox.prompt("New Job Title", function(result) {
if (result != null) {
var job = {};
job.title = result;
$.ajax({
type: 'POST',
data: JSON.stringify(job),
contentType: 'application/json',
url: 'jds/addJob',
success: function(data) {
// this just prints that data is an object. Is that because I'm returning a promise? How would I unpack that here?
console.log('in success:' + data);
// I want to use the returned value here for a page redirect
//window.location.href = "jds/edit/?jobID=" + data;
return false;
},
error: function(err){
console.log('Unable to add job: ' + err);
}
});
} else {
}
});
});
And finally here is the express router code calling the function:
const express = require('express');
//....
const app = express();
//....
app.post('/jds/addJob', function(req, res){
let dataJSON = JSON.stringify(req.body)
let parsedData = JSON.parse(dataJSON);
const addJob = require("../models/jds/addJob");
let statusResult = addJob(parsedData.title);
statusResult.then(result => {
res.send(req.body);
});
});
I've been reading up on promises and trying to figure out what needs to change here, but I'm having no luck. Can anyone provide any tips?
You need to actually return a value from your function for things to work. Due to having nested Promises you need a couple returns here. One of the core features of promises is if you return a Promise it participates in the calling Promise chain.
So change the following lines
jobID = result['recordset'][0]['JobID'];
to
return result['recordset'][0]['JobID']
and
req.query(`INSERT INTO Jobs (Title, ActiveJD) VALUES ('${title}', 0) ; SELECT ##IDENTITY AS JobID`).then(function (result) {
to
return req.query(`INSERT INTO Jobs (Title, ActiveJD) VALUES ('${title}', 0) ; SELECT ##IDENTITY AS JobID`).then(function (result) {
and
conn.connect().then(function () {
to
return conn.connect().then(function () {
You may need to move code around that is now after the return. You would also be well served moving conn.close() into a single .finally on the end of the connect chain.
I recommend writing a test that you can use to play around with things until you get it right.
const jobId = await addJob(...)
console.log(jobId)
Alternatively rewrite the code to use await instead of .then() calls.
I want to return data to function that calls a function with firebase code, because of the asynchronous and nested structure of firebase queries it is not able to return values, I intend to use this logic to set tool tips in chart.js
Here is my code:
window.onload = function() {
get_data();
}
function get_data() {
var data = get_val();
console.log("...." + data);
}
function get_val() {
var label = "10/2/2017";
var Name = localStorage.getItem("VName");
console.log("Name:::" + Name);
var at_val;
var dbref = new Firebase("https://apraisalstaging.firebaseio.com/EmployeeDB/EInfo/");
dbref.once("value").then(function(snapshot) {
snapshot.forEach(function(childsnapshot) {
var data = childsnapshot.val();
var Nameval = data.Name;
if (Nameval == Name) {
console.log("Success");
Ikey = childsnapshot.key();
console.log("Key:::" + Ikey);
var dxRef = new Firebase("https://apraisalstaging.firebaseio.com/EmployeeDB/EApraise/" + Ikey);
dxRef.once("value").then(function(snapshot) {
snapshot.forEach(function(childsnapshot) {
var data = childsnapshot.val();
console.log(data);
if (label == data.Dateval) {
console.log("-------> bingo");
at_val = data.Attitude;
console.log("got value:" + at_val);
}
});
}).then(function() {
console.log("In then:" + at_val);
return at_val;
});
}
})
})
}
Data is loaded from the Firebase Database asynchronously. You cannot return a value now that hasn't been loaded yet. And until the async and await keywords are commonplace, you cannot make JavaScript wait for the async value.
The closest you can get today is to return a promise from get_val(). Then your calling code will be:
get_val().then(function(data) {
console.log("...." + data);
});
To do this you'll have to implement get_val() as:
function get_val() {
var label = "10/2/2017";
var Name = localStorage.getItem("VName") || "delta";
console.log("Name:::" + Name);
var at_val;
var dbref = firebase.database().ref("EmployeeDB/EInfo/").orderByChild("Name").equalTo(Name);
return dbref.once("value").then(function(snapshot) {
var promises = [];
snapshot.forEach(function(childsnapshot) {
var data = childsnapshot.val();
var Nameval = data.Name;
Ikey = childsnapshot.key;
var dxRef = firebase.database().ref("EmployeeDB/EApraise/" + Ikey).orderByChild("Dateval").equalTo(label);
promises.push(
dxRef.once("value").then(function(snapshot) {
snapshot.forEach(function(childsnapshot) {
var data = childsnapshot.val();
at_val = data.Attitude;
});
}).then(function() {
console.log("In then:" + at_val);
return at_val;
})
);
})
return Promise.all(promises);
})
}
I made a few changes to get_val():
it uses the new 3.x versions of the Firebase SDK. The solution could also work with 2.x, I just didn't feel like setting up a jsbin for that.
it populates a list of promises, one for each of the EApraise records
it returns a promise that resolves when all the EApraise records are loaded.
it uses Firebase queries to find the correct record. This removes the need for checking the values in the then() callbacks. But more importantly it ensures that only the necessary data is downloaded.
To make that last point true, you need to add a few index definitions to the rules in your Firebase Database console:
{
"rules": {
"EmployeeDB": {
"EInfo": {
".indexOn": "Name"
}
"EApraise": {
"$eid": {
".indexOn": "Dateval"
}
}
}
}
}
Here a jsbin with the working code: http://jsbin.com/jawamilawo/edit?js,console
i have an app the automatically insert data to mongodb after calling an api in jive, my problem is the response is paginated.
var getAPICall=function(apiLink){
console.log(apiLink);
var response = HTTP.get(apiLink, {
headers: {
Authorization: getAPIToken()
}
});
return response;
}
var getAPIToken = function(){
var token = HTTP.post("https://api.jivesoftware.com/analytics/v1/auth/login?clientId=fakeclientid.i&clientSecret=fakesecret.s");
//console.log(token);
return token.content;
}
the response looks like this.
{"paging":{"next":"https://cloudalytics-api-phx.prod.jivehosted.com:443/analytics/v2/export/activity/lastday?startIndex=100&count=100&show-all=true","itemsPerPage":100,"totalCount":164406,"currentPage":1,"totalPages":1645}, list: [{....}]}
which has next, currentPage and totalPage field
im planning to do a loop for the next page function and stop when totalPage is equal to currentPage
im doing this
var res = getAPICall(baseline);
_.each(res.data.list, function(item) {
console.log(item.uuid);
Reports.upsert({"uuid":item.uuid},{$set :item});
});
var totalPage = res.data.paging.totalPage;
for(var current = res.data.paging.currentPage ; current <= totalPage ;) {
var res = getAPICall(res.data.paging.next );
_.each(res.data.list, function(item) {
console.log(item.uuid);
Reports.upsert({"uuid":item.uuid},{$set :item});
});
}
but this statement only runs until the next page with infinite loop. any ways i can do this better with functional approach?
im new in javascript please hammer me.
thanks!
this should be pretty simple with recursion.
var myFunction = function(baseline){
var res = getAPICall(baseline);
_.each(res.data.list, function(item) {
console.log(item.uuid);
Reports.upsert({"uuid":item.uuid},{$set :item});
});
if (res.data.paging.currentPage != res.data.paging.totalPage) {
myFunction(res.data.paging.next)
}
}
myFunction(baseline);
How can I refractor my code to get rid of this error from JSLinter?
I tried moving the entire function out to a var but the code wasn't able to run after that.
for (i = 0; i < timeDifference; i++) {
timestamp ++;
console.log(timestamp);
energyDatum.find({timestamp: timestamp}).toArray(function(err, result) {
var data = {};
result.forEach(function(element) {
data[element.deviceId] = element;
});
var roomRawData = [];
mappings.forEach(function(room) {
var hash = {};
hash.floor = room.floor;
hash.name = room.name;
hash.room_type = room.room_type;
hash.energy_ac = sumApplianceEnergy('energy_ac', room, data);
hash.energy_light = sumApplianceEnergy('energy_light', room, data);
hash.energy_socket_1 = sumApplianceEnergy('energy_socket_1', room, data);
hash.energy_socket_2 = sumApplianceEnergy('energy_socket_2', room, data);
hash.energy_socket_3 = sumApplianceEnergy('energy_socket_3', room, data);
hash.energy_total = hash.energy_ac + hash.energy_light + hash.energy_socket_1 + hash.energy_socket_2 + hash.energy_socket_3;
hash.timestamp = timestamp;
roomRawData.push(hash);
});
roomRaw.insert(roomRawData, {w:1}, function(err, result) { console.log('done'); });
});
lastTimestamp.update({_id: timestampId}, {timestamp: timestamp});
}
JSLinter shows this message because your code has potential errors.
Take a look at this line:
energyDatum.find({timestamp: timestamp}).toArray(...);
This method is async, right? It means that the callback of toArray method
is called after the for loop finishes its iterations, and therefore timestamp
variable (when you use it inside this callback) doesn't have a value of current iteration,
but instead it has value incremented for timeDifference times.
To solve this problem you could move this callback to another function:
var getIterationFunc = function(timestamp) {
return function(err, result) {
var data = {};
// rest of function ...
}
}
and then use it:
energyDatum.find({timestamp: timestamp}).toArray(getIterationFunc(timestamp));
I believe this error should be fixed now. Hope this helps.
P.S. sorry for my English
What I have is simple CRUD operation. Items are listed on page, when user clicks button add, modal pops up, user enters data, and data is saved and should automatically (without refresh)be added to the list on page.
Service:
getAllIncluding: function(controllerAction, including) {
var query = breeze.EntityQuery.from(controllerAction).expand(including);
return manager.executeQuery(query).fail(getFailed);
},
addExerciseAndCategories: function(data, initialValues) {
var addedExercise = manager.createEntity("Exercise", initialValues);
_.forEach(data, function(item) {
manager.createEntity("ExerciseAndCategory", { ExerciseId: addedExercise._backingStore.ExerciseId, CategoryId: item.CategoryId });
});
saveChanges().fail(addFailed);
function addFailed() {
removeItem(items, item);
}
},
Controller:
$scope.getAllExercisesAndCategories = function() {
adminCrudService.getAllIncluding("ExercisesAndCategories", "Exercise,ExerciseCategory")
.then(querySucceeded)
.fail(queryFailed);
};
function querySucceeded(data) {
$scope.queryItems = adminCrudService.querySucceeded(data);
var exerciseIds = _($scope.queryItems).pluck('ExerciseId').uniq().valueOf();
$scope.exerciseAndCategories = [];
var createItem = function (id, exercise) {
return {
ExerciseId: id,
Exercise : exercise,
ExerciseCategories: []
};
};
// cycle through ids
_.forEach(exerciseIds, function (id) {
// get all the queryItems that match
var temp = _.where($scope.queryItems, {
'ExerciseId': id
});
// go to the next if nothing was found.
if (!temp.length) return;
// create a new (clean) item
var newItem = createItem(temp[0].ExerciseId, temp[0].Exercise);
// loop through the queryItems that matched
_.forEach(temp, function (i) {
// if the category has not been added , add it.
if (_.indexOf(newItem.ExerciseCategories, i.ExerciseCategory) < 0) {
newItem.ExerciseCategories.push(i.ExerciseCategory);
}
});
// Add the item to the collection
$scope.items.push(newItem);
});
$scope.$apply();
}
Here is how I add new data from controller:
adminCrudService.addExerciseAndCategories($scope.selectedCategories, { Name: $scope.NewName, Description: $scope.NewDesc });
So my question is, why list isn't updated in real time (when I hit save I must refresh page).
EDIT
Here is my querySuceeded
querySucceeded: function (data) {
items = [];
data.results.forEach(function(item) {
items.push(item);
});
return items;
}
EDIT 2
I believe I've narrowed my problem !
So PW Kad lost two hours with me trying to help me to fix this thing (ad I thank him very very very much for that), but unfortunately with no success. We mostly tried to fix my service, so when I returned to my PC, I've again tried to fix it. I believe my service is fine. (I've made some changes as Kad suggested in his answer).
I believe problem is in controller, I've logged $scope.items, and when I add new item they don't change, after that I've logged $scope.queryItems, and I've noticed that they change after adding new item (without refresh ofc.). So probably problem will be solved by somehow $watching $scope.queryItems after loading initial data, but at the moment I'm not quite sure how to do this.
Alright, I am going to post an answer that should guide you on how to tackle your issue. The issue does not appear to be with Breeze, nor with Angular, but the manner in which you have married the two up. I say this because it is important to understand what you are doing in order to understand the debug process.
Creating an entity adds it to the cache with an entityState of isAdded - that is a true statement, don't think otherwise.
Now for your code...
You don't have to chain your query execution with a promise, but in your case you are returning the data to your controller, and then passing it right back into some function in your service, which wasn't listed in your question. I added a function to replicate what yours probably looks like.
getAllIncluding: function(controllerAction, including) {
var query = breeze.EntityQuery.from(controllerAction).expand(including);
return manager.executeQuery(query).then(querySucceeded).fail(getFailed);
function querySucceeded(data) {
return data.results;
}
},
Now in your controller simply handle the results -
$scope.getAllExercisesAndCategories = function() {
adminCrudService.getAllIncluding("ExercisesAndCategories", "Exercise,ExerciseCategory")
.then(querySucceeded)
.fail(queryFailed);
};
function querySucceeded(data) {
// Set your object directly to the data.results, because that is what we are returning from the service
$scope.queryItems = data;
$scope.exerciseAndCategories = [];
Last, let's add the properties we create the entity and see if that gives Angular a chance to bind up properly -
_.forEach(data, function(item) {
var e = manager.createEntity("ExerciseAndCategory");
e.Exercise = addedExercise; e.Category: item.Category;
});
So I've managed to solve my problem ! Not sure if this is right solution but it works now.
I've moved everything to my service, which now looks like this:
function addCategoriesToExercise(tempdata) {
var dataToReturn = [];
var exerciseIds = _(tempdata).pluck('ExerciseId').uniq().valueOf();
var createItem = function (id, exercise) {
return {
ExerciseId: id,
Exercise: exercise,
ExerciseCategories: []
};
};
// cycle through ids
_.forEach(exerciseIds, function (id) {
// get all the queryItems that match
var temp = _.where(tempdata, {
'ExerciseId': id
});
// go to the next if nothing was found.
if (!temp.length) return;
// create a new (clean) item
var newItem = createItem(temp[0].ExerciseId, temp[0].Exercise);
// loop through the queryItems that matched
_.forEach(temp, function (i) {
// if the category has not been added , add it.
if (_.indexOf(newItem.ExerciseCategories, i.ExerciseCategory) < 0) {
newItem.ExerciseCategories.push(i.ExerciseCategory);
}
});
// Add the item to the collection
dataToReturn.push(newItem);
});
return dataToReturn;
}
addExerciseAndCategories: function (data, initialValues) {
newItems = [];
var addedExercise = manager.createEntity("Exercise", initialValues);
_.forEach(data, function (item) {
var entity = manager.createEntity("ExerciseAndCategory", { ExerciseId: addedExercise._backingStore.ExerciseId, CategoryId: item.CategoryId });
items.push(entity);
newItems.push(entity);
});
saveChanges().fail(addFailed);
var itemsToAdd = addCategoriesToExercise(newItems);
_.forEach(itemsToAdd, function (item) {
exerciseAndCategories.push(item);
});
function addFailed() {
removeItem(items, item);
}
}
getAllExercisesAndCategories: function () {
var query = breeze.EntityQuery.from("ExercisesAndCategories").expand("Exercise,ExerciseCategory");
return manager.executeQuery(query).then(getSuceeded).fail(getFailed);
},
function getSuceeded(data) {
items = [];
data.results.forEach(function (item) {
items.push(item);
});
exerciseAndCategories = addCategoriesToExercise(items);
return exerciseAndCategories;
}
And in controller I have only this:
$scope.getAllExercisesAndCategories = function () {
adminExerciseService.getAllExercisesAndCategories()
.then(querySucceeded)
.fail(queryFailed);
};
function querySucceeded(data) {
$scope.items = data;
$scope.$apply();
}