Below is the code that I'm using to implement pagination for data retrieved from the firebase realtime database. Basically, I'm trying to get the first n content according to page number, and then getting the last n content from the data retrieved in the first query.
function getUserSnapshotOrVerifyUserId(username, idToken, cb) {
if (username == null || username.length == 0 || idToken == null || idToken.length == 0)
return cb({
status: "error",
errorMessage: "Missing params."
}, null);
admin.auth().verifyIdToken(idToken).then(decodedToken => {
let uid = decodedToken.uid;
admin.database().ref().child("users").orderByChild("username").equalTo(username).once('value', snapshot => {
if (!snapshot.exists())
return cb({
status: "error",
message: "invalid-profile"
});
snapshot.forEach(child => {
const id = child.val().id;
if (id !== uid)
return cb({
status: "error",
message: "Invalid ID"
});
admin.database().ref("users/" + id).once("value", snapshot => {
if (!snapshot.exists())
return cb({
status: "error",
errorMessage: "user not found."
});
return cb(null, id, snapshot);
});
});
});
}).catch(err => cb({
status: "error",
message: err
}));
}
exports.getUserContentTestPagination = functions.https.onRequest((req, res) => {
corsHandler(req, res, async () => {
try {
const username = req.body.username || req.query.username;
const idToken = req.body.idToken;
const limit = 2;
const page = req.body.page || 1;
const limitToFirst = page * limit;
const limitToLast = limit;
getUserSnapshotOrVerifyUserId(username, idToken, async (err, id) => {
if(err) return res.json(err);
const uploadsRef = admin.database().ref('uploads').orderByChild('createdBy').equalTo(id)
const firstnquery = uploadsRef.limitToFirst(limitToFirst);
const lastnquery = firstnquery.limitToLast(limitToLast);
lastnquery.once("value", snapshot => {
res.json({
snapshot
})
})
})
} catch (err) {
res.json({
status: "error",
message: err
})
}
});
});
This is returning a function timeout, however, when I try to get the first n data using firstnquery, it is returning the first n data as expected. So the problem is with lastnquery. Any help would be appreciated.
UPDATE 1:
exports.getUserContentTestPagination = functions.https.onRequest((req, res) => {
corsHandler(req, res, async () => {
try {
const username = req.body.username || req.query.username;
const idToken = req.body.idToken;
const limit = 2;
const page = req.body.page || 1;
let lastKnownKeyValue = null;
getUserSnapshotOrVerifyUserId(username, idToken, async (err, id) => {
if(err) return res.json(err);
const uploadsRef = admin.database().ref('uploads');
const pageQuery = uploadsRef.orderByChild('createdBy').equalTo(id).limitToFirst(limit);
pageQuery.once('value', snapshot => {
snapshot.forEach(childSnapshot => {
lastKnownKeyValue = childSnapshot.key;
});
if(page === 1){
res.json({
childSnapshot
})
} else {
const nextQuery = uploadsRef.orderByChild('createdBy').equalTo(id).startAt(lastKnownKeyValue).limitToFirst(limit);
nextQuery.once("value", nextSnapshot => {
nextSnapshot.forEach(nextChildSnapshot => {
res.json({
nextChildSnapshot
})
})
})
}
});
})
} catch (err) {
res.json({
status: "error",
message: err
})
}
});
});
It is incredibly uncommon to use both limitToFirst and limitToLast in a query. In fact, I'm surprised that this doesn't raise an error:
const firstnquery = uploadsRef.limitToFirst(limitToFirst);
const lastnquery = firstnquery.limitToLast(limitToLast);
Firebase queries are based on cursors. This means that to get the data for the next page, you must know the last item on the previous page. This is different from most databases, which work based on offsets. Firebase doesn't support offset based queries, so you'll need to know the value of createdBy and the key of the last item of the previous page.
With that, you can get the next page of items with:
admin.database().ref('uploads')
.orderByChild('createdBy')
.startAt(idOfLastItemOfPreviousPage, keyOfLastItemOfPreviousPage)
.limitToFist(pageSize + 1)
I highly recommend checking out some other questions on implementing pagination on the realtime database, as there are some good examples and explanations in there too.
Related
I don't know If I'm checking for the value of the boolean correctly
what this code does: the user creates a note for himself, his ID is on the note and it needs to belong to a category name that has to be in the category schema ( where my error happens )
exports.postAddNote = (req, res) => {
const errors = validationResult(req);
if (!errors.isEmpty()) {
const error = new Error("validation failed, entered data is incorrect");
throw error;
}
const content = req.body.content;
const tags = req.body.tags;
const categoryName = req.body.categoryName;
let creator;
const note = new Note({
content: content,
categoryName: categoryName, // work
tags: tags,
creator: req.userId,
});
Category.find()
.select("-_id")
.select("-__v")
.select("-notesId")
.then((categories) => {
console.log(categories); //stripping everything but names off categories
const CategoryExists = categories.some(
(category) => category.name === categoryName
);
console.log(CategoryExists); // ~~~~~~~~~~ this logs correctly
if (CategoryExists === -0) { // ~~~~~~~~~~ what i want: if the value is false
return res.json({ Error: "The category you entered does not exist" });
}
note // ~~~~~~~~~~ the code stops here :/ it doesn't save the note
.save()
.then((note) => {
console.log("saved note");
User.findById(req.userId);
})
.then((user) => {
creator = user;
user.notes.push(note);
return user.save();
})
.then((result) => {
res.status(201).json({
info: {
dateCreated: new Date().toISOString(),
status: "Note Created Successfully",
creator: { _id: creator._id, email: creator.email },
},
});
})
.catch((err) => {
if (!err.statusCode) {
err.statusCode = 500;
}
});
})
.catch((err) => {
console.log(err);
next();
});
};
if (CategoryExists === -0)
should be
if (CategoryExists === false)
or just
if (!CategoryExists)
i believe. did you try that? not sure why you are using -0. the return value for some() is either going to be true or false.
try this:
if (!CategoryExists) {
return res.json({ Error: 'The category you entered does not exist' });
}
I'm using passport strategies for different socialMedia logins and getting the following two errors
InternalOAuthError: Failed to fetch user profile
Cannot set headers after they are sent to the client
I have doubt there somewhere I have returned a callback or response so getting 2nd error but for 1st don't know reasons scope seems to be correct!
strategy code
passport.use(new GoogleStrategy({
clientID: GOOGLE_CLIENT_ID,
clientSecret: GOOGLE_SECRET_KEY,
callbackURL: GOOGLE_CALLBACK_URL
}, async (acessToken, refreshToken, profile, done) => {
await User.findOne({ email: profile._json.email }, async (err, user) => {
if (err) {
console.log("passport.config --> err", err);
done(err, null);
} else if (user) {
if (user.socialType !== "GOOGLE" || user.socialType === null)
done(`LOGIN_CREDENTIALS_WITH_${(user.socialType || "PASSWORD").toUpperCase()}`, false);
else {
done(null, user);
}
} else {
// console.log(profile);
const user = {
email: profile._json.email,
socialId: profile.id,
socialType: "GOOGLE",
firstName: profile.name.givenName,
lastName: profile.name.familyName,
isActive: profile._json.email_verified,
isVerified: profile._json.email_verified,
socialImageUrl: profile._json.picture,
userType: "CUSTOMER"
};
const newUser = new User({ ...user });
const newUserData = await newUser.save();
done(null, newUserData);
}
});
}));
route code:
router.get('/auth/:socialType', customerCtrl.socialTypeLogin);
router.get('/auth/:socialType/callback', customerCtrl.socialTypeLoginCallback);
controller code:
const socialTypeLogin = async (req, res) => {
await customerService.socialTypeLogin(req, res);
};
const socialTypeLoginCallback = async (req,res) => {
await customerService.socialTypeLoginCallback(req,res);
};
service code:
const socialTypeLogin = async (req, res) => {
try {
const socialType = (req.params.socialType || '').toLowerCase();
const GOOGLE_SCOPE = ['email', 'profile'];
const FACEBOOK_SCOPE = ['email'];
let scope = [];
if (socialType === 'google') {
scope = GOOGLE_SCOPE;
} else if (socialType === 'facebook') {
scope = FACEBOOK_SCOPE;
}
let oauthOptions = { scope: scope};
const { returnUrl } = req.query;
if(returnUrl && returnUrl.trim().length !== 0) {
oauthOptions['state'] =JSON.stringify({ returnUrl: returnUrl });
}
passport.authenticate(socialType, oauthOptions)(req, res);
}
catch (error) {
}
}
/**
* #param {string} socialType
*/
const socialTypeLoginCallback = async (req, res) => {
const socialType = (req.params.socialType || '').toLowerCase();
// return new Promise((resolve, reject) => {
try {
passport.authenticate(socialType, async (err, user) => {
let webappRedirectURL = WEBAPP_LOGIN_URL;
try {
const state = req.query.state;
if(state) {
const stateObj = JSON.parse(state);
webappRedirectURL = stateObj.returnUrl;
}
} catch (err1) {
console.log("customer.service --> parsing error",err1);
}
if (err || !user) {
console.log("customer.service --> !user",err);
res.render('oauth-redirect', {
webappRedirectURL: webappRedirectURL,
success: false,
error: err,
timerCounter: 5,
accessToken: undefined
});
}
else {
console.log("customer.service --> Generating Token",user.generateJWT());
res.render('oauth-redirect', {
webappRedirectURL: webappRedirectURL,
success: true,
timerCounter: 5,
accessToken: user.generateJWT(),
error: undefined
});
}
})(req, res);
}
catch (error) {
console.log("customerService.js ==> socialTypeLoginCallback -->",error);
}
};
Thanks for help in advance!
I have doubt there somewhere I have returned a callback or response so getting 2nd error but for 1st don't know reasons scope seems to be correct!
In socialTypeLogin
add line
oauthOptions['session'] = false;
app.get("/api/users/:_id/logs", (req, res) => {
const id = req.params._id;
const { from, to, limit } = req.query;
** Here I tried to search for the matched user and it works successfully: **
User.findById({ _id: id }, (err, user) => {
if (!user || err) {
res.send("Unknown User Id !!");
} else {
**Then I tried to filter the log array with date **
// const username = user.username;
let responObject = {};
if (from) {
responObject["$gte"] = new Date(from).toDateString();
}
if (to) {
responObject["$lte"] = new Date(to).toDateString();
}
let filter = {
_id: id,
};
if (from || to) {
filter.date = responObject;
}
let nonNullLimit = limit ?? 500;
**try to build the array log and return it to the user but it always be empty and never return the exercises for the user **
Exercise.find(filter)
.limit(+nonNullLimit)
.exec((err, data) => {
if (err || !data) {
res.json([]);
} else {
const count = data.length;
const rowLog = data;
const { username, _id } = user;
const log = rowLog.map((item) => ({
description: item.description,
duration: item.duration,
date: new Date(item.date).toDateString(),
}));
console.log(log)
if (from && to) {
res.json({
username,
from: new Date(from).toDateString(),
to: new Date(to).toDateString(),
count,
_id,
log,
});
} else {
res.json({
username,
count,
_id,
log,
});
}
}
});
}
});
});
this is the result when I try to log all the exercises for the user
{"username":"ahmed","count":0,"_id":"62a9aab2743ddfc9df5165f2","log":[]}
I'm following a restApi course with Node.js, It's a blog API. my problem is when deleting a post for the unAuthorized user it first gives me a 500 error
"error": "Cannot read property 'toString' of undefined"
. but when doing it again it gives me
Post not found with id of.
Of course, it supposed to give me
not authorized to delete this post.
Update the post is also the same, I even tried to copy/paste the code from the course but the same problem.
postController
exports.deletePost = asyncHandler(async (req, res, next) => {
const post = await Post.findByIdAndDelete(req.params.id);
if (!post) {
return next(
new ErrorResponse(`Post not found with id of ${req.params.id}`, 404)
);
}
// Make sure user is post owner
if (post.user.toString() !== req.user.id) {
return next(
new ErrorResponse(
`User ${req.params.id} is not authorized to delete this post`,
401
)
);
}
post.remove();
res.status(200).json({ success: true, data: post});
});
updatePost
exports.updatePost = asyncHandler(async (req, res, next) => {
let post = await Post.findById(req.params.id);
if (!post) {
return next(
new ErrorResponse(`Post not found with id of ${req.params.id}`, 404)
);
}
// Make sure user is post owner
if (post.user.toString() !== req.user.id) {
return next(
new ErrorResponse(
`User ${req.params.id} is not authorized to update this post`,
401
)
);
}
post = await Post.findOneAndUpdate(req.params.id, req.body, {
new: true,
runValidators: true
});
res.status(200).json({ success: true, data: post });
});
You could introduce a type check on the post.user object to ensure that the user exists within the post.
if (typeof post.user == "undefined" || post.user.toString() !== req.user.id)
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/typeof
I have tried the code
if (typeof post.user == "undefined" || post.user.toString() !== req.user.id)
But after using this cause to get me an error 'User not Authorized' in my error handling.
In my case, I have to convert the req.user.id to an integer
if (post.user !== req.user.id.parseInt)
const express = require("express");
const router = express.Router();
const fetchuser = require("../middleware/Fetchuser");
const Notes = require("../models/Notes.js");
const { body, validationResult } = require("express-validator");
router.get("/fetchnotes", fetchuser, async (req, res) => {
try {
const notes = await Notes.find({ user: req.user });
res.json(notes);
} catch (error) {
console.log(error.message);
res.status(500).send("error occured");
}
});
router.post(
"/addnotes",
[
body("title").isLength({ min: 5 }),
body("description").isLength({ min: 3 }),
],
fetchuser,
async (req, res) => {
try {
const { title, description, tag } = req.body;
const errors = validationResult(req);
if (!errors.isEmpty()) {
return res.status(400).json({ errors: errors.array() });
}
const note = new Notes({
title,
description,
tag,
user: req.user.id,
});
const savedNotes = await note.save();
res.json(savedNotes);
} catch (error) {
console.log(error.message);
res.status(500).send("error occured");
}
}
);
router.put("/updatenotes/:id", fetchuser, async (req, res) => {
const { title, description, tag } = req.body;
const newNote = {};
if (title) {
newNote.title = title;
}
if (description) {
newNote.description = description;
}
if (tag) {
newNote.tag = tag;
}
let note = await Notes.findById(req.params.id);
if (!note) {
return res.status(400).send("error occured");
}
if (note.user !== req.user.id.parseInt) {
return res.status(401).json("user not allowed");
}
note = await Notes.findByIdAndUpdate(
req.params.id,
{ $set: newNote },
{ new: true }
);
res.json(note);
});
module.exports = router;
I have been trying to insert large data about(400-1000) json object array to mongodb using mongoose + expressjs When i changed data about(50) items insertMany works great without problem. But if data is more than 100 it giving me an error.
Departed.insertMany(results)
.then(dep => {
console.log(dep)
res.sendStatus(201)
})
.catch(err => {
console.log(err)
})
in morgan console i got following:
creation { active: true,
_id: 5b73e8af19722d1689d863b0,
name: 'TEST DATA 241',
map: '',
created_at: 2018-08-15T08:47:43.196Z,
updated_at: 2018-08-15T08:47:43.196Z,
__v: 0 }
insert read 453
(node:5769) [DEP0079] DeprecationWarning: Custom inspection function on Objects via .inspect() is deprecated
also on client side(chrome, dev tools network tab) status got
(failed)
net::ERR_EMPTY_RESPONSE
I have read mongo's insertMany() has limit about 1000 and i am using mongo 4.0 version. Even i chunked large json into several arrays and tried to insert it but still got same results. Actual snippets are
router.post('/xls', upload.single('file'), async (req, res, next) => {
try {
if (req.body && req.file) {
console.log('req', req.file)
const segments = req.file.originalname.split('.')
let exceltojson = segments[segments.length - 1] === 'xlsx' ? xlsx : xls
exceltojson(
{
input: req.file.path,
output: 'output.json'
},
async (err, result) => {
if (err) console.log(err)
const section = await Section.create({
name: req.body.section,
map: req.body.map
})
const results = await result.map(item => {
return {
branch: req.body.branch,
section: String(section._id),
...item
}
})
await console.log('creation', section)
console.log('insert read', results.length)
if (results.length >= 100) {
console.log('more than 100')
const data = _.chunk(results, 100)
data.forEach(async chunk => {
console.log('foreach')
Departed.insertMany(chunk)
.then(dep => {
console.log(dep)
res.sendStatus(201)
})
.catch(err => {
console.log(err)
})
})
}
}
)
}
} catch (error) {
next(error)
}
})
Your problem is not related to any insertMany limit. You have a race condition in your code where you don't wait for all chunks to be inserted, before sending the status back:
data.forEach(async chunk => {
console.log('foreach')
Departed.insertMany(chunk)
.then(dep => { // this will be called as soon as one of the inserts finish
console.log(dep)
res.sendStatus(201)
})
.catch(err => {
console.log(err)
})
})
Change this in something like (untested):
Promise.all(data.map(chunk => Departed.insertMany(chunk)))
.then(dep => { // this will be called when all inserts finish
console.log(dep)
res.sendStatus(201)
})
.catch(err => {
console.log(err)
})
})
Another alternative is to use the bulkWrite API which is is faster than sending multiple independent operations because with bulkWrite() there is only one round trip to MongoDB:
router.post('/xls', upload.single('file'), async (req, res, next) => {
try {
if (req.body && req.file) {
console.log('req', req.file)
const segments = req.file.originalname.split('.')
let exceltojson = segments[segments.length - 1] === 'xlsx' ? xlsx : xls
exceltojson(
{
input: req.file.path,
output: 'output.json'
},
async (err, result) => {
if (err) console.log(err)
const section = await Section.create({
name: req.body.section,
map: req.body.map
})
let chunk = [];
result.forEach(item => {
chunk.push({
insertOne: {
document: {
branch: req.body.branch,
section: String(section._id),
...item
}
}
});
if (chunk.length === 500) {
const blkResult = await Departed.bulkWrite(chunk);
console.log(blkResult)
res.sendStatus(201)
}
});
if (chunk.length > 0) {
const dep = await Departed.bulkWrite(chunk);
console.log(dep)
res.sendStatus(201)
}
}
)
}
} catch (error) {
next(error)
}
})