I'm creating a web scraper and I need to store my data in MongoDB. I scrape some data and store it in a variable call item. I also don't to add a data to the database if it already exist. so here is my resultAnalysis.mjs file that is supposed to do that. What should I change in this code, because it don't insert data and also throw error :
TypeError: Cannot destructure property 'lotNumber' of 'item' as it is undefined.
at file:///Users/AlainMolleyres/Desktop/scraper_V1/resultsAnalysis.mjs:48:11
at runMicrotasks (<anonymous>)
at processTicksAndRejections (internal/process/task_queues.js:93:5)
Here is my code
resultsAnalysis.mjs
mongoose
.connect(mongoURI, { useNewUrlParser: true })
.then(() => console.log("MongoDB connected"))
.catch((err) => console.error(err));
export const compareAndSaveResults = (item) => {
try {
const Schema = mongoose.Schema;
const lotSchema = new Schema({
lotNumber: {},
artistName: {},
artworkNameOriginal: {},
pictureUrl: {},
artworkDate: {},
signed: {},
titled: {},
technic: {},
literature: {},
provenance: {},
dimension: {},
lowEstimation: {},
highEstimation: {},
currency: {},
soldPrice: {},
lotUrl: {},
});
const lot = mongoose.model("lot", lotSchema);
lot
.find({}, function (err, lotList) {
return lotList;
})
.then((lotList) => {
if (lotList == "") {
console.log(`A new data was created:\n${JSON.stringify(item)}`);
const newLot = new lot(item);
return newLot.save().catch((err) => console.log(err));
}
const {
lotNumber,
artistName,
artworkNameOriginal,
pictureUrl,
artworkDate,
signed,
titled,
technic,
literature,
provenance,
lowEstimation,
highEstimation,
currency,
soldPrice,
dimensionInCm,
lotUrl,
} = item;
const dbId = lotList[0]._id;
const dbArtworkNameOriginal = lotList[0].artworkNameOriginal;
const dbLotUrl = newsList[0].lotUrl;
let catchDifference = false;
if (dbArtworkNameOriginal !== artworkNameOriginal) {
catchDifference = true;
} else {
dbLotUrl.forEach((elem, i) => {
if (elem !== lotUrl[i]) catchDifference = true;
});
}
if (catchDifference) {
console.log("A new evidence was found, updating database...");
mongoose.set("useFindAndModify", false);
return News.findOneAndUpdate({ _id: dbId }, item);
}
console.log("File is equal to page, no lot to report");
})
.then(() => {
mongoose.disconnect();
})
.catch((err) => console.log(err));
} catch (err) {
console.error(err);
}
};
``
Related
I am a student I am trying to create a booking api and I need to authenticate if the user that log in is an admin that is allowed to add courses and if not an admin would return access denied..
module.exports.addCourse = (reqBody) => {
let newCourse = new Course({
name : reqBody.name,
description : reqBody.description,
price : reqBody.price
});
return newCourse.save().then((course,error) => {
if (!req.user.isAdmin) {
return res.status(401).send({ message: "Access denied" });;
} else if (reqBody.isAdmin = true){
return true;
};
});
};
router.post("/",auth.verify,(req,res)=>{
const data ={
course: req.body,
isAdmin: auth.decode(req.headers.authorization).isAdmin
}
courseController.addCourse(data).then(resultFromController=>res.send(resultFromController));
});
module.exports = router;
I wanted to add the course in my database if the user is an admin and does not allow if not an admin, I always get the error:
(node:17308) UnhandledPromiseRejectionWarning: ValidationError: Course validation failed: name: Course is required, description: Description is required, price: Price is required
even though I have given the name description and price and also added the token in the postman.
why are you using callback and promises? your syntax is not right...
// Alter 1
module.exports.addCourse = (reqBody) => {
const newCourse = new Course({
name : reqBody.name,
description : reqBody.description,
price : reqBody.price
});
return newCourse.save()
.then((course) => {
// ...code
}).catch((err) => {
// code error
})
}
router.post("/",auth.verify,(req,res)=>{
const data ={
course: req.body,
isAdmin: auth.decode(req.headers.authorization).isAdmin
}
courseController.addCourse(data).then(resultFromController=>res.send(resultFromController));
});
module.exports = router;
// alter 2
module.exports.addCourse = async (reqBody) => {
const newCourse = new Course({
name : reqBody.name,
description : reqBody.description,
price : reqBody.price
});
return await newCourse.save()
}
router.post("/",auth.verify, async (req,res)=>{
const data ={
course: req.body,
isAdmin: auth.decode(req.headers.authorization).isAdmin
}
try {
const response = await courseController.addCourse(data)
// response
} catch (err) {
// error response
}
});
module.exports = router;
// alter 3
module.exports.addCourse = (reqBody) => {
const newCourse = new Course({
name : reqBody.name,
description : reqBody.description,
price : reqBody.price
});
return new Promise((resolve, reject) => {
newCourse.save()
.then((course) => {
resolve(course)
}).catch((err) => {
reject(err)
})
})
}
router.post("/",auth.verify,(req,res)=>{
const data ={
course: req.body,
isAdmin: auth.decode(req.headers.authorization).isAdmin
}
return new Promise((resolve, reject) => {
courseController.addCourse(data)
.then((resultFromController) => {
resolve(res.send(resultFromController))
}).catch((error) => {
reject(res.send('error'))
})
})
});
I have this code in App.js
const getPlayers = async()=>{
const players = await API.getPlayers();
setPlayers(players)
}
getPlayers()
This code in my API.js file
const getPlayers = async () => {
return getJson(
fetch(SERVER_URL + 'users', { credentials: 'include'})
).then( json => {
return json.map((user) => {
return {
id: user.id,
name: user.name,
rank: user.rank
}
})
})
}
This code in my server.js file
app.get('/api/players',
(req, res) => {
riddleDao.getPlayers()
.then(async players => {
res.json(players)
})
.catch((err) => res.status(500).json(err));
});
and finally, this in my DataAccessObject.js file
exports.getPlayers = () => {
return new Promise((resolve, reject) => {
const sql = 'SELECT * FROM users';
db.all(sql, [], (err, rows) => {
if (err) { reject(err); return; }
else {
const players = rows.map(row => {
return {
id: row.id,
name: row.name,
rank: row.rank
}
})
resolve(players);
}
});
});
};
but i am getting this error:
I am expecting to get an array of object in my App.js when i call the getPlayer() function and the objects in the array should have id, name and rank of the players in my db table
I think you've got "users" in your fetch URL when it should be "players".
fetch(SERVER_URL + 'users', { credentials: 'include'})
should be
fetch(SERVER_URL + 'players', { credentials: 'include'})
your api endpoint differs from the url you are sending requests
app.get('/api/players',
you are listening to "players" but
fetch(SERVER_URL + 'users', { credentials: 'include'})
you are fetching "users"
I'm trying to build the following use case of DataLoader together with Mongoose:
export const PurchaseOrderType = new GraphQLObjectType({
name: "PurchaseOrder",
description: "PurchaseOrder",
interfaces: () => [NodeInterface],
isTypeOf: value => value instanceof PurchaseOrderModel,
fields: () => ({
id: {
type: new GraphQLNonNull(GraphQLID),
resolve: obj => dbIdToNodeId(obj._id, "PurchaseOrder")
},
name: {
type: new GraphQLNonNull(GraphQLString)
},
customer: {
type: CustomerType,
resolve: (source, args, context) => {
return context.customerLoader.load(source.customer_id);
}
}
})
});
export default () => {
return graphqlHTTP((req, res, graphQLParams) => {
return {
schema: schema,
graphiql: true,
pretty: true,
context: {
customerLoader: customerGetByIdsLoader()
},
formatError: error => ({
message: error.message,
locations: error.locations,
stack: error.stack,
path: error.path
})
};
});
};
export const customerGetByIdsLoader = () =>
new DataLoader(ids => {
return customerGetByIds(ids);
});
export const customerGetByIds = async ids => {
let result = await Customer.find({ _id: { $in: ids }, deletedAt: null }).exec();
let rows = ids.map(id => {
let found = result.find(item => {
return item.id.equals(id);
});
return found ? found : null; << === found always undefined
});
return rows;
};
I'm facing the following problems when loading several PurchaseOrders:
A single customer_id is being called more than once in the ids parameter of the DataLoader. So an example id 5cee853eae92f6021f297f45 is being called on several requests to my loader, in successive calls. That suggests that the cache is not working properly.
My found variable when processing the read result is always being set to false, even comparing the right ids.
You can use findOne
export const customerGetByIds = async ids => {
let result = await Customer.find({ _id: { $in: ids }, deletedAt: null }).exec();
const rows = []
let promiseAll = ids.map(async (id) => {
let found = result.filter(item => item.id.toString() === id.toSring());
if(found) {
rows.push(found[0])
return found[0]
}
return null;
});
await Promise.all(promiseAll);
return rows;
};
I'm trying to format the data i got from YouTube Data API v3 but i'm unable to change anything of it.
const videoIds = youtubeResponse.items.map(item => item.id);
VideoRepo.getById(videoIds, (err, videos) => {
/*
videos is an array of objects that contain youtube videos from YT API and MongoDB(mongoose)
*/
console.log(videos.map((v) => {
v.time = moment(v.time).fromNow();
v.duration = moment('1900-01-01 00:00:00').seconds(v.duration).format('HH:mm:ss');
return v;
}));
});
VideoRepo class:
static getById(id, callback) {
if (Array.isArray(id)) {
// Multiple ids were specified
async.waterfall([
(next) => {
// Get existing videos' data
Video.find({ _id: { $in: id } }).select('-__v').sort({ createdAt: 1 }).exec((err, data) => {
if (err) return next(err);
next(null, data);
});
},
(existingData, next) => {
if (existingData.length === 0) {
// All videos are new, skip to the next step
return next(null, [], id);
}
// Remove existing data from ID array
const obj = existingData.map(el => el._id);
next(null, existingData, id.filter(el => !obj.includes(el)));
},
(existingData, newIDs, next) => {
if (newIDs.length === 0) {
return next(null, existingData);
}
// Get new videos' data from YT API
youtube.videos.list({ id: newIDs.join(','), part: 'snippet,contentDetails,statistics' }, (err, videoResp) => {
if (err) return next(err);
// Final data
const data = id;
// New videos' data
const newData = videoResp.data.items.map(item => this.fixVideoData(item));
// Add new videos to the DB
Video.insertMany(newData, (err) => {
if (err) return next(err);
// Merge new data with existing data
const merged = existingData.concat(newData);
// Fix order
for (let i = 0; i < merged.length; i += 1) {
const d = merged[i];
data[data.indexOf(d._id)] = d;
}
// Success!
next(null, data);
});
});
},
], (err, data) => callback(err, data));
}
}
static fixVideoData(videoData) {
const data = {
_id: videoData.id,
channelId: videoData.snippet.channelId,
title: videoData.snippet.title,
description: videoData.snippet.description,
slug: slugify(videoData.snippet.title, { lower: true }),
views: videoData.statistics.viewCount,
duration: moment.duration(videoData.contentDetails.duration).asSeconds(),
tags: videoData.snippet.tags,
thumbnail: null,
preThumbnail: null,
time: videoData.snippet.publishedAt,
};
const possibleThumbs = ['maxres', 'standard', 'high', 'medium', 'default'];
for (let j = 0; j < possibleThumbs.length; j += 1) {
if (Object.prototype.hasOwnProperty.call(videoData.snippet.thumbnails, possibleThumbs[j])) {
data.thumbnail = videoData.snippet.thumbnails[possibleThumbs[j]].url;
break;
}
}
if (videoData.snippet.thumbnails.medium) {
data.preThumbnail = videoData.snippet.thumbnails.medium.url;
} else if (videoData.snippet.thumbnails.high) {
data.preThumbnail = videoData.snippet.thumbnails.high.url;
} else {
data.preThumbnail = data.thumbnail;
}
return data;
}
This is what videos array contains:
// videoData: https://developers.google.com/youtube/v3/docs/videos#resource
{
_id: videoData.id,
channelId: videoData.snippet.channelId,
title: videoData.snippet.title,
description: videoData.snippet.description,
views: videoData.statistics.viewCount,
duration: moment.duration(videoData.contentDetails.duration).asSeconds(),
tags: videoData.snippet.tags,
thumbnail: null,
preThumbnail: null,
time: videoData.snippet.publishedAt,
};
Expected results:
[...{ [..other keys] duration: "00:05:43", time: "3 days ago" }]
Actual output (nothing is changed, exactly the same array as videos):
[...{ [..other keys] duration: 343, time: 2018-12-26T13:37:32.000Z }]
Why is it not working and how can i fix it?
You can return a new object where you override only those 2 specific keys,
video.map(v => ({
...v,
duration: moment('1900-01-01 00:00:00').seconds(v.duration).format('HH:mm:ss'),
time: moment(v.time).fromNow()
}))
So apparently Model.find() returns mongoose documents instead of javascript objects and i should have used Query.lean().
Video.find({ _id: { $in: id } }).lean().select('-__v').sort({ createdAt: 1 })
.exec()
I was trying to stub an arrow function removeUserEntry, but when executing acctRmRouter in the test, my stub seems being ignored. I have to explicitly stub the UserModel's deleteOne method to get the test successfully, I am wondering why the ignorance happens, thank you
acctRoute.js
const removeUserEntry = (username) => {
const condition = {username: username};
return UserModel.deleteOne(condition)
.then((res) => {
if (res.n < 1) {
throw new Error('User not exists');
}
return true;
}, (err) => {throw err})
.catch(err => err);
};
const acctRmRouter = function(httpReq, httpRes, next) {
if (!argValidate(httpReq.body, 'string')) {
httpRes.locals = {api: { success: false }};
// return to avoid running downwards
return next(new Error('bad argument'));
}
// perform DB rm user
return removeUserEntry(httpReq.body).then((res) => {
if (res === true) {
httpRes.locals = {api: { success: true }};
next();
} else {
httpRes.locals = {api: { success: false }}
next(res);
}
});
};
acctRoute.spec.js
it('should remove user handler pass success request', async () => {
shouldDbReset = false;
const mockRequestURL = "/api/account/rm-user";
const mockRequest = httpMocks.createRequest({
method: "POST",
url: mockRequestURL,
headers: {
"Content-Type": "text/plain"
},
body: 'validRmUser',
});
const mockResponse = httpMocks.createResponse();
const spyNext = sinon.spy();
const stubRemoveUserEntry = sinon.stub(accountRouterHelper, 'removeUserEntry');
stubRemoveUserEntry.callsFake(function(){
return Promise.resolve(true);
}); // Expecting this function to be stubbed, and always return true
await accountRouterHelper.acctRmRouter(mockRequest, mockResponse, spyNext);
/* But when running the function, it returns error object with "User not exists"
which is not what intended */
const firstCallArgs = spyNext.getCall(0).args[0];
expect(spyNext.called).to.be.true;
console.log(`firstCallArgs: ${firstCallArgs}`)
expect(firstCallArgs instanceof Error).to.be.false;
expect(spyNext.args[0].length).to.equal(0);
expect(mockResponse.statusCode).to.equal(200);
expect(mockResponse.locals.api.success).to.be.true;
stubRemoveUserEntry.resetHistory();
stubRemoveUserEntry.restore();
});
The following indeed stubbed successfully with similar pattern to removeUserEntry.
acctRoute.js
const createUserEntry = (userData) => {
const updatedUserData = filterInput(userData);
const userDoc = new UserModel(updatedUserData);
return userDoc.save()
.then((userObj) => userObj._doc
,(err) => { throw err;})
.catch(err => err);
};
const acctCreateRouter = function (httpReq, httpRes, next) {
// do something in mongodb
return createUserEntry(userCondition)
.then((response) => {
if (!(response instanceof Error)) {
httpRes.locals = {api: { success: true}};
next();
} else {
httpRes.locals = {api: { success: false}};
next(response);
}
}, (err) => {
httpRes.locals = {api: { success: false}};
next(err);
})
.catch((err) => {
httpRes.locals = {api: { success: false}};
next(err);
});
};
const acctOutputRouter = function(req, res, next) {
if (res.locals) {
res.send(res.locals.api);
} else {next()}
};
acctRoute.spec.js
it("should return and save the success result to response locals for next route", () => {
shouldDbReset = false;
const mockResponse = httpMocks.createResponse();
const stubCreateUserEntry = sinon.stub(accountRouterHelper, 'createUserEntry');
const mockNext = sinon.spy();
stubCreateUserEntry.callsFake(function(){
return Promise.resolve();
}); // Unlike removeUserEntry, stubbing neatly with desired output
return accountRouterHelper.acctCreateRouter(mockRequest, mockResponse, mockNext)
.then(() => {
expect(mockNext.called).to.be.true;
expect(mockResponse.locals.api.success).to.be.true;
})
.finally(() => {
mockNext.resetHistory();
stubCreateUserEntry.restore();
});
});
Issue
sinon.stub(accountRouterHelper, 'removeUserEntry') replaces the module export.
acctRmRouter() is not calling the module export, it is calling removeUserEntry() directly so stubbing the module export does nothing.
Solution
Refactor acctRmRouter() to call the module export for removeUserEntry().
ES6
// import module into itself
import * as self from './acctRoute';
...
const acctRmRouter = function(httpReq, httpRes, next) {
...
// call the function using the module
return self.removeUserEntry(httpReq.body).then((res) => {
...
Node.js module
...
const acctRmRouter = function(httpReq, httpRes, next) {
...
// call the function using module.exports
return module.exports.removeUserEntry(httpReq.body).then((res) => {
...