I am a novice in JS/nodeJS world and I have some difficulties with the asynchronous way of thinking...
app.post("/upload", upload.array("files"), uploadFiles);
function uploadFiles(req, res) {
req.files.forEach(function(file) {
var linesInserted = 0;
var linesInError = 0;
fs.createReadStream(file.destination + file.filename)
.pipe(parse({ delimiter: ",", columns: false, fromLine: 2 }))
.on("data", function (row) {
Model.findOneAndUpdate(
{ code: row[0] },
{
$set: {
a: row[1],
b: row[2],
c: row[4],
d: row[5],
e: row[6],
f: moment(row[7], "DD/MM/YYYY").toISOString(),
g: moment(row[8], "DD/MM/YYYY").toISOString(),
h: row[12].split(/\s/)[0].replace(',','.')
},
$setOnInsert: {
i: row[0],
j: "airbnb",
k: row[3],
l: moment(row[10], "DD/MM/YYYY").toISOString()
},
$push: {
connectedTo: [{ m : "xxx" }, { service: "n", serviceId: "o" }]
}
},
{
upsert: true,
runValidators: true
},
function(err, res) {
if (err) {
console.log(err);
linesInError++;
} else if (res) {
console.log(linesInserted);
linesInserted++;
}
}
);
})
.on("end", function () {
File.create({
file: file.destination + file.filename,
originalName: file.originalname,
linesInserted: linesInserted,
linesInError: linesInError
});
console.log(`File ${file.originalname} - ${file.destination + file.filename} imported`);
})
.on("error", function (error) {
console.log(error.message);
});
});
res.json({ counterFilesImported: req.files.length });
}
With this code, the problem is that, at the end the values for File are 0 for linesInserted and linesInError. These counters increment during the reading of the file but at the "end" their value is 0. I know that this is a problem of asynchronous calls but I can't find any solution.
I would like to retrieve the "real" value of the counters.
Thank you for your help !
Your issue is that the stream pushes data/rows as fast as it can, and when it's done it calls end but it doesn't wait for the rows to be processed before checking the lineCount.
I have mixed feelings about streams, but according to the docs they are async iterable; that I know how to work with.
I've changed the code to Promises.
app.post("/upload", upload.array("files"), function (req, res) {
processFiles(req.files);
res.json({ counterFilesImported: req.files.length });
});
async function processFiles(files) {
for (const file of files) {
try { // to catch stream errors
let linesInserted = 0;
let linesInError = 0;
const stream = fs.createReadStream(file.destination + file.filename)
.pipe(parse({ delimiter: ",", columns: false, fromLine: 2 }));
for await (const row of stream) {
try {
await Model.findOneAndUpdate(
{ code: row[0] },
{
$set: {
a: row[1],
b: row[2],
c: row[4],
d: row[5],
e: row[6],
f: moment(row[7], "DD/MM/YYYY").toISOString(),
g: moment(row[8], "DD/MM/YYYY").toISOString(),
h: row[12].split(/\s/)[0].replace(',', '.')
},
$setOnInsert: {
i: row[0],
j: "airbnb",
k: row[3],
l: moment(row[10], "DD/MM/YYYY").toISOString()
},
$push: {
connectedTo: [{ m: "xxx" }, { service: "n", serviceId: "o" }]
}
},
{
upsert: true,
runValidators: true
}
);
++linesInserted;
} catch (err) {
console.log(err);
++linesInError;
}
}
File.create({
file: file.destination + file.filename,
originalName: file.originalname,
linesInserted: linesInserted,
linesInError: linesInError
});
console.log(`File ${file.originalname} - ${file.destination + file.filename} imported`);
} catch (error) {
// error with the stream/file
console.log(error.message);
}
}
}
Related
An Action is calling the helper function inside the loop. If the helper function raise some error then it exits with a specific code queryFailed like follows:
helpers/a/execute.js
module.exports = {
friendlyName: '',
description: '',
inputs: {},
exits: {
queryError: {
description: 'Query error'
},
success: {
description: 'yayyy!! success!'
}
},
fn: async function ({ conditions }, exits) {
let records = [],
MYSQL_QUERY = `SELECT * FROM model WHERE COLUMN = $1`;
try {
records = await Model.getDatastore().sendNativeQuery(MYSQL_QUERY, [['true']]);
}
catch (error) {
return exits.queryFailed(error);
}
return exits.success(records);
}
};
I have an action as follows that calls the above mentioned helper function.
controllers/action.js:
module.exports = {
friendlyName: 'Action',
description: 'Performs some action',
inputs: {
param1: {
description: 'param 1',
type: 'string'
},
param2: {
description: 'param 2',
type: 'ref'
}
},
exits: {
invalid: {
description: 'Invalid request',
responseType: 'invalid',
statusCode: 400
},
unexpected: {
description: 'Unexpected error',
responseType: 'unexpected',
statusCode: 500
},
success: {
description: 'success',
statusCode: 200,
outputType: 'ref'
}
},
fn: async function (inputs, exits) {
// Helper Ids
const arr = ['a', 'b'];
let response = [];
for (const element of arr) {
try {
records = await sails.helpers[element].execute.with({
conditions: conditions
});
}
catch (err) {
if (err.code === 'queryError') {
LOGGER.error('Database Error', err);
return exits.unexpected();
}
return exits.unexpected();
}
response.push(records);
}
return exits.success(response);
}
};
The issue with this is in case of an invalid query the helper function exits with queryError code as follows:
return exits.queryFailed(error);
Assuming helper a is executed successfully, if there is an error in helper b then ideally the action should not exit itself. It should continue executing and show the error in the final response for that block.
Expected Response:
{
"rows": [
{
"value": {
"id": "a",
"data": {},
"meta": {},
}
},
{
"error": {
"name": "serverError",
"statusCode": 500,
"message": "Internal server error.",
"id": 2
}
},
Current Behaviour: It's catching the queryError in the action and doing an exit with the error response:
{
"trace": "",
"error": {
"name": "serverError",
"statusCode": 500,
"message": "Internal server error"
}
}
Thank you in advance!
I want to get the array of rooms and assign it to each property wrt their property_id, but the value returned is a pending promise. Not sure what's wrong. Although when I log the rooms inside the then it does log the value correctly. The result of console.log(property) is given below.
const vendorProfile = catchAsync(async (req, res, next) => {
await passport.authenticate("vendor-jwt", { session: false }, (err, user, info) => {
if (err) {
res.error = err || info.message;
return next(401);
}
if (!user) {
res.error = info.message;
return next(401);
}
return Promise.resolve(
getVendorProfileInfo(user._id)
.then((result) => {
if (result == "error") {
res.error = "Failed to fetch Vendor Profile";
next(500);
}
return getPropertyByVendorId(result._id).then((prop) => {
for (const property of prop) {
property._doc.property_rooms = getAllRooms(property._id).then((rooms) => rooms);
console.log(property);
}
res.message = "Vendor Profile fetched successfully";
res.data = {
vendor_info: result,
vendor_properties: prop,
};
return next(200);
});
})
.catch((err) => {
Logger.error(err);
res.error = "Failed to get vendor profile";
return next(500);
})
).catch((err) => {
Logger.error(err);
res.error = "Failed to get vendor profile";
return next(500);
});
})(req, res, next);
});
This is the function to get all the rooms for that property_id:
const getAllRooms = (propertyId) => {
return Promise.resolve(Room.find({ property_id: propertyId }).then((result) => result)).catch((err) => {
Logger.error(err);
return "error";
});
};
Here is my console.log(property):
{
property_basic_info: {
property_name: 'Welcome',
property_star_rating: 1,
property_booking_since: 2021,
property_channel_manager: ''
},
property_location: {
property_geo_loc: { coordinates: [Array], type: 'Point' },
property_locality: 'bhandup',
property_address: 'MAHAVIR UNIVERSE',
property_country: 'India',
property_state: 'Maharashtra',
property_city: 'Mumbai',
property_zip_code: '400078'
},
property_contact_details: { phone_no: '7059462868', email: 'roy.srijan#outlook.com' },
property_amenities: {
basic_facilities: [ 'Electricity', 'Air Conditioning', 'Elevator/ Lift', 'Bathroom' ],
general_services: [ 'Food', 'Bellboy service' ],
outdoor_activities_sports: [],
common_area: [],
food_drink: [],
health_wellness: [],
business_center_conference: [],
beauty_spa: [],
security: []
},
property_policies: {
checkin_time: '10:06',
checkout_time: '22:06',
cancellation_policy: 'Free cancellation upto 48 hrs'
},
property_rules: {
id_proof: {
acceptable_identity_proofs: 'Adhaar',
unacceptable_identity_proofs: 'Adhaar',
allow_same_id: true
},
guest_profile: [
[Object], [Object],
[Object], [Object],
[Object], [Object],
[Object]
],
general_safety_hygiene_guidelines: [],
room_safety_hygiene: [],
social_distancing: [],
food_drinks_hygiene: [],
property_restrictions: [],
pet_policy: [],
guest_suitabilty: [],
checkin_checkout_policy: [],
extra_bed_policy: [ [Object] ],
custom_policy: []
},
property_finance_legal: { gst_details: '29AAACR4849R2ZG' },
property_status: 1,
property_photo_id: [],
_id: 61607791b1af193c7b8b9f08,
vendor_id: 61607775b1af193c7b8b9f07,
createdAt: 2021-10-08T16:53:37.734Z,
updatedAt: 2021-10-08T16:53:37.734Z,
__v: 0,
property_rooms: Promise { <pending> }
}
Thanks in advance.
That's because you are logging the promise outside the then method.
The promise is resolved async so outside then it is not resolved yet.
you have to change this line:
property._doc.property_rooms = getAllRooms(property._id).then((rooms) => rooms);
console.log(property);
to
property._doc.property_rooms = getAllRooms(property._id).then((rooms) => console.log(rooms));
or use async/await to work with it like sync values
I have a product schema like
quantity: { type: Number, required: true },
batch_no: {
type: [
{
batch_no: { type: String, required: true },
quantity: { type: Number, required: true },
created_at: { type: Date, default: Date.now() },
}
],
default: []
}
I am trying to update both the quantity fields in one query.
The code goes something like :
var expression = { $and: [{ "$inc": { "quantity": -10 } }, { "$inc": {"batch_no.$.batch_no": -10 } }] }
await ProductModel.findOneAndUpdate({ "sku": "TestSKU", "batch_no.$.batch_no":"Batch 1" }, { expression }, (err, response) => {
if (response) {
console.log("Update success")
} else if (err) {
res.status(400).send(err);
}
});
This nested query does not work.
Is there no way that I can update both the quantites at once?
$and is a query operator. If you just wanted to update multiple fields via $inc, you can pass them as key-value pairs object argument to $inc, like this:
var expression = { $inc: { "quantity": -10, "batch_no.$.batch_no": -10 } }
await ProductModel.findOneAndUpdate({ "sku": "TestSKU", "batch_no.$.batch_no": "Batch 1" }, expression, (err, response) => {
if (response) {
console.log("Update success")
} else if (err) {
res.status(400).send(err);
}
});
Also, you can just pass in expression directly as the 2nd argument, without wrapping it in another object.
I'm trying to map a json response from mysql query, but i receive ho response: data: NULL
This is my code:
const audience = rows.map((row) => {
db.query(CountAudiences, [row.campaign], function(err, count, fields) {
if (err) throw err;
console.log('Query result: ', count[0].audience);
return {
id: row.id,
title: row.title,
campaign: row.campaign,
action: row.action,
date: row.date,
audiences: count[0].audience
}
});
});
res.json({
count: rows.length,
data: audience
})
Response:
{
"count":1,
"data":[
null
]
}
Do you know how solve this?
Thanks :)
In your code as you are placing a query, so it is a Async hit. Try this
function getResponse(){
let rows = [{ id: 1, title: 5 }, { id: 2, title: "ggg" }]
const audience = rows.map(async (row) => {
return new Promise((resolve,reject)=> {
db.query(CountAudiences, [row.campaign], function (err, count, fields) {
if (err) throw err;
console.log('Query result: ', count[0].audience);
resolve( {
id: row.id,
title: row.title,
campaign: row.campaign,
action: row.action,
date: row.date,
audiences: count[0].audience
})
})
})
});
return Promise.all(audience)
}
getResponse().then((reponseData)=>{
res.json({
count: rows.length,
data: reponseData
})
I have and map structure stored at dynamodb, an I would like to add another attribute inside school object
something like:
{
name: 'Felipe'
uid: 112233,
data: {
structure: {
school: {
name: 'beta'
}
}
}
}
previously the add_year did not was a part of the structure, so this part is new
school: {
name: 'beta'
add_year: '2020'
}
How can I accomplised that?
I've tried the following solutions, without success
(async ()=>{
try {
let teste = await dynamoDb.updateItem({
TableName: 'test',
Key: {
uid: "112233"
},
UpdateExpression: "SET data.#structure.#school = list_append(#structure, :attrValue)",
ExpressionAttributeNames: {
"#data": "data",
"#structure": "structure",
"#school": "school",
},
ExpressionAttributeValues: {
":school":{
"add_year": 2020
}
},
ReturnValues:"UPDATED_NEW "
})
console.log('update',teste)
} catch (error) {
console.log(error)
}
Felipe, did you see the AWS documentation about this topic?
I think this code can work for you:
(async () => {
try {
var params = {
TableName: 'test',
Key: {
"uid": "112233"
},
UpdateExpression: "SET data.structure.school.add_year = :year)",
ExpressionAttributeValues: {
":add_year": 2020
},
ReturnValues: "UPDATED_NEW"
}
dynamoDb.update(params, (err, data) => {
if (err) {
console.error("Unable to update item. Error JSON:", JSON.stringify(err, null, 2));
} else {
console.log("UpdateItem succeeded:", JSON.stringify(data, null, 2));
}
})
} catch (error) {
console.log(error)
}
})
const { DocumentClient } = require('aws-sdk/clients/dynamodb');
const documentClient = new DocumentClient({
region: 'us-east-1',
});
try {
let add_year= '2020'
let teste = documentClient.update({
TableName: 'test',
Key: {
uid: "112233"
},
UpdateExpression: `SET #data.structure.school= :add_year`,
ExpressionAttributeValues: {
':add_year': add_year
},
ExpressionAttributeNames: {
"#data": "data"
},
ReturnValues:"ALL_NEW"
}).promise()
teste.then(t => console.log(t));
} catch (error) {
console.log(error)
}