Hi I am trying to work out the best way to achieve something. I am essentially making two database calls
const [emails] = await dbConnection.execute('SELECT name, programme, timestamp FROM emails');
const [emailsCancelled] = await dbConnection.execute('SELECT data FROM emails where name = "email.cancelled"');
The reason I am making two calls is that I am processing over hundred thousand rows, and the data field contains quite a bit of JSON data, so don't want to retrieve that for all the rows.
So with the emails, I get data back in the following format
[
{
name: 'email.sent',
programme: 'Email One',
timestamp: 2022-03-24T18:06:02.000Z
},
{
name: 'email.sent',
programme: 'Email Two',
timestamp: 2022-03-24T18:06:02.000Z
},
{
name: 'email.sent',
programme: 'Email One',
timestamp: 2022-03-24T18:06:02.000Z
},
...
]
So what I needed to do is group by programme, to identify how many were sent and the total count. I do obtain some other details but reduced for this post. To do this I do
const emailsReduced = await emails.reduce((acc, o) => {
const name = o.name?.replace('email.', '');
if (!acc[o.programme]) {
acc[o.programme] = {
count: 0,
sent: 0,
};
}
acc[o.programme].count = (acc[o.programme].count || 0) + 1;
acc[o.programme][name] = (acc[o.programme][name]) + 1;
return acc;
}, {});
And that will return something like this
'Email One': {
count: 2,
sent: 2,
},
'Email Two': {
count: 1,
sent: 1,
},
Now emailsCancelled returns JSON data. So what I can do is loop it and show an example out the part I need
Object.entries(emailsCancelled).forEach(([key, value]) => {
const data = JSON.parse(value.data);
if (data?.payload?.body?.toUpperCase() === 'STOP') {
console.log(data?.payload?.correlation?.metadata);
}
});
And that will produce rows like this
[
{ customerId: '12345', programName: 'Email One' },
{ customerId: '2321', programName: 'Email Two' },
{ customerId: '33321', programName: 'Email Two' }
]
Now what I need to do is get that into the original array as a count. So you can see that there was 1 cancelled for Email One, and 2 for Two. So I need to add this in like so, matching it based on the programme name.
'Email One': {
count: 2,
sent: 2,
cancelled: 1,
},
'Email Two': {
count: 1,
sent: 1,
cancelled: 2,
},
How can I achieve something like this?
Thanks
Actual format
{
"name":"email.cancelled",
"payload":{
"body":"STOP",
"correlation":{
"metadata":{
"customerId":"232131232113",
"programName":"Email One"
}
},
"id":"123454323343232",
"receivedOn":"2022-05-15T12:51:54.403Z"
},
}
From emailsCancelled, you can reduce your array to a lookup Map before your perform your .reduce() on on emails. The lookup will store the programName as the keys, and the count of that program as the values:
const emails = [
{ customerId: '12345', programName: 'Email One' },
{ customerId: '2321', programName: 'Email Two' },
{ customerId: '33321', programName: 'Email Two' }
];
const lut = emails.reduce((map, {programName}) =>
map.set(programName, (map.get(programName) || 0) + 1)
, new Map);
console.log(lut.get("Email One"));
console.log(lut.get("Email Two"));
You can build this Map directly from your .forEach() loop also, note that I'm using Object.values() instead of .entries() as you're only intrested in the values and not the keys:
const lut = new Map();
Object.values(emailsCancelled).forEach(value => {
const data = JSON.parse(value.data);
if (data?.payload?.body?.toUpperCase() === 'STOP') {
const programName = data.payload.correlation?.metadata?.programName; // if `correcltation`, or `metadata` or `programName` don't exist, use optional chaining and an if-statement to check for `undefined` before updating the map.
lut.set(programName, (map.get(programName) || 0) + 1)
}
});
You can then use this lookup lut Map when you use .reduce() on emails to work out the cancelled value, defaulting cancelled to 0 if the programme can't be found in the Map:
const emailsReduced = await emails.reduce((acc, o) => {
const name = o.name?.replace('email.', '');
if (!acc[o.programme]) {
acc[o.programme] = {
count: 0,
sent: 0,
cancelled: lut.get(o.programme) || 0 // default to zero if program can't be found
};
}
acc[o.programme].count = acc[o.programme].count + 1;
acc[o.programme][name] = acc[o.programme][name] + 1;
return acc;
}, {});
Assuming your data structures look like these, you can map and filter according to emails keys:
const emails = [
{ 'Email One': {
count: 2,
sent: 2,
}},
{'Email Two': {
count: 1,
sent: 1,
}}
]
const canceled = [
{ customerId: '12345', programName: 'Email One' },
{ customerId: '2321', programName: 'Email Two' },
{ customerId: '33321', programName: 'Email Two' }
]
const newmails = emails.map(mail => {
let strmail = Object.keys(mail)
let ncanceled = canceled.filter(item => {
return item.programName == strmail
}).length
mail[strmail].canceled = ncanceled
return mail
})
console.log(newmails)
Try this!
const emails = [{
'Email One': {
count: 2,
sent: 2,
cancelled: 0,
},
},
{
'Email Two': {
count: 1,
sent: 1,
cancelled: 0,
},
},
];
const cancelled_emails = [{
customerId: '12345',
programName: 'Email One'
},
{
customerId: '2321',
programName: 'Email Two'
},
{
customerId: '33321',
programName: 'Email Two'
},
];
for (let cancelled_email of cancelled_emails) {
let prg_name = cancelled_email.programName;
for (email of emails) {
if (Object.keys(email)[0] === prg_name) {
email[prg_name].cancelled += 1;
}
}
}
console.log(emails);
Related
Assuming that I have this multiple data and I need to filter set of elements.
How could I only filter the data with only text and createdAt will produce:
[
createdAt: "2021-07-07",
text: "No answer found."
]
Data:
var getMessage = [
0: {
status: 'SENT',
type: 'text',
createdAt: "2021-07-07T08:11:51.686Z",
web: {
message: {
text: "Get Started"
}
}
},
1: {
status: 'SENT',
type: 'text',
createdAt: "2021-07-07T08:11:53.547Z",
web: {
message: {
text: "Etrt"
}
}
},
2: {
status: 'SENT',
type: 'text',
createdAt: "2021-07-07T08:12:07.785Z",
web: {
message: {
text: "No answer found."
}
}
}
];
const findKeywords = "O"
const messageData = getMessage.map(x => x);
const findMessage = messageData.filter(x => x.web.message.text.toLowerCase().includes(findKeywords.toLowerCase()));
console.log(findMessage);
to get specific key value try this
const findMessage = messageData.filter(x => x.web.message.text.toLowerCase().includes(findKeywords.toLowerCase())).map(function (obj) {
return {
createdAt: obj.createdAt,
text: obj.web.message.text
};
});;
console.log(findMessage);
BasicData {
userid:123123,
email:something#gmail.com
},
BusData {
Dropping: ....,
Boarding:......,
.....
......
},
Traveller1 {
name:....,
seatno0.....,
age:....,
},
Traveller2 {
name:....,
seatno0.....,
age:....,
},
Traveller3 {
name:....,
seatno0.....,
age:....,
},
Traveller4 {
name:....,
seatno0.....,
age:....,
}
In BookedBusdata there is table contains BasicData BusData and TravellerData. I want to access the the Data only consist of Travellers, But it may Traveller may be 2, 4 , 10 any number The Documents consist of only Map. But i want to all the data of Traveller which are in the form of Traveller1, Traveller2 whatever Traveller are there
var docRef = db.collection("BookedTicketData").doc(orderid);
docRef.get().then(function(doc) {
if (doc.exists) {
data = doc.data()
console.log(data.Traveller1.Age) // I am getting the data if i am accessing only one
} else {
// console.log("No such document!");
}
}).catch(function(error) {
console.log("Error getting document:", error);
});
This code i have performed
If you're happy to filter the data client side, it's a fairly straight forward task
const data = {
BasicData: {},
BusData: {},
Traveller1: {
name: 'Name1',
email: 'Email1'
},
Traveller2: {
name: 'Name2',
email: 'Email2'
},
Traveller3: {
name: 'Name2',
email: 'Email2'
},
}
// returns an array of travelers
console.log(Object.keys(data).reduce((acc, key) => {
if (key.includes('Traveller')) {
acc.push(data[key])
}
return acc;
}, []))
// returns an object of travelers
console.log(Object.keys(data).reduce((acc, key) => {
if (key.includes('Traveller')) {
acc[key] = data[key];
}
return acc;
}, {}))
In your case you can just use Object.keys(doc.data()).reduce...
I have example input
[[1,2],[3,2],[1,3],...,[4,5]]
How to write model schema in mongoose?
This is my Schema
const SubproductSchema = new Schema({
...
positions: [{
type: [Number],
validate: {
validator: function(value){
return value.length == 2;
},
message: 'Positions should be 2'
}
}]
}, {
timestamps: true
});
And this does not work.
Input should be array with fixedSize with 2 length in array like this [[1,2],[3,2],[1,3],...,[4,5]]
If input is [[1,2,4],[3,2],[1,3],...,[4,5]], it should validate with 'Position should be 2'
UPDATED
I also tried this code (logically correct I guess):
const SubproductSchema = new Schema({
...
positions: {
type: [{
type: [Number],
validate: {
validator: function(value){
return value.length == 2;
},
message: 'Positions should be 2'
}
}],
}
}, {
timestamps: true
});
And my post is
{
...
"positions": [[2,3], [1,4], [4, 5]]
}
And it output error:
Subproduct validation failed: positions: Cast to Array failed for
value \"[ [ 2, 3 ], [ 1, 4 ], [ 4, 5 ] ]\" at path \"positions\""
model should look like
This is what you were looking...
const SubproductSchema = new Schema({
...
positions: [{
type: [Number],
validate: [limit, 'Positions should be 2']
}]
}, { timestamps: true });
const limit = (val) => {
let subElementsValidated = true;
val.forEach(el => {
if (el.length != 2){
subElementsValidated = false;
return;
}
});
return subElementsValidated;
}
You can change the validate options like this
const SubproductSchema = new Schema({
...
positions: [{
type: [Number],
validate: [limit, 'Positions should be 2']
}]
}, {
timestamps: true
});
const limit = (val) => {
return val.length == 2;
}
I'm querying customer orders for a specified customer using Sequelize relationships.
index.js
var results2 = await customerService.getOrders(1);
console.log(results2);
service.js
exports.getOrders = function (id) {
return customerModel.findAll({
raw: true,
include: [{
model: orderModel,
where: { customer_idcustomer: id }
}],
}).then(r => r);
};
results
[ { idcustomer: 1,
customername: 'hello world',
'orders.idorder': 1,
'orders.orderdesc': 'order description 1',
'orders.customer_idcustomer': 1 },
{ idcustomer: 1,
customername: 'hello world',
'orders.idorder': 2,
'orders.orderdesc': 'Test 456',
'orders.customer_idcustomer': 1 },
{ idcustomer: 1,
customername: 'hello world',
'orders.idorder': 3,
'orders.orderdesc': 'Test 123',
'orders.customer_idcustomer': 1 } ]
expected
[ { idcustomer: 1,
customername: 'hello world',
'orders: [{
'orders.idorder': 1,
'orders.orderdesc': 'order description 1',
'orders.customer_idcustomer': 1 },
},
{
'orders.idorder': 2,
'orders.orderdesc': 'order description 2',
'orders.customer_idcustomer': 1 },
},
{
'orders.idorder': 3,
'orders.orderdesc': 'order description 3',
'orders.customer_idcustomer': 1 },
}]
]
All you need is to remove raw: true, from query ,
as it will return plain/flat object , and that will convert your object as it looks now.
exports.getOrders = function (id) {
return customerModel.findAll({
// raw: true, // <------ Just remove this line
include: [{
model: orderModel,
where: { customer_idcustomer: id }
}],
}).then(r => r);
};
Note : You should put the where condition in upper level as per your
logic
exports.getOrders = function (id) {
return customerModel.findAll({
where: { id: id } ,
// raw: true, // <------ Just remove this line
include: [{
model: orderModel
}]
}).then(r => r);
};
Try removing raw key value from your query.
Finder methods are intended to query data from the database. They do
not return plain objects but instead return model instances. Because
finder methods return model instances you can call any model instance
member on the result as described in the documentation for instances.
If you want to get the data without meta/model information then map your results using
{ plain: true }
Good sequelize examples in docs
Example:
const getPlainData = records => records.map(record =>
record.get({ plain: true }));
// Your code
return customerModel.findAll({
// raw: true, <= remove
include: [{
model: orderModel,
where: { customer_idcustomer: id }
}],
}).then(getPlainData);
In my case, having
raw: true
in the options didn't make any difference.
I added
distinct: true
and the issue disappeared.
I was using findAndCountAll, though.
Documentation: https://sequelize.org/master/class/lib/model.js~Model.html
I am trying to implement a way to upload files asynchronously.
I have a process I want to apply to every item of my array.
I am taking the name of each item, call a API to get additinal information about it, then I am sending it to a text to speech utility, and upload the resultingwav file to a S3 instance.
I can't find a way to do this asynchronously, and wait for all of them to finish.
I can do it in serie, but it take lots of time (12 minutes for 30 files (2mb each file)).
I tried to implement a asynchronous way, which takes around 5 minutes (7 minutes less), but I fear the problem is on the net line?
Function to apply to each item:
function doAll(c, lan, country, fileName, callback){
getNews(c, lan)
.then(function(newsResults){
getWavFile(newsResults, lan, fileName)
.then(function(wavResults){
uploadToS3(country,lan,fileName)
.then(function(s3Results){
return callback("done");
}, function(s3err){
console.log('s3 error: ',s3err);
return callback("done");
})
}, function(waverr){
console.log('wav error: ',waverr);
})
}, function(newserr){
console.log('news error: ',newserr);
})
}
Array example :
var arr = [
{
_id: '5769369ba2d42fd82ca4d851',
Name: 'Sports',
Order: 1,
Color: 'White',
Description: 'ספורט',
UpdatedDate: '2016-07-28T07:44:47.906Z',
CreatedDate: '2016-06-21T12:44:11.468Z',
Country: 'IL',
Langs: [
{
Name: 'Sports',
IsoCode: 'en',
Url: 'SportsJSON',
_id: '576b93486c7a9ff025275836'
},
{
Name: 'ספורט',
IsoCode: 'iw',
Url: 'HebrewSportsJSON',
_id: '576be6ad56126ccc25852613'
}
]
},
{
_id: '576bf4eb28176a3e5ce15afa',
Name: 'Top Stories',
Description: 'הכותרות',
Color: 'ww',
Order: 1,
UpdatedDate: '2016-07-10T12:01:26.713Z',
CreatedDate: '2016-06-23T14:40:43.435Z',
Country: 'IL',
Langs: [
{
Name: 'כותרות',
Url: 'HebrewTopStoriesJSON',
IsoCode: 'iw',
_id: '576bf52228176a3e5ce15afb'
},
{
Name: 'Top Stories',
IsoCode: 'en',
Url: 'TopStoriesJSON',
_id: '576bf94d28176a3e5ce15afd'
}
]
},
{
_id: '5756d5d6c4a3dfe478b16aa2',
Description: 'Nation Channel',
Order: 1,
Color: 'blue',
Code: 'Nation',
Name: 'Nation',
UpdatedDate: '2016-06-24T22:23:07.198Z',
CreatedDate: '2016-06-07T14:10:30.699Z',
Country: 'US',
Langs: [
{
Name: 'Nation',
IsoCode: 'en',
Url: 'NationJson',
_id: '576db2cb28176a3e5ce15b02'
}
]
}
]
My asynchronous way:
var array = [] // see the example how array look like
var newArray= [];
console.log('start uploading files time:', new Date());
for (var i = 0; i < array.length; i++) {
var list = array[i].Langs;
for (var j= 0; j < list.length; j++) {
var c = list[j];
var lan = convertIsoCode(c.IsoCode);
var fileName = array[i].Name + "_" + lan;
var country = array[i].Country;
doAll(c,lan,country,fileName, function(){
newArray.push(array[i]);
if (array.length == newArray.length) {
console.log('done');
defer.resolve('done');
}
})
}
}
EDIT:
I tried to do it with async.each and async.parallel, but didn't succeed, can anyone show me the right way to implement it?
Removed newArray since you don't need it for anything useful, it was wasting CPU time and was a horrendous way of tracking what was done. A simple counter would have done the tricks.
Gone ES6 since it's 2016. Also added semi colon because you were using them inconstitently.
Also, doAll is not a meaningful name.
'use strict';
const async = require('async');
let array = [/*data*/];
console.log('START ' + (new Date()));
//Asynchronously iterate throught the array
async.each(array, (item, callback) => {
//item is your array[i]
async.each(item.Langs, (lang, callback) => {
//lang is your array[i].Langs[j]
let lan = convertIsoCode(item.IsoCode),
fileName = item.Name + '_' + lan,
country = item.Country;
//Apply your functions
getNews(c, lan).then((newsResults) => {
getWavFile(newsResults, lan, fileName).then((wavResults) => {
uploadToS3(country,lan,fileName).then((s3Results) => {
//Everything is OK, callback without error
callback();
}, (s3err) => {
//Raise the error
callback(s3err);
});
}, (waverr) => {
console.log('wav error: ',waverr);
//Raise the error
callback(waverr);
});
}, (newserr) => {
console.log('news error: ',newserr);
//Raise the error
callback(newserr);
});
}, (error) => {
callback(error);
});
}, (error) => {
//If a error was raised, everything pending will be aborted and the error will be displayed
if(error) {
console.log(error);
//Else, just report it did fine
} else {
console.log('OK');
}
});