I'm pushing DynamoDB rows into a Elasticsearch cluster. The date fields are unix timestamps and not recognized by Kibana as a date.
I read about Elasticsearch mapping types and found this post, but don't know where to implement the mapping in my Lambda script:
/* ... requires and config ... */
exports.handler = (event, context, callback) => {
event.Records.forEach((record) => {
var dbRecord = JSON.stringify(record.dynamodb);
postToES(dbRecord, context, callback);
});
};
function postToES(doc, context, lambdaCallback) {
var req = new AWS.HttpRequest(endpoint);
req.method = 'POST';
req.path = path.join('/', esDomain.index, esDomain.doctype);
req.region = esDomain.region;
req.headers['presigned-expires'] = false;
req.headers['Host'] = endpoint.host;
req.body = doc;
// Maybe here?
var signer = new AWS.Signers.V4(req , 'es');
signer.addAuthorization(creds, new Date());
var send = new AWS.NodeHttpClient();
send.handleRequest(req, null, function(httpResp) {
var respBody = '';
httpResp.on('data', function (chunk) {
respBody += chunk;
});
httpResp.on('end', function (chunk) {
lambdaCallback(null,'Lambda added document ' + doc);
});
}, function(err) {
console.log('Error: ' + err);
lambdaCallback('Lambda failed with error ' + err);
});
}
Elasticsearch document
{
_index: "posts",
_type: "post",
_id: "6YKF2AAV06RSSRrzv6R-",
_version: 1,
found: true,
_source: {
ApproximateCreationDateTime: 1499922960,
Keys: {
id: {
S: "7asda8b0-628a-11e7-9e5e-25xyc7179dx7"
}
},
NewImage: {
posted_at: {
N: "1499922995401"
},
id: {
S: "7asda8b0-628a-11e7-9e5e-25xyc7179dx7"
}
},
SequenceNumber: "2442423900000000003279639454",
SizeBytes: 221,
StreamViewType: "NEW_AND_OLD_IMAGES"
}
}
Dynamoose Schema
var Schema = dynamoose.Schema;
var s = new Schema({
id: {
type: String,
hashKey: true,
required: true
},
posted_at: {
type: Date,
required: true
}
});
module.exports = dynamoose.model('posts', s);
Example: in my DynamoDB table I've the field posted_at. The content is a unix timestamp. In Kiabana it's indexed as
NewImage.posted_at.N (type: string, searchable, analyzed) and
NewImage.posted_at.N.keyword (type: string, searchable, aggregateable)
I'm confused by the N and type: string.
Any ideas?
Thanks!
Ok it turns out that the N is there to denote the DynamoDB attribute type (i.e. N stands for Number).
The problem is that the number gets stringified and thus indexed as a string in ES (i.e. what you currently see in your mapping).
We can get around this using a dynamic template definition. First delete your index in ES and the corresponding index pattern in Kibana. Then run this command:
curl -XPUT localhost:9200/_template/post_template -d '{
"template": "posts",
"mappings": {
"post": {
"dynamic_templates": [
{
"dates": {
"path_match": "NewImage.posted_at.N",
"mapping": {
"type": "date"
}
}
},
{
"strings": {
"match_mapping_type": "string",
"mapping": {
"type": "text",
"fields": {
"raw": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
]
}
}
}'
Finally you can reindex your data through Dynamoose and you should be able to find a date field in Kibana afterwards.
Related
I have a product schema like
quantity: { type: Number, required: true },
batch_no: {
type: [
{
batch_no: { type: String, required: true },
quantity: { type: Number, required: true },
created_at: { type: Date, default: Date.now() },
}
],
default: []
}
I am trying to update both the quantity fields in one query.
The code goes something like :
var expression = { $and: [{ "$inc": { "quantity": -10 } }, { "$inc": {"batch_no.$.batch_no": -10 } }] }
await ProductModel.findOneAndUpdate({ "sku": "TestSKU", "batch_no.$.batch_no":"Batch 1" }, { expression }, (err, response) => {
if (response) {
console.log("Update success")
} else if (err) {
res.status(400).send(err);
}
});
This nested query does not work.
Is there no way that I can update both the quantites at once?
$and is a query operator. If you just wanted to update multiple fields via $inc, you can pass them as key-value pairs object argument to $inc, like this:
var expression = { $inc: { "quantity": -10, "batch_no.$.batch_no": -10 } }
await ProductModel.findOneAndUpdate({ "sku": "TestSKU", "batch_no.$.batch_no": "Batch 1" }, expression, (err, response) => {
if (response) {
console.log("Update success")
} else if (err) {
res.status(400).send(err);
}
});
Also, you can just pass in expression directly as the 2nd argument, without wrapping it in another object.
I am trying to remove an array item with "updateOne" method but my query is not matching the right record in the model structure that I have. Given an email, I would like to find the array item with the provided email and pulls it out, remove it. (There is no array item with the same email)
My model is like so:
var mongoose = require('mongoose');
var teamMemberModelSchema = new mongoose.Schema({
_id: false,
"email": {
"type": String,
"required": true,
"minlenght": 5,
"maxheight": 50
},
"name": {
"type": String,
"required": true,
"minlenght": 5,
"maxheight": 256
},
"role": {
"type": String,
"required": true,
"minlenght": 20,
"maxheight": 256
},
"twitter": {
"type": String,
"required": true,
"minlenght": 1,
"maxheight": 100
},
"facebook": {
"type": String,
"required": true,
"minlenght": 1,
"maxheight": 100
},
"linkedin": {
"type": String,
"required": true,
"minlenght": 1,
"maxheight": 100
},
});
var teamModelSchema = new mongoose.Schema({
"title": {
"type": String,
"required": true,
"minlenght": 5,
"maxheight": 20
},
"headline": {
"type": String,
"required": true,
"minlenght": 5,
"maxheight": 30
},
"description": {
"type": String,
"required": true,
"minlenght": 5,
"maxheight": 80
},
"members": [teamMemberModelSchema]
}, { collection: 'team' });
teamModelSchema.set('collection', 'team');
mongoose.model('team', teamModelSchema)
And the approach that I am trying is the following:
module.exports.removeMember = function (req, res) {
const email = req.params.email;
const query = { "members.email": email };
const pull = { $pull: { "members.$.email": email } };
try {
var message = teamMsg.teamMemberRemoveSuccess;
TeamModel.updateOne(query, pull);
responseUtilities.sendJSON(res, false, { message: message });
} catch (err) {
console.log(err.message);
responseUtilities.sendJSON(res, err, { message: err.message });
}
};
It executes with no errors, but nothing is updated.
I have tried some others alternatives with "FindOneAndUpdate" and "FindOneAndRemove" but, I could not find a solution.
Any ideas?
You can use findOneAndUpdate with $pull operator for this task.
For removing items from an array of documents you can check MongoDb docs
You need to use await or then block to query. I used await, and made the function asynchronous by adding async keyword. We also need empty query object.
I also added the new: true option, to return the updated object to check if the item is deleted.
You need to handle the case where no document matches, I added a TODO for you.
module.exports.removeMember = async function(req, res) {
const email = req.params.email;
const query = {};
const pull = {
$pull: {
members: {
email: email
}
}
};
const options = {
new: true
};
try {
var message = teamMsg.teamMemberRemoveSuccess;
const result = await TeamModel.updateOne(query, pull, options);
console.log(result);
if (!result) {
//TODO: return 400-Bad Request or 404 - Not Found
} else {
responseUtilities.sendJSON(res, false, { message: message });
}
} catch (err) {
console.log(err.message);
responseUtilities.sendJSON(res, err, { message: err.message });
}
};
give this query a try:
db.collection.update(
{ 'members.email': 'email#address' },
{ $pull: { members: { email: 'email#address' } } },
{ multi: true }
)
Try using the update() method and async/await:
module.exports.removeMember = async function (req, res) {
const email = req.params.email;
console.log(`email = ${email}`) // Make sure correct value is coming thru
const query = { "members.email": email };
const pull = { $pull: { "members.$.email": email } };
const options = { multi: true }
try {
var message = teamMsg.teamMemberRemoveSuccess;
await TeamModel.update( query, pull, options );
responseUtilities.sendJSON(res, false, { "message": message });
} catch (err) {
console.log(err.message);
responseUtilities.sendJSON(res, err, { "message": err.message });
}
};
[Previously titled "How to get 1 record from a list..."]
I am very new to GraphQL and trying to understand how to get 1 record from query.
This is the result of my current query:
{
"data": {
"todos": null
}
}
I am not sure what is wrong. I would like the result to be this:
{
"data": {
"todos": {
"id": 1,
"title": "wake up",
"completed": true
}
}
}
Here is my code that I've created as I try to learn GraphQL.
schema.js:
var graphql = require('graphql');
var TODOs = [
{
"id": 1,
"title": "wake up",
"completed": true
},
{
"id": 2,
"title": "Eat Breakfast",
"completed": true
},
{
"id": 3,
"title": "Go to school",
"completed": false
}
];
var TodoType = new graphql.GraphQLObjectType({
name: 'todo',
fields: function () {
return {
id: {
type: graphql.GraphQLID
},
title: {
type: graphql.GraphQLString
},
completed: {
type: graphql.GraphQLBoolean
}
};
}
});
var queryType = new graphql.GraphQLObjectType({
name: 'Query',
fields: function () {
return {
todos: {
type: new graphql.GraphQLList(TodoType),
args: {
id: { type: graphql.GraphQLID }
},
resolve: function (source, args, root, ast) {
if (args.id) {
return TODOs.filter(function(item) {
return item.id === args.id;
})[0];
}
return TODOs;
}
}
}
}
});
module.exports = new graphql.GraphQLSchema({
query: queryType
});
index.js:
var graphql = require ('graphql').graphql;
var express = require('express');
var graphQLHTTP = require('express-graphql');
var Schema = require('./schema');
var query = 'query { todos(id: 1) { id, title, completed } }';
graphql(Schema, query).then( function(result) {
console.log(JSON.stringify(result,null," "));
});
var app = express()
.use('/', graphQLHTTP({ schema: Schema, pretty: true }))
.listen(8080, function (err) {
console.log('GraphQL Server is now running on localhost:8080');
});
To run this code I just run node index from the root directory. How can I get one specific record returned by the records id?
You have the wrong type for the todos field of your queryType. It should be TodoType, not a list of TodoType. You're getting an error because GraphQL expects to see a list, but your resolver is just returning a single value.
By the way, I suggest passing the graphiql: true option to graphqlHTTP, which will let you use GraphiQL to explore your schema and make queries.
I'm trying to test a post to a mongoose schema that has nested fields, but I'm only able to level 1 and the first field in level 2. For example:
I have a network model that can contain multiple ip addresses and multiple subnets. When I place the query in to Postman it allows me to create multiple ip addresses and multiple subnets (great) but I can't define a type field for example?
Mongoose Schema:
var mongoose = require('mongoose'), Schema = mongoose.Schema, ObjectId = mongoose.Schema.ObjectId;
var networkSchema = module.exports = mongoose.model('Network', {
network_id:ObjectId,
location: String,
hostname: String,
device: String,
model: String,
ipAddress: [ipaddressSchema],
subnets: [subnetSchema],
iosVersion: String,
softwareImage: String,
serialNumber: String,
});
var ipaddressSchema = Schema ({
ipAddress: String,
type: String,
});
var subnetSchema = Schema ({
range: String,
type: String,
});
Controller:
var Network = require('../models/network');
module.exports.create = function (req, res) {
var network = new Network(req.body);
network.save(function (err, result) {
res.json(result);
});
}
module.exports.list = function (req, res) {
Network.find({}, function (err, results) {
res.json(results);
});
}
Postman Query:
Postman Result:
I would like:
{
"__v": 0,
"location": "London Office",
"hostname": "lon-asa-01",
"device": "Switch-MLS",
"model": "Cisco 3750",
"softwareImage": "1.2",
"serialNumber": "123456",
"_id": "5510495c1d40ef965d7d1cec",
"subnets":[
["range" : "10.0.100.0/24", "type" : "Client" ],
["range" : "10.0.101.0/24", "type" : "Server" ],
],
"ipAddress": [
["ipAddress" : "10.0.100.1", "type" : "Inside" ],
["ipAddress" : "10.0.101.254", "type" : "Outside" ],
]
}
Ok, here you go:
First of all, your schema should look like this:
var networkSchema = module.exports = mongoose.model('Network', {
network_id: ObjectId,
location: String,
hostname: String,
device: String,
model: String,
ipAddress: [{type: ObjectId, ref: 'IpadressModelName'}],
subnets: [{type: ObjectId, ref: 'SubnetModelName'}],
iosVersion: String,
softwareImage: String,
serialNumber: String,
});
In your controller you have to insert first the entities on which your network relies on so you will have an _id to provide to the network model as reference:
module.exports.create = function (req, res) {
var network = new Network(req.body);
var ipAddress = [],
ipIds = [];
req.body.ipAddress.forEach(function(ip){
ipAddress.push(new IpadressModelName(ip));
});
var subnets = [],
subnetsIds = [];
req.body.subnets.forEach(function(sn){
subnets.push(new SubnetModelName(sn));
});
IpadressModelName.create(ipAddress, function () {
// args[0] should be the error
if (args[0]) {
throw args[0]
}else{
for(var i=1; i<args.length; i++ )
ipIds.push(args[i]._id);
}
SubnetModelName.create(subnets, function () {
// args[0] should be the error
if (args[0]) {
throw args[0]
}else{
for(var i=1; i<args.length; i++ )
subnetsIds.push(args[i]._id);
}
network.ipAddress = ipIds;
network.subnets = subnetsIds;
network.save(function (err, result) {
res.json(result);
});
});
});
}
Finally post data as raw JSON:
{
"location": "London Office",
"hostname": "lon-asa-01",
"device": "Switch-MLS",
"model": "Cisco 3750",
"softwareImage": "1.2",
"serialNumber": "123456",
"subnets":[
{"range" : "10.0.100.0/24", "type" : "Client" },
{"range" : "10.0.101.0/24", "type" : "Server" }
],
"ipAddress": [
{"ipAddress" : "10.0.100.1", "type" : "Inside" },
{"ipAddress" : "10.0.101.254", "type" : "Outside" }
]
}
The code in this example is just for demonstrating a approach you can use and it is not conforming to the best practices.
I'm trying to get a sub-document in my User collection using mongoose. I followed the Mongoose Sub Document on its official website. It's written that:
Each document has an _id. DocumentArrays have a special id method for looking up a document by its _id.
var doc = parent.children.id(id);
Here is my code:
exports.editAccount = function(req, res) {
var user = new User(req.user);
var newAccount = new Account(req.body);
console.log("Account:" + newAccount._id); // Gave me 53bf93d518254f880c000009
var account = user.accounts.id(newAccount._id);
console.log("Account" + account); // Never been printed
};
The console.log("Account" + account); has never been printed. I don't know what happen. I tried many different ways, however, still can't figure it out. Any help would be appreciated.
User collection:
{
"__v" : 1,
"_id" : ObjectId("53bcf3e6fbf5adf10c000001"),
"accounts" : [
{
"accountId" : "123456789",
"type" : "Saving account",
"balance" : 100,
"_id" : ObjectId("53bf93d518254f880c000009")
}
]
}
I
Not too sure how you have defined your Schema or basically even model instances, but really all you need is this:
var accountSchema = new Schema({
"accountId": String,
"type": { "type": String, "enum": ["Saving Account", "Checking Account"] },
"balance": { "type": Number, "default": 0 }
]);
var userSchema = new Schema({
"accounts": [accountSchema]
]);
var User = mongoose.model( "User", userSchema );
Then when you want to add an account to the User you just do, presuming you have input that matches the first variable declaration:
var input = {
"accountId": "123456789",
"type": "Savings Account",
};
User.findByIdAndUpdate(
userId,
{ "$push": { "accounts": input } },
function(err,user) {
// work with result in here
}
);
That does bypass things like validation and other hooks, but is more efficient in communicating with MongoDB.
If you really need the validation and/or other features then you and using a .find() variant and issuing a .save() method.
User.findById(userId,function(err,user) {
if (err) throw err; // or handle better
user.accounts.push( input );
user.save(function(err, user) {
// more handling
});
]);
And to modify the document then you are doing much the same. Either by the most efficient MongoDB way:
var input = {
accountId: "123456789",
amount: 100
};
User.findOneAndUpdate(
{ "_id": userId, "accounts.accountId": input.accountId },
{ "$inc": { "accounts.$.balance": input.amount } },
function(err,user) {
// handle result
}
);
Or again where you need the Mongoose hooks and or validation to apply:
User.findById(userId,function(err,user) {
if (err) throw err; // or handle otherwise
user.accounts.forEach(function(account) {
if ( account.accountId === input.accountId )
account.balance += input.balance;
});
user.save(function(err,user) {
// handle things
});
);
Remember that these things are "arrays", and you can either handle them the MongoDB way or the JavaScript way. It just depends on where you choose to "validate" your input.
More code to illustrate where the usage is not correct:
var async = require('async'),
mongoose = require('mongoose'),
Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/child');
var accountSchema = new Schema({
"accountId": String,
"type": { "type": String },
"balance": { "type": Number, "default": 0 }
});
var userSchema = new Schema({
"accounts": [accountSchema]
});
var User = mongoose.model( "User", userSchema );
async.waterfall([
function(callback) {
User.create({},function(err,user) {
if (err) throw err;
console.log(
"Created:\n%s\n",
JSON.stringify( user, undefined, 4 )
);
callback(null,user);
});
},
function(user,callback) {
var account = user.accounts.create({
"accountId": "123456789",
"type": "Savings"
});
console.log(
"Account is:\n%s\n",
JSON.stringify( account, undefined, 4 )
);
console.log(
"User is still:\n%s\n",
JSON.stringify( user, undefined, 4 )
);
user.accounts.push( account );
console.log(
"User Changed:\n%s\n",
JSON.stringify( user, undefined, 4 )
);
User.findById(user.id,function(err,saved) {
if (err) throw err;
console.log(
"Persisted is still:\n%s\n",
saved
);
user.save(function(err,user) {
if (err) throw err;
callback(null,user,account);
});
});
},
function(user,account,callback) {
User.findById(user.id,function(err,saved) {
if (err) throw err;
console.log(
"Persisted is now:\n%s\n",
saved
);
var item = user.accounts.id(account.id);
console.log(
"Item is:\n%s\n",
item
);
callback();
});
}
],function(err) {
process.exit();
});
Results:
Created:
{
"__v": 0,
"_id": "53c08ab51083d1fe3852becc",
"accounts": []
}
Account is:
{
"accountId": "123456789",
"type": "Savings",
"_id": "53c08ab51083d1fe3852becd",
"balance": 0
}
User is still:
{
"__v": 0,
"_id": "53c08ab51083d1fe3852becc",
"accounts": []
}
User Changed:
{
"__v": 0,
"_id": "53c08ab51083d1fe3852becc",
"accounts": [
{
"accountId": "123456789",
"type": "Savings",
"_id": "53c08ab51083d1fe3852becd",
"balance": 0
}
]
}
Persisted is still:
{ _id: 53c08ab51083d1fe3852becc, __v: 0, accounts: [] }
Persisted is now:
{ _id: 53c08ab51083d1fe3852becc,
__v: 1,
accounts:
[ { accountId: '123456789',
type: 'Savings',
_id: 53c08ab51083d1fe3852becd,
balance: 0 } ] }
Item is:
{ accountId: '123456789',
type: 'Savings',
_id: 53c08ab51083d1fe3852becd,
balance: 0 }