I need a data filtered by date but I am getting an error
Error ValidationException: The provided key element does not match the schema
My table has a primary key (only partition key) of id.
async function fetchDatafromDatabase() { // get method fetch data from dynamodb
var date = todayDate();
var params = {
TableName: table,
Key: {
"date": date
}
};
let queryExecute = new Promise((res, rej) => {
dynamoDB.get(params, function (err, data) {
if (err) {
console.log("Error", err);
rej(err);
} else {
console.log("Success! get method fetch data from dynamodb");
res(JSON.stringify(data, null, 2));
}
});
});
const result = await queryExecute;
console.log(result);
}
For getting an item from DynamoDB, we must pass primary key, in this case, its just partition key 'id' (assuming it is numeric and storing epoc date)
var documentClient = new AWS.DynamoDB.DocumentClient();
var date = Date.now();
console.log("date", date);
var params = {
TableName: "test2",
Key: {
id: date,
},
};
documentClient.get(params, function (err, data) {
if (err) {
console.log("Error", err);
} else {
console.log("Success", data);
}
});
Complete Example to put an item and get it.
var documentClient = new AWS.DynamoDB.DocumentClient();
var date = Date.now();
documentClient.put(
{
TableName: "test2",
Key: {
id: date,
},
},
function (err, data) {
if (err) console.log("err", err);
if (data) {
documentClient.get(
{
TableName: "test2",
Key: {
id: date,
},
},
function (errGet, dataGet) {
if (errGet) {
console.log("Error", errGet);
} else {
console.log("Success", dataGet);
}
}
);
}
}
);
Related
I have written this function to do update in dynamo table
const updateTask = async (req, res) => {
try {
const { existingTaskText,updatedTaskText } = req.body;
console.log(existingTaskText,updatedTaskText );
UPDATE({
TableName: "todos",
Key:{ task: existingTaskText},
UpdateExpression:"set task = :task",
ExpressionAttributeValues: {":task": updatedTaskText},
});
res.status(200).json({ data: "this is controller" });
} catch (error) {
res.status(400).json({ message: error.message });
}
};
this is calling UPDATE
const UPDATE = async (payload) => {
try {
console.log(payload);
const updateDoc = await dbClient
.update({
TableName: payload.TableName,
Key: payload.Key,
UpdateExpression: payload.UpdateExpression,
ExpressionAttributeNames:payload.ExpressionAttributeNames,
ReturnValues: "UPDATED_NEW",
})
.promise();
console.log(updateDoc);
} catch (error) {
console.log(error);
}
};
When I am testing this in postman, I am getting this error
ValidationException: Invalid UpdateExpression: An expression attribute value used in expression is not defined; attribute value: :task
this is payload log getting passed
{
TableName: 'todos',
Key: { task: 'see its done' },
UpdateExpression: 'set task = :task',
ExpressionAttributeValues: { ':task': 'edited' }
}
I made below common functions for the update, get, and create a table.use the same.
const AWS = require('aws-sdk');
AWS.config.update({ region: "us-east-1",accessKeyId : process.env.AWS_ACCESS_KEY_ID, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY });
const dynamoDB = new AWS.DynamoDB()
const documentClient = new AWS.DynamoDB.DocumentClient();
const Dynamo = {
async get(id, TableName) {
const params = {
TableName,
Key: {
id,
},
};
const data = await documentClient.get(params).promise();
if (!data || !data.Item) {
throw Error(`There was an error fetching the data for ID of ${id} from ${TableName}`);
}
console.log(data);
return data.Item;
},
async getall(TableName) {
const params = {
TableName: TableName,
};
const data = await documentClient.scan(params).promise();
if (!data || !data.Item) {
throw Error(`There was an error fetching the data for ID of ${ID} from ${TableName}`);
}
console.log(data);
return data.Items;
},
async getMany(params) {
const data = await documentClient.scan(params).promise();
console.log(data);
if (!data || !data.Items) {
throw Error(`There was an error fetching the data`);
}
return data.Items;
},
async write(data, TableName) {
console.log('write dynamo',data, TableName);
if (!data.id) {
throw Error('no ID on the data');
}
const params = {
TableName,
Item: data,
};
const res = await documentClient.put(params).promise();
if (!res) {
throw Error(`There was an error inserting ID of ${data.id} in table ${TableName}`);
}
console.log('res of write dynamo ',res);
return data;
},
async createTable(TableName) {
documentClient
.scan({
TableName: TableName,
})
.promise()
.catch(error => {
return new Promise(resolve => {
dynamoDB
.createTable({
AttributeDefinitions: [
{
AttributeName: "id",
AttributeType: "S",
},
],
KeySchema: [
{
AttributeName: "id",
KeyType: "HASH",
},
],
BillingMode: "PAY_PER_REQUEST",
TableName: TableName,
})
.promise()
.then(data => console.log("Success!", data))
.catch(console.error)
})
});
},
};
module.exports = Dynamo;
When you call the dbClient.update method, you are declaring the parameter ExpressionAttributeNames. It should be ExpressionAttributeValues. This is why the error message indicates that expression attribute value used in expression is not defined.
So you can try it changing the dbClient.update call in this way:
const updateDoc = await dbClient
.update({
TableName: payload.TableName,
Key: payload.Key,
UpdateExpression: payload.UpdateExpression,
ExpressionAttributeValues:payload.ExpressionAttributeValues,
ReturnValues: "UPDATED_NEW",
})
.promise();
Here as you are setting 'ExpressionAttributeNames:', you have to set 'ExpressionAttributeValues' as well.
I have one question about a problem that I'm not able to fix. I try to update push a string passed via Query in my mongoose collection.
My collection are like this:
{
"_id": {
"$oid": "6199288597e42bf84d017f9e"
},
"name": "Lisa",
"surname": "Bianchi",
"ID_school": "afbH598U3",
"classes": [
{
"class": "5A",
"activities": {
"in_progress": [],
"finisched": []
},
"_id": {
"$oid": "6199288597e42bf84d017f9f"
}
},
{
"class": "1A",
"activities": {
"in_progress": [],
"finisched": []
},
"_id": {
"$oid": "6199288597e42bf84d017fa0"
}
}
],
"email": "insegnante#a.com",
"__v": 0
}
and I try to push a string in in_progress array that match, for example, with class:"5A" using this way:
import db from "../models/index.js";
const Teacher = db.teacher
const updateActivity = (req, res) => {
const query = { _id: req.query.id};
const update = {$push:{'classes.$[group].activities.in_progress': req.query.data } };
const options = {arrayFilters: { 'group.class': req.query.class }};
Teacher.findOneAndUpdate(query, update, options).exec((err, data) => {
if (err) {
res.status(400).send({ message: err });
return;
} else {
res.status(200).send(data);
}
})
}
const API = {
updateActivity
}
export default API
The query works fine, but nothing was pushed. I tested whit Insomnia passing in the Query field
id = 6199288597e42bf84d017f9e;
class:'5A';
data:"pushed"
Any suggestion? Thanks!
try this way by passing classes.class in query and also change push to $push:{'classes.$.activities.in_progress': req.query.data }
const updateActivity = (req, res) => {
const query = { _id: req.query.id ,'classes.class': req.query.class};
const update = {$push:{'classes.$.activities.in_progress': req.query.data } };
Teacher.updateOne(query,update).exec((err, data) => {
if (err) {
res.status(400).send({ message: err });
return;
} else {
res.status(200).send(data);
}
})
}
There are two ways of doing this:
Option 1: arrayFilters - more flexible Docu
The option you are using.
You have a syntax error - arrayFilters should be an array of documents.
const updateActivity = (req, res) => {
const query = { _id: req.query.id };
const update = {
$push:{ 'classes.$[group].activities.in_progress': req.query.data }
};
// This MUST be an array of filter documents!
const options = { arrayFilters: [{ 'group.class': req.query.class }] };
Teacher
.findOneAndUpdate(query, update, options)
.exec((err, data) => {
if (err) {
res.status(400).send({ message: err });
return;
} else {
res.status(200).send(data);
}
});
}
Option 2: Via Query (as answered by #Saurabh Mistry)
Repeating his answer for completeness
By specifying a query that targets a particular element in an array within result documents.
const updateActivity = (req, res) => {
const query = {
_id: req.query.id,
'classes.class': req.query.data,
};
const update = {
$push:{ 'classes.$.activities.in_progress': req.query.data }
};
Teacher
.findOneAndUpdate(query, update, options)
.exec((err, data) => {
if (err) {
res.status(400).send({ message: err });
return;
} else {
res.status(200).send(data);
}
});
}
I'm trying to format the data i got from YouTube Data API v3 but i'm unable to change anything of it.
const videoIds = youtubeResponse.items.map(item => item.id);
VideoRepo.getById(videoIds, (err, videos) => {
/*
videos is an array of objects that contain youtube videos from YT API and MongoDB(mongoose)
*/
console.log(videos.map((v) => {
v.time = moment(v.time).fromNow();
v.duration = moment('1900-01-01 00:00:00').seconds(v.duration).format('HH:mm:ss');
return v;
}));
});
VideoRepo class:
static getById(id, callback) {
if (Array.isArray(id)) {
// Multiple ids were specified
async.waterfall([
(next) => {
// Get existing videos' data
Video.find({ _id: { $in: id } }).select('-__v').sort({ createdAt: 1 }).exec((err, data) => {
if (err) return next(err);
next(null, data);
});
},
(existingData, next) => {
if (existingData.length === 0) {
// All videos are new, skip to the next step
return next(null, [], id);
}
// Remove existing data from ID array
const obj = existingData.map(el => el._id);
next(null, existingData, id.filter(el => !obj.includes(el)));
},
(existingData, newIDs, next) => {
if (newIDs.length === 0) {
return next(null, existingData);
}
// Get new videos' data from YT API
youtube.videos.list({ id: newIDs.join(','), part: 'snippet,contentDetails,statistics' }, (err, videoResp) => {
if (err) return next(err);
// Final data
const data = id;
// New videos' data
const newData = videoResp.data.items.map(item => this.fixVideoData(item));
// Add new videos to the DB
Video.insertMany(newData, (err) => {
if (err) return next(err);
// Merge new data with existing data
const merged = existingData.concat(newData);
// Fix order
for (let i = 0; i < merged.length; i += 1) {
const d = merged[i];
data[data.indexOf(d._id)] = d;
}
// Success!
next(null, data);
});
});
},
], (err, data) => callback(err, data));
}
}
static fixVideoData(videoData) {
const data = {
_id: videoData.id,
channelId: videoData.snippet.channelId,
title: videoData.snippet.title,
description: videoData.snippet.description,
slug: slugify(videoData.snippet.title, { lower: true }),
views: videoData.statistics.viewCount,
duration: moment.duration(videoData.contentDetails.duration).asSeconds(),
tags: videoData.snippet.tags,
thumbnail: null,
preThumbnail: null,
time: videoData.snippet.publishedAt,
};
const possibleThumbs = ['maxres', 'standard', 'high', 'medium', 'default'];
for (let j = 0; j < possibleThumbs.length; j += 1) {
if (Object.prototype.hasOwnProperty.call(videoData.snippet.thumbnails, possibleThumbs[j])) {
data.thumbnail = videoData.snippet.thumbnails[possibleThumbs[j]].url;
break;
}
}
if (videoData.snippet.thumbnails.medium) {
data.preThumbnail = videoData.snippet.thumbnails.medium.url;
} else if (videoData.snippet.thumbnails.high) {
data.preThumbnail = videoData.snippet.thumbnails.high.url;
} else {
data.preThumbnail = data.thumbnail;
}
return data;
}
This is what videos array contains:
// videoData: https://developers.google.com/youtube/v3/docs/videos#resource
{
_id: videoData.id,
channelId: videoData.snippet.channelId,
title: videoData.snippet.title,
description: videoData.snippet.description,
views: videoData.statistics.viewCount,
duration: moment.duration(videoData.contentDetails.duration).asSeconds(),
tags: videoData.snippet.tags,
thumbnail: null,
preThumbnail: null,
time: videoData.snippet.publishedAt,
};
Expected results:
[...{ [..other keys] duration: "00:05:43", time: "3 days ago" }]
Actual output (nothing is changed, exactly the same array as videos):
[...{ [..other keys] duration: 343, time: 2018-12-26T13:37:32.000Z }]
Why is it not working and how can i fix it?
You can return a new object where you override only those 2 specific keys,
video.map(v => ({
...v,
duration: moment('1900-01-01 00:00:00').seconds(v.duration).format('HH:mm:ss'),
time: moment(v.time).fromNow()
}))
So apparently Model.find() returns mongoose documents instead of javascript objects and i should have used Query.lean().
Video.find({ _id: { $in: id } }).lean().select('-__v').sort({ createdAt: 1 })
.exec()
I'm trying to get an item from my DynamoDB but get the following error
ValidationException: The provided key element does not match the
schema
The create item piece of the code works. But no the Get item.
Table Info:
Table Name: movieTable
Primary Partition Key: itemID
Primary Sort Key: sortKey
Here's the code for the create and update:
var fbUserId;
var params;
var keyText;
var attText;
var valText;
var dynamodb = null;
var docClient = null;
var appId = '405140756489952'; //from facebook
var roleArn = 'arn:aws:iam::042765862882:role/Verzosa'; //from AWS IAM
var resultData = null;
document.getElementById('putThis').onclick = function () {
dynamodb = new AWS.DynamoDB({ region: 'us-west-2' });
docClient = new AWS.DynamoDB.DocumentClient({ service: dynamodb });
keyText = document.getElementById("keyValue").value;
attText = document.getElementById("attributeText").value;
valText = document.getElementById("valueText").value;
console.log("Key Value: ", keyText);
console.log("Attribute: ", attText);
console.log("Value: ", valText);
params = {
TableName: 'movieTable',
Item: {
itemID: keyText,
sortKey: valText
}
};
docClient.put(params, function(err, data){
if (err) console.log(err);
else
{
resultData = data;
console.log(resultData);
}
})
};
document.getElementById('getThis').onclick = function () {
dynamodb = new AWS.DynamoDB({ region: 'us-west-2' });
docClient = new AWS.DynamoDB.DocumentClient({ service: dynamodb });
keyText = document.getElementById("keyValue").value;
attText = document.getElementById("attributeText").value;
console.log("Key Value: ", keyText);
console.log("Attribute: ", attText);
params = {
TableName: 'movieTable',
Key: {
itemID: keyText,
},
ProjectionExpression: "#a",
ExpressionAttributeNames: {
'#a': attText
}
};
docClient.get(params, function (err, data)
{
if (err)
{
console.log(err, err.stack);
}
else
{
console.log("success, logging data: ");
console.log(data);//shows keys
console.log("attribute 1 is " + data.Item.sortKey)
//var output = data.Item.attribute1;
l = document.getElementById("output");
l.innerHTML = data.Item.sortKey;
}
})
};
Any help would be appreciated.
You are getting this error because when using AWS.DynamoDB.DocumentClient.get method, you must specify both hash and sort key of an item. But you have only hash key specified (itemId), and sort key is missing.
Here is how your get params should look like:
...
params = {
TableName: 'movieTable',
Key: {
itemID: keyText,
sortKey: valText // <--- sort key added
},
ProjectionExpression: "#a",
ExpressionAttributeNames: {
'#a': attText
}
};
docClient.get(params, function (err, data) {
...
If you'd like to get a record with a hash key only, without specifying its sort key, you should use query method instead of get:
...
params = {
TableName: 'movieTable',
KeyConditionExpression: '#itemID = :itemID',
ProjectionExpression: "#a",
ExpressionAttributeNames: {
'#a': attText,
'#itemID': 'itemID'
},
ExpressionAttributeValues: {
':itemID': keyText
}
};
dynamodbDoc.query(params, function(err, data) {
...
Be aware that while get method always returns 1 or no records, query can possibly return multiple records, so you would have to revisit your current implementation of get callback (e.g. instead of accessing data.Item you should use data.Items array, see query method docs)
You need to pass both primary key and sort key in the params.
You can find these keys from the table UI.
and you should pass those as parameters when making the request
this.ProjectsModel.delete({pk1:"project#giri-test#appsc", sk1:"metadata#giri-test#appsc"}, (error) => {
if (error) {
console.error(error);
} else {
console.log("Successfully deleted item");
}
});
const AWS = require('aws-sdk')
const docClient = new AWS.DynamoDB.DocumentClient({ region: 'eu-central-1' })
const createDocument = (text, callback) => {
const createParams = {
Item: {
text: text
},
TableName: 'ToDoItems'
}
docClient.put(createParams, (err, data) => {
if(err) {
callback(err, null)
} else {
callback(null, data)
}
})
}
exports.handle = (event, context, callback) => {
createDocument(event.text, (err, data) => {
if(err) {
callback(err, null)
} else {
callback(null, data)
}
})
}
That's my AWS Lambda function, the issue is that when I get a callback, data object is empty, even though document is inserted into DynamoDB. What could the issue be here?
You can't. You have to separately query. On put, if you set ReturnValues: 'ALL_NEW', then you'll get "ReturnValues can only be ALL_OLD or NONE"
Note the 'possible' in AWS's documentation:
http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB/DocumentClient.html#put-property
ReturnValues — (String) Possible values include:
"NONE"
"ALL_OLD"
"UPDATED_OLD"
"ALL_NEW"
"UPDATED_NEW"
Also, instead of separately querying, you can also just use the params value. If it was saved, then what you have in createParams.Item is basically what's returned if you were to separately query.
There is a workaround - You can use update method of DynamoDB.DocumentClient.
TableName: "table",
Key: {
id: randomId
},
AttributeUpdates: {
authorId: {Action: "PUT", Value: event.authorId},
date: {Action: "PUT", Value: event.date},
description: {Action: "PUT", Value: event.description},
title: {Action: "PUT", Value: event.title}
},
ReturnValues: "ALL_NEW"
This method will create new item and return all what you need
You have to request the return values, like this:
const createParams = {
Item: {
text: text
},
TableName: 'ToDoItems',
ReturnValues: 'ALL_NEW'
}
This is documented here.
I did have to implementing that the return in .then() was params.Item, like this:
var params = {
TableName:table,
Item:{
"name": value,
"email": value2,
}
};
console.info("Adding a new item...");
await docClient.put(params)
.promise()
.then(data => {
return params.Item;
}).catch(error => {
console.error(error)
throw new Error(error)
})