We Keep Coding

How to scan dynamoDB in lambda?

so I have a table of 10 items, each item has about 5 keys (name,experience,level, etc). Now, I want to scan that table, get each item as an object and add it to an array and then JSON stringify that array and return it.
I just need help with the scanning code and getting all items and putting it into an array.
Here's my code I have currently.
var dynamodb = new AWS.DynamoDB.DocumentClient();
exports.handler = function(event, context, callback)
{
var returnArray = {
"cards": {}
}
getCards();
function getCards() {//Not sure how to write this function
var params = {
TableName : "toBeApprovedTable",
Key: {//not sure what to put here, since I want all items, and not searching through keys.
},
};
dynamodb.scan(params,function(err,data)
{
if(err)
{
console.log("error in scanning");
}
else
{
console.log("scanning success!");
//Not sure what to do here.
}
});
}
};

I figured it out after scrapping through Google + AWS docs.
Here is how to scan a table for all elements in the table. My return is a map, which contains an array of elements. Each element is a map of my object.
exports.handler = function(event, context, callback)
{
var returnArray = {
"cardsToBeApproved":[]
};
getCards();
function getCards() {//Not sure how to write this function
var params = {
TableName : "toBeApprovedTable"
};
dynamodb.scan(params,onScan);
}
function onScan(err,data)
{
if(err)
{
console.log("unable to scan table");
}
else
{
console.log("scan succeeded");
data.Items.forEach(function(card)
{
console.log("my card name is " + card.name);
var cardStringified = JSON.stringify(card);
returnArray.cards.push(card);
});
callback(null,JSON.stringify(returnArray));
}
}
};

Integrating asynchronous mongo call within an inner forEach loop

I got two loops, the outer loops over the users and the inner one loops over the venueID's of each user. Within the inner loop I want to look up the venue and attach it to an array defined in the outer look (userItem). However because forEach is synchronous and the mongo database look up is asynchronous the result always remains empty. I've tried to integrate this answer but to no avail. How to do this?
ret = [];
users.forEach(function(user) {
var userItem = user.getSanitised('ADM');
userItem.venues = [];
var tmp = [];
userItem.adminVenueIds.forEach(function(adminVenueId){
tmp.push(function(callback) {
Venue.findOne({_id:adminVenueId}, function(error, venue) {
callback(null, venue.toObject());
});
});
});
async.parallel(userItem.venues, function(err, result) {
/* this code will run after all calls finished the job or
when any of the calls passes an error */
if (err)
return console.log(err);
userItem.venues.push(result);
});
ret.push(userItem);
});
Tried the following as well but doesn't work also
users.forEach(function(user) {
var userItem = [];
async.series({
setUserItem : function(callback)
{
userItem = user.getSanitised('ADM');
callback(null, 'OK');
},
setUserVenues : function(callback)
{
userItem.venues = [];
user.adminVenueIds.forEach(function(adminVenueId,index) {
Venue.findOne({_id:adminVenueId}, function(error, venue) {
userItem.venues.push(venue.toObject());
if((index+1) == user.adminVenueIds.length)
callback(null, 'OK');
});
});
}
},
function(error, results) {
if(error)
winston.error(error);
ret.push(userItem);
}
);
});

You could simply put an if statement (in your case put the conditional as the array length) then when the loop is done you could then make it continue doing its thing by calling a function to continue (or put your code in there, but it will start to look messy)
var ret = [];
var test = [];
for (var i = 0; i < 20; i++) {
for (var x = 0; x < 20; x++) {
setTimeout(function() {
test.push("Test"+x);
if (x === 20) {
finishIt();
}
}, 300)
}
}
function finishIt() {
console.log(test);
ret.push(test);
}

I think you might want to look into using Mongoose. It is a NodeJS application layer on top of MongoDB that provides a more SQL like experience.
http://mongoosejs.com

I ended up with the following solution. It's dirty but I guess that's just nodejs being nodejs.
users.forEach(function(user) {
var userItem = user.getSanitised('ADM');
userItem.venues = [];
user.adminVenueIds.forEach(function(adminVenueId) {
Venue.findOne({_id:adminVenueId}, function(error, venue) {
userItem.venues.push(venue.toObject());
});
});
(function(){
if(userItem.venues.length == user.adminVenueIds.length) {
ret.push(userItem);
} else {
setTimeout(arguments.callee, 30);
}
})();
});

node.js return data from find collection in loop

I am trying to return data from this function. Console.log(documents) successfully shows the data in console. But this works only in body of the function. I can't return this data to the template. What should I do? Should I use some async package for node.js, or can be accomplished somehow like this?
Thank you.
var projects = req.user.projects;
var docs = [];
db.collection('documents', function(err, collection) {
for (i = 0; i < projects.length; i++) {
collection.find({'_projectDn': projects[i].dn},function(err, cursor) {
cursor.each(function(err, documents) {
if(documents != null){
console.log(documents);
//or docs += documents;
}
});
});
}
});
console.log(documents); // undefined
res.render('projects.handlebars', {
user : req.user,
documents: docs
});

Those db functions are async, which means that when you try to log it, the function hasn't finished yet. You can log it using a callback, for example:
function getDocuments(callback) {
db.collection('documents', function(err, collection) {
for (i = 0; i < projects.length; i++) {
collection.find({
'_projectDn': projects[i].dn
}, function(err, cursor) {
cursor.each(function(err, documents) {
if (documents !== null) {
console.log(documents);
callback(documents);// run the function given in the callback argument
}
});
});
}
});
}
//use the function passing another function as argument
getDocuments(function(documents) {
console.log('Documents: ' + documents);
});

Using async with multiple functions javascript

How to use async correctly with multiple dependent functions...
Here was my attempt which is not working out, it is within an async.waterfall function:
function (urlsCreated, cb) {
var z, artist, title, added_on;
z = [];
async.mapSeries(urlsCreated, function (url, next) {
scrape_music.total_pages(50, url, function (array, total, extra) {
scrape_music.each(artist, title, added_on, array, url, function (result) {
});
});
}, function (z) {
console.log(z);
});
}
Everything is working fine up to this part...
basically urlsCreated is an array of 2 urls...
Then I called a mapSeries assuming that it will iterate between them...
The way it should be working is, it iterates through each url in the array, then for each url it should get a total page count for the url, and then for each page count which is added to the array parameter/callback of the total_pages, should be iterated within...
so basically the arrays are: urlsCreated (containing 2 links) --> array (containing total pages in total_pages method) --> result (.each method should grab off each page, number of pages included in the array beforehand) and then repeat for amount of urls in urlsCreated...
Any help would be wonderful, currently nothing gets printed for z, and essentially I just want an array filled with objects that are returned in result from scrape_music.each.
EDIT----
Here is the code for those functions.
//loop thrugh each page and find jquery elements that match
Scrape.prototype.each = function (artist, title, added_on, array, urls, cb) {
console.log('entered each');
console.log(array);
var $trs, list;
list = [];
this.page(array, urls, function ($page) {
//$trs selects all the rows from 1-50
$trs = $page('tr').slice(11, -3);
$trs.map(function (i, item) {
var result;
result = {};
result.artist = $page(item).find('td').eq(1).text();
result.title = $page(item).find('td').eq(2).text();
result.added_on = $page(item).find('td').eq(3).text();
list.push(result);
});
cb(list);
});
};
Scrape.prototype.total_pages = function (divide, url, cb) {
return request("" + url + config.url.pageQ + 0, function (err, res, body) {
if (err) { throw err; }
var page, select, match, total, matches, array, extra;
array = [];
page = cheerio.load(body);
select = page('tr').slice(9, 10);
match = page(select).find('td').eq(1).text();
matches = match.slice(-18, -14).trim();
total = Math.round(matches / divide);
extra = matches % divide;
for(x = 0; x < total; x++) {
array.push(x);
}
cb(array, total, extra);
});
};
//used to loop through all pages
Scrape.prototype.page = function (array, urls, cb) {
return array.forEach(function (i) {
return request("" + urls + config.url.pageQ + i, function (err, res, body) {
//console.log(urls + config.url.pageQ + i);
if (err) { throw err; }
cb(cheerio.load(body));
});
});
};

function (urlsCreated, cb) {
var artist, title, added_on;
async.mapSeries(urlsCreated, function (url, next) {
scrape_music.total_pages(50, url, function (array, total, extra) {
// 1:
scrape_music.each(artist, title, added_on, array, url, function (result) {
// 2:
next(null, result);
});
});
}, function (err, z) {
// 3:
console.log(z);
});
}
each() here can't be an iterator (not sure what it does) as you can only call next() for asyncMap once per iteration. If the callback is called when the iterating is done then it's fine
Tell async this iteration is done. The first argument is any error
The second argument is the new array

javascript callback not pushing data up to the top of the calls

I am slowly learning how to use callbacks and I am running into trouble. I think the code below should work byt it is not.
From what I can tell it is not descending in to the recursive function as when it hit another 'tree' entry or outputting anything at the end.
One of the issues I ran into is the callback in the for loop. I was not sure that there is a better way to do it: I just used a counter to see if I was at the end of the loop before calling the callback.
Some guidance would be really appreciated.
(function () {
'use strict';
var objectsList = [];
function makeAJAXCall(hash, cb) {
$.ajaxSetup({
accept: 'application/vnd.github.raw',
dataType: 'jsonp'
});
$.ajax({
url: hash,
success: function (json) {
if (cb) {
cb(json);
}
},
error: function (error) {
console.error(error);
throw error;
}
});
}
function parseBlob(hash, cb) {
makeAJAXCall(hash, function (returnedJSON) { // no loop as only one entry
if (cb) {
cb(returnedJSON.data);
}
});
}
function complete(cb, loopLength, treeContents) {
concole.info(loopLength);
if (cb && loopLength === 0) {
objectsList.push(treeContents);
cb();
}
}
function parseTree(hash, treeName, cb) {
var treeContents = {'tree': treeName, 'blobs': []}, loopLength, i, entry;
var tree = 'https://api.github.com/repos/myusername/SVG-Shapes/git/trees/' + hash;
makeAJAXCall(tree, function (returnedJSON) {
loopLength = returnedJSON.data.tree.length;
for (i = 0; i < returnedJSON.data.tree.length; i += 1) {
entry = returnedJSON.data.tree[i];
if (entry.type === 'blob') {
if (entry.path.slice(-4) === '.svg') { // we only want the svg images not the ignore file and README etc
parseBlob(entry.url, function (json) {
treeContents.blobs.push(json.content);
loopLength -= 1;
complete(hash, loopLength, cb);
});
}
} else if (entry.type === 'tree') {
parseTree(entry.sha, entry.path, function () {console.info(objectsList);});
}
}
});
}
$(document).ready(function () {
parseTree('master', 'master', function () { // master to start at the top and work our way down
console.info(objectsList);
});
});
}());

Your recursion will not work properly because the variable where you are collecting blob data treeContents is a local variable of the recursive function so it's created new and destroyed on every call to parseTree(). It will not be accumulating data. You must create this variable outside the scope of the parseTree() function so a single instance of this variable can live from one call to the next and you can correctly accumulate data in it.
There are several ways to fix this:
You can pass the current state of treeContents into the parseTree() function
You can make the recursive part of the function be a local function that shares a common treeContents variable.
You can make the treeContents variable be a global variable.
The second is my choice here something like this:
function parseTree(topHash, topTreeName, topCb) {
var treeContents = {'tree': toptreeName, 'blobs': []};
function parse(hash, treeName, cb) {
var loopLength, i, entry;
var tree = 'https://api.github.com/repos/myusername/SVG-Shapes/git/trees/' + hash;
makeAJAXCall(tree, function (returnedJSON) {
loopLength = returnedJSON.data.tree.length;
for (i = 0; i < returnedJSON.data.tree.length; i += 1) {
entry = returnedJSON.data.tree[i];
if (entry.type === 'blob') {
if (entry.path.slice(-4) === '.svg') { // we only want the svg images not the ignore file and README etc
parseBlob(entry.url, function (json) {
treeContents.blobs.push(json.content);
loopLength -= 1;
complete(hash, loopLength, cb);
});
}
} else if (entry.type === 'tree') {
parse(entry.sha, entry.path, function () {console.info(objectsList);});
}
}
});
}
parse(topHash, topTreeName, topCb);
}
Assuming the ajax call is asynchronous, you will still need to find a way to know when you're done with all the parsing and call some function that you pass the treeContents data to because otherwise, that data is not available to any other function. It can't be simply returned from parseTree because of the asynch nature of the ajax calls.