Amazon S3 Node.js SDK deleteObjects - javascript

I am trying to delete several objects after copying them to a different folder.
My code is like:
var deleteParam = {
Bucket: 'frontpass-test',
Delete: {
Objects: [
{Key: '1.txt'},
{Key: '2.txt'},
{Key: '3.txt'}
]
}
};
s3.deleteObjects(deleteParam, function(err, data) {
if (err) console.log(err, err.stack);
else console.log('delete', data);
});
and the returned data is:
delete { Deleted: [ { Key: '1.txt' }, { Key: '3.txt' }, { Key: '2.txt' } ],
Errors: [] }
so I assume the deletion is completed. But the objects are still exist in the folder, is there something wrong with my code?
I also tried to delete objects using for loop and s3.deleteObject, but it only delete the last object in my list of files.
for (var i = 0; i < files.length; i++) {
var copyParams = {
Bucket: 'frontpass-test',
CopySource: 'frontpass-test/unsold/' + files[i].filename,
Key: 'sold/' + files[i].filename
};
var deleteParam = {
Bucket: 'frontpass-test',
Key: 'unsold/' + files[i].filename
};
s3.copyObject(copyParams, function(err, data) {
if (err) console.log(err, err.stack);
else {
s3.deleteObject(deleteParam, function(err, data) {
if (err) console.log(err, err.stack);
else console.log('delete', data);
});
}
});
}
Any idea on how to delete objects in my case? Thanks in advance.

Well the first example looks good. Do you have object versioning turned on in the bucket? That would keep a copy of a file even after you delete it.
The second example actually contains some bugs that would explain why only the last one gets deleted. Because Node.js is asynchronous, when you hit the copyObject function call, the loop iteration ends and goes to the next iteration, not waiting for the callback on copyObject to be called. You try to define the params variables for each iteration of the loop with the var keyword, but because Javascript has function level scope not block level scope, you aren't actually creating new variables on each iteration. You only have one instance of copyParmas and deleteParams. So you quickly run through the loop and deleteParams stays on the value it receives in the last iteration of the loop. Then eventually the callbacks to the copyObject calls start firing, and they all call deleteObject with deleteParams which by now is the last one. In order to make multiple asynchronous calls in a loop, I like to use the async library. Using it, you could do the following:
async.each(files, function iterator(file, callback) {
var copyParams = {
Bucket: 'frontpass-test',
CopySource: 'frontpass-test/unsold/' + file.filename,
Key: 'sold/' + file.filename
};
var deleteParam = {
Bucket: 'frontpass-test',
Key: 'unsold/' + file.filename
};
s3.copyObject(copyParams, function(err, data) {
if (err) callback(err);
else {
s3.deleteObject(deleteParam, function(err, data) {
if (err) callback(err)
else {
console.log('delete', data);
callback();
}
});
}
});
}, function allDone(err) {
//This gets called when all callbacks are called
if (err) console.log(err, err.stack);
});

Just had to implement folder rename on top of s3, I did it as follows: (promise api)
_getDataForItemRename(from, to) {
return s3.listObjectsV2({Bucket: services.conf.workspace, Prefix: from}).promise()
.then((data) => {
const toCopy = [];
const toRemove = [];
const s3Contents = Object.assign([], data.Contents);
// in case of a single dir (with no children)
if (s3Contents.length === 0) {
s3Contents.push({Key: from});
}
s3Contents.forEach((item) => {
const copyPromise = s3.copyObject({
Bucket: services.conf.workspace,
Key: to,
CopySource: services.conf.workspace + '/' + item.Key
}).promise();
const deletePromise = s3.deleteObjects({
Bucket: services.conf.workspace,
Delete: {Objects: [{Key: from}]}
}).promise();
toCopy.push(copyPromise);
toRemove.push(deletePromise);
});
return {copy: toCopy, remove: toRemove};
}).catch((err) => {
return Promise.reject(err);
});
}
return this._getDataForItemRename(_from, _to).then((files) => {
return Promise.all(files.copy).then(() => {
return Promise.all(files.remove).then(result => {
return result;
});
});
}).catch((err) => {
return Promise.reject(err);
});

Related

value not setting in the variable which is defined outside in NodeJS

I'm new to NodeJS so please apologize if below code is not up-to the standard. I would like to access isSuccess value outside of this function stepfunctions.listExecutions
I tried below code but I'm getting the value is undefined not getting the expected output. I did some internet search and came to know in NodeJS we can't set the value which is defined in globally but I've use case and I'm pretty sure this is a common case for others too - where I would like to access this isSuccess value after my execution.
const AWS = require('aws-sdk');
const stepfunctions = new AWS.StepFunctions({
region: process.env.AWS_REGION
});
var params = {
stateMachineArn: 'arn:aws:states:us-west-1:121:stateMachine:test',
maxResults: '2',
nextToken: null,
statusFilter: 'SUCCEEDED'
};
var isSuccess
stepfunctions.listExecutions(params, function (err, data) {
if (err) console.log(err, err.stack);
else
data.executions.forEach(function (result) {
let params = {
executionArn: result.executionArn
};
stepfunctions.describeExecution(params, function (err, data) {
if (err) console.log(err, err.stack);
else {
isSuccess = 'true'
}
});
});
console.log('isSuccess: ' +isSuccess)
});
Expected output:
isSuccess: true
But I'm getting
isSuccess: undefined
Could you please help me to resolve this issue. Appreciated your help and support on this.
This is how you can wrap it on promise
let isSuccess;
const listExecute = function(params) {
return new Promise((resolve, reject) => {
stepfunctions.listExecutions(params, function (err, data) {
if (err) reject(err);
else
data.executions.forEach(function (result) {
let params = {
executionArn: result.executionArn
};
stepfunctions.describeExecution(params, function (err, data) {
if (err) reject(err);
else {
resolve(true)
}
});
});
});
})
}
async function getOutout(params) {
try {
isSuccess = await listExecute(params);
console.log(isSuccess, 'Output')
} catch(e) {
console.log(e)
}
}
getOutout(params)
Also you can export the listExecute so that you can use this function outside of this file.
module.exports = {listExecute}

Node.js - Cannot append global variable when using fs

Im trying to read multiple xml files and parse data from them and i managed to do that but now new problem appeared.
allData variable is never changed, no matter what i do. What am i supposed to do here?
I dont know what to do or what to try, this is my first time working with files and im honestly surprised ive managed to come this far.
var parseString = require('xml2js').parseString;
var fs = require('fs')
var allData = {
store: []
}
function readFiles(__dirname, onFileContent, onError) {
fs.readdir(__dirname + '\\parse\\', function (err, filenames) {
if (err) {
return;
}
filenames.forEach(function (filename) {
console.log(filename)
fs.readFile(__dirname + '\\parse\\' + filename, 'utf-8', function (err, content) {
if (err) {
console.log(err)
return;
}
parseString(content, function (err, result) {
let tempObj = {}
let data = result.storeD[0]
if (data.name) {
tempObj['name'] = data.name[0];
}
if (data.price) {
tempObj['price'] = data.price[0];
}
//more of the same type of code
console.log(tempObj)
//output: { name: 'Data1', price: '1000' }
allData.store.push(tempObj)
})
})
})
});
console.log("All data: ",allData)
//Outputs once at the begining
//output: All data: { store: [] }
}
readFiles(__dirname)
SOLVED
adjusted code to use.readFileSync()(removed callback function) and now it works.
var parseString = require('xml2js').parseString;
var fs = require('fs')
var allData = {
store: []
}
function readFiles(__dirname, onFileContent, onError) {
fs.readdir(__dirname + '\\parse\\', function (err, filenames) {
if (err) {
return;
}
filenames.forEach(function (filename) {
console.log(filename)
let file = fs.readFileSync(__dirname + '\\parse\\' + filename, 'utf-8')
parseString(file, function (err, result) {
let tempObj = {}
let data = result.storeD[0]
if (data.name) {
tempObj['name'] = data.name[0];
}
if (data.price) {
tempObj['price'] = data.price[0];
}
//more of the same type of code
console.log(tempObj)
//output: { name: 'Data1', price: '1000' }
allData.store.push(tempObj)
})
})
console.log("All data: ",allData)
});
//Outputs once at the begining
//output: All data: { store: [] }
}
readFiles(__dirname)
The .readdir() and .readFile() methods are async, so in fact the console.log() is executed before all of the readFile operations.
In order to access the allData variable after these operations are complete, you have to either make them sync using .readFileSync() instead or you need to promisify the .readFile() method and wait for all of the promises to resolve.

Javascript code not running in order while getting data from database

I have two functions as shown below. It is essentially just getting data from a database.
function processRequest(query){
let dynamoData = getDynamoData(query);
console.log('abc')
}
function getDynamoData(key){
var params = {
TableName: 'test_table',
Key: {
'TWEET_KEY' : {S: String(key)}
}
};
// Call DynamoDB to read the item from the table
ddb.getItem(params, function(err, data) {
if (err) {
console.log("Error");
} else {
console.log("Successfully got data from table")
return data.Item;
}
});
}
Currently when I run the code, within the console it prints the following:
abc
Successfully got data from table
However, I need it to print Successfully got data from table before it prints abc.
I know I have to possibly use async within the function however am really struggling to get the code running in order. Would really appreciate it if anyone could help me get the code running in order. Thank you!
You should move both functions into a separate module (if this wasn't done yet) and make them async ones like this:
async function processRequest(query){
let dynamoData = await getDynamoData(query);
console.log('abc')
}
async function getDynamoData(key){
var params = {
TableName: 'test_table',
Key: {
'TWEET_KEY' : {S: String(key)}
}
};
return new Promise((resolve, reject) => {
// Call DynamoDB to read the item from the table
ddb.getItem(params, function(err, data) {
if (err) {
console.log("Error");
reject(err);
} else {
console.log("Successfully got data from table")
resolve(data.Item);
}
});
});
}
You need to make processRequest asynchronous:
async function processRequest(query){
let dynamoData = await getDynamoData(query);
console.log('abc')
}
function getDynamoData(key){
var params = {
TableName: 'test_table',
Key: {
'TWEET_KEY' : {S: String(key)}
}
};
// Call DynamoDB to read the item from the table
ddb.getItem(params, function(err, data) {
if (err) {
console.log("Error");
} else {
console.log("Successfully got data from table")
return data.Item;
}
});
}

array push results in empty array javascript

I am trying to store API results into an array.
The data is displayed in console, but on pushing the data into an array, the array is still empty.
Here's the code:
app.post('/fetchFavoriteTweets/', verifyToken, function(req, res) {
var favorites = [];
dbConn.then( function (database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne(
{ _id: ObjectId(req.userId) }, function(err, result) {
if(err) throw err;
if(!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({msg:'record not found'});
}
else {
result.favorite_tweets.forEach(function (tweet) {
T.get('statuses/show', {id: tweet.id}, function(err, data, response) {
if(!err){
favorites.push(data);
console.log(data); //this returns data
} else {
console.log(err);
}
});
});
console.log(favorites);
// res.status(200).json({msg:'success', data:favorites});
}
});
}).catch(function(e){console.log(e)})
});
It looks like you're defining the favorites array within the scope of the function callback. Try putting var favorites = []; above you app.post() call instead.
Also, keep in mind that it will only have a value after the callback is complete, so any synchronous code later down the line will only see the empty array value.
I've updated your code to get favorites by storing separately the promise and call it afterwards:
UPDATE
As you can see in the demo, i have 2x console.log at the bottom, the first one(C1) is contained in the promise favoritesPromise () and the second (C2) is after the promise.
Synchronous actions will never wait for asynchronus actions to take place, therefore in my example C2 will always be outputted before C1, even if console.log(1 ... ) is before console.log(2 ... ), they'll appear reversed in the console.
In the promise i added a setTimeout of 1ms to mock a request, it was all it took to achieve the current output. Another thing you can test is removing the setTimeout then output will change a bit, your promise becomes synchronus until it reaches resolve(favorites), that means favorites has all the data by now, but when resolve takes place, it becomes async, and in your console you will still see C2 first (but now with data) and C1 second.
In my earlier answer i tried to implement this reasoning to your code.
Keep it async folks!
var favorites = [];
var favoritesPromise = () => {
return new Promise((resolve, reject) => {
console.log('Retrieving data from the internet.');
// This timeout mocks your request to anything that is async or promie
setTimeout(() => {
console.log('Request done')
let resultFavorite_tweets = [{
id: 1,
name: 'a dog'
}, {
id: 2,
name: 'a cat'
}];
resultFavorite_tweets.forEach(item => {
favorites.push(item.name);
})
resolve(favorites);
// if you have an error use
// reject(err)
}, 1);
});
}
favoritesPromise().then(favList => {
console.log(1, 'this will always contain data from the internet, but will always be last', favList);
})
console.log(2, 'this will be empty (unless you remove setTimeout), but will always be first', favorites);
app.post('/fetchFavoriteTweets/', verifyToken, function(req, res) {
const favoritesPromise = () => {
return new Promise((resolve, reject) => {
var favorites = [];
dbConn.then(function(database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne({
_id: ObjectId(req.userId)
}, function(err, result) {
if (err) reject(err);
if (!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({
msg: 'record not found'
});
} else {
result.favorite_tweets.forEach(function(tweet) {
T.get('statuses/show', {
id: tweet.id
}, function(err, data, response) {
if (!err) {
favorites.push(data);
console.log(data); //this returns data
} else {
console.log(err);
reject(err);
}
});
resolve(data);
});
console.log(favorites);
// res.status(200).json({msg:'success', data:favorites});
}
});
}).catch(function(e) {
reject(e)
})
});
}
// Here you call the promise to retrieve "favorites"
favoritesPromise().then(favoritesList => {
console.log('your favorites array', favoritesList)
})
})
Try next code
app.post('/fetchFavoriteTweets/', verifyToken, function (req, res) {
var favorites = [];
dbConn.then(function (database) {
var dbo = database.db("twitter_search");
dbo.collection('users').findOne(
{ _id: ObjectId(req.userId) }, function (err, result) {
if (err) throw err;
if (!result.hasOwnProperty('favorite_tweets')) {
res.status(404).json({ msg: 'record not found' });
}
else {
// Counter
let count = result.favorite_tweets.length;
result.favorite_tweets.forEach(function (tweet) {
T.get('statuses/show', { id: tweet.id }, function (err, data, response) {
// Decrease count
count -= 1;
if (!err) {
favorites.push(data);
// Check if count is zero
if (count === 0) {
console.log(favorites);
res.status(200).json({msg:'success', data:favorites});
}
} else {
console.log(err);
}
});
});
}
});
}).catch(function (e) { console.log(e) })
});

Callback NodeJS & insert data with mongoose

I know this topic as already asked many times before but I didn't find the right answer to do what I want.
Actually, I try to save two different list of JSON object in MongoDB via Mongoose. To perform both at the same time I use 'async'.
However, when I save it with the command insertMany() I get an error because he calls the callback of async before finishing the insertMany(). Therefore answer[0] is not defined.
What will be the proper way of doing it ?
Here is my code with the async:
const mongoose = require("mongoose");
const async = require("async");
const utils = require("../utils");
const experimentCreate = function(req, res) {
let resData = {};
let experimentList = req.body.experiment;
let datasetList = req.body.datasetList;
async.parallel(
{
dataset: function(callback) {
setTimeout(function() {
answer = utils.createDataset(datasetList);
callback(answer[0], answer[1]);
}, 100);
},
experiment: function(callback) {
setTimeout(function() {
answer = utils.createExp(experimentList);
callback(answer[0], answer[1]);
}, 100);
}
},
function(err, result) {
if (err) {
console.log("Error dataset or metadata creation: " + err);
sendJSONresponse(res, 404, err);
} else {
console.log("Experiment created.");
resData.push(result.dataset);
resData.push(result.experiment);
console.log(resData);
sendJSONresponse(res, 200, resData);
}
}
);
};
Then the two functions called createExp and createDataset are the same in another file. Like this:
const createDataset = function(list) {
let datasetList = [];
for (item of list) {
let temp = {
_id: mongoose.Types.ObjectId(),
name: item.name,
description: item.description,
type: item.type,
};
datasetList.push(temp);
}
Dataset.insertMany(datasetList, (err, ds) => {
if (err) {
console.log("Error dataset creation: " + err);
return [err, null];
} else {
console.log("All dataset created.");
return [null, ds];
}
});
};
There's a few problems with your code. For one, you're not returning anything in your createDataset function. You're returning a value in the callback of insertMany but it doesn't return that value to the caller of createDataset as it's within another scope. To solve this issue, you can wrap your Dataset.insertMany in a promise, and resolve or reject depending on the result of Data.insertMany like this:
const createDataset = function(list) {
let datasetList = [];
for (item of list) {
let temp = {
_id: mongoose.Types.ObjectId(),
name: item.name,
description: item.description,
type: item.type,
};
datasetList.push(temp);
}
return new Promise((resolve, reject) => {
Dataset.insertMany(datasetList, (err, ds) => {
if (err) {
console.log("Error dataset creation: " + err);
reject(err);
} else {
console.log("All dataset created.");
resolve(ds);
}
});
});
};
Now your return object is no longer going to be an array so you won't be able to access both the error and the result via answer[0] and answer[1]. You're going to need to chain a then call after you call createDataset and use callback(null, answer) in the then call (as that means createDataset executed successfully) or use callback(err) if createDataset throws an error like below:
dataset: function(callback) {
setTimeout(function() {
utils.createDataset(datasetList).then(answer => {
callback(null, answer);
}).catch(err => callback(err)); // handle error here);
}, 100);
}
Note: You'll most likely need to alter your createExp code to be structurally similar to what I've produced above if it's also utilizing asynchronous functions.

Categories