We Keep Coding

JavaScript: Converting RSS into JSON in AWS Lambda Node app

I'm currently working on writing a function on AWS Lambda. I want to convert a RSS feed into a JSON and give that as a response in the body when making to the Lambda endpoint.
I'm using an npm package to convert the RSS to JSON. However, when I run the code. I see that I get undefined in the conversion of the RSS URL. Here is the following code:
const feed = require('rss-to-json');
exports.handler = async (event) => {
let rssFeed = event.queryStringParameters.rssFeed;
let rssAsJsonData = convertRssIntoJson(rssFeed);
return sendRes(200, rssAsJsonData);
};
const sendRes = (status, body) => {
var response = {
isBase64Encoded: true|false,
statusCode: status,
headers: {
"Content-Type": "application/json"
},
body: body,
};
return response;
};
function convertRssIntoJson (rssFeed) {
console.log(rssFeed);
return feed.load(rssFeed, function(err, rss){
if(err) {
console.log("Error: ${err}");
return;
}
console.log(rss)
return rss;
});
};
However, in the logs I get undefined when console.log(rssAsJsonData).
However, when debugging I was able to see console.log(rss) working when I change body to be body: json.stringify("TESTING")
However, it only worked when logging to the console not when I tried to pass it to the body body: body, I can't seem to find what the error is. I'm moving from Ruby to JavaScript for this project maybe I'm missing something.
I'm using Postman to make the calls:

function convertRssIntoJson (rssFeed) {
console.log(rssFeed);
return feed.load(rssFeed, function(err, rss){
if(err) {
console.log("Error: ${err}");
return;
}
console.log(rss)
return rss;
});
};
The piece of code above is a callback. Under the hood, feed.load is asynchronous, which makes your callback be executed asynchronously.
Now, when you invoke your function like this
let rssAsJsonData = convertRssIntoJson(rssFeed);
your rss object inside convertRssIntoJson does not hold any value yet, because the callback hasn't been populated up to now. This is where your undefined comes from.
Callbacks themselves don't make code asynchronous by default, but NodeJS works with a non-blocking IO model and, since feed.load is an IO call, it will be executed asynchronously.
You have a few options now, but I will list only two. A not-so-nice and a nice solution:
1) The not-so-nice way to fix it is to add a callback as argument to your convertRssIntoJson function and pass the value of that rss object upstream. The not-so-nice full code can be found below:
const feed = require('rss-to-json');
exports.handler = async (event) => {
let rssFeed = event.queryStringParameters.rssFeed;
convertRssIntoJson(rssFeed, (err, data) => {
if (err) {
return sendRes(500, { message: 'There was an err: ' + err.message })
}
return sendRes(200, data)
})
};
const sendRes = (status, body) => {
var response = {
isBase64Encoded: true | false,
statusCode: status,
headers: {
"Content-Type": "application/json"
},
body: body,
};
return response;
};
const convertRssIntoJson = (rssFeed, callback) => {
console.log(rssFeed);
feed.load(rssFeed, function (err, rss) {
if (err) {
console.log("Error: ${err}");
callback(err, undefined)
}
console.log(rss)
callback(undefined, rss)
});
};
2) The nice, clean, elegant and recommended solution is this one. Wrap your callback in a Promise, like this
function convertRssIntoJson(rssFeed) {
console.log(rssFeed);
return new Promise((res, rej) => {
feed.load(rssFeed, function (err, rss) {
if (err) {
console.log("Error: ${err}");
return rej(err)
}
console.log(rss)
return res(rss)
});
})
};
Since your handler is async, it means it can just await on Promises.
So your client code is now as simple as:
return sendRes(200, await convertRssIntoJson(rssFeed));
Your final code will look like (I have refactored a little bit to make use of arrow functions):
const feed = require('rss-to-json');
exports.handler = async (event) => {
let rssFeed = event.queryStringParameters.rssFeed;
return sendRes(200, await convertRssIntoJson(rssFeed));
};
const sendRes = (status, body) => {
var response = {
isBase64Encoded: true | false,
statusCode: status,
headers: {
"Content-Type": "application/json"
},
body: body,
};
return response;
};
const convertRssIntoJson = (rssFeed) => {
console.log(rssFeed);
return new Promise((res, rej) => {
feed.load(rssFeed, (err, rss) => {
if (err) {
console.log("Error: ${err}");
return rej(err)
}
console.log(rss)
return res(rss)
});
})
};
If you want to know more about async/await, you can see it in here.
EDIT: Code refactor and code added for solution 1)

Figuring the complexity of Promise.all

I have been struggling for quite some time to get this multiple async nodejs request apis to work but unfortunately i am not able to get them work.
Index.js Code:
service.get(
"/restraunts",
versionRoutes({
"1.0.0": getRestrauntsList
})
);
function getRestrauntsList(req, res, next) {
console.log("Started getRestrauntsList");
file1
.appEnvironment(req, res, next)
.then(function(result) {
return file2.getRestrauntsList(req, res, next);
})
.then(function(result) {
res.status(200).send(result);
return;
})
.catch(function(errorResult) {
res.status(500).send(errorResult);
return;
});
}
File2.js
module.exports = {
getRestrauntsList: function(req, res, next) {
console.log("getRestrauntsList started..");
var cities = [1, 2, 3, 4, 5];
let restrauntsList = [];
let urlArray = [];
var restrauntsListPromise = cities.map(function(id) {
return new Promise(function(resolve, reject) {
var options = {
method: "GET",
url: "someurl/" + id + "/restaurants",
headers: {
"AUTH-TOKEN": "TOKEN"
}
};
request(options, function(error, response, body) {
if (error) {
if ("message" in error) {
errorMsg = error.message;
var result = {
status: "error",
message: errorMsg
};
} else {
var result = {
status: "error",
message: "Resource Timeout."
};
}
reject(result);
return promise;
}
console.log(
"Response: " + JSON.stringify(response)
);
if (response.statusCode === 200 || response.statusCode === 201) {
body = JSON.parse(body);
if (body.success) {
let result = {
status: "success",
data: body.result
};
resolve(result);
} else {
let result = {
status: "error",
message: body.error
};
reject(result);
}
} else {
let result = {
status: "error",
message: body.error
};
reject(result);
}
});
});
});
console.log('restrauntsListPromise:' + JSON.stringify(restrauntsListPromise));
Promise.all(restrauntsListPromise).then(function(result) {
var content = result.map(function(restraunts) {
return restrauntsList.push(restraunts.body);
});
// res.send(content);
resolve({
restrauntsList: restrauntsList
});
return promise;
});
},
};
Ideally i expect to get the response of all the apis in the
restrauntsListPromise
and then using Promise.all i should iterate all the promises and formulate my required object.
The response of my code however is
restrauntsListPromise:[{},{},{},{},{}]
and then
Response: {"statusCode":200,"body":"{\"success\":true,\"res
Response: {"statusCode":200,"body":"{\"success\":true,\"res
Response: {"statusCode":200,"body":"{\"success\":true,\"res
Response: {"statusCode":200,"body":"{\"success\":true,\"res
Response: {"statusCode":200,"body":"{\"success\":true,\"res
Ideally what should happen is i should be able to pass the combined result of all the five apis calls as a single object back to the calling promise here
.then(function(result) {
res.status(200).send(result);
return;
})
The problem being the method getRestrauntsList finishes execution and then after some time, i get the responses of the apis.

The problem being the method getRestrauntsList finishes execution and then after some time, i get the responses of the apis.
This is because you're not returning a promise from the getRestrauntsList().
There are few items that needs to addressed to make it work
1. Remove the unused variables
return promise; // both inside promise.all[] and request()
There is no declared variable named promise. So, you can remove it.
2. Accessing .body instead of .data
You're resolving as resolve({status: "success", data: body.result}); But When you are iterating, you are accessing using .body instead of .data. You need to be using .data. Also, you can eliminate restrauntsList array since you're using a .map()
3. Calling resolve() to return values.
You can't use resolve() to return value within Promise.all[] since you didn't create a promise using new Promise((resolve, reject) => { ... });. By default, a return within a promise will be a promise. so, a simple return will suffice. But if you want to be explicit, you can also return using Promise.resolve()
Making those changes,
return Promise.all(restrauntsListPromise).then(function (result) {
return {
restrauntsList: result.map(function (restraunts) {
return restraunts.data;
})
};
//or using Promise.resolve();
// return Promise.resolve({
// restrauntsList: result.map(function (restraunts) {
// return restraunts.data;
// })
// });
});

You are looking for
return Promise.all(restrauntsListPromise).then(function(result) { /*
^^^^^^ */
var contents = result.map(function(restaurants) {
return restaurants.body;
// ^^^^^^^^^^^^^^^^^^^^^^^^
});
return {restaurantsList: contents};
// ^^^^^^
});
You need to return the promise chain from the getRestrauntsList method, you should return the value from the map callback instead of using push on an array, and you will need to return from the then callback - there is no resolve function as you're not inside a new Promise constructor that you only need for callback APIs.

Node js Sequential Request with setTimeout not able to get output array to proccess further

sequential request with synchronous request with
Using Rest Api get method called sequential request of settimeout on 1 sec.
after each request data massaging and put in array.
Now I would like to get this array after all request were done.
async function sequentialMF(mfIds) {
var mfData = [];
try {
var timeout = 0;
return await Promise.all(mfIds.map(mfId => {
console.log(mfId)
var options = {
//https://www.quandl.com/api/v3/datasets/AMFI/103504.json?api_key=WfUR65SA5p1PzpBysgK4
method: 'GET',
url: 'https://www.quandl.com/api/v3/datasets/AMFI/' + mfId + '.json',
qs: { api_key: 'WfUR65SA5p1PzpBysgK4' },
};
setTimeout(function () {
request(options, function (error, response, body) {
var obj = JSON.parse(body);
var mfobj = {
"code": obj.dataset.dataset_code,
"name": obj.dataset.name,
"date": obj.dataset.end_date,
"nav": obj.dataset.data[0][1]
}
mfData.push(mfobj);
console.log(obj.dataset.dataset_code + ' ' + obj.dataset.data[0][1]);
console.log('lngth ' + mfData.length + ' ' + mfIds.length)
})
}, timeout);
timeout += 1000;
new Promise(function (resolve, reject) {
if (mfData.length === mfIds.length) {
console.log('testing')
resolve(mfData);
} else reject('errrrrrrrrrrrrror')
})
}))
} catch (err) {
console.log(err)
}
}

Instead of map and Promise.all it's easier to use reduce and .then to serialize all the request to one result. You can also catch any errors for failed requests so you'll get the result in the end.
//create Fail type instead of rejecting promise
function Fail(details){this.details=details;};
const isFail = item => (item&&item.constructor)===Fail
const sequentialMF = mfIds =>
mfIds.reduce(//reduce ids to one promise
(acc,mfId) =>
acc.then(//acc is the one promise resolving to array of result or Fail items
results => {
var options = {
//https://www.quandl.com/api/v3/datasets/AMFI/103504.json?api_key=WfUR65SA5p1PzpBysgK4
method: 'GET',
url: 'https://www.quandl.com/api/v3/datasets/AMFI/' + mfId + '.json',
qs: { api_key: 'WfUR65SA5p1PzpBysgK4' },
};
return new Promise(//make the request as promise
(resolve,reject)=>
request(options, function (error, response, body) {
if(err){
reject(err);//reject if there is an error
}
var obj = JSON.parse(body);
resolve({//resolve if there is no error
"code": obj.dataset.dataset_code,
"name": obj.dataset.name,
"date": obj.dataset.end_date,
"nav": obj.dataset.data[0][1]
});
})
)
.then(
result=>{//add result item to results if successfull
results.push(result);
return results;
}
)
.catch(
err=>{//add Fail item to results if failed
results.push(new Fail([err,mfId]));
return results;
}
)
.then(
results=>
new Promise(//wait for one second to make the next request
(resolve,reject)=>
setTimeout(
_=>resolve(results)
,1000
)
)
)
}
)
,Promise.resolve([])
)
//example of how to ue this function
sequentialMF(ids)
.then(
results=>{
const successes = results.filter(result=>!isFail(result));
const failed = results.filter(isFail);
}
)

How to use json response to create another URL. Nodejs

I have the question, below code:
The problem is:
How can I send each line from response promiseGetCitiesData to promiseGetInformationDataPerCity.
Can I do it in one async.each functions?
Now, I created multiple Promise functions. One general function, which one start the program - getDataAndCloseDb().
Also I used async.each to call promise function with array parameter - locationArray.
Now, I would like to send each line from json response to next promise function (create get url), and collect the general response.
const MongoClient = require("mongodb").MongoClient;
const request = require("request");
const async = require("async");
var locationsArray = [
'location1',
'location2',
'location3'
];
function promiseConnectToDatabase(urldb) {
return new Promise(function(resolve, reject) {
MongoClient.connect(urldb, (err, db) => {
if (err) {
console.log("MongoDb connection error.");
reject(err);
}
console.log("Connected to MongoDb.");
resolve(db);
});
});
}
function promiseGetCitiesData(location) {
return new Promise(function(resolve, reject) {
request({
url: `https://example.com/${location}`,
json: true
}, (error, response, body) => {
if (error) {
console.log("Error connection to url.");
reject();
}
console.log("location: " + location);
console.log({location: location, cities: body.result.cities});
resolve({location: location, cities: body.result.cities});
});
});
}
/*
Example response from promiseGetCitiesData:
Location: location1
{ location: 'location1',
cities:
[ 'information1',
'information2',
'information3',
'information4'' ] }
*/
function promiseGetInformationDataPerCity(location, cities) {
return new Promise(function(resolve, reject) {
request({
url: `https://example.com/${location}/${cities}`,
//f.e https://example.com/location1/information1 etc.
json: true
}, (error, response, information) => {
if (error) {
console.log("Error connection to url.");
reject();
}
console.log(information);
resolve(information);
});
});
}
function promiseSaveDataToDatabase(db, body) {
return new Promise(function(resolve, reject) {
db.collection("testlocation").insert(body, function(dbError) {
if (dbError) {
reject(dbError);
}
resolve()
});
});
}
function promiseDisconnectDatabase(db) {
return new Promise(function(resolve, reject) {
db.close((err) => {
if (err) {
console.log("MongoDb disconnect error.");
reject(err);
}
console.log("MongoDb disconnected.");
resolve();
});
});
}
function promiseProvideDataFromEach(locationsArray, db) {
return new Promise(function(resolve, reject) {
async.each(locationsArray, function(loc, locProcessedCb) {
promiseGetcitiesData(loc).then(function(resultscities) {
promiseGetInformationDataPerCity(loc, resultscities).then(function(resultDetails) {
promiseSaveDataToDatabase(db, resultDetails).then(function() {});
locProcessedCb();
});
});
}, function(err) {
if (err) {
locProcessedCb(err);
reject(err);
}
console.log("All locations have been processed.");
resolve();
});
});
}
function getDataAndCloseDb() {
return new Promise(function(resolve, reject) {
promiseConnectToDatabase("mongodb://127.0.0.1:27017/testApp").then(function(db) {
promiseProvideDataFromEach(locationsArray, db).then(function() {
promiseDisconnectDatabase(db).then(function() {});
});
});
});
}
getDataAndCloseDb();

I think this is a lot simpler than the code in the question makes it appear. In particular, new Promise(...) can be completely avoided by :
using require('async-request') instead of require('request').
allowing MongoDb methods to return Promise, as many of them will do if no callback is passed.
Also
by using the Promise.all(array.map(...)) pattern the need for require('async') disappears.
https://stackoverflow.com/a/28915678/3478010 - provides a great little reusable disposer utility, which is useful here.
Remember to return a promise/value from every .then() callback that is itself asynchronous and/or should deliver data.
With some guesswork, I think you want something like this :
const MongoClient = require('mongodb').MongoClient;
const request = require('async-request'); // just like `request()` but returns a promise
var locationsArray = [
'location1',
'location2',
'location3'
];
function promiseGetCitiesData(loc) {
return request({
url: `https://example.com/${loc}`,
json: true
}).then(body => body.result.cities);
}
function promiseGetInformationDataPerCity(loc, cities) {
return Promise.all(cities.map(city => {
return request({
'url': `https://example.com/${loc}/${city}`,
'json': true
}).then(cityInfo => ({ 'name':city, 'info':cityInfo }));
}));
}
function promiseProvideDataFromEach(locationsArray, db) {
return Promise.all(locationsArray.map(loc => {
return promiseGetCitiesData(loc)
.then(cities => promiseGetInformationDataPerCity(loc, cities)
.then(citiesWithCityInfo => ({ 'location':loc, 'cities':citiesWithCityInfo }));
}))
.then(resultDetails => db.collection('testlocation').insertMany(resultDetails));
}
// disposer utility - credit: https://stackoverflow.com/a/28915678/3478010
function withDb(work) {
var _db;
return MongoClient.connect("mongodb://127.0.0.1:27017/testApp")
.then((db) => {
_db = db; // keep reference
return work(db); // perform work on db
}).finally(() => {
if (_db)
_db.close();
});
}
withDb(db => promiseProvideDataFromEach(locationsArray, db))
.then(() => {
// connection released here
});
The guesswork centres mainly around what is to be inserted at db.collection('testlocation').insertMany(resultDetails). The code in the question gives no more than a clue. My attempt seems reasonable but may not be exactly what you want. Be prepared to make some changes in promiseProvideDataFromEach() and promiseGetInformationDataPerCity().

you can do something like this. Its a simpler code but I think you can map it to your current code.
const Promise = require('bluebird')
const cities = ['citya', 'cityb', 'cityc']
function resolveCities() {
return new Promise(function(resolve, reject) {
resolve(cities)
})
}
function logCity(city) {
console.log('city ', city)
}
return resolveCities()
.then(function(cities) {
return Promise.mapSeries(cities, function(city) {
logCity(city);
});
})

How do I call three requests in order?

I am learning Node. I have a console app that must make requests to web services in order. Specifically, I need to make three requests in order. In an attempt to make these requests, I'm using the built-in HTTPS module. I have one request successfully executing. But, I need to make three in succession. I'm not sure how to do this. Right now, I have:
console.log('Running Request #1...');
var options = {
host: 'example.com',
path: '/api/service',
port: 443,
method: 'GET',
headers: {
'api-key': '[Hidden]'
}
};
var req = https.request(options, (res) => {
res.on('data', (d) => {});
});
req.end();
req.on('error', (e) => {
console.error(e);
});
I'm not sure how to call my three requests in order. Yet, at the same time, gracefully handling an error. If I had promises, I would know how to chain them together and just use the catch handler. But, I'm not sure how to chain together requests since the HTTPS module uses the Arrow function syntax.
Any help is appreciated it.

Try this:
var https = require('https');
var urls = ['url1', 'url2', 'url3'];
var request = function(url) {
console.log(url);
return new Promise((resolve, reject) => {
https.get(url, (res) => {
res.on('end', () => {
resolve('what');
});
res.on('data', data =>{
});
}).on('error', e => {
reject(e);
});
});
};
var promise = request(urls.shift());
while(urls.length > 0) {
let url = urls.shift();
promise = promise.then(function() {
return request(url);
});
}
promise.catch(e => console.log);

consider use promise with reduce,something like this
var urls=['u1','u2','u3'];
var er=0
function getPromise(url) {
return new Promise(function (resolve,reject) {
setTimeout(function () {
console.log(url+ " is resolved in 2 sec")
er++
if(er==1)
{
reject(url)
}else{
resolve(url)
}
},2000)
})
}
urls.reduce(function (pre,cur) {
return pre.then(function () {
return getPromise(cur)
})
},new Promise(function (resolve,reject) {
resolve(null)
}))
.then(function (result) {
console.log("final result is "+result)
},function (e) {
console.log("something wrong happens : "+e)
})
Play with the code,I think it is want you want