I have a pretty simple node script which parses a json file from an external url. I'm attempting to have the script loop over each record returned and make a decision to add it to the DB (using nedb) if we haven't previously got it. At the moment my script below looks to be only processing the last record in the json file.
var Datastore = require('nedb')
, db = new Datastore({ filename: 'foo.db', autoload: true });
var request = require('request');
;
var cgrecentsalesurl = "http://ffo.com.json";
request({
url: cgrecentsalesurl,
headers: {
'User-Agent': 'cgapistats'
},
json: true
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
var cgrecentsales = body["recent-sales"];
for (var i in cgrecentsales) {
console.log( "processing record " + i );
(function() {
var query = { saletimestamp : cgrecentsales[i].sold_at };
db.find( query , function( err, docs ) {
console.log( docs.length );
if ( docs.length == 0 ) {
db.insert( { item: cgrecentsales[i].item, saletimestamp: cgrecentsales[i].sold_at, amount: cgrecentsales[i].amount } );
console.log( "db record inserted" );
} else {
console.log( "record exists!" )
};
});
})();
}
} else {
console.log(response);
}
})
Any ideas what I'm doing wrong?
Thanks!
Related
Firstly, I'm sorry if this has been posted before. I searched but couldn't find any credible solution.
So I'm working on this route in nodejs where I make an API call for a piece of information and then using that info in an if statement to check if it's the correct info(the server sometimes sends wrong info).
If I get the correct info then I use that in another API to get more info about it and render it into my template. Everything works fine.
But I want the first API call to take place again if the info doesn't match or it's wrong. How can I initiate the API call again from the start(like a loop) and it will break only if the info is correct. Please check the "comment" in the code below. That is where I don't know what to put. Your help would be highly appreciated.
PS. I am a beginner in nodejs and javascript.
Route
router.get("/check", (req, res) => {
if(req.query.search) {
var input = req.query.search;
var url = "http://firstapi.com/json/" + input + "?fields=query";
request(url, function(error, response, body) {
if(!error && response.statusCode === 200) {
var data = JSON.parse(body);
if(data.query.match(/((^|\.)((25[0-5])|(2[0-4]\d)|(1\d\d)|([1-9]?\d))){4}$/)){
var url = "https://secondapi.com/" + data.query + "?key=something";
request(url, function(error, response, body) {
if(!error && response.statusCode === 200) {
var Data = JSON.parse(body);
res.render("index", {data: Data});
}
});
}else{
//want to use the input at the top and check the firstapi again. All the code above should run again until its the correct one which I will use in my template.
}
}
});
}else{
res.render("index", {data: null});
}
});
I would probably do it this way:
router.get('/check', (req, res) => {
if (req.query.search) {
var input = req.query.search;
// Put this logic away in a `checkData` function
checkData(input)
.then(data => {
res.render('index', { data }); // Short version of {data: data}
})
.catch(err => {
console.error(err);
res.render('index', { data: null });
});
} else {
res.render('index', { data: null });
}
});
// Here, we have a `retries` parameter, set to 0 initially
function checkData (input, retries = 0) {
const maxRetries = 3;
// Return a promise (you could also use callbacks)
return new Promise((resolve, reject) => {
// Define a retry method
const retry = () => {
if (retries < maxRetries) {
// Increment the retries count and go for another try
checkData(input, retries + 1).then(resolve).catch(reject);
} else {
reject(`Could not get the data after ${retries} retries.`);
}
};
var url = `http://firstapi.com/json/${input}?fields=query`;
request(url, function (error, response, body) {
if (!error && response.statusCode === 200) {
var data = JSON.parse(body);
if (data.query.match(/((^|\.)((25[0-5])|(2[0-4]\d)|(1\d\d)|([1-9]?\d))){4}$/)) {
var url = 'https://secondapi.com/' + data.query + '?key=something';
request(url, function (error, response, body) {
if (!error && response.statusCode === 200) {
var Data = JSON.parse(body);
// If it worked, resolve with the data
resolve(Data);
} else {
retry();
}
});
} else {
retry();
}
}
});
});
}
I have a node.js app that has a few files that I am working with, but the main two javascript files are a RestController and AuthController. The RestController is supposed to call the AuthController to pull a new access token from Salesforce, the server that I am attempting to hit.
I currently set up my AuthController.js to work with promises so that I can wait to get my access token. The problem is I have no idea how to get my RestController.js file to wait for the access token from the AuthController.js file.
I am also very very new to Javascript and Promises, so I am not sure if I even set up my functions correctly. Basically, I want my AuthController to handle getting the access token and the RestController to handle the Rest request to our server.
AuthController.js
var fs = require('fs');
var jwt = require('jsonwebtoken');
var request = require('request');
var querystring = require('querystring');
var config = require('../../configs/config.json');
var filename = __dirname + '/../../' + config.key_path;
var access_token;
var readFilePromise = function(file) {
return new Promise(function(ok, notOk) {
fs.readFile(file, function(err,data) {
if (err) {
notOk(err);
} else {
ok(data);
}
});
});
}
var getAccessToken = function(key) {
return new Promise(function(ok,notOk) {
var jwtparams = {
iss : config.client_id,
sub: config.username,
aud: 'https://' + config.host,
exp : Date.now() + 300
};
var token = jwt.sign(jwtparams, key, {algorithm: 'RS256'});
var data = querystring.stringify({
grant_type : 'urn:ietf:params:oauth:grant-type:jwt-bearer',
assertion : token
});
request.post({
headers: {
'Content-Type' : 'application/x-www-form-urlencoded',
'Content-Length' : data.length
},
url: 'https://' + config.host + '/services/oauth2/token',
body: data
}, function(error, response, body) {
if (error) {
return notOk(error);
}
try {
ok(JSON.parse(body).access_token);
} catch (e) {
notOk(e);
}
});
});
}
function main() {
readFilePromise(filename).then(function(data) {
getAccessToken(data.toString()).then(function(data) {
access_token = data.toString();
});
});
}
module.exports = {main};
RestController.js
var https = require('https');
var request = require('request');
var auth = require('./authController.js');
class SystemController {
doProcessPostStatus (req,res) {
if (!req.body.systemId) {
return res.status(400).send([
{
'errorCode' : 'INVALID_REQUEST_BODY',
'message' : 'System Id "systemId" variable is required.'
}
]);
} else if (typeof req.body.success === 'undefined') {
return res.status(400).send([
{
'errorCode' : 'INVALID_REQUEST_BODY',
'message' : 'Success "success" variable is required'
}
]);
} else if (!req.body.message) {
return res.status(400).send([
{
'errorCode' : 'INVALID_REQUEST_BODY',
'message' : 'Message "message" variable is required'
}
]);
}
var access_token = auth.main();
console.log(access_token);
}
}
var systemControllerVar = new SystemController();
module.exports = systemControllerVar;
Any help is greatly appreciated as I am currently stuck, thanks!
I am trying to create a dictionary object of the response json's using postman's pm api sendRequest.
Wrote a recursive function to get all the responses but the problem is the response dictionary object population happens way before even the response comes back.
Is there any way to wait the dictionary population before each of the respective response is received so as to capture the response within the dictionary object ?
var respDictionary = {};
getResponses (listOfUrls);
console.log("respDictionary: ");
console.log(respDictionary);
function getResponses(urlList) {
if (typeof urlList === 'string') {
urlList = urlList.split(' ');
}
_url = urlList[0];
var call = {
url: _url ,
method: 'GET',
header: {
"Authorization": `Bearer ${token}`,
"Content-Type": "application/json"
}
};
urlList.splice(0, 1);
pm.sendRequest(
call,
function (err, res) {
if (err) {
console.log(err);
} else {
if (urlList.length === 0) {
return;
}
try {
respDictionary[_url] = res.json();
} catch (e) {
console.log(err);
}
getResponses(urlList);
}
});
console.log(respDictionary);
}
Output is:
respDictionary:
Object:{}
//further, pm request responses are listed
You do not understand JavaScript asynchrounous handling. Maybe the following will help:
Event loop video
Promises
Event loop documentation (video is easier)
Your code will work if you use promises:
function getResponses(urlList) {
if (typeof urlList === 'string') {
urlList = urlList.split(' ');
}
return Promise.all(
urlList.map(
function(url){
return {
url: url ,
method: 'GET',
header: {
//not sure where token comes from
"Authorization": `Bearer ${token}`,
"Content-Type": "application/json"
}
};
}
).map(
function(call){
return new Promise(
function(resolve,reject){
pm.sendRequest(
call,
function (err, res) {
if (err) {
reject(err);
} else {
resolve([call.url,res.json()]);
}
});
}
)
.then(
undefined
,function(err){
//if something goes wrong we will still return something
return [call.url,{error:err}];
}
)
}
)
)
.then(
function(results){
return results.reduce(
function(acc,result){
acc[result[0]] = result[1];
}
,{}
);
}
)
}
getResponses (listOfUrls)
.then(//this will always succeed, failed items have {error:something}
function(results){
console.log("results:",results);
}
);
console.log("this comes before results");
The code above will cause all request to happen at once, this may not be the desired behavior, I have written a throttle method that is part of some library functions that may come in handy. You can apply the throttle to your code so it will only have max amount of connections:
const max = 10;//maximum 10 active connections
function getResponses(urlList) {
const throttled = throttle(max);
if (typeof urlList === 'string') {
urlList = urlList.split(' ');
}
return Promise.all(
urlList.map(
function(url){
return {
url: url ,
method: 'GET',
header: {
//not sure where token comes from
"Authorization": `Bearer ${token}`,
"Content-Type": "application/json"
}
};
}
).map(
throttled(//only max amount of connections active at any time
function(call){
return new Promise(
function(resolve,reject){
pm.sendRequest(
call,
function (err, res) {
if (err) {
reject(err);
} else {
resolve([call.url,res.json()]);
}
});
}
)
.then(
undefined
,function(err){
//if something goes wrong we will still return something
return [call.url,{error:err}];
}
)
}
)
)
)
.then(
function(results){
return results.reduce(
function(acc,result){
acc[result[0]] = result[1];
}
,{}
);
}
)
}
getResponses (listOfUrls)
.then(//this will always succeed, failed items have {error:something}
function(results){
console.log("results:",results);
}
);
console.log("this comes before results");
Apart from the other answer(s), following simple approach also works well instead of recursion:
_.forEach (
urls,
function (urls) {
getAPI(url,function (url,schema,err) {
if (!err) {
respDictionary[url]=resp.json();
console.log(schema);
} else {
console.log(err);
}
pm.environment.set('respDictionary', JSON.stringify(respDictionary));
});
}
);
function getAPI(url, callback) {
var _url = '{{BP_SERVER}}/' + urls;
var call = {
url: _url,
method: 'GET',
header: {
"Authorization": `Bearer ${token}`,
"Content-Type": "application/json"
}
};
pm.sendRequest(
call,
function (err, res) {
if (!err) {
callback(urls,res.json());
} else {
callback(urls,'',err);
}
}
);
}
Im trying to do a post from a .ejs page to a nodejs api.
this is the call i do
var dataSend={
from:from,
to:to,
datesStart: startDates,
datesEnd: endDates,
price: totalPriceOften,
places:JSON.stringify(placesArray),
package: pSizeOften,
transport: $('.activeFilter:eq(1) a').html().toLowerCase(),
pickupRange: pFlexOften,
dropRange: dFlexOften,
};
$.ajax({
url: location.origin + '/publishTravelOften',
headers: {
apiKey: APIKEYWEB
},
type: "POST",
data: dataSend,
dataType: "json",
async: true,
success: function (data) {
//do stuff
},
error: function (err) {
//do other stuff
}
});
the data i send is a object with this information
and in the nodejs api, i have this:
router.post('/publishTravelOften', function (req, res) {
if(req.user == null || req.user == undefined){
res.status(403).send(Error.ERROR_NO_USER);
}
else {
var time = Date.now();
var userLanguage = Geo.getCountry(req);
console.log("LOAD TIME /publishTravelOften: " + (Date.now() - time));
Config.setCountry(Geo.getCountry(req));
publishTravelOften(req, function (error, data) {
if (!error) {
data.lang = Lang.getLang(userLanguage);
data.socketUrl = Config.getSocketUrl();
data.apiUrl = Config.getApiUrlWeb();
res.send(data);
}
else {
res.status(500).send(error);
}
});
}
});
function publishTravelOften(req,callback){
console.log("########");
console.log(req.body);
console.log("########");
var url = Config.getApiUrlWeb() + "travel/often/create?user="+req.user._id+"¤cy="+Geo.getCountry(req);
var options = {
url: url,
form:req.body,
headers: {
'apikey': Config.getAPIKeyWeb()
},
};
request.post(options, function (error, response, body) {
var parsedResponse = JSON.parse(body);
if (parsedResponse.success == false)
callback(parsedResponse, parsedResponse);
else {
var data = {
newTravel: parsedResponse.data
};
callback(error, data);
}
});
}
my problem is that when i print the data i get on the nodejs part, i have this
{ from: 'ChIJO_PkYRozGQ0R0DaQ5L3rAAQ',
to: 'ChIJ9xUaAY73vEcRUt8vzFOSk1w',
'datesStart[]': [ '1471683240000', '1471596840000' ],
'datesEnd[]': [ '1471683240000', '1471596840000' ],
price: '134.93',
places: '[]',
package: 'medium',
transport: 'airplane',
pickupRange: '15',
dropRange: '5' }
the number arrays get converted to string arrays, also the field names change from datesStart and datesEnd to 'datesStart[]' and 'datesEnd[]'
is this normal? and how can i prevent the arrays from change from number to string arrays?
I have a modified code in react-native for fetching data with server, that works fine. I want to add NetInfo to always check before fetching if telephone has connection to internet. Is it posible inside promise? How to connect this async function to my code?
'use strict';
var MAX_WAITING_TIME = 30000
var processStatus = function (response) {
// status "0" to handle local files fetching (e.g. Cordova/Phonegap etc.)
if (response.status === 200 || response.status === 0 || response.status === 201 || response.status === 422 || response.status === 302 ) {
return Promise.resolve(response)
} else if(response.status === 413) {
return Promise.reject(alert(____mobile.connection_error.large_file))
} else {
//return Promise.reject(alert("Process status: "+JSON.stringify(response )))
return Promise.reject(alert(____mobile.connection_error.top));
console.log("Process status: "+JSON.stringify(response ));
}
};
var parseJson = function (response) {
return response.json();
};
var getWrappedPromise = function () {
var wrappedPromise = {},
promise = new Promise(function (resolve, reject) {
wrappedPromise.resolve = resolve;
wrappedPromise.reject = reject;
});
wrappedPromise.then = promise.then.bind(promise);
wrappedPromise.catch = promise.catch.bind(promise);
wrappedPromise.promise = promise;// e.g. if you want to provide somewhere only promise, without .resolve/.reject/.catch methods
return wrappedPromise;
};
/* #returns {wrapped Promise} with .resolve/.reject/.catch methods */
var getWrappedFetch = function () {
var wrappedPromise = getWrappedPromise();
var args = Array.prototype.slice.call(arguments);// arguments to Array
fetch.apply(null, args)// calling original fetch() method
.then(function (response) {
wrappedPromise.resolve(response);
}, function (error) {
// wrappedPromise.reject(alert("Fetch status: " + error));
wrappedPromise.reject(____mobile.connection_error.top);
console.log("Fetch status: " + error);
})
.catch(function (error) {
wrappedPromise.catch(error);
});
return wrappedPromise;
};
/**
* Fetch JSON by url
* #param { {
* url: {String},
* [cacheBusting]: {Boolean}
* } } params
* #returns {Promise}
*/
var postJSON = function (params) {
var headers1 = {}
if (params.json){
headers1 = {
'Accept': 'application/json',
'Content-Type': 'application/json'}
}
if (params.headersIn){
headers1 = params.headersIn
}
var methodTmp = 'POST'
if (params.methodIn) {
methodTmp = params.methodIn
}
console.log(methodTmp)
var wrappedFetch = getWrappedFetch(
params.cacheBusting ? params.url + '?' + new Date().getTime() : params.url,
{
method: methodTmp,//'POST',// optional, "GET" is default value
headers: headers1,
body: params.send_data
});
var timeoutId = setTimeout(function () {
wrappedFetch.reject(alert(____mobile.connection_error.timeout, ____mobile.connection_error.check_connection));// reject on timeout
}, MAX_WAITING_TIME);
return wrappedFetch.promise// getting clear promise from wrapped
.then(function (response) {
clearTimeout(timeoutId);
return response;
})
.then(processStatus)
.then(parseJson);
};
module.exports = postJSON;
What would be the bast way to implement: NetInfo.isConnected.fetch() so fetched would only worked when there is internet connection?
EDIT:
I want to use:
NetInfo.isConnected.fetch()
Yeah I have to rewrite this code, not to use getWrappedPromise and now I think is good time for it.
EDIT2: Ok I refactored this code fragment, hope its better. Any comments welcome. I tested and I'm not sure if I still need this NetInfo.isConnected.fetch(). Now there is no errors where there is no connection or am I missing something?
New code:
var processStatus = function (response) {
if (response == undefined) {
return null
}
// status "0" to handle local files fetching (e.g. Cordova/Phonegap etc.)
if (response.status === 200 || response.status === 0 || response.status === 201 || response.status === 422 || response.status === 302 ) {
return Promise.resolve(response)
} else if(response.status === 413) {
return Promise.reject(alert(____mobile.connection_error.large_file))
} else {
//return Promise.reject(alert("Process status: "+JSON.stringify(response )))
console.log("Process status: "+JSON.stringify(response ));
return Promise.reject(alert(____mobile.connection_error.top));
}
};
var parseJson = function (response) {
if (response == undefined) {
return null
}
return response.json();
};
var postJSON = function (params) {
var headers1 = {}
if (params.json){
headers1 = {
'Accept': 'application/json',
'Content-Type': 'application/json'}
}
if (params.headersIn){
headers1 = params.headersIn
}
var methodTmp = 'POST'
if (params.methodIn) {
methodTmp = params.methodIn
}
console.log(methodTmp)
var fetchPromise = fetch(params.cacheBusting ? params.url + '?' + new Date().getTime() : params.url,
{
method: methodTmp,//'POST',// optional, "GET" is default value
headers: headers1,
body: params.send_data
})// calling original fetch() method
.then(function (response) {
return response;
}, function (error) {
console.log("Fetch status: " + error);
return fetch
}).then(processStatus)
.then(parseJson);
// timeoutId = setTimeout(function () {
// wrappedFetch.reject(alert(____mobile.connection_error.timeout, ____mobile.connection_error.check_connection));// reject on timeout
// }, MAX_WAITING_TIME);
return fetchPromise
};