I want to use async/await inside this promise.allSettled to convert currency by fetching some api or databased stored currency rates.
like i want to use await here like this, but not sure where to put async.
arrcars["price_client"] = await forexService.convertCurrency(item.regular_price, "EUR", userCurrency);
I have posted my entire code. need some guidance here.
This is controller
const httpStatus = require('http-status');
const pick = require('../utils/pick');
const catchAsync = require('../utils/catchAsync');
const async = require('async');
const axios = require('axios');
const { talixoService } = require('../services');
const { iwayService } = require('../services');
const { forexService } = require('../services');
const searchCars = function (req, res) {
const talixoData = talixoService.findTalixoCars(req.body) //call talixo api in services
const iwayData = iwayService.findiWayCars(req.body) //call iwaytransfers api in services
if (req.query.usercurrency) {
var userCurrency = req.query.usercurrency;
} else {
var userCurrency = "INR";
}
return Promise.allSettled([talixoData, iwayData]).then(([restalixo, resiway]) => {
const taxiresults = [];
if (restalixo.status === "fulfilled") {
//console.log(restalixo.value.data);
if (restalixo.value.data.taxis.length) {
restalixo.value.data.taxis.forEach(function (item) {
arrcars = {};
arrcars["carname"] = item.car_model;
arrcars["originprice"] = item.regular_price;
arrcars["price_client"] = forexService.convertCurrency(item.regular_price, "EUR", userCurrency);
arrcars["discountprice_client"] = forexService.convertCurrency(item.discount_price, "EUR", userCurrency);
arrcars["packageId"] = item.id;
arrcars["image"] = item.image_url;
arrcars["freewaittime"] = item.included_waiting_time;
arrcars["maxluggage"] = item.luggage;
arrcars["maxpassengers"] = item.seats;
arrcars["discountprice"] = item.discount_price;
arrcars["vehicletype"] = item.vehicle_type;
arrcars["vendor"] = "Talixo";
arrcars["vehicleremarks"] = "Or Similar";
taxiresults.push(arrcars);
});
}
if (restalixo.value.data.limousines.length) {
restalixo.value.data.limousines.forEach(function (item) {
arrcars = {};
arrcars["carname"] = item.car_model;
arrcars["originprice"] = item.regular_price;
arrcars["price_client"] = forexService.convertCurrency(item.regular_price, "EUR", userCurrency);
arrcars["discountprice_client"] = forexService.convertCurrency(item.discount_price, "EUR", userCurrency);
arrcars["packageId"] = item.id;
arrcars["image"] = item.image_url;
arrcars["freewaittime"] = item.included_waiting_time;
arrcars["maxluggage"] = item.luggage;
arrcars["maxpassengers"] = item.seats;
arrcars["discountprice"] = item.discount_price;
arrcars["vehicletype"] = item.vehicle_type;
arrcars["vendor"] = "Talixo";
arrcars["vehicleremarks"] = "Or Similar";
taxiresults.push(arrcars);
});
}
}
//iwaytransfers supplier data
if (resiway.status === "fulfilled") {
//console.log(resiway.value.data);
if (resiway.value.data.result.length) {
resiway.value.data.result.forEach(function (item) {
var imgsrc = "https://iwayex.com/images/cars/";
arrcars = {};
arrcars["carname"] = item.car_class.models[0];
arrcars["originprice"] = item.price;
arrcars["price_client"] = forexService.convertCurrency(item.price, "EUR", userCurrency);
arrcars["discountprice_client"] = forexService.convertCurrency(item.price, "EUR", userCurrency);
arrcars["packageId"] = item.price_uid;
arrcars["image"] = imgsrc + item.car_class.photo;
arrcars["freewaittime"] = item.allowable_time;
arrcars["maxluggage"] = "";
arrcars["maxpassengers"] = item.car_class.capacity;
arrcars["discountprice"] = item.price;
arrcars["vehicletype"] = item.vehicle_type;
arrcars["vendor"] = "iway Transfers";
arrcars["vehicleremarks"] = "Or Similar";
taxiresults.push(arrcars);
});
}
}
if (taxiresults.length) {
sortedresult = taxiresults.sort(function (a, b) {
return a.discountprice - b.discountprice;
});
res.status(200).send(sortedresult)
}else{
res.status(200).send({})
}
});
}
module.exports = {
searchCars
}
this is the result sample i am getting.
{
carname: "Toyota Prius"
discountprice: 27.5
discountprice_client: {}
freewaittime: 45
image: "https://static.talixo.de/images/vehicles/economy.png"
maxluggage: 3
maxpassengers: 3
originprice: 27.5
packageId: "16021"
price_client: {}
vehicleremarks: "Or Similar"
vehicletype: "limo"
vendor: "Talixo"
}
here the problem is,
i expect
discountprice_client: Converted Number,
but i am getting blank object there as can be seen in above result sample.
i have been using this in another function with async await, then it works fine. but in this case, i am not sure how to put async/await.
this is inside
forex.service.js
const convertCurrency = async (amount, basecurrency, reqCurrency) => {
if (basecurrency === reqCurrency) {
let result = Math.round(amount) + ' ' + reqCurrency;
return result;
} else {
const reqForexData = await getForexbyISO(reqCurrency);
const baseForexData = await getForexbyISO(basecurrency);
const amountInUSD = amount / baseForexData.forexRate;
const amountInreqC = amountInUSD * reqForexData.forexRate;
let result = Math.round(amountInreqC) + ' ' + reqCurrency;
return result;
}
}
module.exports = {
convertCurrency
}
You could do:
// notice the `async` before `function`
resiway.value.data.result.forEach(async function (item) {
// ...
arrcars = {};
arrcars["price_client"] = await forexService.convertCurrency(item.price, "EUR", userCurrency);
arrcars["discountprice_client"] = await forexService.convertCurrency(item.price, "EUR", userCurrency);
// ...
});
But it can lead to unexpected behaviours
I would do this
// add missing `async`
return Promise.allSettled([talixoData, iwayData]).then(async ([restalixo, resiway]) => {
// ...
// await for all loop at once to avoid odd side effects
await Promise.all(resiway.value.data.result.map(async function (item) {
// ...
arrcars = {};
arrcars["price_client"] = await
forexService.convertCurrency(item.price, "EUR", userCurrency);
arrcars["discountprice_client"] = await
forexService.convertCurrency(item.price, "EUR", userCurrency);
// ...
}));
// ...
});
simplify your services
You are writing the Talixo and Iway services but using the result is difficult. Instead of returning the entire response, return the fields that matter to your consumer -
// services/talixo_service.js
async function findTalixoCars(query) {
const result = await // ... perform request
return result.value.data ?? [] // <- return relevant data
}
Notice we use return _____ ?? [] to ensure that an array is always returned -
// services/iway_service.js
async function findiWayCars(query) {
const result = await // ... perform request
return result.value.data.result ?? [] // <- return relevant data
}
searchCars
With simplified services, we write a high-level implementation of searchCars. Instead of using Promise.allSettled we can use .catch(_ => []) on potential failures to guarantee we have an array result. This reduces the need for logical checks and code branches to make the code more declarative and less prone to errors -
async function searchCars(req, res) {
const [talixo, iway] = await Promise.all([
talixoService.findTalixoCars(req.body).catch(_ => []),
iwayService.findiWayCars(req.body).catch(_ => [])
])
const userCurrency = req.query.usercurrency ?? "INR"
const taxis = [
...await Promise.all(talixo.taxis.map(item => fromTalixo(item, userCurrency))),
...await Promise.all(talixo.limousines.map(item => fromTalixo(item, userCurrency))),
...await Promise.all(iway.map(item => fromIway(item, userCurrency)))
]
res.status(200).send(
taxis.sort((a,b) => a.discountprice - b.discountprice)
)
}
data mapping
The search function above depends on fromTalixo and fromIway. These mapping functions reduce code complexity and repitition and gives us a good conceptual model for how data gets mapped from the remote services to our local data -
async function fromTalixo(data, userCurrency) {
return {
carname: data.car_model,
originprice: data.regular_price,
price_client: await forexService.convertCurrency(data.regular_price, "EUR", userCurrency),
discountprice_client: await forexService.convertCurrency(data.discount_price, "EUR", userCurrency),
packageId: data.id,
image: data.image_url,
freewaittime: data.included_waiting_time,
maxluggage: data.luggage,
maxpassengers: data.seats,
discountprice: data.discount_price,
vehicletype: data.vehicle_type,
vendor: "Talixo",
vehicleremarks: "Or Similar",
}
}
async function fromIway(data, userCurrency) {
return {
carname: data.car_class.models[0],
originprice: data.price,
price_client: await forexService.convertCurrency(data.price, "EUR", userCurrency),
discountprice_client: await forexService.convertCurrency(data.price, "EUR", userCurrency),
packageId: data.price_uid,
image: `https://iwayex.com/images/cars/${data.car_class.photo}`,
freewaittime: data.allowable_time,
maxluggage: "",
maxpassengers: data.car_class.capacity,
discountprice: data.price,
vehicletype: data.vehicle_type,
vendor: "iway Transfers",
vehicleremarks: "Or Similar",
}
}
Related
I'm trying to call balanceOf() method on a specific Tron contract (TYukBQZ2XXCcRCReAUguyXncCWNY9CEiDQ), but what I get is Failed to execute . If I do not provide the parameters I get Invalid argument count provided - meaning at some level it works for this contract.
What is interesting it works well on contracts other than the ones created with JustSwap Factory contract eg. https://tronscan.io/#/contract/TYukBQZ2XXCcRCReAUguyXncCWNY9CEiDQ/code .
The code includes the standard TRC20 methods - including balanceOf() - I'm stuck and tried all that's possible form my side, but let's just say I'm not fluent in tronweb api.
My code:
export const getDataToken = async (contractAddress, account, account2) => {
try {
const instance = await window.tronWeb.contract(
[
{
constant: true,
inputs: [{ name: "owner", type: "address" }],
name: "balanceOf",
outputs: [{ name: "", type: "uint256" }],
payable: false,
stateMutability: "view",
type: "function"
}
],
contractAddress
);
console.log(instance);
if (instance.balanceOf) {
console.log("dadadad if");
const tokenBalance = await instance.balanceOf(account).call();
const tokenBalance2 = await instance.balanceOf(account2).call();
return {
tokenBalance: (tokenBalance / Math.pow(10, 18)).toString(),
tokenContract: instance,
tokenBalance2: (tokenBalance2 / Math.pow(10, 18)).toString()
};
}
} catch (message) {
console.log("error getData :" + message);
}
};
const { tokenBalance, tokenContract, tokenBalance2 } = getDataToken(
"TYukBQZ2XXCcRCReAUguyXncCWNY9CEiDQ",
"TL4HzzxGMc1LMfs3XCi4yTJikaBVubz5y4",
"TTFp171XD4JdUB33sDq2ydXJyUEEZjNhLD"
);
This function can help (example for getting JustSwap pair price):
async function takePrice(contractAddress, token){
var functionSelector = 'getTokenToTrxInputPrice(uint256)';
var parameter = [
{
type: 'uint256',
value: token
}
]
var options = {};
transaction = await window.tronWeb.transactionBuilder.triggerConstantContract(contractAddress, functionSelector, options, parameter);
return window.tronWeb.BigNumber("0x"+transaction['constant_result'][0]);
}
priceUSDTTRX = window.tronWeb.fromSun(await takePrice(USDTTRX_ADDRESS, "1000000"));
priceSomeTone18TRX = window.tronWeb.fromSun(await takePrice(SomeTone18TRX_ADDRESS, "1"+"0".repeat(18)));
I have good result:
const TronWeb = require("tronweb");
const ethers = require('ethers')
const MAINNET_RPC = "https://api.trongrid.io";
const PLACEHOLDER_PRIVATE_KEY = "YOUR_PRIVATE_KEY";
const HttpProvider = TronWeb.providers.HttpProvider;
const fullNode = new HttpProvider(MAINNET_RPC);
const solidityNode = new HttpProvider(MAINNET_RPC);
const eventServer = new HttpProvider(MAINNET_RPC);
const tronWeb = new TronWeb(fullNode,solidityNode,eventServer,PLACEHOLDER_PRIVATE_KEY);
const startJustSwap = async () => {
try {
const contractTokenExchangeUSDT = 'TQn9Y2khEsLJW1ChVWFMSMeRDow5KcbLSE'; //S-USDT-TRX Token
const parameter = [{type:`uint256`,value: 10000000}];
const tx = await tronWebLocal.transactionBuilder.triggerConstantContract(contractToken, `trxToTokenSwapInput(uint256,uint256)`, {}, parameter);
console.log(tx);
} catch (e) {
console.log(e);
}
};
startJustSwap();
try const tokenBalance = await instance.balanceOf(account).call({ _isConstant: true });
It works for me.
transactionBuilder.triggerConstantContract works too, but mine is simpler
I'm having two issues this moment
I am having trouble displaying the data from my firestore document after the user signs in and they do have a document with data
I have a button that lets the user upload a document and I will parse it to extract all important data. That data is then put into the document with Ids, which is basically the userId + what number doc it is belonging to the user. When they upload the data it seems that all data extracted uses the same ID and only the last one extracted gets displayed.
this is my code:
const loansRef = firebase.firestore().collection('goals');
const [ courseGoals, setCourseGoals ] = useState([]);
const [goalCounter, setGoalCounter] = useState(0)
//let the user upload doc, what gets called when they press button
const pickDocument = async () => {
try {
let input = await DocumentPicker.getDocumentAsync({
type: "text/plain",
});
setUserOut(await FileSystem.readAsStringAsync(input.uri));
} catch (error) {
console.log(error);
}
createLoans();
};
//extracts important info and returns array with all info for one loan
const fileParser = () => {
const parsedLoans = [];
var newUserOut = userOut;
if (newUserOut.length == 0) {
return;
}
//remove the grants
var grantPos = newUserOut.search("Grant Type:");
var pos = newUserOut.search("Loan Type:");
//hopefully just the loans now
newUserOut = newUserOut.slice(pos, grantPos);
while (newUserOut.length > 0) {
var lastPos = newUserOut.lastIndexOf("Loan Type:");
parsedLoans.push(newUserOut.slice(lastPos, newUserOut.length));
newUserOut = newUserOut.slice(0, lastPos);
}
//console.log('parsed loans: ' + parsedLoans)
return parsedLoans;
};
//where we actually create loans and get the important data for each loan
const createLoans = () => {
const newLoans = fileParser();
const title= 'Loan Amount:$'
const interest = 'Loan Interest Rate:'
for(let i =0; i < newLoans.length; i++){
let loan = newLoans[i]
let goalTitle=loan.substring(loan.indexOf(title)+title.length,loan.indexOf('Loan Disbursed Amount:'))
//console.log("goalTitle: " + goalTitle)
let interestRate = loan.substring(loan.indexOf(interest)+interest.length,loan.indexOf('Loan Repayment Plan Type'))
//console.log("Interest rate: "+ interestRate)
let years = 0
let paidOff = 0
addGoalHandler(goalTitle,interestRate,years,paidOff)
}
return
};
useEffect(() => {
getPW()
let isMounted = true;
if (isMounted) {
loansRef.doc(userId).onSnapshot(
(docSnapshot) => {
if(!docSnapshot.exists){console.log('doc doesnt exist, start from scratch')}
else{
console.log('loaded successfully '+docSnapshot.data())
setGoalCounter(docSnapshot.data().loans.length)
console.log(goalCounter)
}
},
(error) => {
console.log(error);
}
);
}
return () => {
isMounted = false;
};
}, []);
const addGoalHandler = (goalTitle, interestRate, years, paidOff) => {
//setCourseGoals([...courseGoals, enteredGoal])
setGoalCounter(goalCounter+1)
setCourseGoals((prevGoals) => [
...courseGoals,
{
id:userId.toString() + goalCounter.toString(),
value: goalTitle,
interest: interestRate,
years: years,
paidOff: paidOff
}
]);
addToFB(goalTitle, interestRate,years,paidOff)
//var oldIDS = docIDS
//oldIDS.push(userId.toString() + goalCounter.toString())
//setDocIDS(oldIDS)
setIsAddMode(false);
};
const cancelGoalAdditionHandler = () => {
setIsAddMode(false);
};
const addToFB = async (goalTitle, interestRate, years, paidOff) => {
//adding data to firebase, takes into account if doc exists already
const loadDoc = await loansRef.doc(userId).get()
.then((docSnapshot)=> {
if(docSnapshot.exists){
loansRef.doc(userId).onSnapshot((docu)=>{
const updateLoansArr = loansRef.doc(userId).update({
loans: firebase.firestore.FieldValue.arrayUnion({
id: userId+goalCounter.toString(),
value: goalTitle,
interest: interestRate,
years: years,
paidOff: paidOff
})
})
//setGoalCounter(goalCounter+1)
})
}
else{
const addDoc = loansRef.doc(userId).set({
loans: firebase.firestore.FieldValue.arrayUnion({
id: userId+goalCounter.toString(),
value: goalTitle,
interest: interestRate,
years: years,
paidOff: paidOff
})
})
//setGoalCounter(goalCounter+1)
}})
//setGoalCounter(goalCounter+1)
}
const removeGoalHandler = async (goalId) => {
/*
setCourseGoals((currentGoals) => {
loansRef.doc(goalId).delete().then(console.log('removed correctly'))
return currentGoals.filter((goal) => goal.id !== goalId);
});
setGoalCounter(goalCounter-1)
*/
//need to use the value and not the goalId
const removeGoal = await loansRef.doc(goalId).update({
goals: firebase.firestore.FieldValue.arrayRemove(goalId)
})
setCourseGoals((currentGoals)=> {
return currentGoals.filer((goal)=> goal.id !== goalId)
})
};
I am having some issues with my a particular call in my cloud function that doesn't seem to be resolving correctly.
This is the code that doesn't want to resolve:
console.log('Getting Search Patterns');
let searchPatterns: FirebaseFirestore.QuerySnapshot;
try {
searchPatterns = await admin
.firestore()
.collection('FYP_LOCATIONS')
.get();
} catch (error) {
console.error(error);
}
console.log(`Search Patterns Received: ${searchPatterns}`);
LOG:
As you can see in the log, my function runs up until the console log before the try block, then stops until the function times out. I'm not sure what it is that is causing this issue.
EDIT: I have reformatted my code by separating out each of the different parts in my cloud function into separate functions that I can call; the resulting getSearchTerms() function is as follows:
async function getSearchTerms(): Promise<FirebaseFirestore.DocumentData[]> {
try {
const snapshot = await admin
.firestore()
.collection('FYP_LOCATIONS')
.get();
console.log('Get Returned');
return snapshot.docs.map(doc => doc.data());
} catch (e) {
console.error(e);
return [];
}
}
This still stops at the same point in the function execution, the full function is here, this has been updated to the latest version:
import * as functions from 'firebase-functions';
import * as admin from 'firebase-admin';
import * as path from 'path';
import * as WordExtractor from 'word-extractor';
import * as textract from 'textract';
import suffixArray from './suffixArray';
// interface Location {
// lid: string;
// location_name: string;
// location_type: string;
// sentimental_value: number;
// }
// interface Context {
// lid: string;
// context_string: string;
// fid: string;
// }
export const processFile = functions.storage.object().onFinalize(async file => {
const serviceAccount = require(__dirname + '/../config/serviceAccount.json');
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
databaseURL: 'https://fyp-alex.firebaseio.com',
});
const firestore = admin.firestore();
const fileBucket: string = file.bucket;
const filePath: string = file.name;
const fileDet: string = path.basename(filePath);
const fileNameSplit: string[] = fileDet.split('.');
const fileExt: string = fileNameSplit.pop();
const fileName: string = fileNameSplit.join('.');
const bucket = admin.storage().bucket(fileBucket);
const fileRef = bucket.file(filePath);
const _path: string = `/tmp/${fileName}.${fileExt}`;
console.log(`File path ${filePath}`);
console.log('Getting Download URL');
try {
console.log(`Downloading to: ${_path}`);
await fileRef.download({ destination: _path });
console.log('File Saved');
console.log(`Getting Details: ${_path}`);
const text: string = await getText(_path, fileExt);
console.log(`Processing: ${fileName}`);
console.log('Creating Suffix Array');
const suffix_array = suffixArray(text);
console.log(`Suffix Array Created: ${suffix_array}`);
console.log('Getting Search Patterns');
const searchTerms: FirebaseFirestore.DocumentData[] = await getSearchTerms();
console.log('Search Patterns Received');
const promises = [];
const allContexts: Object[] = [];
for (const searchDoc of searchTerms) {
const searchTerm = searchDoc.location_name.toLowerCase();
console.log(searchTerm);
const matchedIndexes = search(text, searchTerm, suffix_array);
const contexts = createContexts(matchedIndexes, searchDoc, text, fileName);
allContexts.concat(contexts);
}
for (const context of allContexts) {
const p = admin
.firestore()
.collection('FYP_CONTEXTS')
.add(context);
promises.push(p);
}
await Promise.all(promises);
const data = {
processed: 1,
};
return firestore.doc(`FYP_FILES/${fileName}`).update(data);
} catch (e) {
console.error(e);
const data = {
processed: 2,
};
return firestore.doc(`FYP_FILES/${fileName}`).update(data);
}
});
async function getText(_path: string, fileExt: string) {
let text: string = '';
switch (fileExt) {
case 'docx':
case 'doc':
const extractor = new WordExtractor();
const extracted = await extractor.extract(_path);
text = extracted.getBody();
break;
case 'pdf':
break;
case 'txt':
textract.fromFileWithPath(_path, function(extractedError: any, string: string) {
if (extractedError) {
console.error(extractedError);
}
if (string !== null) {
text = string;
}
});
break;
default:
console.log('Unsupported File Type');
}
return text;
}
async function getSearchTerms(): Promise<FirebaseFirestore.DocumentData[]> {
try {
const snapshot = await admin
.firestore()
.collection('FYP_LOCATIONS')
.get();
console.log('Get Returned');
return snapshot.docs.map(doc => doc.data());
} catch (e) {
console.error(e);
return [];
}
}
function createContexts(
matchedIndexes: number[],
searchDoc: FirebaseFirestore.DocumentData,
text: string,
fileName: string
) {
console.log('Creating Contexts');
const contexts = [];
const searchTerm = searchDoc.location_name.toLowerCase();
for (const index of matchedIndexes) {
let left = index - 25;
let right = index + searchTerm.length + 25;
if (left < 0) {
left = 0;
}
if (right > text.length) {
right = text.length;
}
const context = text.substring(left, right);
contexts.push({
lid: searchDoc.lid,
context_string: context,
fid: fileName,
});
}
return contexts;
}
function search(text: string, searchTerm: string, suffix_array: number[]) {
console.log(`Beginning search for: ${searchTerm}`);
let start = 0;
let end = suffix_array.length;
const matchedIndexes: Array<number> = [];
while (start < end) {
const mid: number = (end - 1) / 2;
const index: number = suffix_array[mid];
const finalIndex: number = index + searchTerm.length;
if (finalIndex <= text.length) {
const substring: string = text.substring(index, finalIndex);
const match: number = searchTerm.localeCompare(substring);
if (match === 0) {
console.log(`Match Found at Index: ${index}`);
matchedIndexes.push(index);
} else if (match < 0) {
end = mid;
} else if (match > 0) {
start = mid;
}
console.log(matchedIndexes);
}
}
if (matchedIndexes.length === 0) {
console.log(`No matches found for search term: ${searchTerm}`);
}
return matchedIndexes;
}
Hopefully the full function provides a bit more context.
I have watched Doug's videos through a few times but I am still coming up against this, I did notice that removing the await that seems to be failing (as in removing the promise all together) seemed to cause an earlier await to fail. This is indicative of it being an issue with promises later on in the function but I cannot for the life of me find the issue, I will keep trying but hopefully that provides some useful context.
You aren't letting the function know when the async operations have finished.
I would guess that you want to collect all of the async operations in an array and wait for all of them to finish before letting the function exit.
(Those youtube videos by Doug, mentioned in the comments above are quite good and do a more thorough job of explaining why)
ie.
const requests = [];
const things = [1,2,3];
for (let index = 0; index < things.length; index++) {
const element = things[index];
const promise = firebase.firestore().push(element);
requests.push(promise);
}
return Promise.all(requests);
I have an API call in api.js:
export const getGraphData = (domain, userId, testId) => {
return axios({
url: `${domain}/api/${c.embedConfig.apiVersion}/member/${userId}/utests/${testId}`,
method: 'get',
});
};
I have a React helper that takes that data and transforms it.
import { getGraphData } from './api';
const dataObj = (domain, userId, testId) => {
const steps = getGraphData(domain, userId, testId)
.then((result) => {
return result.attributes;
});
console.log(steps);
// const steps = test.get('steps');
const expr = /select/;
// build array of steps that we have results in
const resultsSteps = [];
steps.forEach((step) => {
// check for types that contain 'select', and add them to array
if (expr.test(step.get('type'))) {
resultsSteps.push(step);
}
});
const newResultsSteps = [];
resultsSteps.forEach((item, i) => {
const newMapStep = new Map();
const itemDescription = item.get('description');
const itemId = item.get('id');
const itemOptions = item.get('options');
const itemAnswers = item.get('userAnswers');
const newOptArray = [];
itemOptions.forEach((element) => {
const optionsMap = new Map();
let elemName = element.get('value');
if (!element.get('value')) { elemName = element.get('caption'); }
const elemPosition = element.get('position');
const elemCount = element.get('count');
optionsMap.name = elemName;
optionsMap.position = elemPosition;
optionsMap.value = elemCount;
newOptArray.push(optionsMap);
});
newMapStep.chartType = 'horizontalBar';
newMapStep.description = itemDescription;
newMapStep.featured = 'false';
newMapStep.detailUrl = '';
newMapStep.featuredStepIndex = i + 1;
newMapStep.id = itemId;
newMapStep.isValid = 'false';
newMapStep.type = 'results';
const listForNewOptArray = List(newOptArray);
newMapStep.data = listForNewOptArray;
newMapStep.userAnswers = itemAnswers;
newResultsSteps.push(newMapStep);
});
return newResultsSteps;
};
export default dataObj;
The issue is steps, when logged outside the .then() returns a Promise {<pending>}. If I log results.attributes inside the .then(), I see the data fully returned.
You need to wait until your async call is resolved. You can do this by chaining on another then:
getGraphData(domain, userId, testId)
.then((result) => {
return result.attributes;
})
.then(steps => {
// put the rest of your method here
});
You can also look at async/await if your platform supports it which would allow code closer to your original
const steps = await getGraphData(domain, userId, testId)
.then((result) => {
return result.attributes;
});
// can use steps here
You have 2 options to transform your fetched data :
1st option : create a async function that returns a promise with the modified data :
const dataObj = (domain, userId, testId) => {
return getGraphData(domain, userId, testId).then((result) => {
const steps = result.attributes;
const expr = /select/;
// build array of steps that we have results in
const resultsSteps = [];
steps.forEach((step) => {
// check for types that contain 'select', and add them to array
if (expr.test(step.get('type'))) {
resultsSteps.push(step);
}
});
const newResultsSteps = [];
resultsSteps.forEach((item, i) => {
const newMapStep = new Map();
const itemDescription = item.get('description');
const itemId = item.get('id');
const itemOptions = item.get('options');
const itemAnswers = item.get('userAnswers');
const newOptArray = [];
itemOptions.forEach((element) => {
const optionsMap = new Map();
let elemName = element.get('value');
if (!element.get('value')) {
elemName = element.get('caption');
}
const elemPosition = element.get('position');
const elemCount = element.get('count');
optionsMap.name = elemName;
optionsMap.position = elemPosition;
optionsMap.value = elemCount;
newOptArray.push(optionsMap);
});
newMapStep.chartType = 'horizontalBar';
newMapStep.description = itemDescription;
newMapStep.featured = 'false';
newMapStep.detailUrl = '';
newMapStep.featuredStepIndex = i + 1;
newMapStep.id = itemId;
newMapStep.isValid = 'false';
newMapStep.type = 'results';
const listForNewOptArray = List(newOptArray);
newMapStep.data = listForNewOptArray;
newMapStep.userAnswers = itemAnswers;
newResultsSteps.push(newMapStep);
});
return newResultsSteps;
});
};
With es7 async/await syntax it should be :
const dataObj = async (domain, userId, testId) => {
const result = await getGraphData(domain, userId, testId);
const steps = result.attributes;
... modify the data
}
Then keep in mind that this function returns a promise, you'll need to wait for it to get the result, example in a react component :
componentDidMount(){
dataObj('mydomain', 'myuserId', 'mytestId').then((res) => {
this.setState({ data: res });
}
}
The component will update when the promise is resolve, you can then use the data (you'll need to handle the undefined data state in render method)
2nd option : Create a sync function to modify the data :
const dataObj = (steps) => {
const expr = /select/;
const resultsSteps = [];
steps.forEach((step) => {
...
}
return newResultsSteps;
};
To have the same result as option 1 in our component we'll use it like this :
componentDidMount(){
getGraphData('mydomain', 'myuserId', 'mytestId').then((res) => {
const modifiedData = dataObj(res);
this.setState({ data: modifiedData });
}
}
That's how promises work. The data is not ready when you are trying to use it so you should move all your processing into the .then. The reason your variable is a Promise {<pending>} is because you can chain other things onto it.
Something like:
steps.then((steps) => {
...
});
This is the module that collections and exports async data: scraper.js
const express = require('express')
const cheerio = require('cheerio')
const request = require("tinyreq")
const fs = require('fs')
const _ = require('lodash')
const uuid = require('uuid/v4')
const async = require('async')
const mental_models = {
url: 'https://www.farnamstreetblog.com/mental-models/',
data: {}
}
const decision_making = {
url: 'https://www.farnamstreetblog.com/smart-decisions/',
data: {}
}
const cognitive_bias = {
url: 'https://betterhumans.coach.me/cognitive-bias-cheat-sheet-55a472476b18',
data: {}
}
const DATA_URLS = [mental_models, decision_making, cognitive_bias]
const filterScrape = async (source, params) => {
let filtered_data = {
topics: [],
content: [],
additional_content: []
}
let response = await scrape(source)
try {
let $ = cheerio.load(response)
params.forEach((elem) => {
let headers = ['h1', 'h2', 'h3']
if ($(elem) && headers.includes(elem)) {
let topic = {}
let content = {}
let id = uuid()
topic.id = id
topic.text = $(elem).text()
if ($(elem).closest('p')) {
content.text = $(elem).closest('p').text()
content.id = id
}
filtered_data.topics.push(topic)
filtered_data.content.push(content)
} else if ($(elem) && !headers.includes(elem)) {
let content = {}
let id = uuid()
content.text = $(elem).text()
content.id = id
filtered_data.additional_content.push(content)
} else {
}
})
}
catch (err) {
console.log(err)
}
return filtered_data
}
const scrape = (source) => {
return new Promise((resolve, reject) => {
request(source.url, function (err, body) {
if (err) {
reject(err)
return
}
resolve(body)
})
})
}
const DATA = _.map(DATA_URLS, async (source) => {
let params = ['h1', 'h2', 'h3', 'p']
let new_data = await filterScrape(source, params)
try {
source.data = new_data
}
catch (err) {
console.log(err)
}
})
module.exports = DATA
This is the module that imports the data: neural.js
const brain = require('brain')
const neural_net = new brain.NeuralNetwork()
const DATA = require('./scraper')
console.log(DATA)
Obviously not much going on, I've removed the code since the variable doesn't resolve. When logged it logs a promise but the promise does not resolve. However in the imported module, the promise is logged and then resolves. What gives? Should I import a function that resolves the data?
Of course it would be best to import that function, however it won't change the issue in your code which is here:
const DATA = _.map(DATA_URLS, async (source) => {
Lodash doesn't support async iteration - so you need to have some other method, one would be to use the newest nodejs version (10.x) and make use of async iteration - but that won't use the full power of asynchronous code.
You can also use scramjet - a framework my company is supporting. The code above would take the following form:
const {DataStream} = require("scramjet");
const DATA_URLS = [mental_models, decision_making, cognitive_bias];
module.exports = async () => DataStream.fromArray(DATA_URLS)
.setOptions({maxParallel: 2}) // if you need to limit that at all.
.map(async ({url}) => {
let params = ['h1', 'h2', 'h3', 'p']
let data = await filterScrape(source, params);
return { url, data };
})
.toArray();
The other file would take the following form:
const brain = require('brain')
const neural_net = new brain.NeuralNetwork()
const scraper = require('./scraper')
(async (){
const DATA = await scraper();
console.log(DATA); // or do whatever else you were expecting...
})();