I have a requirement to execute several parallel functions
First functions:
Database get operation ie find inventory details from database
Second functions:
Process db results and save the result as a file
I am using two promise all, back to back to execute the above functions, I don't feel like this is the correct way to do this. Is there a
better way of handling these function calls. I am getting the result as per below code but want to know any other way.
Doing it following way:
let allQuery = {
sql: "SELECT * from inventory",
};
let inventoryQuery = {
sql: "SELECT * from inventory where inventory='1'",
};
let nearbyQuery = {
sql: "SELECT * from inventory where inventory='2",
};
let firstPromises = [dbService.fetch(allQuery),
dbService.fetch(inventoryQuery),
dbService.fetch(nearbyQuery)
];
Promise.all(firstPromises)
.then((values) => {
let all = values[0];
let inventory = values[1];
let nearby = values[2];
let fileKey1 = folderName + '/' + all.QueryExecutionId + '.csv';
let fileName1 = all.QueryExecutionId + '.csv';
let fileKey2 = folderName + '/' + inventory.QueryExecutionId + '.csv';
let fileName2 = inventory.QueryExecutionId + '.csv';
let fileKey3 = folderName + '/' + nearby.QueryExecutionId + '.csv';
let fileName3 = nearby.QueryExecutionId + '.csv';
let secondPromises = [s3Service.s3StreamDownload(bucketName, fileKey1, fileName1),
s3Service.s3StreamDownload(bucketName, fileKey2, fileName2),
s3Service.s3StreamDownload(bucketName, fileKey3, fileName3)
];
Promise.all(secondPromises)
.then((values) => {
console.log('Do later operation');
}).catch((error) => {
debug(`Error in promises ${error}`);
});
}).catch((error) => {
debug(`Error in promises ${error}`);
});
I think it can be more readable to extract the inner function and then chain them together:
Promise.all(firstPromises)
.then(transformToSecondPromises)
.then(Promise.all)
.then(values => {/* do later operation */})
.catch(error => { debug(`Error in promises ${error}`) })
function transformToSecondPromises ([all, inventory, nearby]) {
const fileKey1 = folderName + '/' + all.QueryExecutionId + '.csv';
const fileName1 = all.QueryExecutionId + '.csv';
const fileKey2 = folderName + '/' + inventory.QueryExecutionId + '.csv';
const fileName2 = inventory.QueryExecutionId + '.csv';
const fileKey3 = folderName + '/' + nearby.QueryExecutionId + '.csv';
const fileName3 = nearby.QueryExecutionId + '.csv';
return [
s3Service.s3StreamDownload(bucketName, fileKey1, fileName1),
s3Service.s3StreamDownload(bucketName, fileKey2, fileName2),
s3Service.s3StreamDownload(bucketName, fileKey3, fileName3)
];
}
Related
const express = require("express");
const bodyParser = require("body-parser");
const https = require("https");
const app = express();
app.get("/", function (request, response) {
//it is the endpoint of the api
const url =
"https://api.cricapi.com/v1/currentMatches?apikey=acdcba6a-5e79-4d7d-8efc-650c785096e2&offset=0";
https.get(url, function (res) {
console.log(res);
console.log(res.statusCode);
response.write("<h1><center>Cricket Score</center></h1><hr>");
response.write;
res.on("data", function (data) {
const cricket = JSON.parse(data);
for (var i = 0; i < cricket.data.length; i++) {
//it is stringify the r component of data.score[0].r
const cricketscore0 = JSON.stringify(cricket.data[i].score[0].r);
const cricketscore1 = JSON.stringify(cricket.data[i].score[1].r);
const cricketwickets0 = JSON.stringify(cricket.data[i].score[0].w);
const cricketwickets1 = JSON.stringify(cricket.data[i].score[1].w);
const cricketover0 = JSON.stringify(cricket.data[i].score[0].o);
const cricketover1 = JSON.stringify(cricket.data[i].score[1].o);
response.write("<h2>\n" + cricket.data[i].name + "</h2>");
response.write("<h3>\nMatch type :- " + cricket.data[0].matchType);
response.write(
"<h3>\n" +
cricket.data[i].score[0].inning +
" :-" +
cricketscore0 +
"/" +
cricketwickets0 +
"(" +
cricketover0 +
")</h3>"
);
response.write(
"<h3>\n" +
cricket.data[i].score[1].inning +
" :-" +
cricketscore1 +
"/" +
cricketwickets1 +
"(" +
cricketover1 +
")</h3>"
);
response.write("<h3><em>" + cricket.data[i].status + "</em></h3>");
}
response.send();
console.log();
});
});
});
app.listen(3000, function (req, res) {
console.log("The server is running on the port 3000");
});
I am getting into rendering the components of the data.score[1]. It is giving an error that they are undefined so I want you to please help me out in the above problem. I am unable to render it using for loop but I can get it if specify the value of I individually. when I am using for loop it is saying that components of data.score[1] does not exit. please help me out in this case. Please specify the answer and if possible please tell the problem with a solution code so that I can understand my problem and know what I am doing and avoid it in the future
This question will resolve the problems related to the rendering of things through APIs also. Please help me out with this problem.
I have made use of axios for this. You may use https. some of the objects are undefined for score index 1 that is why you are getting that error.
I have made use of axios for this. You may use https. some of the objects are undefined for score index 1 that is why you are getting that error.
const express = require('express');
const axios = require("axios");
const app = express();
const port = 3000;
app.use(express.json())
app.use(express.urlencoded())
const url = "https://api.cricapi.com/v1/currentMatches?apikey=acdcba6a-5e79-4d7d-8efc-650c785096e2&offset=0";
// middleware
app.use((req, res, next) => {
next()
});
const getCricketScors = async () => {
const resData = await axios.get(url)
// console.log(resData.data)
const { data } = resData
// console.log(data)
return data
}
app.get('/', async (req, res, next) => {
try {
const { data } = await getCricketScors()
// res.status(200).json(data)
if (!data) {
res.write("<h1><center>Cricket Score</center></h1><hr>");
res.write("<h4><center>Somthing went wrong</center></h4>");
res.send();
return
}
res.write("<h1><center>Cricket Score</center></h1><hr>");
res.write;
for (var i = 0; i < data.length; i++) {
const score0 = JSON.stringify(data[i].score[0])
const cricketscore0 = score0.r;
const cricketwickets0 = score0.w;
const cricketover0 = score0.o;
if (data[i].score.length > 1) {
const score1 = JSON.stringify(data[i].score[1])
const cricketscore1 = score1.r;
const cricketwickets1 = score1.w;
const cricketover1 = score1.o;
} else {
const cricketscore1 = '-'
const cricketwickets1 = '-'
const cricketover1 = '-'
}
res.write("<h2>\n" + data[i].name + "</h2>");
res.write("<h3>\nMatch type :- " + data[0].matchType);
res.write(
"<h3>\n" +
score0.inning +
" :-" +
cricketscore0 +
"/" +
cricketwickets0 +
"(" +
cricketover0 +
")</h3>"
);
if (score1) {
res.write(
"<h3>\n" +
score1?.inning +
" :-" +
cricketscore1 +
"/" +
cricketwickets1 +
"(" +
cricketover1 +
")</h3>"
);
}
res.write("<h3><em>" + data[i].status + "</em></h3>");
}
res.send();
} catch (error) {
console.log({ error })
res.status(500).json(error.message)
}
});
app.listen(port, '0.0.0.0', () => {
console.log(`Example app listening on port ${port}`);
});
I need to get urls of all files in 'bucket/loads/:loadID' path. I'm able to get these files in array called 'files'. Then I filter it (I get endFiles array). And now I just need a new array called url to push all the urls in (getSignedUrl). But I don't know how to do It. I need to get signed urls inside a loop (endFiles.forEach) and push it to urls array or something like that.
exports.testCloudFunc = functions.storage.object().onFinalize(async object => {
const filePath = object.name;
const { Logging } = require('#google-cloud/logging');
console.log(`Logged: FILEPATH: ${filePath}`);
const id = filePath.split('/');
console.log(`Logged: ID: ${id[0]}/${id[1]}`);
const bucket = object.bucket;
console.log(`Logged: BUCKET: ${object.bucket}`);
async function listFilesByPrefix() {
const options = {
prefix: id[0] + '/' + id[1]
};
const [files] = await storage.bucket(bucket).getFiles(options);
const endFiles = files.filter(el => {
return (
el.name === id[0] + '/' + id[1] + '/' + 'invoiceReport.pdf' ||
el.name === id[0] + '/' + id[1] + '/' + 'POD.pdf' ||
el.name === id[0] + '/' + id[1] + '/' + 'rateConfirmation.pdf'
);
});
for (let i = 0; i < endFiles.length; i++) {
console.log(endFiles[i].name);
}
}
listFilesByPrefix().catch(console.error);
});
I'm stuck and need help. Any help is highly appreciated.
The getSignedUrl() method is asynchronous and returns a Promise.
Since you want to concurrently execute multiple calls to this method, you need to use Promise.all() as follows:
async function listFilesByPrefix() {
const options = {
prefix: id[0] + '/' + id[1]
};
const [files] = await storage.bucket(bucket).getFiles(options);
const endFiles = files.filter(el => {
return (
el.name === id[0] + '/' + id[1] + '/' + 'invoiceReport.pdf' ||
el.name === id[0] + '/' + id[1] + '/' + 'POD.pdf' ||
el.name === id[0] + '/' + id[1] + '/' + 'rateConfirmation.pdf'
);
});
const config = {
action: 'read',
expires: '03-17-2025'
};
const promises = [];
for (let i = 0; i < endFiles.length; i++) {
console.log(endFiles[i].name);
promises.push(endFiles[i].getSignedUrl(config));
}
const urlsArray = await Promise.all(promises);
return urlsArray;
}
listFilesByPrefix()
.then(results => {
//results is an array of signed URLs
//It's worth noting that values in the array will be in order of the Promises passed with promises.push()
//do whatever you need, for example:
results.forEach(url => {
//....
});
})
I created this function to obtain GitHub issues:
retrieveEnerpriseIssues: function(repoOrg, repoName, callback) {
let data = null;
// token auth
octokit.authenticate({
type: 'basic',
username: config.githubEnterprise.username,
password: config.githubEnterprise.token
});
async function paginate(method) {
let response = await method({
q: "repo:" + repoOrg + "/" + repoName + " is:issue",
per_page: 100
});
data = response.data.items;
var count = 0;
while (octokit.hasNextPage(response)) {
count++;
console.log(`request n°${count}`);
response = await octokit.getNextPage(response);
data = data.concat(response.data.items);
}
return data;
}
paginate(octokit.search.issues)
.then(data => {
callback(data);
})
.catch(error => {
console.log(error);
});
}
It is called in this function which takes the issues, filters out all of the unwanted keys into json format and puts it in my db.
extractToDb: function() {
let gitIssues = null;
for(var i = 0; i < config.githubEnterprise.orgs.length; i++) {
for(var j = 0; j < config.githubEnterprise.orgs[i].repos.length; j++) {
gitHubService.retrieveEnerpriseIssues(
config.githubEnterprise.orgs[i].owner,
config.githubEnterprise.orgs[i].repos[j].repoName,
function(data, err) {
if(err) {
console.log('err: ', err);
} else {
gitIssues = data;
}
gitIssues = JSON.stringify(gitIssues);
gitIssues = JSON.parse(gitIssues);
let issueFormatForDb = null;
for(var i = 0; i < gitIssues.length; i++) {
issueFormatForDb = gitIssues[i];
const body = '{' +
'"github_id": "' + issueFormatForDb.id + '",' +
'"issue_title": "' + issueFormatForDb.title + '",' +
'"issue_number": "' + issueFormatForDb.number + '",' +
'"issue_url": "' + issueFormatForDb.url + '",' +
'"issue_state": "' + issueFormatForDb.state + '"' +
'}';
console.log('Body: ', body);
getGitHubIssues.postToDb(body);
}
});
}
}
}
I'd like to take this a step further by filtering out any issues where the state is closed. How is this done and should it be handled in my retrieveEnerpriseIssues function or my extractToDb?
Possible solution
I tried this in my extractToDb function:
gitIssues = JSON.parse(gitIssues);
gitIssues = _.where(gitIssues, {state: "open"});
let issueFormatForDb = null;
Is it the best solution or is there a better way?
As #givehug stated:
Better use _.filter, or native filter method like
gitIssues = gitIssues.filter(i => i.state === 'open')
I think .where was deprecated in later versions of lodash github.com/lodash/lodash/wiki/Deprecations. Other than that its perfectly fine.
I just realsied I can filter the state in my paginate function with this:
let response = await method({
q: "repo:" + repoOrg + "/" + repoName + " is:issue" + " label:issue_label" + " state:open",
per_page: 100
});
I am writing a function calling an API to fetch URLs. These are the steps that I wish to accomplish:
Parsing in an array of objects (restaurants) as arguments
For each object, call the Google Search API to get some imageURLs
Store those imageURLs in an array
Add imageURLs as an attribute called imageURLs to each object within the array in the argument
The code is able to log the imageURLs within the GET request, but outside of the request, imageURLs is just an empty array.
var googleSearch = function(restaurants, cb){
console.log("google starts");
const apiKey = google_apiKey;
const cseKey = cseID;
Array.from(restaurants).forEach(function(restaurant){
var keyWord = restaurant.name + " "+ restaurant.location.city
+ " "+ restaurant.location.state + " food";
var googleURL = "https://www.googleapis.com/customsearch/v1?key="+ apiKey +
"&q="+ keyWord +
"&searchType=image" +
"&cx=" + cseKey +
"&num=7" +
"&safe=medium"
;
//image URLs of each restaurants to be displayed in the front end
var imageURLs = [];
request
.get(googleURL,
{
json : true, headers: {
'User-Agent' : 'thaorell'
}
})
.then(function(response){
Array.from(response.items).forEach(function(item){
imageURLs.push(item.link)
});
})
.catch(e => {
console.log(e);
})
restaurant.imageURLs = imageURLs
})
cb(null, restaurants);
}
You're misunderstanding the Promise API:
var googleSearch = function (restaurants, cb) {
console.log("google starts");
const apiKey = google_apiKey;
const cseKey = cseID;
return Promise.all(Array.from(restaurants).map(function (restaurant) {
var keyWord = restaurant.name + " " + restaurant.location.city
+ " " + restaurant.location.state + " food";
var googleURL = "https://www.googleapis.com/customsearch/v1?key=" + apiKey +
"&q=" + keyWord +
"&searchType=image" +
"&cx=" + cseKey +
"&num=7" +
"&safe=medium"
;
return request
.get(googleURL,
{
json: true, headers: {
'User-Agent': 'thaorell'
}
}
)
.then(function (response) {
restaurant.imageURLs = Array.from(response.items).map(function (item) {
return item.link;
});
return restaurant;
})
})
)
.then(restaurants2 => cb(null, restaurants2))
.catch(cb)
}
As you can see you need to wait for all of the requests to finish before you pass the data back to the callback.
I was having this problem when trying to pull from firebase. My database structure as such:
I am trying to pull the merchantName and its relevant branches details:
function getAllMerchant(){
var query = firebase.database().ref('merchants');
return query.once('value').then(data => {
var result = [];
data.forEach(snapshot => {
var merchantData = snapshot.val();
var merchantName = merchantData.merchantName;
var branchName = merchantData.branches.branchName;
var branchAddress = merchantData.branches.branchAddress;
console.log('check ' + merchantName + ' ' + branchName + ' ' + branchAddress);
result.push({merchantName: merchantName, branchName: branchName, branchAddress: branchAddress});
});
return result;
});
resolve(result);
}
However, when I printed out the branchName and branchAddress, I am getting undefined. I managed to print out merchantName though. Any ideas?
Thanks!
You're not iterating over the branches of the merchant.
data.forEach(merchantSnapshot => {
var merchantData = snapshot.val();
var merchantName = merchantData.merchantName;
console.log('check ' + merchantName);
merchantSnapshot.child("branches").forEach(brancheSnapshot => {
var branchName = brancheSnapshot.val().branchName;
var branchAddress = brancheSnapshot.val.branchAddress;
console.log(' ' + branchName + ' ' + branchAddress);
});
});