nodejs recursively call same api and write to excel file sequentially - javascript

I need to call an API recursively using request promise after getting result from API need to write in an excel file , API sample response given below
{
"totalRecords": 9524,
"size": 20,
"currentPage": 1,
"totalPages": 477,
"result": [{
"name": "john doe",
"dob": "1999-11-11"
},
{
"name": "john1 doe1",
"dob": "1989-12-12"
}
]
}
Now I want to call this API n times, here n is equal to totalPages, after calling each API I want to write response result to the excel files.
First write page 1 response result to excel then append page 2 response result to excel file and so on..
I have written some sample code given below
function callAPI(pageNo) {
var options = {
url: "http://example.com/getData?pageNo="+pageNo,
method: 'GET',
headers: {
'Content-Type': 'application/json'
},
json: true
}
return request(options)
}
callAPI(1).then(function (res) {
// Write res.result to excel file
}).catch(function (err) {
// Handle error here
})
But facing problem calling recursively API and maintaining sequentially like write page 1 result first to excel file then page 2 result append to excel and so on..
Any code sample how to achieve in nodejs

You want to do something like this:
function getAllPages() {
function getNextPage(pageNo) {
return callAPI(pageNo).then(response => {
let needNextPage = true;
if (pageNo === 1) {
// write to file
} else {
// append to file
}
if (needNextPage) {
return getNextPage(pageNo+1);
} else {
return undefined;
}
});
}
return getNextPage(1);
}
Obviously change that 'needNextPage' to false to stop the recursion when you're done

So you want to do 477 requests in sequence? How long do you wanna wait for this to finish? Even in paralell, this would be still too long for me.
Best: write an API that can return you a batch of pages at once. Reducing the number of requests to the backend. Maybe something like http://example.com/getData?pages=1-100 and let it return an Array; maybe like
[
{
"totalRecords": 9524,
"currentPage": 1,
"totalPages": 477,
"result": [...]
},
{
"totalRecords": 9524,
"currentPage": 2,
"totalPages": 477,
"result": [...]
},
...
]
or more compact
{
"totalRecords": 9524,
"totalPages": 477,
"pages": [
{
"currentPage": 1,
"result": [...]
},
{
"currentPage": 2,
"result": [...]
},
...
]
}
Sidenote: writing the size of the results array into the json is unnecessary. This value can easily be determined from data.result.length
But back to your question
Imo. all you want to run in sequence is adding the pages to the sheet. The requests can be done in paralell. That already saves you a lot of overall runtime for the whole task.
callApi(1).then(firstPage => {
let {currentPage, totalPages} = firstPage;
//`previous` ensures that the Promises resolve in sequence,
//even if some later request finish sooner that earlier ones.
let previous = Promise.resolve(firstPage).then(writePageToExcel);
while(++currentPage <= totalPages){
//make the next request in paralell
let p = callApi(currentPage);
//execute `writePageToExcel` in sequence
//as soon as all previous ones have finished
previous = previous.then(() => p.then(writePageToExcel));
}
return previous;
})
.then(() => console.log("work done"));
or you wait for all pages to be loaded, before you write them to excel
callApi(1).then(firstPage => {
let {currentPage, totalPages} = firstPage;
let promises = [firstPage];
while(++currentPage < totalPages)
promises.push(callApi(currentPage));
//wait for all requests to finish
return Promise.all(promises);
})
//write all pages to excel
.then(writePagesToExcel)
.then(() => console.log("work done"));
or you could batch the requests
callApi(1).then(firstPage => {
const batchSize = 16;
let {currentPage, totalPages} = firstPage;
return Promise.resolve([ firstPage ])
.then(writePagesToExcel)
.then(function nextBatch(){
if(currentPage > totalPages) return;
//load a batch of pages in paralell
let batch = [];
for(let i=0; i<batchSize && ++currentPage <= totalPages; ++i){
batch[i] = callApi(currentPage);
}
//when the batch is done ...
return Promise.all(batch)
//... write it to the excel sheet ...
.then(writePagesToExcel)
//... and process the next batch
.then(nextBatch);
});
})
.then(() => console.log("work done"));
But don't forget to add the error handling. Since I'm not sure how you'd want to handle errors with the approaches I've posted, I didn't include the error-handling here.
Edit:
can u pls modify batch requests, getting some error, where you are assigning toalPages it's not right why the totalPages should equal to firstPage
let {currentPage, totalPages} = firstPage;
//is just a shorthand for
let currentPage = firstPage.currentPage, totalPages = firstPage.totalPages;
//what JS version are you targeting?
This first request, callApi(1).then(firstPage => ...) is primarily to determine currentIndex and totalLength, as you provide these properties in the returned JSON. Now that I know these two, I can initiate as many requests in paralell, as I'd want to. And I don't have to wait for any one of them to finish to determine at what index I am, and wether there are more pages to load.
and why you are writing return Promise.resolve([ firstPage ])
To save me some trouble and checking, as I don't know anything about how you'd implement writePagesToExcel.
I return Promise.resolve(...) so I can do .then(writePagesToExcel). This solves me two problems:
I don't have to care wether writePagesToExcel returns sync or a promise and I can always follow up with another .then(...)
I don't need to care wether writePagesToExcel may throw. In case of any Error, it all ends up in the Promise chain, and can be taken care of there.
So ultimately I safe myself a few checks, by simply wrapping firstPage back up in a Promise and continue with .then(...). Considering the amounts of data you're processing here, imo. this ain't too much of an overhead to get rid of some potential pitfalls.
why you are passing array like in resolve
To stay consistent in each example. In this example, I named the function that processes the data writePagesToExcel (plural) wich should indicate that it deals with multiple pages (an array of them); I thought that this would be clear in that context.
Since I still need this seperate call at the beginning to get firstPage, and I didn't want to complicate the logic in nextBatch just to concat this first page with the first batch, I treat [firstPage] as a seperate "batch", write it to excel and continue with nextBatch

function callAPI(pageNo) {
var options = {
url: "http://example.com/getData?pageNo="+pageNo,
method: 'GET',
headers: {
'Content-Type': 'application/json'
},
json: true
}
return request(options)
}
function writeToExcel(res){console.log(res)} //returns promise.
callAPI(1).then(function (res) {
if(res){
writeToExcel(res).then(() => {
var emptyPromise = new Promise(res => setTimeout(res, 0));
while(res && res.currentPage < res.totalPages){
emptyPromise = emptyPromise.then(() => {
return callAPI(res.currentPage).then(function (res){
if(res){
writeToExcel(res)
}
});
}
}
return emptyPromise;
});
}
}).catch(function (err) {
// Handle error here
})

Related

NodeJS request times out when sending a archiver file

I have a NextJS app where I have to generate loads of QR codes, 300,400,500 at a time. I want to put them inside a zip and let the user download them. Here's the code which puts the codes into a zip using the archiver library`:
const archive = archiver.create("zip", {});
let index = 1;
for (const code of codes) {
// #ts-expect-error
const qrCode = new QRCodeStyling({
nodeCanvas: canvas,
width: 300,
height: 300,
data: code.url
});
archive.append(
await qrCode.download({
name: "testName",
extension: "png",
skipDownload: true,
buffer: true,
}),
{
name: `${code.name}-${index}.png`,
},
);
index++;
}
console.log("before finalizing");
await archive.finalize();
console.log("after finalizing");
return archive;
The request takes like 10s on a local machine, and when in production it just times out every time. The program sometimes doesn't even respond, it just hangs on before finalizing..
This is how I'm sending my codes to the frontend:
res.setHeader("Content-Disposition", "attachment");
return res.status(200).send(await exportCodes());
Please note that these zip files are usually around 3-5mb, so size should not be a problem I don't think
When calling qrCode.download(...), you create a promise, which is then awaited, and the result is passed to archive.append(...), – only after that you get to download the next QR code. Let's say, downloading a QR code takes 100 msec. Multiply that by 300, and you get 30000 msec, or 30 seconds, – on average. Of course, it will time out.
You have to parallelize downloading. Create a bunch of promises, await them all together at once, then archive.append(...) the results one by one:
interface QrCodeDownloaded {
name: string;
index: number;
// I don't know the type of result here, sorry; must be something like Buffer | Stream | string
result: any;
}
const archive = archiver.create("zip", {});
const downloads: Promise<QrCodeDownloaded>[] = [];
// We have to use classic for-loop here, because we want to have
// an independent `index` variable for each iteration
for (let index = 0; index < codes.length; index++) {
// counting from 0 here
const code = codes[index];
const qrCode = new QRCodeStyling({
nodeCanvas: canvas,
width: 300,
height: 300,
data: code.url
});
const download = qrCode.download({
name: "testName",
extension: "png",
skipDownload: true,
buffer: true,
});
// We have to use .then() here, because we want to hold to
// code names and indexes, but we also don't want to re-iterate through
// the array again. We can't use `await` here, because this will negate
// the whole thing, – we want to initiate the next download without
// awaiting the current download
downloads.push(download.then((result) => ({
result,
name: code.name,
index: index + 1, // counting from 1 here
})));
}
// This is the most important line, it does the parallelizing
const results = await Promise.all(downloads);
for (const { result, name, index } of results) {
archive.append(result, { name: `${name}-${index}.png` });
}
console.log("before finalizing");
await archive.finalize();
console.log("after finalizing");
return archive;

Define response structure in Adonisjs with Middleware

I want to define the response structure of my requests in the simplest way, and the first thing that comes in my mind to do this is a middleware.
My endpoints are returning the response content correctly:
{{base_url}}/users returns a list of users:
{
[
{
"id": 44,
"name": "some name"
[...]
}
]
}
What I want to do (in all requests) is to add the fields status and data (or any other I'd like to add), like this:
{
"status": 200,
"data": [
{
"id": 44,
"name": "some name"
[...]
}
]
}
I've created a middleware that waits for the resolution but I'm not able to get the content nor add some property to it.
[...]
async handle ({request, response}, next) {
await next()
const content = response._lazyBody.content
content.status = response.response.statusCode
}
[...]
I know this will not work but I want something similar to this. I've looked in Adonis docs and forum, but no answers fit to my needs.
Any help will be welcome
You can extend Response By extending the core. The simplest way is to create a file inside start folder and name it hooks.js and copy and paste the content below inside it:
const { hooks } = use('#adonisjs/ignitor')
const Response = use('Adonis/Src/Response')
hooks.after.providersBooted(() => {
Response.macro('customJson', function (status, data) {
this.status(status).json({
status,
data
})
})
})
this piece of code extends the Response module and add customJson method to it which takes two arguments, status and data, and send them back to the client.
And here you can see how to use it:
Route.get('/users', async ({ response }) => {
let status = ''// whatever you want
let data = ''// whatever you want
return response.customJson(status, data)
})

Problem w/state of a vue data object in component

I am updating my original vue project and am getting an error w/data object sports_feeds_boxscores_*. The site has three tabs to pull down scores for the three major leagues. I am adding the player stats for each game now. I first did baseball and all worked fine. Now I am doing football and the problem arises. I have three objects setup for the stats for each league. The nfl also contains an object with the three days of the week they play. What is happening is the stats for Sunday get pulled down ok but then Thursday's stats which should only be one game instead has all sunday's games plus the one thursday game. And then Monday has both Sunday & Thursdays results in it besides Mondays. I have made all the components separate as well as three separate data objects for the component props. And if I first click the nfl tab and then go to the mlb tab all results from nfl data object are in sports_feeds_boxscores_mlb. I setup a site here to better understand whats going on in using Vue.js devtools. Here is the pertinent code:
index.html:
<component
v-if="currentTabComponent === 'tab-mlb'"
v-bind:is="currentTabComponent"
v-bind:props_league_data="sports_feeds_data"
v-bind:props_league_standings="standings"
v-bind:props_baseball_playoffs="baseball_playoffs"
v-bind:props_end_of_season="end_of_season[this.currentTab.toLowerCase()]"
v-bind:props_box_game_scores_mlb="sports_feeds_boxscores_mlb"
class="tab"
>
</component>
<component
v-if="currentTabComponent === 'tab-nfl'"
v-bind:is="currentTabComponent"
v-bind:props_league_data="sports_feeds_data"
v-bind:props_league_data_nfl="nfl_feeds"
v-bind:props_league_standings="standings"
v-bind:props_nfl_playoffs="nfl_playoffs"
v-bind:props_end_of_season="end_of_season[this.currentTab.toLowerCase()]"
v-bind:props_box_game_scores_nfl="sports_feeds_boxscores_nfl"
class="tab"
>
</component>
vue.js:
data() {
return {
sports_feeds_boxscores_mlb: null,
sports_feeds_boxscores_nfl: {
sun: null,
mon: null,
thurs: null
},
sports_feeds_boxscores_nba: null,
etc
/* Component Code */
// First let's get the Game and BoxScores Data
const nflScores = async () => {
this.nfl_feeds.sunday_data = await getScores(
nflDate.sundayDate,
config
);
this.nfl_feeds.thurs_data = await getScores(
nflDate.thursdayDate,
config
);
this.nfl_feeds.mon_data = await getScores(nflDate.mondayDate, config);
// Next we need the gameid's to retrieve the game boxscores for each day
this.nfl_feeds.sunday_data.forEach(function(item, index) {
if (item.isCompleted === "true") {
nflGameIDs.sunday[index] = item.game.ID;
}
});
this.nfl_feeds.thurs_data.forEach(function(item, index) {
if (item.isCompleted === "true") {
nflGameIDs.thursday[index] = item.game.ID;
}
});
this.nfl_feeds.mon_data.forEach(function(item, index) {
if (item.isCompleted === "true") {
nflGameIDs.monday[index] = item.game.ID;
}
});
// Check if boxscores have been retrieved on previous tab click for each day
// if not retrieve the boxscores
this.sports_feeds_boxscores_nfl.sun =
this.sports_feeds_boxscores_nfl.sun ||
(await getBoxScores(nflGameIDs.sunday, url, params));
this.sports_feeds_boxscores_nfl.thurs =
(await getBoxScores(nflGameIDs.thursday, url, params));
this.sports_feeds_boxscores_nfl.mon =
this.sports_feeds_boxscores_nfl.mon ||
(await getBoxScores(nflGameIDs.monday, url, params));
}; /* End nflScores Async function */
getBoxScores.js:
try {
const getBoxScores = async (gameIDs, myUrl, params) => {
gameIDs.forEach(function(item) {
promises.push(
axios({
method: "get",
headers: {
Authorization:
"Basic &&*&&^&&=="
},
url: myUrl + item,
params: params
})
);
});
// axios.all returns a single Promise that resolves when all of the promises passed
// as an iterable have resolved. This single promise, when resolved, is passed to the
// "then" and into the "values" parameter.
await axios.all(promises).then(function(values) {
boxScores = values;
});
console.log(`boxScores is ${boxScores.length}`)
return boxScores;
};
module.exports = getBoxScores;
} catch (err) {
console.log(err);
}
I have split up all the sports_feeds_boxscores objects and at a loss as to why they are sharing state??? Sorry for verbosity of the question but it is somewhat complex. That is why I provided the site where you can see devtools that for instance this.sports_feeds_boxscores_nfl.thurs has 14 elements instead of one after the call to API. And if mlb tab is clicked after nfl tab then mlb results include the nfl results. I would really appreciate help in figuring this out. Thanks in advance...
Update:
I have added getBoxScores.js cause it seems as if I am returning the extra stats from this call.
This was my bad. I didnt realize I had created a closure in getBoxScores.js:
let boxScores = [];
let promises = [];
try {
const getBoxScores = async (gameIDs, myUrl, params) => {
gameIDs.forEach(function(item) {
promises.push(
axios({
method: "get",
headers: {
Authorization:
"Basic &&^^&^&&^FGG="
},
url: myUrl + item,
params: params
})
);
});
Moving declarations inside async function quickly solved trouble. URRRRGGGHHH!!!

Can't change my array, because I've got Scope Problems

I don't know why but I have some problems with my Dashboard.
So basically I want to create some fancy Donut Charts.
For that I've prepared a dataset-Array where I put my numbers in. All that works.
But when I get my data from the database I want to change the array, to update the Chart.
This is where I have problems.
So my data() looks like this:
data() {
return {
disturbances_category_0: [],
disturbances_category_1: [],
disturbances_category_2: [],
disturbances_category_3: [],
datasets: [
{
data: [20, 20, 10, 50], //HERE I HAVE TO CHANGE THE NUMBERS <-------------
backgroundColor: ["#A40000", "#580000", "#EC4A3B", "#179C7D"],
hoverBackgroundColor: ["#ff1a1a", "#b30000", "#f4948b", "#66bfac"]
}
],
labels: ["Banana", "Apple", "Strawberry", "Cherry"],
option: {}
};
},
And then there is my created()-Block, where I use Axios + Sequelize and Feathers to get my data:
created() {
axios.get('http://localhost:3030/disruptions/', {
params: {
DisruptionCategory: 0
}
})
.then((response) => {
this.disturbances_category_0 = response.data.data; //HERE IS THE COMPLETE ARRAY
this.datasets[0].data[0] = this.disturbances_category_0.length; //HERE I WANT TO SET THE LENGTH
})
.catch((error) => {
console.log(error.data);
});
//imagine that for the other fruits as well...
console.log(this.datasets[0].data[0]);
}
If I test this script I always get "20" as printout.
I don't know why it doesn't change the datasets.data-Array ... I also tried out to use Array.push but... nothing happened..
I'm sure I forgot something obvious...
It is because the console log likely happened long before your then block executed. Its initial value is an array of four integers before you overwrite it with a length. Try making the created function async and await the axios promise chain to resolve.
async function created() {
await axios.get('http://localhost:3030/disruptions/', { // await the resolve
params: {
DisruptionCategory: 0
}
})
.then((response) => {
this.disturbances_category_0 = response.data.data; //HERE IS THE COMPLETE ARRAY
this.datasets[0].data[0] = this.disturbances_category_0.length; //HERE I WANT TO SET THE LENGTH
})
.catch((error) => {
console.log(error.data);
});
//imagine that for the other fruits as well...
console.log(this.datasets[0].data[0]); // now this should be updated
}
console.log(this.datasets[0].data[0]);
The above will run before the response to your request has been handled since it is asynchronous. Your code will just keep executing while the .then() part will execute on another thread once you get a response from the server.

Firebase Real Time Database Structure for File Upload

I am working on my first Firebase project using AngularFire2. Below is the overall design of my learning project.
Users uploads photos and it's stored in the Firebase storage as images.
The uploaded photos are listed in the homepage sorted based on timestamp.
Below is the structure that I have now when I started. But I feel difficulty when doing joins. I should be able to get user details for each uploads and able to sort uploads by timestamp.
User:
- Name
- Email
- Avatar
Uploads:
- ImageURL
- User ID
- Time
I read few blogs de-normalising the data structure. For my given scenario, how best can i re-model my database structure?
Any example for creating some sample data in the new proposed solution will be great for my understanding.
Once the image upload is done, I am calling the below code to create an entry in the database.
addUpload(image: any): firebase.Promise<any> {
return firebase.database().ref('/userUploads').push({
user: firebase.auth().currentUser.uid,
image: image,
time: new Date().getTime()
});
}
I am trying to join 2 entities as below. i am not sure how can I do it efficiently and correctly.
getUploads(): any {
const rootDef = this.db.database.ref();
const uploads = rootDef.child('userUploads').orderByChild('time');
uploads.on('child_added',snap => {
let userRef =rootDef.child('userProfile/' + snap.child('user').val());
userRef.once('value').then(userSnap => {
???? HOW TO HANDLE HERE
});
});
return ?????;
}
I would like to get a final list having all upload details and its corresponding user data for each upload.
This type of join will always be tricky if you write it from scratch. But I'll try to walk you through it. I'm using this JSON for my answer:
{
uploads: {
Upload1: {
uid: "uid1",
url: "https://firebase.com"
},
Upload2: {
uid: "uid2",
url: "https://google.com"
}
},
users: {
uid1: {
name: "Purus"
},
uid2: {
name: "Frank"
}
}
}
We're taking a three-stepped approach here:
Load the data from uploads
Load the users for that data from users
Join the user data to the upload data
1. Load the data uploads
Your code is trying to return a value. Since the data is loaded from Firebase asynchronously, it won't be available yet when your return statement executes. That gives you two options:
Pass in a callback to getUploads() that you then call when the data has loaded.
Return a promise from getUploads() that resolves when the data has loaded.
I'm going to use promises here, since the code is already difficult enough.
function getUploads() {
return ref.child("uploads").once("value").then((snap) => {
return snap.val();
});
}
This should be fairly readable: we load all uploads and, once they are loaded, we return the value.
getUploads().then((uploads) => console.log(uploads));
Will print:
{
Upload1 {
uid: "uid1",
url: "https://firebase.com"
},
Upload2 {
uid: "uid2",
url: "https://google.com"
}
}
2. Load the users for that data from users
Now in the next step, we're going to be loading the user for each upload. For this step we're not returning the uploads anymore, just the user node for each upload:
function getUploads() {
return ref.child("uploads").once("value").then((snap) => {
var promises = [];
snap.forEach((uploadSnap) => {
promises.push(
ref.child("users").child(uploadSnap.val().uid).once("value")
);
});
return Promise.all(promises).then((userSnaps) => {
return userSnaps.map((userSnap) => userSnap.val());
});
});
}
You can see that we loop over the uploads and create a promise for loading the user for that upload. Then we return Promise.all(), which ensures its then() only gets called once all users are loaded.
Now calling
getUploads().then((uploads) => console.log(uploads));
Prints:
[{
name: "Purus"
}, {
name: "Frank"
}]
So we get an array of users, one for each upload. Note that if the same user had posted multiple uploads, you'd get that user multiple times in this array. In a real production app you'd want to de-duplicate the users. But this answer is already getting long enough, so I'm leaving that as an exercise for the reader...
3. Join the user data to the upload data
The final step is to take the data from the two previous steps and joining it together.
function getUploads() {
return ref.child("uploads").once("value").then((snap) => {
var promises = [];
snap.forEach((uploadSnap) => {
promises.push(
ref.child("users").child(uploadSnap.val().uid).once("value")
);
});
return Promise.all(promises).then((userSnaps) => {
var uploads = [];
var i=0;
snap.forEach((uploadSnap) => {
var upload = uploadSnap.val();
upload.username = userSnaps[i++].val().name;
uploads.push(upload);
});
return uploads;
});
});
}
You'll see we added a then() to the Promise.all() call, which gets invoked after all users have loaded. At that point we have both the users and their uploads, so we can join them together. And since we loaded the users in the same order as the uploads, we can just join them by their index (i). Once you de-duplicate the users this will be a bit trickier.
Now if you call the code with:
getUploads().then((uploads) => console.log(uploads));
It prints:
[{
uid: "uid1",
url: "https://firebase.com",
username: "Purus"
}, {
uid: "uid2",
url: "https://google.com",
username: "Frank"
}]
The array of uploads with the name of the user who created that upload.
The working code for each step is in https://jsbin.com/noyemof/edit?js,console
I did the following based on Franks answer and it works. I am not sure if this is the best way for dealing with large number of data.
getUploads() {
return new Promise((resolve, reject) => {
const rootDef = this.db.database.ref();
const uploadsRef = rootDef.child('userUploads').orderByChild('time');
const userRef = rootDef.child("userProfile");
var uploads = [];
uploadsRef.once("value").then((uploadSnaps) => {
uploadSnaps.forEach((uploadSnap) => {
var upload = uploadSnap.val();
userRef.child(uploadSnap.val().user).once("value").then((userSnap) => {
upload.displayName = userSnap.val().displayName;
upload.avatar = userSnap.val().avatar;
uploads.push(upload);
});
});
});
resolve(uploads);
});
}

Categories