We Keep Coding

I got my JSON, now I need it to download inot an specific directory

I have an API that periodically makes a request to RESTAPI and stores the data into my database, now I want to add a new feature for certain data: download some data by a request to my API, then, my API makes another request to the RESTAPI, but I dont want this data to store in my database I want it to download as JSON or CSV.
I managed to create the request, I coded the request to the RESTAPI and managed to get the JSON into a variable with all the data, Im stuck there, How do I make this data to get downloaded into a direcory?
Im using javascript nodeJS with bent ,getJSON ,mssql ,https.
the code of the function:
async function descargarDatosOpendata(anioIni, anioFin, Indicativo){
try{
while(anioIni == anioFin || anioIni < anioFin){
console.log("first");
var http = require("https");
var options = {
"method": "GET",
"hostname": "opendata.aemet.es",
"path": "/opendata/api/valores/climatologicos/mensualesanuales/datos/anioini/"+ anioIni +"/aniofin/"+ anioIni +"/estacion/"+ Indicativo +"",
"headers": {
"cache-control": "no-cache",
"api_key": "MYKEY"
}
};
console.log("second");
var req = http.request(options, function (res) {
console.log("tercera");
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", async function () {
console.log("endChunk");
var body = Buffer.concat(chunks);
console.log(body);
var bodyString = body.toString();
bodyJSON = JSON.parse(bodyString);
console.log(bodyJSON);
console.log(bodyJSON.datos);
if (bodyJSON.estado == 200 && bodyJSON.descripcion == "exito") {
let obj = await getJSON(bodyJSON.datos);
console.log(obj)
}
});
});
anioIni++;
req.end();
}
}
catch (err) {
console.log(err);
}
}
obj log is the data in json format: [{data}]

If this code is running in Node, you should use the Node fs module. The appendFile method will create a new file or append to it if it already exists. Learn more about fs
Example code
var fs = require('fs');
fs.appendFile('mynewfile1.txt', 'Hello content!', function (err) {
if (err) throw err;
console.log('Saved!');
});

Cloud Functions working in emulator but not when deployed

I am trying to create a collection and a document once a document is created in Firebase.
It is working fine in emulator, but when I deploy my functions to Firebase project, they don't create that collection and document.
But if I create fields in the existing document (snapshot) it works.
In the function I get data from an API and write it to the new document (which is not being created at the moment).
Function:
exports.quoteEndPoint = functions.firestore.document('users/{userID}/followedStocks/{stockID}')
.onCreate((snap, context) => {
const stock_id = context.params.stockID;
const user_id = context.params.userID;
var request = require('request');
var http = require('https');
const options = {
"method": "GET",
"hostname": "alpha-vantage.p.rapidapi.com",
"port": null,
"path": '/query?function=GLOBAL_QUOTE&symbol='+stock_id+'&datatype=json',
"headers": {
"x-rapidapi-host": "%API_HOST%",
"x-rapidapi-key": "%API_KEY%",
"useQueryString": true
}
};
const req = http.request(options, function(res){
const chunks = [];
res.on('data', function(chunk){
chunks.push(chunk);
});
res.on('end', function(){
const body = Buffer.concat(chunks);
//console.log(body.toString());
const result = JSON.parse(body.toString());
console.log(result);
//set values from json responso to Firebase
return snap.ref.collection('quoteEndPoint').doc('data').set(
{
'symbol': result['Global Quote']['01. symbol'],
'open': result['Global Quote']['02. open'],
}, { merge: true }).then(()=>{
console.log('New quoteEndPoint fields for ' + stock_id + ' added to Firebase');
})
.catch(err => {
console.log(err);
});
});
})
.on('error',(err) => {
console.log('Error: '+err.message);
});
req.end();
return true;
});
I tried to make the function: function() async, but it didn't work.
In emulator is creating and populating values to the right path : /users/7nDGdHmZDuoDiJkxixgz/followedStocks/AMD/quoteEndPoint/data
Anyone can help?
Thanks

Your problem most probably comes from the fact that a call with request does not return a promise, while in Cloud Functions triggered by background events (like .onCreate() for Firestore) you must return a Promise. Watch this official video series for more details: in particular the 3 videos titled "Learn JavaScript Promises".
In addition, request is deprecated.
You can use axios, which returns a Promise, or node-fetch. You need to chain the promises returned by the asynchronous operations, i.e. axios and Firestore asynchronous calls, as illustrated in the following code "skeleton":
const functions = require('firebase-functions');
const admin = require('firebase-admin');
const axios = require('axios');
exports.quoteEndPoint = functions.firestore.document('users/{userID}/followedStocks/{stockID}')
.onCreate((snap, context) => {
const stock_id = context.params.stockID;
const user_id = context.params.userID;
return axios({
method: 'get',
url: 'http://....'
// ... See the doc
})
.then(response => {
// ...
return snap.ref.collection('quoteEndPoint').doc('data').set(...);
});
});

Unable to receive POST data unless I refresh client. why? How can I wait for post data to be received before page loads?

My app starts with a simple html form. the inputs are PIN# and Date Of Birth.
My express server runs on the same port 3000, when the user submits their data, puppeteer starts and logs into a specific webpage. Then I scrape the image on that webpage. Google Api takes the text from that image and saves it in an array. I then post that array string to src/results.html. But as soon as the user hits submit, they are redirected to /resuts route right immediately and the page says cannot post the data. but when I see in the console (roughly a minute later) that the post was successful, I refresh the page and I get the array of text I wanted to see.
How can I await for the data to finish being posted to the route before the page loads the data? Im using react for client side. below is my server side code. client side is just a basic react page login and a static /results page meant for the data.
const puppeteer = require("puppeteer");
const express = require("express");
const app = express();
const morgan = require("morgan");
const fs = require("fs");
const cors = require("cors");
const request = require("request-promise-native").defaults({ Jar: true });
const poll = require("promise-poller").default;
app.use(morgan("combined"));
const port = 3000;
// Imports the Google Cloud client library
const vision = require("#google-cloud/vision");
require("dotenv").config();
app.use(cors());
const textArray = [];
const App = (pinNum, dateOfB) => {
const config = {
sitekey: process.env.SITEKEY,
pageurl: process.env.PAGEURL,
apiKey: process.env.APIKEY,
apiSubmitUrl: "http://2captcha.com/in.php",
apiRetrieveUrl: "http://2captcha.com/res.php",
};
const chromeOptions = {
executablePath: "/Program Files/Google/Chrome/Application/chrome.exe",
headless: true,
slowMo: 60,
defaultViewport: null,
};
async function main() {
const browser = await puppeteer.launch(chromeOptions);
const page = await browser.newPage();
console.log(`Navigating to ${config.pageurl}`);
await page.goto(config.pageurl);
try {
const requestId = await initiateCaptchaRequest(config.apiKey);
// const pin = getPIN();
console.log(`Typing PIN ${pinNum}`);
await page.type("#PIN", pinNum);
// const dob = getDOB();
console.log(`Typing DOB ${dateOfB}`);
const input = await page.$("#DOB");
await input.click({ clickCount: 3 });
await input.type(dateOfB);
const response = await pollForRequestResults(config.apiKey, requestId);
console.log(`Entering recaptcha response ${response}`);
await page.evaluate(
`document.getElementById("g-recaptcha-response").innerHTML="${response}";`
);
console.log(`Submitting....`);
page.click("#Submit");
} catch (error) {
console.log(
"Your request could not be completed at this time, please check your pin number and date of birth. Also make sure your internet connection is working and try again."
);
console.error(error);
}
await page.waitForSelector(
"body > div.container.body-content > div:nth-child(1) > div:nth-child(2) > p"
);
const image = await page.$(
"body > div.container.body-content > div:nth-child(1) > div:nth-child(2) > p"
);
await image.screenshot({
path: "testResults.png",
});
await getImageText();
await page.close(); // Close the website
await browser.close(); //close browser
await deleteImage();
}
main();
//This section grabs the text off the image that was gathered from the web scraper.
async function getImageText() {
// Creates a client
const client = new vision.ImageAnnotatorClient();
console.log(`Looking for text in image`);
// Performs label detection on the image file
const [result] = await client.textDetection("./testResults.png");
const [annotation] = result.textAnnotations;
const text = annotation ? annotation.description : "";
console.log("Extracted text from image:", text);
//Pushed the text into a globally available array.
textArray.push(text);
//Sent a NOTIFICATION ALERT to the client with the text gathered from the image.
var axios = require("axios");
var data = JSON.stringify({
to: "dp8vGNkcYKb-k-72j7t4Mo:APA91bEfrI3_ht89t5X1f3_Y_DACZc9DbWI4VzcYehaQoXtD_IHIFSwm9H1hgXHNq46BQwDTlCKzkWNAHbBGauEXZNQtvhQc8glz4sHQr3JY3KM7OkUEcNB7qMMpCPxRe5GzzHbe3rkE",
notification: {
body: text,
title: "AverHealth Schedule",
},
});
var config = {
method: "post",
url: "https://fcm.googleapis.com/fcm/send",
headers: {
"Content-Type": "application/json",
Authorization: `key=${process.env.FCM_SERVER_KEY}`,
},
data: data,
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
}
//Captcha Solver for the web scraper
async function initiateCaptchaRequest(apiKey) {
const formData = {
key: apiKey,
method: "userrecaptcha",
googlekey: config.sitekey,
json: 1,
pageurl: config.pageurl,
};
console.log(
`Submitting recaptcha request to 2captcha for ${config.pageurl}`
);
const response = await request.post(config.apiSubmitUrl, {
form: formData,
});
console.log(response);
return JSON.parse(response).request;
}
async function pollForRequestResults(
key,
id,
retries = 90,
interval = 5000,
delay = 1500
) {
console.log(`Waiting for ${delay} milliseconds....`);
await timeout(delay);
return poll({
taskFn: requestCaptchaResults(key, id),
interval,
retries,
});
}
function requestCaptchaResults(apiKey, requestId) {
const url = `${config.apiRetrieveUrl}?key=${apiKey}&action=get&id=${requestId}&json=1`;
console.log(url);
return async function () {
return new Promise(async function (resolve, reject) {
console.log(`Polling for response...`);
const rawResponse = await request.get(url);
console.log(rawResponse);
const resp = JSON.parse(rawResponse);
console.log(resp);
if (resp.status === 0) return reject(resp.request);
console.log("Response received");
console.log(resp);
resolve(resp.request);
});
};
}
// DELETES THE FILE CREATED BY GOOGLEAPI
function deleteImage() {
const path = "./testResults.png";
try {
fs.unlinkSync(path);
console.log("File removed:", path);
} catch (err) {
console.error(err);
}
}
const timeout = (ms) => new Promise((res) => setTimeout(res, ms));
};
app.use(express.urlencoded({ extended: false }));
// Route to results Page
app.get("/results", (req, res) => {
res.sendFile(__dirname + "/src/results.html");
res.send(textArray);
});
app.post("/results", (req, res) => {
// Insert Login Code Here
let username = req.body.username;
let password = req.body.password;
App(username, password);
});
app.listen(port, () => {
console.log(`Scraper app listening at http://localhost:${port}`);
});

I think I got the problem.
In the react app, maybe you are not using e.preventDefault() when you click submit. The browser, by default, redirects to a page where the form action is directing, if the action attribute is empty then the browser reloads the same page. I would recommend you to use e.preventDefault() on form submission and then use fetch API to make the request.
In the express server, on the route POST "results", you are not sending any response back to the user. You should always send a response to the user. In your case you are calling the App function - which has many async functions, but you are not awaiting for App() to complete in the POST route, express is sending default response to the user as soon as it parses App() - it is not waiting for the App() to complete - express will get to this later.
You can make the (req, res) => { ... } function in the route as async function async (req, res) => { ... }, then you can make the App as async function as well. Then you can await App(...) in the route function. Also, you need to await for the main() function as well inside the App() function. Then once App() call has finished, you can send redirect response to the user.

Requesting Discord Attachments

What I'm trying to do:
I'm trying to scrape all images in a discord channel and getting their URL by requesting attachments but I can't seem to find a way to request it
Code
const fs = require("fs");
const fetch = require("node-fetch");
function readFileString(path) {
return fs.readFileSync(path, {encoding: "utf8"}).replace(/\r?\n|\r/g, "");
}
const token = readFileString("token.txt");
const channel = process.argv[2];
if(!channel) {
console.error("Usage: node index.js <channel id>");
process.exit(1);
}
const headers = {authorization: token};
async function request(before) {
const options = {
method: "GET",
headers: headers
};
const request = await fetch(
`https://discord.com/api/channels/${channel}/attachments`,
options
);
return await request.json();
}
let result;
async function go() {
let page = await request();
result = page;
while(page.length >= 100) {
page = await request(page[page.length - 1].id);
result = result.concat(page);
}
console.log(`Fetched ${result.length} images`);
fs.writeFileSync("links.json", JSON.stringify(result, null, 2));
}
go();
Output: Console
Fetched undefined images
Output: links.json
{
"message": "404: Not Found",
"code": 0
}
Any help of how I would get all image links in the links.json file would be appreciated

It seems at looking at the Docs It does not allow you to make a GET request for message attachments.

Uploading PDF to external API results in blank pages inside PDF

I'm new to Node.js and I need to upload some PDFs to an external API (Zip Forms).
Right now I have the code below but the PDF pages are blank when they arrive at the destination. I tried saving the PDF locally, using the same binary data that I'm sending to the API, and the PDFs are correctly saved.
I am also using setTimeout method here because I cannot find a method that waits for the PDF to read, before sending it to the API.
Also tried binary instead of latin-1 in readFileSync method, but it doesn't change anything.
Code:
const aws = require('aws-sdk');
const https = require('https');
const request = require('request');
const { createWriteStream, readFileSync, writeFileSync } = require('fs');
const s3 = new aws.S3(); // Pass in opts to S3 if necessary
// Look up order and related info.
var order = await Order.findOne({ id })
.populate('agent');
if (createZiplogixTransaction) {
ziplogixTransactionId = await sails.helpers.ziplogix.createZiplogixTransaction.with({
ziplogixContextId: ziplogixContextId,
transactionName: order.propertyStreetAddress + ', ' + order.propertyCity,
// FUTURE: if the transaction helper is updated, include actual order information
// e.g. Primary seller name, property street address, etc.
});
}
if (!order) {
throw 'noSuchOrder';
}
// Permissions
if (this.req.me && this.req.me.accountType !== 'agent' && !ziplogixContextId) {
throw 'forbidden';
}
let savedPdfs = await PdfOrderExternalId.find({ orderId: id });
await PdfOrderExternalId.destroy({
where: { orderId: id }
});
for (const pdf of pdfs) {
let url = await s3.getSignedUrl('getObject', {
Bucket: 'disclosure-pdfs',
Key: pdf.uploadFd,
Expires: 60 * 5
});
let file = createWriteStream(`/tmp/${pdf.slug}.pdf`);
await https.get(url, async (response) => {
await response.pipe(file);
// Need to wait for file to write on disk :|. Doesn't work with await or Promise (Why? IDK)
setTimeout(async () => {
let postData = await readFileSync(`/tmp/${pdf.slug}.pdf`, 'latin1');
let queryString = `Name=${pdf.displayName}&Description=${pdf.displayName}`;
savedPdfs.forEach(item => {
if (item.pdfTemplate === pdf.pdfTemplate) {
queryString += `Id=${item.externalId}`;
}
});
request({
method: 'POST',
url: `${sails.config.custom.ziplogixApiBaseUrl}/transactions/${ziplogixTransactionId}/documents/file?${queryString}`,
headers: {
'X-Auth-ContextID': ziplogixContextId,
'X-Auth-SharedKey': sails.config.custom.ziplogixSharedKey,
'Content-Type': ['application/pdf', 'application/pdf']
},
body: postData
}, async (error, response, body) => {
// code here ...
});
}, 1000);
});
}
await exits.success(Date.now());
Any ideas what I'm doing wrong?
Thank you