Here's the task:
You need to make a GET request for the resource: https://jsonplaceholder.typicode.com/posts using fetch method
Save the response to response.json file
Save only those items, where id < 20
What I wrote:
const fetch = require('node-fetch');
const fs = require('fs');
const path = require('path');
const filePath = path.join(__dirname, 'response.json');
fetch('https://jsonplaceholder.typicode.com/posts')
.then(res => res.json())
.then(data => {
const refined = data.filter(item => item.id < 20);
const stringified = JSON.stringify(refined);
fs.appendFile(filePath, stringified, err => {
if (err) {
throw err;
}
});
});
How to write the same fetch, but with async/await syntax?
await keyword can only be used inside an async function, so you need to write an async function that makes the API request to fetch the data
async function fetchData() {
const response = await fetch('https://jsonplaceholder.typicode.com/posts');
const data = await response.json();
const refined = data.filter(item => item.id < 20);
const stringified = JSON.stringify(refined);
// promise version of appendFile function from fs.promises API
await fs.appendFile(filePath, stringified);
}
fs module of nodeJS has functions that use promises instead of callbacks. if you don't want to use callback version, you will need to use promise version of appendFile function.
You can import the promise version of fs module as require('fs').promises or require('fs/promises').
To handle errors, make sure that code that calls this function has a catch block to catch and handle any errors that might be thrown from this function. You could also wrap the code in this function with try-catch block to handle the errors inside this function.
Side tip: If you want to write data in the file in easily readable format, change
const stringified = JSON.stringify(refined);
to
const stringified = JSON.stringify(refined, null, 4);
Below snippet could help you (tested in node v14)
const fetch = require("node-fetch")
const fs = require("fs")
const path = require("path")
const filePath = path.join(__dirname, "response.json")
async function execute() {
const res = await fetch("https://jsonplaceholder.typicode.com/posts")
const data = await res.json()
const refined = data.filter((item) => item.id < 20)
const stringified = JSON.stringify(refined)
fs.appendFile(filePath, stringified, (err) => {
if (err) {
throw err
}
})
}
execute()
Related
I'm working on a local Node.js app that needs to access the Google Chrome cookies. I've found the chrome-cookies-secure library that seems to do the job but I just can't figure out what's wrong with the code below.
const chrome = require('chrome-cookies-secure');
const domains = [
"google.com"
];
const resolveCookies = async () => {
let result = [];
for(domain of domains) {
await chrome.getCookies(`https://${domain}/`, (err, cookies) => {
result.push(cookies);
// console.log(cookies); //? This is going to correctly print the results
})
}
return result;
}
const final = resolveCookies();
console.log(final); //! This is going to return a Promise { <pending> } object
The idea is that I just want to store the cookies from all the domains in a list but no matter what I cannot resolve the Promise.
I didn't see any examples with the async call for this module but if I don't use it it's going to return me an empty list after the script execution.
My Node Version: v14.4.0
What am I doing wrong?
It looks like the implementation of getCookies is not correctly awaiting the asynchronous processes. You can see in the implementation that although getCookies itself is async, it calls getDerivedKey without awaiting it (and that function isn't async anyway).
Rather than trying to rely on this implementation, I'd suggest using Util.promisify to create a proper promise via the callback:
const util = require('util');
const chrome = require('chrome-cookies-secure');
const getCookies = util.promisify(chrome.getCookies);
// ...
const cookies = await getCookies(`https://${domain}/`);
Note that, as Reece Daniels pointed out in the comments, the getCookies implementation actually takes a profile parameter after the callback; if you need to use that parameter, you can't use the built-in promisify. You'd have to wrap it yourself instead, this could look like e.g.:
const getCookies = (url, format, profile) => new Promise((resolve, reject) => {
chrome.getCookies(url, format, (err, cookies) => {
if (err) {
reject(err);
} else {
resolve(cookies);
}
}, profile);
});
They already tried to fix the promise upstream, but the PR hasn't been merged in nearly nine months.
Note that once you have a working function to call you can also convert:
const resolveCookies = async () => {
let result = [];
for(domain of domains) {
await chrome.getCookies(`https://${domain}/`, (err, cookies) => {
result.push(cookies);
// console.log(cookies); //? This is going to correctly print the results
})
}
return result;
}
to simply:
const resolveCookies = () => Promise.all(domains.map((domain) => getCookies(`https://${domain}/`)));
An async function returns a Promise.
So your resolveCookies function will also return a Promise as you noticed.
You need to either chain the console.log with a .then e.g.
resolveCookies().then(console.log);
Or if you need to set it to a variable like final you need to await that Promise too. In that case you need an async IIFE:
(async () => {
const final = await resolveCookies();
console.log(final);
})();
try this.
const chrome = require('chrome-cookies-secure');
const domains = [
"www.google.com"
];
const resolveCookies = async() => {
let result = [];
for (domain of domains) {
const cookies = await getCookies(domain)
result.push(cookies)
}
return Promise.resolve(result);
}
const getCookies = async (domain) => {
chrome.getCookies(`https://${domain}/`, (err, cookies) => {
return Promise.resolve(cookies);
})
}
resolveCookies().then((resp) => {
console.log('FINAL ',resp)
}).catch((e) => {
console.log('ERROR ', e)
})
I need some help with this helper I'm writing. For some reason using reduction within an async on a readFile, when trying to write results to a file it won't advance to the next item of the array. However, if I use a console.log, it works just fine.
const neatCsv = require('neat-csv');
const fetch = require('node-fetch');
const fs = require('fs');
fs.readFile('./codes.csv', async (err, data) => {
if (err) { throw err; }
let baseUrl = 'https://hostname/orders?from=2019-10-21T00:00:00.001Z&to=2019-12-31T23:59:59.000Z&promo=';
const starterPromise = Promise.resolve(null);
const promos = await neatCsv(data);
const logger = (item, result) => console.log(item, result);
function write (item, result) {
return new Promise((resolve, reject) => {
fs.writeFile(`./output/${item.PROMO}.json`, JSON.stringify(result), (err) => {
if (err) { throw err; }
console.log(`Wrote file ${item.PROMO}`);
});
})
}
function asyncFetch(item) {
console.log(`runTask <---------${item.PROMO}---------`);
return fetch(`${baseUrl}${item.PROMO}`, { headers: { 'x-apikey': 'xyz' }})
.then(res => (res.json())
.then(json => json))
}
await promos.reduce(
(p, item) => p.then(() => asyncFetch(item).then(result => write(item, result))),
starterPromise
)
});
The csv file is just a basic layout like so..
PROMO
12345
56789
98765
...
The goal is to iterate over these, make a REST call to get the json results and write those to a file with the name of the current promo, then move to the next, making a new call and saving that one into a different file with its respective code.
In the reduce, if you call logger instead of write, it works fine. Calling write, it just makes the same call over and over and overwriting to the same file, forcing me to kill it. Please help, I'm losing my mind here...
You might have a better time using async functions everywhere, the fs promises API and a simple while loop to consume the CSV items. Dry-coded, naturally, since I don't have your CSV or API.
(Your original problem is probably due to the fact you don't resolve/reject in the write function, but the reduce hell isn't needed either...)
const neatCsv = require("neat-csv");
const fetch = require("node-fetch");
const fsp = require("fs").promises;
const logger = (item, result) => console.log(item, result);
const baseUrl = "https://hostname/orders?from=2019-10-21T00:00:00.001Z&to=2019-12-31T23:59:59.000Z&promo=";
async function asyncFetch(item) {
console.log(`runTask <---------${item.PROMO}---------`);
const res = await fetch(`${baseUrl}${item.PROMO}`, { headers: { "x-apikey": "xyz" } });
const json = await res.json();
return json;
}
async function write(item, result) {
await fsp.writeFile(`./output/${item.PROMO}.json`, JSON.stringify(result));
console.log(`Wrote file ${item.PROMO}`);
}
async function process() {
const data = await fsp.readFile("./codes.csv");
const promos = await neatCsv(data);
while (promos.length) {
const item = promos.shift();
const result = await asyncFetch(item);
await write(item, result);
}
}
process().then(() => {
console.log("done!");
});
A version that uses mock data and the JSON Placeholder service, works just fine:
const fetch = require("node-fetch");
const fsp = require("fs").promises;
const baseUrl = "https://jsonplaceholder.typicode.com/comments/";
async function asyncFetch(item) {
console.log(`runTask <---------${item.PROMO}---------`);
const res = await fetch(`${baseUrl}${item.PROMO}`);
return await res.json();
}
async function write(item, result) {
const data = JSON.stringify(result);
await fsp.writeFile(`./output/${item.PROMO}.json`, data);
console.log(`Wrote file ${item.PROMO}: ${data}`);
}
async function getItemList() {
return [
{PROMO: '193'},
{PROMO: '197'},
{PROMO: '256'},
];
}
async function process() {
const promos = await getItemList();
while (promos.length) {
const item = promos.shift();
const result = await asyncFetch(item);
await write(item, result);
}
}
process().then(() => {
console.log("done!");
});
I'm new to javaScript and trying to crawl a website with node.js. I could check the data in console log, but want to use the data in another javaScript file. How can I fetch the data?
The problem is I've never used node.js. I do javaScript so I know how to write the code, but I don't know how the back-end or server works.
I've tried to open it in my local host but the node method (e.g., require()) didn't work. I found out it's because node doesn't work in browser.(See? very new to js)
Should I use bundler or something?
The steps I thought were,
somehow send the data as json
somehow fetch the json data and render
Here is the crawling code file.
const axios = require("axios");
const cheerio = require("cheerio");
const log = console.log;
const getHtml = async () => {
try {
return await axios.get(URL);
} catch (error) {
console.error(error);
}
};
getHtml()
.then(html => {
let ulList = [];
const $ = cheerio.load(html.data);
const $bodyList = $("div.info-timetable ul").children("li");
$bodyList.each(function(i, elem) {
ulList[i] = {
screen: $(this).find('a').attr('data-screenname'),
time: $(this).find('a').attr('data-playstarttime')
};
});
const data = ulList.filter(n => n.time);
return data;
})
.then(res => log(res));
Could you please explain what steps should I take?
Also, it would be great if I can get understood WHY the steps are needed.
Thanks alot!
you can try writing your data to a JSON file and proceed, that's one way, then you can use the data as an object in any js file
const appendFile = (file, contents) =>
new Promise((resolve, reject) => {
fs.appendFile(
file,
contents,
'utf8',
err => (err ? reject(err) : resolve()),
);
});
getHtml()
.then(html => {
let ulList = [];
const $ = cheerio.load(html.data);
const $bodyList = $("div.info-timetable ul").children("li");
$bodyList.each(function(i, elem) {
ulList[i] = {
screen: $(this).find('a').attr('data-screenname'),
time: $(this).find('a').attr('data-playstarttime')
};
});
const data = ulList.filter(n => n.time);
return data;
})
.then(res => {
return appendFile('./data.json',res.toString())
}))
.then(done => {log('updated data json')});
Inside a function, I would like to set the value of a variable (foldersInDir) to the results of getting the contents of a directory using fs.readdir();
I thought using await would force the console.log line to wait for a response, but it's not.
How can I set foldersInDir = the return value?
/*Begin function*/
const listContents = async (myPath) => {
var fs = require('fs');
let foldersInDir = await fs.readdir(myPath, function(err, items) {
console.log(items); //works
return items;
});
console.log(foldersInDir); //does not work, undefined
}
You need to convert readdir to a promise, e.g.:
const foldersPromised = (path) =>
new Promise((resolve, reject) =>
fs.readdir(path, (err, items) =>
err !== undefined ? reject(err) : resolve(items)
)
);
try {
let foldersInDir = await foldersPromised(myPath);
} catch(err) {
console.log(err);
}
const fs = require('fs');
const test = () => {
let folders = fs.readdirSync('.');
return folders;
}
console.log(test());
Edit: sorry, need to promisify() the function
const fs = require('fs');
const { promisify } = require('util') // available in node v8 onwards
const readdir = promisify(fs.readdir)
async function listContents() {
try { // wrap in try-catch in lieu of .then().catch() syntax
const foldersInDir = await readdir(myPath) // call promised function
console.log('OK, folders:', foldersInDir) // success
} catch (e) {
console.log('FAIL reading dir:', e) // fail
}
}
listContents('path/to/folder') // run test
I recommend using the promisify function provided by Node.js to fix the problem. This function will convert a callback-based function to a promise-based function, which can then be used using the await keyword.
const fs = require('fs');
const {
promisify
} = require('util');
const readdirAsync = promisify(fs.readdir);
/*Begin function*/
const listContents = async(myPath) => {
let foldersInDir = await readdirAsync(myPath);
console.log(foldersInDir);
}
I have a function, startSurvey, which, when run, checks if there are questions in a .json file. If there are no questions, it fetches some questions from Typeform and writes them to the .json file using saveForm. After it writes, I would like to continue executing some code that reads the .json file and logs its contents. Right now, await saveForm() never resolves.
I have promisified the fs.readFile and fs.writeFile functions.
//typeform-getter.js
const fs = require('fs')
const util = require('util')
const fetch = require('cross-fetch')
require('dotenv').config()
const conf = require('../../private/conf.json')
const typeformToken = conf.tokens.typeform
const writeFile = util.promisify(fs.writeFile)
const getForm = async () => {
const form = await fetch(`https://api.typeform.com/forms/${process.env.FORM_ID}`, {
headers: {
"Authorization": `bearer ${typeformToken}`
}
}).then(res => res.json())
const fields = form.fields
return fields
}
const saveForm = async () => {
const form = await getForm()
return writeFile(__dirname + '/../data/questions.json', JSON.stringify(form))
.then((e) => {
if (e) console.error(e)
else console.log('questions saved')
return
})
}
module.exports = saveForm
//controller.js
const fs = require('fs')
const util = require('util')
const request = require('request')
require('dotenv').config()
const typeformGetter = require('./functions/typeform-getter')
const readFile = util.promisify(fs.readFile)
const saveForm = util.promisify(typeformGetter)
let counter = 1
const data = []
const getQuestions = async() => {
console.log('called')
try {
let data = await readFile(__dirname + '/data/questions.json')
data = JSON.parse(data)
return data
} catch (e) {
console.error('error getting questions from read file', e)
}
}
const startSurvey = async (ctx) => {
try {
const questions = await getQuestions()
if (!questions) await saveForm()
console.log(questions) //NEVER LOGS
} catch (error) {
console.error('error: ', error)
}
}
startSurvey() //function called
I don't know your exact error, but there are multiple things wrong with your code:
You're using incorrectly the promisified version of fs.writeFile, if an error occurs, the promise will be rejected, you won't get a resolved promise with an error as the resolved value, which is what you're doing.
Use path.join instead of concatenating paths.
In startSurvey, you're using console.log(questions) but that wont have any data if questions.json doesn't exists, which should happen the first time you run the program, since it's filled by saveForm, so you probably want to return the questions in saveForm
So saveForm should look something like this:
const saveForm = async () => {
const form = await getForm();
const filePath = path.join(path.__dirname, '..', 'data', 'questions.json');
await writeFile(filePath, JSON.stringify(form));
console.log('questions saved');
return form;
}
And startSurvey
const startSurvey = async (ctx) => {
try {
const questions = await getQuestions() || await saveForm();
// This will be logged, unless saveForm rejects
// In your code getQuestions always resolves
console.log(questions);
} catch (error) {
console.error('error: ', error)
}
}
In your controller.js you're using util.promisify on saveForm when it is already a promise.
So it should be:
const saveForm = require('./functions/typeform-getter')