I'm new to javaScript and trying to crawl a website with node.js. I could check the data in console log, but want to use the data in another javaScript file. How can I fetch the data?
The problem is I've never used node.js. I do javaScript so I know how to write the code, but I don't know how the back-end or server works.
I've tried to open it in my local host but the node method (e.g., require()) didn't work. I found out it's because node doesn't work in browser.(See? very new to js)
Should I use bundler or something?
The steps I thought were,
somehow send the data as json
somehow fetch the json data and render
Here is the crawling code file.
const axios = require("axios");
const cheerio = require("cheerio");
const log = console.log;
const getHtml = async () => {
try {
return await axios.get(URL);
} catch (error) {
console.error(error);
}
};
getHtml()
.then(html => {
let ulList = [];
const $ = cheerio.load(html.data);
const $bodyList = $("div.info-timetable ul").children("li");
$bodyList.each(function(i, elem) {
ulList[i] = {
screen: $(this).find('a').attr('data-screenname'),
time: $(this).find('a').attr('data-playstarttime')
};
});
const data = ulList.filter(n => n.time);
return data;
})
.then(res => log(res));
Could you please explain what steps should I take?
Also, it would be great if I can get understood WHY the steps are needed.
Thanks alot!
you can try writing your data to a JSON file and proceed, that's one way, then you can use the data as an object in any js file
const appendFile = (file, contents) =>
new Promise((resolve, reject) => {
fs.appendFile(
file,
contents,
'utf8',
err => (err ? reject(err) : resolve()),
);
});
getHtml()
.then(html => {
let ulList = [];
const $ = cheerio.load(html.data);
const $bodyList = $("div.info-timetable ul").children("li");
$bodyList.each(function(i, elem) {
ulList[i] = {
screen: $(this).find('a').attr('data-screenname'),
time: $(this).find('a').attr('data-playstarttime')
};
});
const data = ulList.filter(n => n.time);
return data;
})
.then(res => {
return appendFile('./data.json',res.toString())
}))
.then(done => {log('updated data json')});
Related
I am new to Cypress (and naive to JS). I would like to write a JS library as a wrapper to 3rd party APIs.
I write the API wrapper as an individual file (instead of using Cypress Custom functions) because I believe I can share the library with teams NOT using Cypress E2E tool.
The problem I am facing is "I cannot let my code to be executed sequentially in order"
From the result, I can see:
the data didn't return successfully
it looks like the "getTestPlanIdByName:20974" were executed last, but I expect it should be executed before "line 01b testPlanId:{}"
I need to help to know the correct way to handle the flow sequentially in Cypress/Javascript, thanks.
API Library(api-util.js)
let axios = require('axios');
const proxy = "http://10.8.8.8:8080/";
const apiPatToken = 'OmdrvbvvvvvvvvWZqa2E='
let proxyAgentHttps = require('https-proxy-agent');
let proxyAgentHttp = require('http-proxy-agent');
let agentHttps = new proxyAgentHttps(proxy);
let agentHttp = new proxyAgentHttp(proxy);
let config = {
baseURL: 'https://dev.3rdparty.com/mycompany/myaccount/_apis',
url: 'DUMMY_INJECTED_LATER',
httpsAgent: agentHttps,
httpAgent: agentHttp,
proxy:false,
headers: {
'Authorization': `Basic ${apiPatToken}`
}
}
export async function getTestPlanIdByName(testplan_name){
config.url = '/test/plans?api-version=5.0'
let found = ''
axios.request(config).then( resp => {
found = resp.data.value.find(function(item, index, array){
return item.name === testplan_name
})
})
.then(() => {
console.log("getTestPlanIdByName:"+found.id)
return found.id
})
.catch(err => console.log(err))
}
My Cypress code
import * as UTIL from 'api-util.js'
describe('CI-', () => {
let testPlanId = 'none'
it('01 Get TestPlanID', () => {
//use cy.log() get a Promise for flow control
cy.log()
.then(() => {
new Cypress.Promise((resolve, reject) => {
console.log("01a testPlanId:"+JSON.stringify(testPlanId))
testPlanId = UTIL.getTestPlanIdByName("TESTPLAN-Regression")
console.log("01b testPlanId:"+JSON.stringify(testPlanId))
})
})
.then(() => {
console.log("01c testPlanId:"+JSON.stringify(testPlanId))
})
});
it('02 Get TestSuitesList', () => {
console.log("02 testPlanId:"+testPlanId)
// UTIL.getTestSuitesIdList(testPlanId)
});
});
Thank you all. Cypress flow isn't 100% compatible with standard JS Promise (Wait for an own function (which returns a promise) before tests are executed). After relentless testings, I decided to use a Cypress Custom Command wrapper to wrap my in-house JS library. Though adding an extra layer may seem a little cumbersome. But I am satisfied with the result. Share my code here in case anyone might need it. :)
Cypress Code
before('Prepare TestPlanId', () => {
cy.getTestPlanIdByName(testPlanName)
.then((result) => {
testPlanId = result
console.log("#01_SDET_testplan:Prepare TestPlanId# "+testPlanId)
})
});
Cypress Custom Command
Cypress.Commands.add('getTestPlanIdByName', (wk_testplan_name) => {
return new Cypress.Promise((resolve, reject) => {
TESTPLAN_API.getTestPlanIdByName(wk_testplan_name)
.then(function (data) {
resolve(data);
})
});
})
In-house JS library
export async function getTestPlanIdByName(wk_testplan_name){
return new Promise((resolve, reject) => {
config.method = 'get'
config.url = '/test/plans?api-version=5.0'
let found = ''
axios.request(config).then( resp => {
found = resp.data.value.find(function(item, index, array){
return item.name === wk_testplan_name
})
})
.then(() => {
resolve(found.id)
})
.catch(err => console.log(err))
})
}
I'm working on a local Node.js app that needs to access the Google Chrome cookies. I've found the chrome-cookies-secure library that seems to do the job but I just can't figure out what's wrong with the code below.
const chrome = require('chrome-cookies-secure');
const domains = [
"google.com"
];
const resolveCookies = async () => {
let result = [];
for(domain of domains) {
await chrome.getCookies(`https://${domain}/`, (err, cookies) => {
result.push(cookies);
// console.log(cookies); //? This is going to correctly print the results
})
}
return result;
}
const final = resolveCookies();
console.log(final); //! This is going to return a Promise { <pending> } object
The idea is that I just want to store the cookies from all the domains in a list but no matter what I cannot resolve the Promise.
I didn't see any examples with the async call for this module but if I don't use it it's going to return me an empty list after the script execution.
My Node Version: v14.4.0
What am I doing wrong?
It looks like the implementation of getCookies is not correctly awaiting the asynchronous processes. You can see in the implementation that although getCookies itself is async, it calls getDerivedKey without awaiting it (and that function isn't async anyway).
Rather than trying to rely on this implementation, I'd suggest using Util.promisify to create a proper promise via the callback:
const util = require('util');
const chrome = require('chrome-cookies-secure');
const getCookies = util.promisify(chrome.getCookies);
// ...
const cookies = await getCookies(`https://${domain}/`);
Note that, as Reece Daniels pointed out in the comments, the getCookies implementation actually takes a profile parameter after the callback; if you need to use that parameter, you can't use the built-in promisify. You'd have to wrap it yourself instead, this could look like e.g.:
const getCookies = (url, format, profile) => new Promise((resolve, reject) => {
chrome.getCookies(url, format, (err, cookies) => {
if (err) {
reject(err);
} else {
resolve(cookies);
}
}, profile);
});
They already tried to fix the promise upstream, but the PR hasn't been merged in nearly nine months.
Note that once you have a working function to call you can also convert:
const resolveCookies = async () => {
let result = [];
for(domain of domains) {
await chrome.getCookies(`https://${domain}/`, (err, cookies) => {
result.push(cookies);
// console.log(cookies); //? This is going to correctly print the results
})
}
return result;
}
to simply:
const resolveCookies = () => Promise.all(domains.map((domain) => getCookies(`https://${domain}/`)));
An async function returns a Promise.
So your resolveCookies function will also return a Promise as you noticed.
You need to either chain the console.log with a .then e.g.
resolveCookies().then(console.log);
Or if you need to set it to a variable like final you need to await that Promise too. In that case you need an async IIFE:
(async () => {
const final = await resolveCookies();
console.log(final);
})();
try this.
const chrome = require('chrome-cookies-secure');
const domains = [
"www.google.com"
];
const resolveCookies = async() => {
let result = [];
for (domain of domains) {
const cookies = await getCookies(domain)
result.push(cookies)
}
return Promise.resolve(result);
}
const getCookies = async (domain) => {
chrome.getCookies(`https://${domain}/`, (err, cookies) => {
return Promise.resolve(cookies);
})
}
resolveCookies().then((resp) => {
console.log('FINAL ',resp)
}).catch((e) => {
console.log('ERROR ', e)
})
I have some csv data I need to parse into array of objects for an API I am building.
I am able to see that the parser I am using is working as expected this way:
const csv = require('fast-csv')
const fs = require('fs');
const path = require('path');
let results = [];
async function parseCSVFromCSV(_sourceCSVFilePath){
return new Promise((resolve, reject) => {
fs.createReadStream(_sourceCSVFilePath)
.pipe(csv.parse({ headers: true }))
.on('error', error => reject(error))
.on('data', row => {
console.log(row);
results.push(row);
})
resolve(results);
})
}
so the console.log(row) prints out the parsed object as expected but the results object is empty. I am needing this parsed data but no matter what I try I can't get results to populate with parsed csv.
Any guidance on what I a missing here?
So I figured out what I need to do:
const fs = require('fs');
const path = require('path');
const csv = require('fast-csv');
async function getData(){
let arrOfObj = [];
fs.createReadStream(path.resolve(__dirname, 'downloads', 'sampleData.csv'))
.pipe(csv.parse({ headers: true }))
.on('error', error => console.error(error))
.on('data', row =>
{arrOfObj.push(row);
//console.log(row);
})
.on('end', rowCount => { printData(arrOfObj);
});
}
async function printData(_arrOfObj){
console.log(_arrOfObj);
}
getData();
Basically the finished array will not be accessible expect inside of the end event. This allows it be exposed to outside the event. Hope this helps others that were needing to do this same thing.
I have some problems when I try to fetch the data, I didn't get response.
I write the path correctly?
I attached the part of the code and pic of my project hierarchy.
let transportation = [];
const init = () => {
fetch('/data/transportationDataCheck.json')
.then((response) => {
return response.json();
})
.then((data) => {
transportation = data;
}).then(() => {
renderList(transportation);
});
};
try this:
const data = require("../data/transportationDataCheck.json")
console.log(JSON.stringify(data));
Or you may try after changing little URL
let transportation = [];
const init = () => {
fetch('../data/transportationDataCheck.json')
.then((response) => {
return response.json();
})
.then((data) => {
transportation = data;
}).then(() => {
renderList(transportation);
});
};
You are trying to serve a static file with a fetch command, which inherently requires the file to be served by a server.
Someone had a similar issue here: Fetch request to local file not working
Depending on what type of file this is, you may not need to make a fetch. You could probably instead require the file:
var transportationDataCheck = require('./data/transportationDataCheck.json');```
Use ./ at the beginning of the path
fetch('./data/transportationDataCheck.json')
.then(response => {
return response.json()
})
.then(data => {
// Work with JSON data here
console.log(data)
})
.catch(err => {
// Do something for an error here
})
I want to mock the result of a function within a node module so that i can run assertions.
Considering the following node module:
const doPostRequest = require('./doPostRequest.js').doPostRequest;
const normalizeSucessResult = require('./normalizer.js').normalizeSucessResult;
const normalizeErrorResult = require('./normalizer.js').normalizeErrorResult;
exports.doPost = (params, postData) => {
return doPostRequest(params, postData).then((res) => {
const normalizedSuccessResult = normalizeSucessResult(res);
return normalizedSuccessResult;
}).catch((err) => {
const normalizedErrorResult = normalizeErrorResult(err);
return normalizedErrorResult;
})
}
The function doPostRequest returns a promise. How can i fake the return value of this promise so that i can assert if normalizeSucessResult has been called?
So for i have tried:
const normalizeSucessResult = require('./normalizer.js');
const doPostRequest = require('./doPostRequests.js');
const doPost = require('./doPost.js');
it('runs a happy flow scenario', async () => {
let normalizeSucessResultStub = sinon.stub(normalizeSucessResult, 'normalizeSucessResult');
let postData = { body: 'Lorum ipsum' };
let params = { host: 'someUrl', port: 433, method: 'POST', path: '/' };
sinon.stub(doPostRequest, 'doPostRequest').resolves("some response data"); //Fake response from doPostRequest
return doPost.doPost(params, postData).then((res) => { //res should be equal to some response data
expect(normalizeSucessResultStub).to.have.been.calledOnce;
expect(normalizeSucessResultStub).to.have.been.with("some response data");
});
});
The doPostRequest module looks like this:
const https = require('https')
module.exports.doPostRequest = function (params, postData) {
return new Promise((resolve, reject) => {
const req = https.request(params, (res) => {
let body = []
res.on('data', (chunk) => {
body.push(chunk)
})
res.on('end', () => {
try {
body = JSON.parse(Buffer.concat(body).toString())
} catch (e) {
reject(e)
}
resolve(body)
})
})
req.on('error', (err) => {
reject(err)
})
if (postData) {
req.write(JSON.stringify(postData))
}
req.end()
})
}
You can use Promise.resolve to return a promise with any given value.
Promise.resolve(“hello world”);
For stub your func you need to do like this
sinon.stub({doPostRequest}, 'doPostRequest').resolves("some response data")
Okay, i figured it out. The function doPostRequest was loaded using require, on the top of the file using const doPostRequest = require('./doPostRequest.js').doPostRequest;
In order to mock the data that comes back from a function that is loaded using require i had to use a node module called mock-require. There are more modules that can take care of this (proxyquire is a populair one) but i picked mock-require (i did not have a specific reason for choosing mock-require).
For anyone else that is stuck with a similar problem, try mock-require to mock the respose from files that are loaded using require.