How to send scraped website data via Express? - javascript

I have a written a crawler in Axios and trying to send file via Express, I have around 10 crawlers and 10 html forms methods in Express.
But when I press button it downloads blank file and then crawlers start to run.
It should download file when crawler is finished and save data. but how to do that?
Crawler code:
"use strict"
const axios = require("axios").default;
const cheerio = require("cheerio");
const fs = require("fs");
var excel = require('excel4node');
let writeStream = fs.createWriteStream('mpRera_Agents.csv');
writeStream.write(`AgentName,AgentPhone,AgentEmail\n`)
// let writeStream = fs.createWriteStream('mpRera_Promoters.csv');
// writeStream.write(`AgentName,Agentaddress,Agenttype,Agentrera,Agentlink\n`)
const Agenturl = "http://www.rera.mp.gov.in/agentsrcg-loop.php?
show=20&pagenum=1&search_txt=&search_state=&search_dist=&search_tehs=&_=1597665284486";
var workbook = new excel.Workbook();
var worksheet = workbook.addWorksheet('MP Agents');
var worksheet2 = workbook.addWorksheet('MP Promoters');
// var arr = []
class mprera{
makeRequest(urls){
return new Promise((resolve, reject)=>{
let url = axios.get(urls);
let data = url.then((res)=>{
if (res.status==200){
resolve(res.data);
}reject("response not 200");
})
})
}
getlink(url){
return new Promise((resolve, reject)=>{
var arr = []
let soup = this.makeRequest(url);
soup.then((res)=>{
let $ = cheerio.load(res);
let table = $("#example");
let tbody = table.find("tbody");
let tr = tbody.find("tr");
for (var i = 0;i<tr.length;i++){
let td = $(tr[i]).find("td");
let link= $(td[6]).find("a").attr("href");
arr.push(link);
}
resolve (arr);
});
})
};
getAgents(url){
var link = this.getlink(url);
link.then((data)=>{
for (var i = 0;i<data.length;i++){
let soup = this.makeRequest(data[i]);
soup.then((res)=>{
let $ = cheerio.load(res);
let getDetails = $(".col-md-9").toArray();
// let name = $(getDetails[1]).text().trim();
// console.log(name);
let phone = $(getDetails[5]).text().trim();
let email = $(getDetails[6]).text().trim();
// console.log(phone, email);
writeStream.write(`${phone},${email} \n`);
})
}
})
};
}
module.exports = mprera;
My Express code:
const express = require("express");
const hbs = require("hbs");
const bodyParser = require("body-parser");
const path = require("path");
var json2csv = require('json2csv');
const mprera = require("./mprera");
let crawleer = new mprera();
const Agenturl = "http://www.rera.mp.gov.in/agentsrcg-loop.php?show=20&pagenum=1&search_txt=&search_state=&search_dist=&search_tehs=&_=1597665284486";
app = express();
//set path for views
app.set("views",path.join(__dirname,"views"))
app.set("view engine","hbs");
// for handling post requests
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
// for public folders
app.use('/assets',express.static(__dirname + '/public'));
app.get("/", (req, res)=>{
res.render("table");
})
app.get("/mprera", async (req, res)=>{
console.log("starting");
data = await crawleer.getAgents(Agenturl);
res.attachment('filename.csv');
res.send (data);
})
app.listen(8000, ()=>{
console.log("server started");
})

crawler.getAgents needs to return a promise that doesn't resolve until after the file is written.
So something like:
crawler.getAgents = url => {
return new Promise((resolve) => {
stuff().then(() => {
// write file here
resolve()
})
})
}

Related

My http request is sending back a blank map

How do I send a map as a response to an HTTP request? Currently I am sending an array of strings as part of the request. I want to send a nested map along with a bool as a response to a post request. The bool sends find but the map always comes back as empty. I did a bit of research to use the Object.entries method but no dice. Heres the client code:
const { response, request } = require('express');
const express = require('express');
const readline = require('readline/promises');
const app = express();
const port = 3000;
const axios = require('axios');
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
});
if (parsedInput > 0 && parsedInput <= 8) {
let listOfNames = [];
for (let i=0; i<parsedInput; i++) {
let name = await rl.question(`Enter name #${i}\n`);
listOfNames.push(name);
console.log(`Added ${name} to list of players\n`);
}
const serverResponse = axios.post('http://localhost:3000/start', {
params: { names: listOfNames },
}).then(function (response) {
const recievedMap = new Map(Object.entries(response.data.scoreboard));
console.log(recievedMap);
})
.catch(function (error) {
console.log(error);
});
}
Server code:
const express = require('express');
var bodyParser = require('body-parser');
const app = express()
app.use(bodyParser.json());
const port = 3000
var listOfNames = [];
const scoreboard = new Map();
app.post('/start', async (req, res) => {
listOfNames = req.body.params.names;
for(let i = 0; i < listOfNames.length; i++){
scoreboard.set(listOfNames[i],[]);
}
var responseObj = {
gameRunning: true,
scoreboard: JSON.stringify(scoreboard),
};
res.status(200).send(responseObj);
});
app.get('/', function(req, res, next) {
res.send("Welcome to the homepage of bowling");
})
app.listen(port, () => {
console.log('Listening to request on port 3000');
});
Output:
Choose a number:
1)Start a bowling game
2)Exit simulation
1
Ok how many players will there be (MAX: 8 players)
1
Enter name #0
dadada
Added dadada to list of players
{}
EDIT: Axios was sending data fine, however Im having issues accessing it when in my web server. Since I cant access any object, I return nothing. Am I refering to my request.body incorrectly?

Trouble Accessing Shopify Products using shopify-api-node

I'm attempting to access products in my private Shopify app using the shopify-api-node module but I'm getting a 403 error.
Here's the code I've written with the help of another Stackoverflow post:
const express = require('express');
const path = require('path');
const Shopify = require('shopify-api-node');
const https = require('https');
const fetch = require('node-fetch');
const {request, gql, GraphQLClient} = require('graphql-request');
const app = express();
const port = 3000;
const apikey = "*";
const apipassword = "*";
const endpoint = "https://<store>.myshopify.com/admin/api/2021-07/graphql.json"
const shopify = new Shopify({
shopName: '<store>.myshopify.com',
apiKey: '*',
password: '*',
autoLimit: true
});
app.set('view-engine', 'ejs');
app.use(express.static(path.join(__dirname, 'public')));
app.get('/', function(req, res) {
res.render(path.join(__dirname, '/views/index.ejs'));
shopify.product.count()
.then(async (count) => {
if (count > 0) {
const pages = Math.ceil(count / 250);
let products = [];
for (i = 0; i < pages; i++) {
// use Promise.all instead of waiting for each response
const result = await shopify.product.list({
limit: 250,
page: i + 1,
fields: 'id, variants'
});
products = products.concat(result);
}
// products array should have all the products. Includes id and variants
console.log(products);
}
})
.catch(err => {
console.log(err);
});
})
app.get('/treasures', function(req, res) {
res.render(path.join(__dirname, '/views/treasure.ejs'));
});
app.get('/poetry_books', (req, res) => {
res.send('Hello World');
});
app.listen(port);
If anyone could tell me what I'm doing wrong or suggest a better approach that would be amazing.
I don't think you can use the page parameter anymore. Try something like that instead:
let last_id = 0;
do {
const results = await ShopifyAPI.product.list({
limit: 250,
since_id: last_id,
});
// Do something with your product data
last_id = (results.length === 250 ? results[results.length - 1].id : 0);
} while (last_id);

display string on client side by fetching data from server side

I was trying to display a string on the client-side by fetching the result from serverside but for some reason, it is not displaying the fetched data. When I console log the variable straight on the js file the server successfully prints the string. The program is not exporting the variable to the client-side to display it. I can't figure out where I went wrong. Any help is appreciated. Thanks in advance.
const router = require("express").Router();
const {
callName
} = require("pathJs");
router.route("PathRoute").get(async(req, res) => {
const Result = await callName();
return res.json(Result);
});
module.exports = router;
function name() {
const liner = "this works"
console.log(liner)
//updated
return liner;
}
async function callName() {
const data1 = await name()
return data1;
}
callName()
<p id="insertHere" style="color: white;"></p>
<script>
async function caller() {
await fetch(`http://localhost:5000/api/PATH`)
.then((res) => {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(res.json())
}, 1000)
})
}).then((response) => {
console.log(response)
document.getElementById("insertHere").innerHTML = response.liner
}
)
}
</script>
const express = require("express");
const cors = require("cors");
const routePath = require("./routePath");
const {
response
} = require("express");
require("dotenv").config({
debug: process.env.DEBUG
});
const port = process.env.PORT || 5000;
const app = express();
app.use(cors());
app.use(express.json());
app.use("/api", routePath);
app.listen(port, () => {
console.log(`server is running on port: http://localhost:${port}`);
});
There is no export in pathJs and you want name() to return an object containing liner. You need
function name() {
const liner = "this works"
console.log(liner)
//updated
return {liner};
}
async function callName() {
const data1 = await name()
return data1;
}
callName()
module.exports = { callName };
The backend is probably crashing with TypeError: callName is not a function while handling the request and therefore doesn't send a response.

fs database will not save for some reason?

I'm trying to generate a code, add it to a database, and then return it to the request server. I'm getting no errors, but the database remains empty, nothing gets added. I'm using glitch as a temporary host, my json file is just {}
My code:
const Discord = require('discord.js')
const rbx = require('noblox.js')
const fs = require("fs")
const express = require("express")
const app = express()
app.use(express.json())
const client = new Discord.Client()
client.verificationCodes = require("./codes.json")
require("dotenv").config()
const port = process.env.PORT
const serverKey = process.env.SERVER_KEY
const cookie = process.env.COOKIE
function randomString(length, chars) {
var result = '';
for (var i = length; i > 0; --i) result += chars[Math.floor(Math.random() * chars.length)];
return result;
}
client.on("ready", () => {
console.log("Client is ready.")
})
app.post("/getVerificationCode", function(req,res,next) {
console.log("Recieved")
if (req.body.serverKey !== serverKey) {
console.log("Invalid serverKey supplied.")
return res.status(403).json({
error: "You do not have permission to use this."
})
}
let verificationCode = randomString(4,'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ').toUpperCase()
const userID = parseInt(req.body.userid)
console.log(verificationCode)
client.verificationCodes[userID] = {
code: verificationCode
}
fs.writeFile("./codes.json", JSON.stringify(client.verificationCodes,null,4), err => {
if (err) throw err
})
return res.status(200).json({
VerificationCode: verificationCode
})
})
app.get("/*", function(req,res,next) {
return res.status(200).json({})
})
app.listen(port)
console.log(`App listening on port ${port}`)
function rbxLogin(newCookie) {
try {
rbx.setCookie(newCookie)
} catch(err) {
console.log(`Invalid cookie supplied, or expired. ${err}`)
}
}
// rbxLogin(cookie)
client.login(process.env.BOT_TOKEN)
I watched a video on how to use an fs database. I appreciate any help!

Problem with server export when testing React with Jest and Supertest

I'm testing a React app with an Express backend, using Jest and Supertest. In my current test, I need to stub the fetch, which I'm doing with Supertest. The problem is, I never get an answer from the agent get(), and thus never get any data back.
I think there is a problem with how I'm exporting my server. I've tried changing around the exports, from module.exports = app, to module.exports = {app}, to const server = app.listen(port, etc), and module.exports = server. So far none of the solutions I've found are working.
server.js:
const app = require('./app.js');
const port = process.env.PORT || 8080;
app.listen(port, () => console.log("Server running on port " + port));
app.js:
const express = require('express');
const path = require('path');
const bodyParser = require('body-parser');
const app = express();
const ews = require('express-ws')(app);
const WebSocket = require('ws');
...
app.get("/menus", (req, res) => {
const menus = MenuRepo.getMenus();
res.json(menus)
})
...
module.exports = app;
home-test.js
test("Test that dishes displays", async () => {
menuRepo.populateMenus();
overrideFetch(app);
const driver = mount(
<MemoryRouter>
<ShowMenus/>
</MemoryRouter>
);
const predicate = () => {
driver.update();
const tableSearch = driver.find('#menuTable');
const tableIsDisplayed = (tableSearch.length >= 1);
return tableIsDisplayed;
};
const displayedTable = await asyncCheckCondition(predicate, 3000, 200);
expect(displayedTable).toBe(true);
const menus = menuRepo.getMenus();
const html = driver.html();
for(let i=0; i<menus.length; i++){
expect(html).toContain(menus[i].dishes.day);
}
});
function I'm using to stub fetch:
function overrideFetch(app){
const agent = request.agent(app);
global.fetch = async (url, init) => {
let response;
if(!init || !init.method || init.method.toUpperCase() === "GET"){
try {
response = await agent.get(url);
} catch (e) {
console.log(e)
}
} else if(init.method.toUpperCase() === "POST"){
response = await agent.post(url)
.send(init.body)
.set('Content-Type', init.headers ? init.headers['Content-Type'] : "application/json");
} else if(init.method.toUpperCase() === "PUT"){
response = await agent.put(url)
.send(init.body)
.set('Content-Type', init.headers ? init.headers['Content-Type'] : "application/json");
} else if(init.method.toUpperCase() === "DELETE"){
response = await agent.delete(url);
} else {
throw "Unhandled HTTP method: " + init.method;
}
const payload = response.body;
return new Promise( (resolve, reject) => {
const httpResponse = {
status: response.statusCode,
json: () => {return new Promise(
(res, rej) => {res(payload);}
)}
};
resolve(httpResponse);
});
};
}
I'm expecting that the stubbed fetch will return a list of seven json menus.

Categories