Hi I'm used to work with c#, I'm new with node js. I'm trying to create some kind of web service using Node Js. I'm using VS 2017 community with node js version 8. I have next code:
Express App1
apps.js
app.param('phone', function (request, response, next, phone) {
// ... Perform database query and
// ... Store the user object from the database in the req object
req.phone = phone;
return next();});
index.js
'use strict';
var express = require('express');
var router = express.Router();
var Utils = require("./JavaScript1");
/* GET home page. */
router.get('/', function (req, res) {
res.render('index', { title: 'Express' });});
router.get('/byPhone/:phone', function (req, res) {
var t = Utils.Phone(req.params.phone).then(value => { return value });
//At this point if i try to use await or consume by using web
//http://localhost:1337/byPhone/777777 i only get a promise ...
res.send(t);
});
module.exports = router;
JavaScript1.js
function sAdd(sPhone) {
return new Promise((resolve, reject) => { // (A)
setTimeout(() => resolve("01800" + sPhone), 5000); // (B)
});}
var utils = {
//at this point in the temp and temp2 variables only have promises
Phone:
async function (sPhone) {
var temp = await sAdd(sPhone).then(value => { return value });
var temp2 = await temp;
return temp2;
}
};
module.exports = utils;
I'm trying to consume it with the next Node Js Console App:
'use strict';
async function main()
{
console.log('Hello world');
var url = "http://localhost:1337/byPhone/777777";
request(url, function (err, response, body) {
if (err) { console.log(err); callback(true); return; }
var tt = body;
console.log(tt);
});
}
main();
i get the below answer:
(node:19492) UnhandledPromiseRejectionWarning: Unhandled promise
rejection (rejection id: 1): ReferenceError: request is not defined
I forgot to answer myself in this question, but I resolved what I wanted to do at that time this way:
I created a solution with 2 projects:
c# controllers webapp
on the c# side:
[Produces("application/json")]
[Route("test/Mondb")]
public class MondbController : Controller
{
// GET: api/Mondb
[HttpGet]
public string Get()
{
var client = new MongoClient("mongodb://localhost:27017");
var database = client.GetDatabase("upixTest");
var collection = database.GetCollection<BsonDocument>("contacts");
var document = collection.Find(new BsonDocument()).ToList();
return document.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.Strict });
}
// GET: api/Mondb/5
[HttpGet("{id}", Name = "Getmdb")]
public string Get(int id)
{
var filter = Builders<BsonDocument>.Filter.Eq("phone", id.ToString());
var client = new MongoClient("mongodb://localhost:27017");
var database = client.GetDatabase("upixTest");
var collection = database.GetCollection<BsonDocument>("contacts");
var document = collection.Find(filter).FirstOrDefault();
return (document==null) ? "nothing":document.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.Strict });
}
on the other side the node js project
functions.js
thefunctions = {
add: function (a, b) { return a + b; }, // test function
mongo: function () {
var axios = require('axios');
var tresult;
return axios.get('http://localhost:55384/test/Mondb/');
}
};
module.exports = thefunctions;
then in the server.js
'use strict';
var http = require('http');
var fs = require('fs');
var url = require('url');
var port = process.env.PORT || 1337;
var thefunctions = require("./thefunctions");
var dataToShow = "";
var JSON = require('JSON');
http.createServer(function (req, res) {
var hostname = req.headers.host;
var pathname = url.parse(req.url).pathname;
var fullurl = 'http://' + hostname + pathname;
var search = url.parse(req.url).search ? url.parse(req.url).search : "";
if (pathname === "/index.html") {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.write('hi my friend\n');
var tout = thefunctions.mongo();
tout.then(function (response) {
res.write(JSON.stringify(response.data).toString());
res.write('\n');
res.end('\n\nThe End\n');
})
.catch(function (error) {
console.log(error);
res.write("error");
res.end('\n\nThe End\n');
});
}
else {
res.writeHead(400, { 'Content-Type': 'text/plain' });
res.write('error my friend\n');
res.end('404 isnt available');
return;
}
}
).listen(port);
this way you will consume c# wb services from node js app
Related
I am trying to create a middleware that receive a form-data and return the fieldname, contentType and the value. So when I send the firts post the data view in the terminal but if I send the same request again doesn't show me the data in the terminal.
And if a toggle the image, the data come show in the terminal
This is my code:
server:
const express = require("express");
const Upes = require("../upes");
const app = express();
const start = new Upes();
app.post("/", start.setup.bind(start), (req, res) => {
res.send("all right");
});
app.listen(3000, () => {
console.log("The server is active");
});
the index of my middleware:
const getData = require("./utils/getData");
const parseContentType = require("./utils/parseContentType");
class Upes {
setup(req, res, next) {
const contentType = parseContentType(req.headers["content-type"]);
if (!contentType) {
throw new Error("Malformed content-type");
}
const SUBTYPES = ["form-data", "x-www-form-urlencoded"];
if (!SUBTYPES.includes(contentType.subtype)) {
throw new Error(
"The subtypes does not match the following subtypes: " + SUBTYPES
);
}
getData(req, contentType.params.boundary, (data) => {
console.log(data);
});
next();
}
}
module.exports = Upes;
The function that receive the data and processes it:
function getData(req, boundary, callback) {
let chunk = "";
let data = [];
req.on("data", (buffer) => {
chunk += buffer.toString();
});
req.on("end", () => {
// Split the chunk in blocks
const blocks = getBlock(chunk, boundary);
blocks.forEach((block) => {
let [params, value] = block.split("\r\n\r\n");
params = params.split(";");
let fieldname = params[1].split("=")[1].replaceAll('"', "");
let contentType = () => {
const condition = params.length === 3;
if (condition) {
let type = params[2].split(":")[1].replace(" ", "");
return type;
}
return "text-plain";
};
const payload = {
fieldname: fieldname,
contentType: contentType(),
value: "", // value.replace("\r\n", "")
};
data.push(payload);
});
callback(data);
});
}
function getBlock(body, boundary) {
boundary = boundary.replaceAll("-", "");
return body.replaceAll("-", "").split(`${boundary}`).slice(1, -1);
}
module.exports = getData;
Send the same request 20 times
I don't know what happend, please can someone help me?
I am making an experimental web proxy in Node.js. A lot of websites work like normal even Discord.com. The problem is on this website.
For some odd reason, it's showing a lot of weird symbols instead of the website. But if I go on any image on that site the images work fine. I would love the help. I would also prefer to use regular HTTPS / HTTP modules.
By the way, the commented out parts are for a WebSocket proxy so discard that.
const express = require('express'),
app = express(),
WebSocket = require('ws'),
fs = require('fs'),
https = require('https'),
http = require('http');
btoa = (str) => {
str = new Buffer.from(str).toString('base64');
return str;
};
atob = (str) => {
str = new Buffer.from(str, 'base64').toString('utf-8');
return str;
};
const server_options = {
key: fs.readFileSync('./ssl/default.key'),
cert: fs.readFileSync('./ssl/default.crt')
}
const server = https.createServer(server_options, app);
//server.on('upgrade', function upgrade(req, socket, head) {
// const wss = new WebSocket.Server({
// server: server
// });
// wss.on('connection', function connection(ws) {
// console.log(req.url.slice(1))
// const client = new WebSocket(req.url.slice(1));
//ws.on('message', function incoming(message) {
// client.send(message)
// });
// client.on('message', function incoming(data) {
// ws.send(data)
// })
// ws.on('error', function incoming(data) {
// client.send(data)
// });
// client.on('error', function incoming(data) {
// ws.send(data)
//});
// });
//});
app.use('/', async(req, res, next) => {
var proxy = {};
proxy.url = `https://www.startpage.com${req.url}`;
proxy.url = {
href: proxy.url,
hostname : proxy.url.split('/').splice(2).splice(0, 1).join('/'),
origin : proxy.url.split('/').splice(0, 3).join('/'),
encoded_origin : btoa(proxy.url.split('/').splice(0, 3).join('/')),
path : '/' + proxy.url.split('/').splice(3).join('/'),
protocol : proxy.url.split('\:').splice(0, 1).join(''),
}
proxy.requestHeaders = req.headers;
proxy.requestHeaders.host = proxy.url.hostname;
delete proxy.requestHeaders['accept-encoding']
proxy.requestHeaders['referer'] = proxy.url.href;
proxy.requestHeaders['origin'] = proxy.url.origin;
proxy.options = {
method: req.method,
headers: proxy.requestHeaders,
rejectUnauthorized: false
}
if (proxy.url.protocol == 'https') { proxy.protocol = https; }
else {proxy.protocol = http};
proxy.sendRequest = proxy.protocol.request(proxy.url.href, proxy.options, server_res => {
var body = [], redirect = false, redirect_value = '';
server_res.on('data', (chunk) => {
body.push(Buffer.from(chunk, 'binary'))
});
var ct = 'text/plain';
Object.entries(server_res.headers).forEach(([header_name, header_value]) => {
if (header_name.startsWith('content-encoding') || header_name.startsWith('x-') || header_name.startsWith('cf-') || header_name.startsWith('strict-transport-security') || header_name.startsWith('content-security-policy')) {
delete server_res.headers[header_name];
}
if (header_name.startsWith('location') || header_name.startsWith('Location')) {
redirect = true; redirect_value = header_value;
delete server_res.headers[header_name];
}
if (header_name == 'content-type') ct = header_value;
});
if (ct == null || typeof ct == 'undefined') ct = 'text/html';
if (redirect == true) { return res.redirect(307, '/' + redirect_value); };
server_res.on('end', () => {
body = Buffer.concat(body)
res.contentType(ct)
res.set(server_res.headers);
res.status(server_res.statusCode)
res.send(body)
});
});
proxy.sendRequest.on('error', err => {
res.send(err.toString())
});
if (req.method == 'POST') {
req.raw_body = '';
req.str_body = '';
req.on('data', chunk => {
req.raw_body += chunk.toString(); // convert Buffer to string
});
req.on('end', () => {
req.str_body = req.raw_body;
proxy.sendRequest.write(req.str_body);
proxy.sendRequest.end();
});
} else proxy.sendRequest.end();
});
server.listen('9000')
So I'm creating a singleton class and when I require it from my server.js file it works fine, but when I require it from another file it returns as undefined. I'll try to post relevant code but some will have to be cut out due to work.
server.js
const express = require('express');
const bodyParser = require('body-parser');
const path = require('path');
const http = require('http');
const app = express();
const config = require('config');
const FBConfigsListener = require('./server/amq_listeners/fb_configs.listener');
const FBConfigs = require('./server/models/FBConfigs');
//Api file for interacting with mongodb
const api = require('./server/routes/api.routes');
//Parsers
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
//Angular Dist output folder
app.use(express.static(path.join(__dirname, 'dist')));
//Api location
app.use('/api', api);
//Send all other requests to angular
app.get('*', (req, res) => {
res.sendFile(path.join(__dirname, 'dist/index.html'));
});
//set port
var port = config.get('webserver.port');
app.set('port', port);
const server = http.createServer(app);
server.listen(port, () => console.log(`Running on localhost:${port}`));
models/FBConfigs.js
var ConfigModel = require('./config');
var config = require('config');
var _ = require('lodash');
var FBConfigsListener = require('../amq_listeners/fb_configs.listener');
var AMQAdapter = require('../adapters/amq.adapter');
var uniqid = require('uniqid');
const connectionOptions = config.get('activemq.connectionOptions');
class FBConfigs {
constructor() {
console.log(config.get('environments'));
this.listener = FBConfigsListener;
this.configs = {};
this.unique_keys = ['id'];
this.update_topic = '/topic/fusebuilder.update.config.';
console.log(FBConfigsListener);
//AMQ Client
this.amq_client = AMQAdapter.getInstance(connectionOptions.host, connectionOptions.port);
}
add(key, config) {
if (!(key in this.configs)) {
this.configs[key] = new ConfigModel(this.unique_keys);
}
this.configs[key].add(config);
}
get(key) {
let configs_json = {};
if (key) {
configs_json[key] = JSON.parse(this.configs[key].toString());
} else {
for (let key in this.configs) {
configs_json[key] = JSON.parse(this.configs[key].toString());
}
}
return configs_json;
}
updateByID(key, id, input_config) {
let configs = this.configs[key].get();
for (let config of configs) {
if (input_config.id === config.id) {
this.update(key, _.merge(config, input_config));
}
}
}
//Send update to config topic
update(key, config) {
let topic = this.update_topic + key;
var update_object = {};
if (Array.isArray(config)) {
update_object[key] = [...config];
} else {
update_object[key] = [config];
}
console.log(`Sending ${key} update:${JSON.stringify(update_object)}`);
this.amq_client.sendMessage(topic, update_object);
}
copyTo(key, id, env) {
let selected_env = config.get('environments.' + env);
// let tmp_amq_client = new AMQAdapter(selected_env.host, selected_env.port);
let selected_config = this.configs[key].getByID(id);
console.log(this);
if (key === 'fuses') {
console.log('In FBConfig Copy to for fuses');
const get_fuse_topic = '/topic/fusebuilder.get_fuse';
const tran_id = uniqid();
const sendObj = { fuseName: id, tran_id };
this.amq_client.sendMessage(get_fuse_topic, sendObj);
var startTime = process.hrtime()[0];
var timeout = false;
while (!this.listener.get_copy_fuse_data(tran_id)) {
console.log('Waiting for config');
sleep(100);
if (process.hrtime()[0] - startTime > 3) {
console.log('Timed out');
timeout = true;
break;
}
}
console.log(JSON.stringify(FBConfigsListener.get_copy_fuse_data(tran_id)));
} else {
tmp_amq_client.sendMessage(this.update_topic, selected_config);
}
console.log(`Copy ${key} id:${id} to ${env}`);
}
}
module.exports = new FBConfigs();
amq_listener/fb_configs.listener.js
const config = require('config');
var AMQAdapter = require('../adapters/amq.adapter');
var FBConfigs = require('../models/FBConfigs');
**removed for work**
class FBConfigsListener {
constructor() {
this.instance;
this.copy_fuse_data = {};
//AMQ Client
this.amq_client = AMQAdapter.getInstance(connectionOptions.host, connectionOptions.port);
//Subscribe to configs
this.amq_client.subscribe(config_subscribe_topic, this.config_topic_callback.bind(this));
//Request Tables
this.amq_client.sendMessage(config_request_topic, { tables: config_tables });
//Subscribe to Copy Fuse topic
this.amq_client.subscribe(subscribe_fuse_copy_topic, this.copy_fuse_callback.bind(this));
}
config_topic_callback(err, message) {
let dest = this.amq_client.getDestination(message);
let key = this.get_key_from_topic(dest);
this.amq_client.readMessage(message, body => {
let configs = JSON.parse(body);
if (key in configs) {
for (let config of configs[key]) {
FBConfigs.add(key, config);
}
}
});
}
copy_fuse_callback(err, message) {
this.amq_client.readMessage(message, body => {
const config = JSON.parse(body);
this.copy_fuse_data[config.tran_id] = config;
});
}
//Get Key from the topic and convert using key map if needed
get_key_from_topic(topic) {
let key = topic.split('.')[topic.split('.').length - 1];
key = key in key_map ? key_map[key] : key;
return key;
}
get_copy_fuse_data(id) {
if (id in this.copy_fuse_data) {
return this.copy_fuse_data[id];
} else {
return false;
}
}
}
module.exports = new FBConfigsListener();
Error happens in FBConfigs. FBConfigsListener returns {} so all functions in there are undefined. Even if I do console.log(require('../amq_listeners/fb_configs.listener')) it prints {} But doing the same thing in server.js (with updated path) it prints the module.
Also tips on how to improve my coding style would be appreciated too.
Edit
So I found out that I have a circular dependency between these classes. How can this be fixed while allowing me to call one from the other.
I would suggest you to instantiate your dependencies firstly and store them in some object which you can pass then to your dependent classes. The structure can be
factories/services.js
/*
* Instantiates passed services and passes injector object to them
*/
module.exports = function createServices(injector, services) {
return Object.entries(services)
.reduce((aggregator, [name, serv]) => {
const name_ = camelCase(name);
aggregator.set(name_, new serv(injector));
return aggregator;
}, new Map());
};
lib/service.js
/**
* Base class for classes need any injections
*/
module.exports = class Service {
constructor(injector) {
this.injector = injector;
}
get dependencies() {
return this.injector.dependencies;
}
/*
* Background jobs can be ran here
*/
async startService() {}
/*
* Background jobs can be stopped here
*/
async stopService() {}
};
lib/injector.js
const Service = require('./service');
/*
* Contains all dependencies
*/
module.exports = class Injector {
constructor() {
this.services = new Map();
this._dependencies = {};
}
has(name) {
return this.services.has(name);
}
register(name, service) {
if (this.has(name)) {
throw new Error(`Service ${name} already exists`);
}
if (service instanceof Service === false) {
throw new Error('Argument #2 should be an instance of Service');
}
this.services.set(name, service);
this._dependencies[name] = service;
}
unregister(name) {
if (! this.has(name)) {
throw new Error(`Service ${name} not found`);
}
this.services.delete(name);
delete this._dependencies[name];
}
get dependencies() {
return { ...this._dependencies };
}
/*
* Starts all registered services
*/
async start() {
for (let service of this.services.values()) {
await service.startService();
}
}
/*
* Stops all registered services
*/
async stop() {
for (let service of this.services.values()) {
await service.stopService();
}
}
};
Then import, initialize and bind your services in the main file (don't forget to export just a class, not an object like you do it now).
server.js
const createServices = require('./factories/services.js');
const injector = require('./lib/injector');
const Injector = new injector();
const services = createServices(Injector, [require('./server/amq_listeners/fb_configs.listener'), require('./server/models/FBConfigs')]);
services.forEach((service, name) => {
Injector.register(name, service);
});
// Start services
Injector.start();
Inherit required classes to Service class and you will get an access to all dependencies there (don't forget to call super() from constructor). Like
models/FBConfigs.js
const Service = require('../lib/service');
class FBConfigs extends Service {
constructor(injector) {
super(injector);
const { FBConfigsListener } = this.dependencies;
...your code here
}
async startService() {
...run bg job or init some connection
}
async stopService() {
...stop bg job or close some connection
}
}
module.exports = FBConfigs;
Also you can pass some config object to createServices (I didn't include it here) with keys equal to service names and values containing config object and pass config to appropriate service.
It is caused by that circular dependency. You should avoid it or used very carefully.
In your case the fix is probably pretty simple, move the line var FBConfigs = require('../models/FBConfigs'); from listener at the end of the file as the last line (yes, even after the module.exports).
Edit: Actually it maybe is not enough as I checked the code more in detail. As you are not using Listener in FBConfig constructor, you can create method assignListener, remove this.listener from that constructor and call it later in server.js which will do the this.listener
Or the last solution, which is also "best practice". Do not export the instances. Export the classes only. Then in server.js create these instances after both are required.
I try to add an option to my app to toggle using a cache. For cache I am using Redis and made simple wrapper for that. The problem is coming when I try to overwrite redis.get method, it's simply doens't work or cannot found this.
'use strict';
const redis = require('redis');
const config = require('../config');
const REDIS_EMPTY_VALUE = 'NOT_EXIST';
const MINUTE = 60;
let client = redis.createClient({
host: config.get('REDIS_HOST'),
port: config.get('REDIS_PORT')
});
client.on("error", function (err) {
logging.error('Redis Error: ' + err);
throw new Error(err);
});
/**
* Next are custom extensions for Redis module
*/
client.emptyValue = REDIS_EMPTY_VALUE;
client.minute = MINUTE;
client.setAndExprire = function(key, value, expire) {
this.set(key, value);
this.expire(key, expire);
};
// Here is the problem
client.get = function(key, cb) {
if (config.get('disable-cache') === 'true') return cb(null, null);
return client.get(key, cb);
}
module.exports = client;
Since you override client.get, the client.get become the new function that you defined. So that, you can call to the function that come along with redis package. You can use another object (custom) to call the redis function like below:
'use strict';
const redis = require('redis');
const config = require('../config');
const REDIS_EMPTY_VALUE = 'NOT_EXIST';
const MINUTE = 60;
const definedFunctions = [
'hgetall', 'hexists', 'hmset', 'hmget', 'hkeys', 'hvals', 'hget', 'hset', 'hdel',
'mget', 'mset', 'set', 'del', 'exists', 'lpush', 'lrange',
'zrange', 'zrem', 'zadd', 'zrangebyscore', 'zrevrangebyscore',
'expire', 'incrby'
];
let client = redis.createClient({
host: config.get('REDIS_HOST'),
port: config.get('REDIS_PORT')
});
client.on("error", function (err) {
console.error('Redis Error: ' + err);
throw new Error(err);
});
/**
* Next are custom extensions for Redis module
*/
const custom = {
emptyValue: REDIS_EMPTY_VALUE,
minute: MINUTE
};
definedFunctions.map((fn) => {
custom[fn] = (...args) => {
return client[fn](args);
};
});
custom.get = function(key, cb) {
if (config.get('disable-cache') === 'true') return cb(null, null);
return client.get(key, cb);
}
module.exports = custom;
I am writing a web scraper that makes multiple requests based on a list that looks like this
1. Category1
1a. categoryItem1
1b. categoryItem2
2. Category2
2a. categoryItem1
2b. categoryItem2
2c. categoryItem3
3. Category3
3a. categoryItem1
Both Category and categoryItem are links. Only 1 Category can be expanded at a time. The amount of Categories and categoryItems can change so I don't know the exact amount before hand.
I am gathering the data on each categoryItem page to be saved in a json that looks like this
{
"Category1": [
"categoryItem1: {
// Details saved here
},
"categoryItem2: {
// Details saved here
}
],
"Category2": [
"categoryItem1: {
// Details saved here
},
"categoryItem2: {
// Details saved here
},
"categoryItem3: {
// Details saved here
}
],
"Category3": [
"categoryItem1: {
// Details saved here
}
]
}
The only thing left for me is to figure out how to make this act synchronous
Get the opening page
Open each Category list
Open each categoryItem details page
THIS was the web scraper tutorial that I followed, if you would like to know. Due to async calls I don't know when the very last page is parsed, so here is the structure of the script
server.js
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
app.get('/scrape', function (req, res) {
globalJSON = {};
baseUrl = 'http://...';
// 1.) open page with list
request.get(baseUrl, function (error, response, html) {
if (!error) {
var $ = cheerio.load(html);
// select the list
$('#categoryListSelector').filter(function () {
var data = $(this);
var listItem = data.find('#listItemSelector');
var expansionLink = listItem.find('a').attr('href'); //
var category = listItem.find('font').text();
// Save category to global json
globalJSON[category] = [];
// 2.) Expand the list by opening expansionLink
request.get(baseUrl + expansionLink, function (error, response, html) {
if (!error) {
var $ = cheerio.load(html);
// Select the sub items of each list item
$('#subItem selector').filter(function () {
var data = $(this);
var categoryItemPageLinkElement = data.find('a');
var categoryItemName = categoryItemPageLinkElement.text();
var categoryItemLink = $(categoryItemPageLinkElement).attr('href');
if (typeof categoryItemLink != "undefinded" && categoryItemLink != null && categoryItemLink != "") {
categoryItemObject = {}; // { categoryItemName: categoryItemDetails }
categoryItemDetails = {};
// 3.) Open the categoryItem page to start gathering data
request.get(baseUrl + categoryItemLink, function (error, response, html) {
if (!error) {
var $ = cheerio.load(html);
// GATHER and save data here
// Done gathering data save to global json
categoryItemObject[categoryItemName] = categoryItemDetails;
globalJSON[category].push(categoryItemObject);
}
});
}
});
}
});
});
fs.writeFile('output.json', JSON.stringify(globalJSON, null, 4), function (err) {
console.log('File successfully written!');
});
res.send(globalJSON);
}//END if(!error)
});
})//END app.get()
app.listen('8081')
console.log('Magic happens on port 8081');
exports = module.exports = app;
Update
I did get my issue solved with some help from the feller below, and this is what I came up with. Now, there might be a better way, feel free to let me know.
Basic Layout
Promise.all(categoriesArr.map(categoryObj => new Promise((resolve, reject)=>{
request.get(baseUrl + categoryObj.categoryItemLink, (error, response, html)=>{
if(error){
return reject(error);
}
//build an array of ALL the categoryItemLinks
return resolve(res, html);
});
}))).then(function(statesArray) {
Promise.all(allCategoryItems.map(categoryItemObject => new Promise((resolve, reject)=>{
request.get(baseUrl + categoryItemObject.categoryItemPageLink, (error, response, html)=>{
if(error){
return reject(error);
}
// Gather Data and put into dataJson
return resolve(response, html);
});
}))).then(function(data) {
// Do finishing stuff
}).catch(/*error*/);
}).catch(/*error*/);
server.js
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
app.get('/scrape', function (req, res) {
categoriesArr = [];
allCategoryItems = [];
dataJson = {}; // Global json to hold all the data
baseUrl = 'http://www.blahblah.org';
request.get(baseUrl, function(error, response, html) {
if (!error) {
var $ = cheerio.load(html);
$('#categorySelector').filter(function() {
var data = $(this);
var categoryItemLink = data.find('a').attr('href');
categoriesArr.push({
"categoryItemLink": categoryItemLink
});
});
Promise.all(categoriesArr.map(categoryObj => new Promise((resolve, reject)=>{
request.get(baseUrl + categoryObj.categoryItemLink, (error, response, html)=>{
if(error){
return reject(error);
}
var $ = cheerio.load(html);
$('#categoryItemSelector').filter(function() {
var data = $(this);
var categoryItemPageLinkElement = data.find('a');
var categoryItemPageLink = $(categoryItemPageLinkElement).attr('href');
if(typeof categoryItemPageLink != "undefinded" && categoryItemPageLink != null && categoryItemPageLink != "") {
allCategoryItems.push({
"categoryItemPageLink": categoryItemPageLink
});
}
});
return resolve(res, html);
});
}))).then(function(statesArray) {
Promise.all(allCategoryItems.map(categoryItemObject => new Promise((resolve, reject)=>{
request.get(baseUrl + categoryItemObject.categoryItemPageLink, (error, response, html)=>{
if(error){
return reject(error);
}
var $ = cheerio.load(html);
// Gather Data and put into dataJson
return resolve(response, html);
});
}))).then(function(data) {
// Do finishing stuff
}).catch(/*error*/);
}).catch(/*error*/);
}//END if(!error)
});
})//END app.get()
app.listen('8081')
console.log('Magic happens on port 8081');
exports = module.exports = app;
You can use Promise.all(), so something like:
Promise.all(urls.map(url => new Promise((resolve, reject)=>{
request.get(url, (err, res, html)=>{
if(err){
return reject(err);
}
return resolve(res, html);
});
}))).then(/*success*/).catch(/*error*/);
In that code, the .then() executes after all requests have come back with a response.