This issue occurs in both Node 6.9.4 and 7.0.0, and I cannot figure out why. I haven't tested in other versions. See comments in Node.js program below:
const express = require('express');
const adaro = require('adaro');
const app = express();
const tabs = require('./config/tabs.json');
const config = require('./config/locals.js');
function getLocals(name) {
const modifiedTabs = config.tabs.map(tab => {
return Object.assign(tab, {active: tab.name === name});
});
return Object.assign({tab: name}, config, {tabs: modifiedTabs});
}
app.engine('dust', adaro.dust());
app.set('view engine', 'dust');
app.set('x-powered-by', false);
app.use(express.static('static'));
tabs.map(tab => tab.name).forEach(name => {
const locals = getLocals(name);
const tab = locals.tabs.find(tab => tab.active);
// these are always true
console.log(tab === locals.tabs.find(tab => tab.active));
function callback(req, res) {
// const locals = getLocals(name);
// this should be true, but is false unless the line above is commented in
console.log(tab === locals.tabs.find(tab => tab.active));
res.render('main', locals);
}
if (tab.url !== '/' + tab.id) {
app.get(tab.url, callback);
}
app.get('/' + tab.id, callback);
});
app.all('*', function (req, res) {
res.sendStatus(404);
});
app.listen(process.env.PORT || 8000);
Can anyone explain why this is happening and how to fix it?
I found the issue. In getLocals(), I'm executing
const modifiedTabs = config.tabs.map(tab => {
return Object.assign(tab, {active: tab.name === name});
});
The Object.assign(tab, ...) overwrites the existing tab object each time it's called, leaving only the last assignments to each tab. So all the views show the last tab in the array as active, since all their active properties have been overwritten with those of the last tab's configuration.
The solution was to switch the arguments to Object.assign() so that the returned object was created rather than overwritten:
return Object.assign({active: tab.name === name}, tab);
Related
I'm writing an API with express, puppeteer-cluster and cheerio that returns all anchor elements containing one or more words that can be added as query parameters. I want to use puppeteer in order to get elements that are javascript generated too. But for some reason it's not working, I get an empty array as an output printed on the browser.
I'm still trying to understand this library but has been 2 days and I made no progress. Any help is deeply appreciated.
Update: I added async to all my functions and they run now, but the result is still empty :(
Update 2: I started logging everything, every step and found that data.name is being passed to the cheerio function as a Promise. '-' I think that is the problem, but don't know how to fix it yet.
Update 3: One of the issues was that the page content (html code) was not being handled properly to the cheerio function. In the browser, however, the response is empty and the console shows an error:
Error handling response: TypeError: Cannot read properties of
undefined (reading 'innerText').
So, I think the response is not json formatted. Is res.json() not the right way to do it?
My code:
app.js
const PORT = process.env.PORT || 8000;
var path = require("path");
const express = require("express");
// Routes
const indexRouter = require("./routes/index");
const allNews = require("./routes/news");
const clusterRouter = require("./routes/cluster");
const app = express();
app.use(cors());
app.use(express.json());
app.use(express.urlencoded({ extended: false }));
app.use(express.static(path.join(__dirname, "public")));
app.use("/", indexRouter);
app.use("/news", allNews);
app.use("/cluster", clusterRouter);
app.listen(PORT, () => console.log(`server running on PORT ${PORT}`));
cluster.js
const express = require("express");
const { Cluster } = require("puppeteer-cluster");
const puppeteer = require("puppeteer-extra");
const cheerio = require("cheerio");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
var router = express.Router();
const newspapers = [
{
"name": "CNN",
"address": "https://edition.cnn.com/specials/world/cnn-climate",
"base": "https://edition.cnn.com"
},
{
"name": "The Guardian",
"address": "https://www.theguardian.com/environment/climate-crisis",
"base": "https://www.theguardian.com"
}]
const app = express();
puppeteer.use(StealthPlugin());
const result = [];
router.get("/", async (req, res) => {
(async () => {
// Query String
const query = checkForQuery(req);
const wordsToSearch = query ? verifyQuery(query) : "";
console.log("Running tests.."); // This is printed on console
//Functions
function checkForQuery(request) {
if (request.originalUrl.indexOf("?") !== -1) {
console.log(request.query);
return request.query;
} else {
return false;
}
}
// // Validates query and remove invalid values
function verifyQuery(queryString) {
const queryParams = {
only: queryString.only ? queryString.only : "",
also: queryString.also ? queryString.also : "",
};
// Creates new list containing valid terms for search
var newList = {
only: [],
also: [],
};
for (const [key, value] of Object.entries(queryParams)) {
const tempId = key.toString();
const tempVal =
queryParams[tempId].length >= 2
? queryParams[tempId].split(",")
: queryParams[tempId];
console.log(queryParams[tempId], " and ", tempVal);
if (tempVal.length > 1) {
console.log("helloooooo");
tempVal.forEach((term) => {
if (topics.indexOf(term) != -1) {
newList[tempId].push(term);
}
});
} else {
if (topics.indexOf(queryParams[tempId]) != -1) {
newList[tempId].push(queryParams[tempId]);
}
}
}
console.log(newList);
return newList;
}
function storeData(element, base, name) {
const results = [];
element.find("style").remove();
const title = element.text();
const urlRaw = element.attr("href");
const url =
urlRaw.includes("www") || urlRaw.includes("http")
? urlRaw
: base + urlRaw;
// Check for duplicated url
if (tempUrls.indexOf(url) === -1) {
// Check for social media links and skip
if (!exceptions.some((el) => url.toLowerCase().includes(el))) {
tempUrls.push(url);
// Get img if child of anchor tag
const imageElement = element.find("img");
if (imageElement.length > 0) {
// Get the src attribute of the image element
results.push({
title: title.replace(/(\r\n|\n|\r)/gm, ""),
url,
source: name,
imgUrl: getImageFromElement(imageElement),
});
} else {
results.push({
title: title.replace(/(\r\n|\n|\r)/gm, ""),
url: url,
source: name,
});
}
}
}
return results;
}
function getElementsCheerio(html, base, name, searchterms) {
console.log(html, base, name);
const $ = cheerio.load(html);
console.log(searchterms);
const concatInfo = [];
if (searchterms) {
const termsAlso = searchterms.also;
const termsOnly = searchterms.only;
termsAlso.forEach((term) => {
$(`a:has(:contains("climate"):contains(${term}))`).each(function () {
const tempData = storeData($(this), base, name);
tempData.map((el) => concatInfo.push(el));
});
});
termsOnly.forEach((term) => {
// $(`a:has(:contains(${term}))`).each(function () {
$(`a:contains(${term})`).each(function () {
const tempData = storeData($(this), base, name);
tempData.map((el) => concatInfo.push(el));
});
});
} else {
$('a:contains("climate")').each(function () {
const tempData = storeData($(this), base, name);
tempData.map((el) => concatInfo.push(el));
});
}
return concatInfo;
}
const cluster = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_CONTEXT,
maxConcurrency: 2,
puppeteerOptions: {
headless: true,
args: ["--no-sandbox", "--disable-setuid-sandbox"],
userDataDir: "./tmp",
defaultViewport: false,
},
});
await cluster.task(async ({ page, data }) => {
await page.goto(data.address);
await page.waitForSelector("body");
// console.log here prints that data.name is a Promise :(
const elements = await getElementsCheerio(
document.body.innerHTML,
data.base,
data.name,
wordsToSearch
);
result.push(elements);
});
newspapers.map((newspaper) => {
console.log("queue" + newspaper); // This logs correctly: queue[object Object]
cluster.queue(newspaper);
});
await cluster.idle();
await cluster.close();
// Display final object
res.json(result);
})();
});
module.exports = router;
I don't get any errors, but on screen I get an empty [ ]. Anyone can see what I am doing wrong here? :(
In general, it's an antipattern to mix Puppeteer with another selection library like Cheerio. In addition to being redundant, the extra HTML parser doesn't work on the live document as Puppeteer does, so you have to snapshot the HTML at a particular moment with Puppeteer to capture it as a string and plug that string into Cheerio, where it's re-parsed back to a traversible tree structure.
Introducing this extra step creates opportunity for bugs and confusion to creep in, and that's what happened here.
The code
const elements = await getElementsCheerio(
document.body.innerHTML,
data.base,
data.name,
wordsToSearch
);
is problematic. document.body.innerHTML doesn't refer to anything related to Puppeteer. Instead, use Puppeteer's await page.content() to snapshot the HTML.
As a minor point, there's no need for Cheerio functions to be async, because they never use await. It's a fully synchronous API.
Here's a minimal set up for using Cheerio with Puppeteer, assuming you accept the terms and conditions and are sure that intoducing this usually unnecessary layer of indirection is appropriate for your use case:
const cheerio = require("cheerio"); // 1.0.0-rc.12
const puppeteer = require("puppeteer"); // ^19.0.0
let browser;
(async () => {
browser = await puppeteer.launch();
const [page] = await browser.pages();
const url = "https://www.example.com";
await page.goto(url, {waitUntil: "domcontentloaded"});
const html = await page.content();
const $ = cheerio.load(html);
// do cheerio stuff synchronously
console.log($("h1").text()); // => Example Domain
})()
.catch(err => console.error(err))
.finally(() => browser?.close());
It's basically the same for puppeteer-cluster: just drop the lines starting with const html = await page.content(); into the cluster.task callback that operates on page.
I am having trouble wrapping my head around the best way to execute some code below from a separate file. I have it running well in my my app.js but obviously, I do not want to keep a bunch of code in only app.js and want to start moving things into other files but am at a loss on how to do so. My app.js more or less looks like this and is extremely bloated:
//web server
const express = require('express');
const app = express();
const port = 3000;
//raspberryPI
const { ChatClient } = require("dank-twitch-irc");
const { yay } = require('./JavaScript/yay');
const { readPin } = require('./JavaScript/readPin');
let ejs = require('ejs');
//listen on port 3000
app.listen(port, () => console.info('Listening on port', { port }));
app.use(express.static('./'))
app.set('view engine', 'ejs');
app.get('/', (req, res) => {
res.render('index');
})
//Setup raspberry pi stepper motor hat
let spec = {
steppers: [{ W1: 'M1', W2: 'M2' }, { W1: 'M3', W2: 'M4'}],
//steppers: [{ W1: 'M3', W2: 'M4' }],
};
const motorHat = require('motor-hat')(spec);
motorHat.init();
motorHat.steppers[0].setSteps(2048);
motorHat.steppers[0].setSpeed({ rpm: 5 });
let client = new ChatClient({
connection: {
type: "websocket",
secure: true,
}
});
//connected?
client.on("ready", () => console.log("Successfully connected to chat"));
client.on("close", (error) => {
if (error != null) {
console.error("Client closed due to error", error);
}
});
const keywordsList = [];
client.on("PRIVMSG", (msg, error) => {
console.log(`[#${msg.channelName}] ${msg.displayName}: ${msg.messageText}`);
const keywordFinder = /\b(^right)|(^left)|(^fire)\b/iy;
//empty array fixes null value
const keywords = msg.messageText.match(keywordFinder) || [];
if (msg.messageText === keywords[0]){
if (keywordsList.length>10){
keywordsList.shift();
}
keywordsList.push(`${msg.displayName}: ${msg.messageText}`);
console.log(keywordsList);
}
// keywordsList.forEach(keywords)
if (msg.messageText === "right") {
motorHat.steppers[0].stepSync('fwd', 12);
}
if (msg.messageText === "left") {
motorHat.steppers[0].stepSync('back', 12);
}
if (msg.messageText === "yay") {
yay();
}
if (msg.messageText === "pin") {
readPin();
}
if (error != null) {
console.error("Client closed due to error", error);
}
});
//connect to specific Twitch chat
client.connect();
client.join("yuhn");
and the code I am having trouble removing is this (sorry if redundant):
const keywordsList = [];
client.on("PRIVMSG", (msg, error) => {
console.log(`[#${msg.channelName}] ${msg.displayName}: ${msg.messageText}`);t
const keywordFinder = /\b(^right)|(^left)|(^fire)\b/iy;
//empty array fixes null value
const keywords = msg.messageText.match(keywordFinder) || [];
if (msg.messageText === keywords[0]){
if (keywordsList.length>10){
keywordsList.shift();
}
keywordsList.push(`${msg.displayName}: ${msg.messageText}`);
console.log(keywordsList);
}
// keywordsList.forEach(keywords)
if (msg.messageText === "right") {
motorHat.steppers[0].stepSync('fwd', 12);
}
if (msg.messageText === "left") {
motorHat.steppers[0].stepSync('back', 12);
}
if (msg.messageText === "yay") {
yay();
}
if (msg.messageText === "pin") {
readPin();
}
if (error != null) {
console.error("Client closed due to error", error);
}
});
What I thought I could do is just put the code in a separate file, make sure all my modules are connected (export client and import it into the separate files along with motorhat, etc) but it does not run. Any and all help is appreciated. I have attempted to read about this for a few hours now and I keep getting in depth guides to routing/express file structure...which is fine but I need to know if that's the direction to go in and if so, the first steps.
With my current understanding of your code, I would start by creating a file that contains the callback of your websocket event:
ws_callbacks.js
const motorHat = require('./main.js').motorHat
const keywordsList = [];
exports.PRIVMSG_callback = (msg, error) => {
console.log(`[#${msg.channelName}] ${msg.displayName}: ${msg.messageText}`);t
const keywordFinder = /\b(^right)|(^left)|(^fire)\b/iy;
//empty array fixes null value
const keywords = msg.messageText.match(keywordFinder) || [];
// Rest of your code
}
You can then access the callback by requiring ws_callbacks.js in your main file as so:
app.js
// ...
motorHat.init();
motorHat.steppers[0].setSteps(2048);
motorHat.steppers[0].setSpeed({ rpm: 5 });
exports.motorHat = motorHat
// ...
client.on("PRIVMSG", require('./ws_callbacks.js').PRIVMSG_callback )
// ...
Here, The callback accesses motorHat, which is why it is being exported within the app.js file.
Note that you can also export the other websocket callbacks (ready and close) in similar fashion.
If you prefer to keep all websocket-related code in a separate file, this is how I would write it:
chat.js
const { ChatClient } = require("dank-twitch-irc");
let client = new ChatClient({
connection: {
type: "websocket",
secure: true,
}
});
//connected?
client.on("ready", () => console.log("Successfully connected to chat"));
client.on("close", (error) => {
if (error != null) {
console.error("Client closed due to error", error);
}
});
client.on("PRIVMSG", require('./ws_callbacks.js').PRIVMSG_callback )
// Exporting client in case you want to use it somewhere else
exports.client = client
And then import it in app.js.
However, you have mentioned experiencing problems with importing and exporting client so I am unsure if this will work
//Require module
const express = require('express');
const { evaluate, compile, parse } = require('mathjs');
// Express Initialize
const app = express();
const port = 8000;
app.listen(port, () => {
console.log('listen port 8000');
})
//create api
app.get('/hello_world', (req, res) => {
const expression = "A B A";
console.log(expression.length);
let response;
const scope = {
A: 5,
B: 4
}
try {
const parsedExp = parse(expression);
const compiled = parsedExp.compile();
const result = compiled.evaluate(scope);
response = {
"expression": parsedExp.toString(),
"variables": parsedExp.args,
"result": result
}
console.log("success");
res.send(JSON.stringify(response));
} catch (error) {
console.log(error);
res.send(JSON.stringify(error));
}
})
The code and calculation are working fine. but it's taking multiply by default. Is there a way we can stop this default behavior and throw an error message to the user that please enter your desired operator?
I tried even with normal javascript code by splitting with space and tried to check if of +,-,*,/,^ operator but the user can still give multiple spaces then writes another variable
Help appreciated
There is currently no option to disable implicit multiplication, but there is a (currently open) github issue for that. And in the comments of that issue there is a workaround to find any implicit multiplications and throw an error if one is found.
try {
const parsedExp = parse(expression);
parsedExp.traverse((node, path, parent) => {
if (node.type === 'OperatorNode' && node.op === '*' && node['implicit']) {
throw new Error('Invalid syntax: Implicit multiplication found');
}
});
...
} catch (error) {
console.log(error);
res.send(JSON.stringify(error));
}
Issue
Custom validator not giving any response. I want to validate this below object in express-validator. The privilege key must be valid and exist in my array. If privilege object does not exist then the validation not working because it's optional but the value must be valid if the object exists
{
"email" : "john#gmail.com",
"privileges" : {
"topic" : true
}
}
user.js
const User = require('../../models/User');
const { check, validationResult } = require('express-validator');
let privilegesArray = ['topic'];
router.post(
'/',
[
check("privileges")
.custom(async (value, { req }) => {
Object.keys(value).forEach(function(key){
if(!privilegesArray.includes(key)){
return false;
}
if(value[key] === 'true' || value[key] === 'false' || value[key] === ''){
return false;
}
})
}).withMessage("Invalid privileges").optional({checkFalsy: true})
],
async (req, res) => {
const errors = validationResult(req);
if (!errors.isEmpty()) {
return res.status(400).json({ errors: errors.array() });
}
}
);
I am a beginner in node-js and not using any custom validator.
Okay, so there are few problems with your script:
Within the custom handler, you are iterating using forEach:
Object.keys(value).forEach(function(key){
if(!privilegesArray.includes(key)){
return false;
the problem is, however, that you are returning false from within the internal forEach handler, not the custom handler itself.
You shouldn't be returning by the way, according to documentation. You should throw an error instead.
You didn't provide handler if everything goes well, eg. return res.status(200).json({ errors: "[]" }); in case everything goes fine in route handler.
Most important I guess, you didn't register any bodyParser. I'm pretty sure express won't be able to understand application/json in the POST body: app.use(bodyParser.json());
After middleware parses input jsons, you shouldn't be comparing value[key] === 'true', as it's going to be boolean true.
Below is complete code which seems to meet your requirements, tested using fiddler:
const { check, validationResult } = require("express-validator");
const bodyParser = require("body-parser");
const express = require("express");
const app = express();
const port = 3000;
let privilegesArray = ["topic"];
app.use(bodyParser.json());
app.post(
"/",
[
check("privileges")
.custom(async (value, { req }) => {
var keys = Object.keys(value);
for (var k = 0; k < keys.length; k++) {
var key = keys[k];
if (!privilegesArray.includes(key)) continue;
if (
value[key] !== true &&
value[key] !== false &&
value[key] !== ""
) {
throw new Error("Topic is invalid.");
}
}
return true;
})
.withMessage("Invalid privileges")
.optional({ checkFalsy: true }),
],
async (req, res) => {
const errors = await validationResult(req);
console.log(errors);
if (!errors.isEmpty()) {
return res.status(400).json({ errors: errors.array() });
}
return res.status(200).json({ errors: "[]" });
}
);
app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`);
});
The fs.writeFile code in the backend is running twice.
The data appending appears in console twice and data is written twice in JSON file.
Any idea why this happens?
Any advice is greatly appreciated.
EDIT: this seems like a front-end problem. onFavSubmit is running twice...
Front-end
constructor (props) {
super (props)
this.state = {
inputOne: '',
chosenOne: ['Favourite Movie', 'X'],
chosenTwo: ['2nd Favourite Movie', 'X'],
chosenThree: ['3rd Favourite Movie', 'X'],
movies:[],
};
this.onFavSubmit = this.onFavSubmit.bind(this);
this.onReset = this.onReset.bind(this);
}
onFavSubmit = event => {
const newFav = {
first: this.state.chosenOne[0],
second: this.state.chosenTwo[0],
third: this.state.chosenThree[0]
}
if(this.state.chosenOne[1] === 'X' || this.state.chosenTwo[1] === 'X' || this.state.chosenThree[1] === 'X'){
alert ('Need All 3 Favourite Shows')
event.preventDefault();
} else {
axios.post('http://localhost:8001/fav', {newFav})
.then(
alert('Successfully Added'),
this.onReset()
)
.catch(err => console.log(err.response))
}
}
<button className="fav__button" type="button" onClick={this.onFavSubmit}>Click Me</button>
Back-end
const express = require("express");
const favData = require("../data/fav.json")
const fs = require ('fs')
const router = express.Router();
router.get("/", (_, res) => {
res.json(favData);
});
router.post("/", (req, res) => {
const newFirst = req.body.newFav.first;
const newSecond = req.body.newFav.second;
const newThird = req.body.newFav.third;
const newfavData = {
First: newFirst,
Second: newSecond,
Third: newThird,
};
fs.readFile('./data/fav.json', 'utf-8', function (err, data){
var json = JSON.parse(data);
json.push(newfavData);
console.log(newfavData)
fs.writeFile('./data/fav.json', JSON.stringify(json), function(err){
if (err) throw err;
console.log('data appended')
return;
})
})
});
module.exports = router;
I don't have enough reputation so I can't comment.
Have you tried commenting out this.onReset() and see if that fixes the problem?
There had been times when I was sending a request to "/" on reload.