My node.js app aim to scan a list of website an return in a panel, statusCode, and other tests.
Now the app work like that :
The page is empty until the end of the load and if one website in the list is slow the loading is slow, when 100% of the websites are scan the panel show.
How can i show the panel when i load the page, and show other informations progressively ?
var express = require('express');
var fs = require('fs');
var http = require('http');
var ejs = require('ejs');
var async = require('async');
var request = require('request');
var app = express();
var mysql = require('mysql');
var cheerio = require('cheerio');
app.use(express.static(__dirname+"/public"));
// Database connection
app.get('/', function(req, res) {
var connection = mysql.createConnection({
host : 'localhost',
user : 'root',
password : '',
database : 'app'
});
connection.connect();
var jsons = new Array();
var odbc;
connection.query('SELECT * from websites', function(err, rows) {
if (err) throw err;
async.each(rows, function(row, callback) {
http.get(row.url, function(resp) {
row.status = resp.statusCode;
request(row.url, function(error, response, html){
if(!error){
var $ = cheerio.load(html);
str = $('*').text();
var errorODBC = str.indexOf("function.odbc-connect");
var errorTrans = str.indexOf(".WDD");
if(errorODBC != '-1'){
row.odbc = 'Erreur odbc';
}
if(errorTrans != '-1'){
row.aw = 'Erreur transfert';
}
jsons.push(row);
callback();
}
});
}).on('error', function(e) {
console.log("Erreur : " + e.message);
});
},function(){
res.render('index.ejs', {data : jsons});
console.log("Scan done...");
});
});
connection.end();
});
app.listen(8080);
Related
I currently implemented login and membership functions using Express & MySQL.
And I want to add JWT.
I wanted to create an API only through Postman, not on the Web, and I heard that I had to use Passport to search.
And I know there is also Express-generate, but I want to modify my current code.
I am a beginner in Node.js and want a guide.
app.js
var express = require('express');
var http = require('http');
var static = require('serve-static');
var path = require('path');
var bodyParser = require('body-parser');
var cookieParser = require('cookie-parser');
var expressSession = require('express-session');
var expressErrorHandler = require('express-error-handler');
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit:10,
host:'localhost',
user:'root',
password:'password',
database:'test',
debug:false
});
var app = express();
app.set('port', 3000);
app.use('/public', static(path.join(__dirname, 'public')));
app.use(bodyParser.urlencoded({extended:false}));
app.use(bodyParser.json());
app.use(cookieParser());
app.use(expressSession({
secret:'my key',
resave: true,
saveUninitialized:true
}));
var router = express.Router();
router.route('/process/login').post(function(req, res) {
console.log('/process/login');
var paramId = req.body.id;
var paramPassword = req.body.password;
console.log('request parameter:' + paramId + paramPassword);
authUser(paramId, paramPassword, function(err, rows) {
if (err) {
console.log('error');
res.writeHead(200, '{"Content-Type":"text/plain; charset=utf-8"}');
res.write('<h1>error</h1>');
red.end();
return;
}
if (rows) {
console.dir(rows);
res.writeHead(200, '{"Content-Type":"text/plain; charset=utf-8"}');
res.write('<h1>user login success</h1>');
res.write('<div><p>user:' + rows[0].id + ' </p></div>');
res.end();
} else {
res.writeHead(200, '{"Content-Type":"text/plain; charset=utf-8"}');
res.write('<h1>user not found</h1>');
res.end();
}
});
});
router.route('/process/adduser').post(function(req, res) {
console.log('/process/adduser');
var paramId = req.body.id;
var paramPassword = req.body.password;
console.log('request parameter' + paramId + paramPassword);
addUser(paramId, paramPassword, function(err, addedUser) {
if (err) {
console.log('error');
res.writeHead(200, '{"Content-Type":"text/plain; charset=utf-8"}');
res.write('<h1>error</h1>');
red.end();
return;
}
if (addedUser) {
console.dir(addedUser);
res.writeHead(200, '{"Content-Type":"text/plain; charset=utf-8"}');
res.write('<h1>user added</h1>');
res.end();
} else {
res.writeHead(200, '{"Content-Type":"text/plain; charset=utf-8"}');
res.write('<h1>user added fail</h1>');
res.end();
}
});
})
app.use('/', router);
var addUser = function(id, password, callback) {
console.log('addUser');
pool.getConnection(function(err, conn) {
if(err) {
if (conn) {
conn.release();
}
callback(err, null);
return;
}
console.log('db threadid' + conn.threadId);
var data = {id:id, password:password};
var exec = conn.query('insert into users set ?', data,
function(err , result) {
conn.release();
console.log('SQL syntax' + exec.sql);
if (err) {
console.log('SQL error;');
callback(err, null);
return;
}
callback(null, result);
});
});
};
var authUser = function(id, password, callback) {
console.log('authUser' + id + password);
pool.getConnection(function(err, conn) {
if (err) {
if (conn) {
conn.release();
}
callback(err, null);
return;
}
console.log('db threadid:'+ conn.threadId);
var tablename = 'users';
var columns = ['id'];
var exec = conn.query('select ?? from ?? where id = ? and password = ?', [columns, tablename, id, password],
function(err, rows) {
conn.release();
console.log('SQL syntax' + exec.sql);
if (err) {
callback(err, null);
return;
}
if (rows.length >0 ) {
console.log('user find');
callback(null, rows);
} else {
console.log('user not found');
callback(null, null);
}
});
});
};
var errorHandler = expressErrorHandler({
static: {
'404' : './public/404.html'
}
});
app.use(expressErrorHandler.httpError(404));
app.use(errorHandler);
var server = http.createServer(app).listen(app.get('port'), function() {
console.log('server start' + app.get('port'));
});
Hi #yori If you want to create an API, first I will recommend parsing your data as JSON objects and thus eliminating all HTML tags in your code.
In order to use JWT for authentication, you will have to install the jsonwebtoken package as part of your project dependencies: https://www.npmjs.com/package/jsonwebtoken
I will recommend following the in-depth instructions in this post as a guide: https://medium.freecodecamp.org/securing-node-js-restful-apis-with-json-web-tokens-9f811a92bb52
For scope though, here is some explanation:
Express can be used to build robust APIs which can be made available for consumption. POSTMAN is a GUI tool that developers use to query APIs. curl commands is the terminal alternative. JWT is a safe way of representating claims transfered between two parties. I see the need to break down these terms to you so that you will understand the need, relevance and difference between each tool.
Following the guide in the article you will have to refactor your code a little bit.
I'm pretty new to javascript but I have a REST API I'm running on a LAN network. I have a MySQL database I'm trying to save a HTTP POST body to in JSON format. My js code will print the data as "result" for me but will not save it when it is sent from Postman, however if I hard code a fake entry into it as "var result = {JSON format entries}" it will then be able to be saved to MySQL. Here is my code:
var http = require('http');
var express = require('express');
var bodyParser = require('body-parser');
const { parse } = require('querystring');
var mysql = require("mysql");
var app = express();
var port = process.env.port || 8080;
var connection = mysql.createConnection({
host : 'localhost',
user : 'root',
password : '*******',
database : 'userInfo'
});
connection.connect(function(err) {
if (err) throw err
console.log('You are now connected with mysql database...')
})
app.post('/licence', function (req, res) {
collectRequestData(req, result => {
console.log(result);
connection.query('INSERT INTO licence SET ?',result, function (error,
results, fields) {
if (error) throw error;
res.end(JSON.stringify(results));
});
res.end(`Parsed data belonging to ${result.fname}`);
});
});
function collectRequestData(request, callback) {
const FORM_URLENCODED = 'application/json';
if(request.headers['content-type'] === FORM_URLENCODED) {
let body = '';
request.on('data', chunk => {
body += chunk.toString();
});
request.on('end', () => {
callback(parse(body));
});
}
else {
callback(null);
}
}
I'm pretty sure I don't need this line "res.end(Parsed data belonging to ${result.fname});" but what I am trying to figure out here is how to save result to the licence table in mysql. This is what console.log(result) prints: { '{"hash":"TestHaaaash", "size":13, "difficulty":47, "time":null, "timestamp":null, "date":null}': '' } I think this should be slightly different but I'm not sure how it should be or how to change it
In collectRequestData try chaging
request.on('end', () => {
callback(parse(body));
});
to
request.on('end', () => {
callback(JSON.parse(body));
});
I am trying to INSERT INTO my SQL Server DB Table using nodejs (Express).
Everything I have tried will not work.
In my app.js I have the below which executes correctly and connects, however where i am struggling is, how do i get data inserted in to my DB table.
app.js
var Connection = require('tedious').Connection;
var Request = require('tedious').Request;
var TYPES = require('tedious').TYPES;
var async = require('async');
var express = require('express');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var mysql = require('mysql');
var sql = require('mssql');
var index = require('./routes/index');
var users = require('./routes/users');
var homepage = require('./routes/homepage');
var app = express();
////// DB CONF //////
var config = {
userName: 'xxxxxxxxxxxxxx',
password: 'xxxxxxx',
server: 'xxxxxxxxxxx',
options: {
database: 'xxxxxxxxxxx'
}
}
var connection = new Connection(config);
connection.on('connect', function(err) {
if (err) {
console.log(err);
} else {
console.log('Connected');
}
});
You already included mysql library. now you have to create mysql connection. Like:
const writeDB = mysql.createConnection({
host: 'localhost',
user: 'root',
password: '********',
database: 'dbName'
});
after creating connection run query like:
let squerySet = "INSERT INTO weekDays (`id`, `date`, `day`) values ('1','2018-03-03', 'Monday');"
writeDB.query(squerySet,(err, result) => {
if(err){
return reject(err)
}
else {
resolve("carrierbillreceipt Table COPY DONE .....")
}
})
Because writeDB.query will return a promise then you have to handle this by creating this in promise. Like:
const insertDays = () => {
return new Promise((resolve, reject) => {
let squerySet = "INSERT INTO weekDays (`id`, `date`, `day`) values ('1','2018-03-03', 'Monday');"
writeDB.query(squerySet,(err, result) => {
if(err){
return reject(err)
}
else {
resolve("insertDays Insert DONE .....")
}
})
})
}
you can also insert an JSON object. Like:
let data = { "id": '1', "date":'2018-03-03', "day": 'Monday'}
writeDB.query(`INSERT INTO weekDays SET ?;`,[data],(err, result) => {
if(err){
return reject(err)
}
else {
resolve("insertDays Insert DONE .....")
}
})
I'm wondering why req.session.username is undefined in the tag >>>DOESNT WORK<<< while it does work in the tag >>>THIS DOES WORK<<< . I brought in req as an argument to my module but it seems I'm supposed to do something else? The /ajax route is accessed via a ajax call and it does set the session variable in >>>THIS DOES WORK<<<
//index.js file
var express = require('express');
var router = express.Router();
var app = express();
var functions = require('../public/javascripts/functions.js');
router.post('/ajax', function(req, res , next){
var username = req.param("username");
var password = req.param("password");
var operation = req.param("operation");
else if (operation === "validate")
{
async.series([
function()
{
functions.validate(username, password, req);
}
], function(err,result)
{
if (err)
return console.log(err);
console.log(result);
});
//req.session.username = "yaryar"; >>>THIS DOES WORK<<<
}
var strings = ["rad", "bla", "ska"]
console.log('body: ' + JSON.stringify(req.body));
console.log("AJAX RECEIVED");
res.send(strings);
});
module.exports = router;
functions.js file:
module.exports = {
validate: function(username, password, req) {
var url = 'mongodb://localhost';
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var ObjectId = require('mongodb').ObjectID;
MongoClient.connect(url, function(err, db)
{
assert.equal(null, err);
console.log("Connected correctly to server.");
var cursor = db.collection('users').find({username : username});
cursor.each(function(err,doc,req)
{
assert.equal(err, null);
if (doc != null)
{
console.log("user found: " + doc.username);
req.session.username = "ttyy"; // >>>DOESNT WORK<<<
return true
}
else
{
console.log("user not found");
return false;
}
});
//db.close();
});
}
};
you're overwriting req by doing cursor.each(function(err,doc,req) change it to cursor.each(function(err,doc,arr) and it will work
I am trying to run following code on server
var express = require("express"),
app = express(),
bodyParser = require('body-parser'),
errorHandler = require('errorhandler'),
methodOverride = require('method-override'),
hostname = process.env.HOSTNAME || 'localhost',
port = parseInt(process.env.PORT, 10) || 4004,
publicDir = process.argv[2] || __dirname + '/public';
var exec = require('child_process').exec;
var fs = require('fs');
var mongodb = require('mongodb'),
serverdb = new mongodb.Server('127.0.0.1', 27017, {}),
dbName = new mongodb.Db('prisync', serverdb, {});
var url = "urlinfo";
//Show homepage
app.get("/", function (req, res) {
//res.redirect("/index.html");
console.log("shubham ");
dbName.open(function (error, client){
var collection = new mongodb.Collection(client , url); //line 24 ====
collection.find().limit(20).toArray(function (err, dataObjArr){
var data = '';
var dataArr = [];
var i = dataObjArr.length;
//check for error
if(err){return res.end('error!'+err);}
//Data
if(dataObjArr){
while(i--){
dataArr[i] = dataObjArr[i]._id;
}
data = dataArr.join(' ');
res.render('/index.html',{ returnedData : data });
}else{
res.end();
}
});
});
});
app.get("/search", function (req, res){
console.log("shubham batra");
var pro_name = req.query.name;
var pro_code = req.query.code;
var pro_category = req.query.category;
var pro_brand = req.query.brand;
pro_name = pro_name+"";
pro_code = pro_code+"";
pro_brand = pro_brand+"";
pro_category = pro_category+"";
MongoClient.connect('mongodb://127.0.0.1:27017/prisync', function(err, db) {
if (err) throw err;
console.log("Connected to Database");
var documen = {name:pro_name, code:pro_code , category:pro_category, brand:pro_brand };
//documen = JSON.stringify(documen);
//insert record
db.collection('urlinfo').insert(documen, function(err, records) {
if (err) throw err;
});
res.redirect("/index.html");
});
});
//Search page
app.use(methodOverride());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
app.use(express.static(publicDir));
app.use(errorHandler({
dumpExceptions: true,
showStack: true
}));
console.log("Server showing %s listening at http://%s:%s", publicDir, hostname, port);
app.listen(port);
but it gives following error while try to load localhost:4004
Server showing /home/shubham/Music/pricesync/server/public listening at http://localhost:4004
shubham
/home/shubham/node_modules/mongodb/lib/server.js:274
process.nextTick(function() { throw err; })
^
Error: collection name must be a String
at Error (<anonymous>)
at checkCollectionName (/home/shubham/node_modules/mongodb/lib/utils.js:69:11)
at new Collection (/home/shubham/node_modules/mongodb/lib/collection.js:57:3)
at /home/shubham/Music/pricesync/server/server.js:24:24
at /home/shubham/node_modules/mongodb/lib/db.js:221:5
at connectHandler (/home/shubham/node_modules/mongodb/lib/server.js:272:7)
at g (events.js:180:16)
at emit (events.js:95:17)
at /home/shubham/node_modules/mongodb/node_modules/mongodb-core/lib/topologies/server.js:399:23
at /home/shubham/node_modules/mongodb/node_modules/mongodb-core/lib/topologies/server.js:756:13
I had the same error, for me it was caused by my timestamp being incorrectly placed.
INCORRECT CODE:
}),{timestamps:true});
CORRECTED CODE:
var mongoose = require('mongoose');
var ObjectId = mongoose.Schema.Types.ObjectId;
var User = mongoose.model('User', new mongoose.Schema({
email:{type:String, required:true},
password:{type:String, required:true},
listings:[{type:ObjectId,ref:"Listings"}]
},{timestamps:true}));
Assuming the collection name is in 'url' varialbe (i.e. 'urlinfo'), the right way to instantiate the collection would be:
var collection = dbName.collection(url); //line 24
If you look at documentation, it states specifically that Collection class should not be instantiated directly : https://mongodb.github.io/node-mongodb-native/api-generated/collection.html
I got stuck at the same situation, while solving the exercise from Guillermos Smashing Node.JS book...
I followed Richard Andreus directions and after few attempts, succesfully established the server.
I newly defined
var Db = require('mongodb').Db,
Server = require('mongodb').Server;
instead of existing variable definitions.
As next I've defined the variable db.
var db = new Db('test', new Server('localhost', 27017));
Then open the connection to db, or in your case dbName.
db.open(function(err, db) {
var collection = db.collection(url);
// in my case I defined app.users instead of collection
NOTE: Don't forget your definition at dbName