We Keep Coding

Node.js shortened url

I have just started learning to code about 5 days ago and what I'm struggling to achieve, is to have an rssfeed-to-twitter script that posts a shortened url instead of a full website/article feed url. I found a node.js module called TinyURL that could do that but i struggle to get it to work. Here's the full script:
var simpleTwitter = require('simple-twitter');
var fs = require('fs');
var http = require('http');
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type' : 'text/plain'});
res.end('RSS Twitter Bot\n');
}).listen(5693);
var timeInterval = 300000; // run every 30m
var timerVar = setInterval (function () {runBot()}, timeInterval);
function runBot(){
var lastCompleted = Date.parse(new Date(0));
console.log(lastCompleted);
try {
var lastcompletedData = fs.readFileSync('./lastCompleted.json', 'utf8');
var timeData = JSON.parse(lastcompletedData);
var lastCompletedFromFile = Date.parse(new Date(timeData.lastCompleted));
if ( isNaN(lastCompletedFromFile) == false ) {
lastCompleted = lastCompletedFromFile;
}
} catch (e) {
console.log(e);
}
fs.readFile('./config.json', 'utf8', function (err, data) {
if (err) console.log(err); // we'll not consider error handling for now
var configData = JSON.parse(data);
console.log(configData);
var twitter = new simpleTwitter( configData.consumerKey //consumer key from twitter api
, configData.consumerSecret //consumer secret key from twitter api
, configData.accessToken //access token from twitter api
, configData.accessTokenSecret //access token secret from twitter api
, 3600);
var dateNow = Date.parse(new Date());
var FeedParser = require('feedparser');
var request = require('request');
var req = request(configData.feedUrl);
var feedparser = new FeedParser();
req.on('error', function (error) {
console.log(error);
});
req.on('response', function (res){
var stream = this;
if (res.statusCode != 200 ) return this.emit('error', new Error('Bad status code'));
stream.pipe(feedparser);
});
feedparser.on('error', function(error) {
console.log(error);
});
feedparser.on('readable', function() {
var stream = this;
var meta = this.meta;
var item;
while (item = stream.read()) {
var itemDate = Date.parse(item.date);
//check to not publish older articles
if (itemDate > lastCompleted){
var titleLength = item.title.length;
var itemTitle = item.title;
var itemLink = item.link;
if (titleLength > 100) {
itemTitle = itemTitle.substring(0, 100);
}
twitter.post('statuses/update'
, {'status' : itemTitle + ' ' + itemLink + " " + configData.tags}
, function (error, data) {
console.dir(data);
});
console.log(itemTitle + ' ' + item.link + configData.tags);
}
}
//TO KNOW WHEN FROM TO START POSTING
var dateCompleted = new Date();
console.log('loop completed at ' + dateCompleted);
var outputData = {
lastCompleted : dateCompleted
}
var outputFilename = './lastCompleted.json';
fs.writeFile(outputFilename, JSON.stringify(outputData, null, 4), function(err) {
if(err) {
console.log(err);
} else {
console.log("JSON saved to " + outputFilename);
}
});
});
});
}
And this is the TinyURL node.js module
var TinyURL = require('tinyurl');
TinyURL.shorten('http://google.com', function(res) {
console.log(res); //Returns a tinyurl
});
Changing the 'http://google.com' string to itemLink var works just fine and prints it in the terminal as expected.
TinyURL.shorten(itemLink, function(res) {
console.log(res); //Returns a tinyurl
});
What i'm trying to achieve is:
twitter.post('statuses/update', {'status' : itemTitle + ' ' + tinyurlLink + " " + configData.tags}
How can i get the response turned into a e.g var tinyurlLink to replace the itemLink var? Any help would be much appreciated!

As suggested by #zerkms sending a tweet from inside the TinyURL.shorten worked!

S3 connection to application

I wrote an app that displays images by ID and all the images save and located in S3.
I have 2 instances and 1 LB in my AWS machine and the index.js is located in both instances.
In my index.js I wrote the path to my S3 bucket and I get an error message like this :
Blockquote
This XML file does not appear to have any style information associated with it. The document tree is shown below.
The code in index.js :
//Get Images Names by Id / Color from MongoDB
var mongoose = require('mongoose');
var db = mongoose.connect('mongodb://db_usr:db_pass#ds023550.mlab.com:23550/db_ringapp2016_g');
var userSchema = require('./1_define_schema');
var size = 0;
var express = require('express'),
url = require('url'),
app = express();
var fs = require('fs');
var http = require('http');
var Upload = require('s3-uploader');
var restify = require('restify');
app.listen(process.env.PORT || 3000);
// connection error
mongoose.connection.once('error', function (err) {
console.log('connectiob error' + err);
});
//connecting to DB
mongoose.connection.once('open', function () {
console.log("============================");
console.log("Connected Successfully to DB");
console.log("============================");
userSchema.find({}, function(err, user){
if(err) throw err;
//first route - call first get function from WS.
app.get('/AllPictures' ,
function (req, res) {
console.log("DB: Get all Pictures");
res.status(200).json(user[0].Name);
});
})
//second route - call second function from WS (by Id).
app.get('/PicById/:Id/:Size', function (req, res) {
userSchema.find({Id:req.params.Id} , function(err, user){
if(err) throw err;
console.log("DB: Get Pic by ID: "+req.params.Id+" from MongoDB");
console.log("Name Pic: "+user[0].Name);
console.log("Size Pic: "+req.params.Size);
var temp = user[0].Name.split(".");
var result = temp[0]+req.params.Size+"."+temp[1];
console.log(result);
res.send('<!DOCTYPE HTML><html><head></head><body><img src="https://s3-us-west-2.amazonaws.com/galshaharbucket/'+result+'"></body></html>');
console.log("============================");
})
})
//thired route - call thired get from WS (by Color).
app.get('/PicByColor/:Color', function (req, res) {
userSchema.find({Color:req.params.Color}, function(err, user){
if(err) throw err;
var names = [];
console.log("DB: Get Pic by Color: "+req.params.Color+" from MongoDB");
console.log("--------------------");
for(var i = 0; i<user.length; ++i){
var temp = user[i].Name.split(".");
names [i] = temp[0]+"S."+temp[1];
console.log("Name Pic: "+names [i]);
console.log("--------------------");
}
var temp = [];
for(var i = 0; i<user.length; ++i){
temp [i] = '<img src="https://s3-us-west-2.amazonaws.com/galshaharbucket/'+names [i]+'">'
}
var result = temp.join("");
res.send('<!DOCTYPE HTML><html><head></head><body>'+result+'</body></html>');
console.log("============================");
})
})
app.get('/GetAllPictures', function (req, res) {
userSchema.find({}, function(err, user){
if(err) throw err;
var names = [];
console.log("DB: Get All Pictures from MongoDB");
for(var i = 0; i<user.length; ++i){
var temp = user[i].Name.split(".");
names [i] = temp[0]+"S."+temp[1];
}
var temp = [];
for(var i = 0; i<user.length; ++i){
temp [i] = '<img src="https://s3-us-west-2.amazonaws.com/galshaharbucket/'+names [i]+'">'
}
var result = temp.join("");
res.send('<!DOCTYPE HTML><html><head></head><body>'+result+'</body></html>');
console.log("============================");
})
})
// Upload an image
app.get('/Upload/:Path', function (req, res) {
var knox = require('knox').createClient({
key: 'AKIAJ4ROKKJBECGFSYIA'
, secret: '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'
, bucket: 'galshaharbucket'
});
var file = req.params.Path;
console.log(file);
var upload_name = "upload_"+ file; // or whatever you want it to be called
knox.putFile(file, upload_name, {
"Content-Type": "image/jpeg"
}, function (err, result) {
if (err != null) {
return console.log(err);
} else {
console.log("Uploaded to amazon S3");
console.log("--------------------");
}
});
})
});
// The function that recieve the name from mongo and display it
function getImageById(){
Input = document.getElementById("imageId");
Size = document.getElementById("imageSize");
size=Size.value;
alert(size);
if(Input.value==""){
alert("Please Enter Id Number Between 1-33");
return;
}
url = "https://s3-us-west-2.amazonaws.com/galshaharbucket/PicById/"+Input.value+"/"+Size.value;
//url = 'https://s3-us-west-2.amazonaws.com/galshaharbucket/'+name;
if(Size.value=="L"){
popupWindow = window.open(
url,'popUpWindow','height=658,width=1120,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes')
}
if(Size.value=="M"){
popupWindow = window.open(
url,'popUpWindow','height=525,width=820,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes')
}
if(Size.value=="S"){
popupWindow = window.open(
url,'popUpWindow','height=330,width=520,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes')
}
}
function getImageByColor(str){
Input = document.getElementById("imageColor");
if(Input.value==""){
alert("Please Enter a Color: red / green / blue / yellow");
return;
}
else{
path = "https://s3-us-west-2.amazonaws.com/galshaharbucket/PicByColor/"+Input.value;
popupWindow = window.open(
path,'popUpWindow','height=608,width=1020,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes')
}
}
function uploadImage(){
var fileChooser = document.getElementById('path');
var results = document.getElementById('results');
var file = fileChooser.files[0];
alert(file.name);
path = "https://s3-us-west-2.amazonaws.com/galshaharbucket/Upload/"+file.name+"";
popupWindow = window.open(
path,'popUpWindow','height=608,width=1020,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes')
}
function getAllPics(){
path = "https://s3-us-west-2.amazonaws.com/galshaharbucket/GetAllPictures";
popupWindow = window.open(
path,'popUpWindow','height=608,width=1020,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes')
}
How I can display the image correctly without an error ? What can cause to this error ?
Thank you,
Tom

Edit your bucket policy and make sure to have something like (or use AWS policy generator under the Bucket Permission section).
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "PublicReadGetObject",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::galshaharbucket/*"
}
]
}
if galshaharbucket is your bucket name so it grants everyone access to the objects in the specified folder.
you can read more about bucket policies

aws lambda function how to use responses to perform calculation

Trying to create a lambda function that lists tagged ec2 and tagged rds and performs an action thereafter. This is the first time i have used javascript and would appreciate some help.
Please see my example below
var aws = require('aws-sdk');
var ec2 = new aws.EC2();
var rds = new aws.RDS();
aws.config.region = 'us-east-1';
exports.handler = function(event,context) {
if (event) {
console.log(event.id);
}
//setup params for rds call
var rdsparams = {
DBInstanceIdentifier: 'master',
};
//setup params for ec2 call
var ec2params = {
Filters: [
{
Name: 'tag:role',
Values: [
'app'
],
}
]
};
//Get ec2 instances with app tag, may need to add a condition on running so pulled it into hash
ec2.describeInstances(ec2params, function(err, appdata) {
if (err) {
console.log(err);
return;
}
else {
var apparray={};
for(var i = 0; i < appdata.Reservations.length; i++) {
var ins = appdata.Reservations[i].Instances[0];
var id = ins.InstanceId;
var state = ins.State.Name;
apparray[id]=state;
}
console.log(apparray);
context.succeed(apparray);
}
});
rds.describeDBInstances(rdsparams, function(err, data) {
if (err) {
console.log(err, err.stack);
return;
}
else {
var rdsarray={};
var rdsarray=(data);
console.log(rdsarray);
var ins=rdsarray[0];
var name = ins.ReadReplicaDBInstanceIdentifiers[0];
rdsarray[replicant]=name;
}
context.succeed(rdsarray);
});
//context.done();
};
I want to return my filtered (apparray) and (rdsarray) back from my functions and perform a calculation on this within the main body of the script. Any ideas on how to do this.
something like
var replicas = rdsarray.length for example
Thanks in advance

var aws = require('aws-sdk');
var ec2 = new aws.EC2();
var rds = new aws.RDS();
aws.config.region = 'us-east-1';
exports.handler = function(event, context) {
if (event) {
console.log(event.id);
}
//setup params for rds call
var rdsparams = {
DBInstanceIdentifier: 'master',
};
//setup params for ec2 call
var ec2params = {
Filters: [{
Name: 'tag:role',
Values: [
'app'
],
}]
};
//Get ec2 instances with app tag, may need to add a condition on running so pulled it into hash
ec2.describeInstances(ec2params, function(err, appdata) {
if (err)
return context.done(err, null);
var apparray = {};
for (var i = 0; i < appdata.Reservations.length; i++) {
var ins = appdata.Reservations[i].Instances[0];
var id = ins.InstanceId;
var state = ins.State.Name;
apparray[id] = state;
}
console.log(apparray);
var resultData = {};
resultData.apparray = apparray;
rds.describeDBInstances(rdsparams, function(err, data) {
if (err)
return context.done(err, null);
var rdsarray = {};
var rdsarray = (data);
console.log(rdsarray);
var ins = rdsarray[0];
var name = ins.ReadReplicaDBInstanceIdentifiers[0];
rdsarray[replicant] = name;
resultData.rdsarray = rdsarray;
context.done(null, resultData);
});
});
};
and back in the code from you are calling the lambda function
var lambda = new sdk.Lambda();
var params = {
FunctionName: 'arn:aws:lambda:us-west-2:1541546477777:function:MyFunction',
Payload: JSON.stringify(/*your params here*/)
};
lambda.invoke(params, function(err, data) {
if (err) {
console.log('error ===', err);
return ;
}
var lambdaData = JSON.parse(data.Payload);
// do your stuff here
});
Is this what you needed? It may be broken but I hope you get the idea of it

Getting hostname of URL in loop - Nodejs

So I'm currently trying to make an http status code checker. But for some reason the response hostname is returning undefined, yet i can retrieve the response statusCode. Am I grabbing it properly?
var fs = require('fs');
var http = require('http');
var stdin = process.openStdin();
stdin.addListener('data', function (userInput) {
var urlListPath = userInput.toString().trim();
fs.readFile(urlListPath, function (err, data) {
if (err) {
return console.log(err);
}
var urlArray = data.toString().split("\n");
urlArray = urlArray.filter(Boolean);
for (var i = 0; i < urlArray.length; i++){
http.get({hostname: urlArray[i]}, function(res){
console.log(res.hostname + ' | ' + res.statusCode + ' | ' + res.statusMessage);
});
}
});
});

Solved by wrapping in a function as #bluesman suggested:
var fs = require('fs');
var http = require('http');
var stdin = process.openStdin();
stdin.addListener('data', function (userInput) {
var urlListPath = userInput.toString().trim();
fs.readFile(urlListPath, function (err, data) {
if (err) {
return console.log(err);
}
var urlArray = data.toString().split("\n");
urlArray = urlArray.filter(Boolean);
for (var i = 0; i < urlArray.length; i++){
getHttpInfo(urlArray[i]);
}
});
});
function getHttpInfo(url){
http.get({hostname: url}, function(res){
console.log(url + ' : ' + res.statusCode + ' | ' + res.statusMessage);
});
}

How can I get node.js to return data once all operations are complete

I am just learning server-side JavaScript so please bear with any glaring mistakes I've made.
I am trying to write a file parser that operates on HTML files in a directory and returns a JSON string once all files have been parsed. I started it with a single file and it works fine. it loads the resource from Apache running on the same machine, injects jquery, does the parsing and returns my JSON.
var request = require('request'),
jsdom = require('jsdom'),
sys = require('sys'),
http = require('http');
http.createServer(function (req, res) {
request({uri:'http://localhost/tfrohe/Car3E.html'}, function (error, response, body) {
if (!error && response.statusCode == 200) {
var window = jsdom.jsdom(body).createWindow();
jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
// jQuery is now loaded on the jsdom window created from 'body'
var emps = {};
jquery("tr td img").parent().parent().each(function(){
var step = 0;
jquery(this).children().each(function(index){
if (jquery(this).children('img').attr('src') !== undefined) {
step++;
var name = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
var name_parts = name.split(",");
var last = name_parts[0];
var name_parts = name_parts[1].split(/\u00a0/g);
var first = name_parts[2];
emps[last + ",_" + first] = jquery(this).children('img').attr('src');
}
});
});
emps = JSON.stringify(emps);
//console.log(emps);
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end(emps);
});
} else {
res.writeHead(200, {"Content-Type": "text/plain"});
res.end("empty");
//console.log(response.statusCode);
}
});
}).listen(8124);
Now I am trying to extend this to using the regular file system (fs) and get all HTML files in the directory and parse them the same way and return a single combined JSON object once all files have been parsed. Here is what I have so far but it does not work.
var sys = require("sys"),
fs = require("fs"),
jsdom = require("jsdom"),
emps = {};
//path = '/home/inet/www/media/employees/';
readDirectory = function(path) {
fs.readdir(path, function(err, files) {
var htmlfiles = [];
files.forEach(function(name) {
if(name.substr(-4) === "html") {
htmlfiles.push(name);
}
});
var count = htmlfiles.length;
htmlfiles.forEach(function(filename) {
fs.readFile(path + filename, "binary", function(err, data) {
if(err) throw err;
window = jsdom.jsdom(data).createWindow();
jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
jquery("tr td img").parent().parent().each(function(){
var step = 0;
jquery(this).children().each(function(index){
if (jquery(this).children('img').attr('src') !== undefined) {
step++;
var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
var name_parts = empname.split(",");
var last = name_parts[0];
var name_parts = name_parts[1].split(/\u00a0/g);
var first = name_parts[2]
emps[last + ",_" + first] = jquery(this).children('img').attr('src');
}
});
});
});
});
});
});
}
readDirectory('/home/inet/www/media/employees/', function() {
console.log(emps);
});
In this particular case, there are 2 html files in the directory. If i console.log(emps) during the htmlfiles.forEach() it shows me the results from the first file then the results for both files together the way I expect. how do I get emps to be returned to readDirectory so i can output it as desired?
Completed Script
After the answers below, here is the completed script with a httpServer to serve up the detail.
var sys = require('sys'),
fs = require("fs"),
http = require('http'),
jsdom = require('jsdom'),
emps = {};
var timed = setInterval(function() {
emps = {};
readDirectory('/home/inet/www/media/employees/', function(emps) {
});
}, 3600000);
readDirectory = function(path, callback) {
fs.readdir(path, function(err, files) {
var htmlfiles = [];
files.forEach(function(name) {
if(name.substr(-4) === "html") {
htmlfiles.push(name);
}
});
var count = htmlfiles.length;
htmlfiles.forEach(function(filename) {
fs.readFile(path + filename, "binary", function(err, data) {
if(err) throw err;
window = jsdom.jsdom(data).createWindow();
jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
var imagecount = jquery("tr td img").length;
jquery("tr td img").parent().parent().each(function(){
var step = 0;
jquery(this).children().each(function(index){
if (jquery(this).children('img').attr('src') !== undefined) {
step += 1;
var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
var name_parts = empname.split(",");
var last = name_parts[0];
var name_parts = name_parts[1].split(/\u00a0/g);
var first = name_parts[2]
emps[last + ",_" + first] = jquery(this).children('img').attr('src');
}
});
});
count -= 1;
if (count <= 0) {
callback(JSON.stringify(emps));
}
});
});
});
});
}
var init = readDirectory('/home/inet/www/media/employees/', function(emps) {
});
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end(JSON.stringify(emps));
}).listen(8124);

That sure is a lot of code a couple of mistakes.
You're never calling the callback function you supply to readDirectory
You need to keep track of the files you have parsed, when you parsed all of them, call the callback and supply the emps
This should work:
var sys = require("sys"),
fs = require("fs"),
jsdom = require("jsdom"),
//path = '/home/inet/www/media/employees/';
// This is a nicer way
function readDirectory(path, callback) {
fs.readdir(path, function(err, files) {
// make this local
var emps = {};
var htmlfiles = [];
files.forEach(function(name) {
if(name.substr(-4) === "html") {
htmlfiles.push(name);
}
});
// Keep track of the number of files we have parsed
var count = htmlfiles.length;
var done = 0;
htmlfiles.forEach(function(filename) {
fs.readFile(path + filename, "binary", function(err, data) {
if(err) throw err;
window = jsdom.jsdom(data).createWindow();
jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
jquery("tr td img").parent().parent().each(function(){
var step = 0;
jquery(this).children().each(function(index){
if (jquery(this).children('img').attr('src') !== undefined) {
step++;
var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
var name_parts = empname.split(",");
var last = name_parts[0];
var name_parts = name_parts[1].split(/\u00a0/g);
var first = name_parts[2]
emps[last + ",_" + first] = jquery(this).children('img').attr('src');
}
});
});
// As soon as all have finished call the callback and supply emps
done++;
if (done === count) {
callback(emps);
}
});
});
});
});
}
readDirectory('/home/inet/www/media/employees/', function(emps) {
console.log(emps);
});

You seem to be doing this a tad wrong
readDirectory('/home/inet/www/media/employees/', function() {
console.log(emps);
});
But you've defined your function as:
readDirectory = function(path) {
Where is the callback argument? Try this:
readDirectory = function(path, callback) {
then under emps[last + ",_" + first] = jquery(this).children('img').attr('src'); put
callback.call(null, emps);
Your callback function will be called however many times your loop goes on for. If you want it to return all of them at once, you'll need to get a count of how many times the loop is going to run for, count up until that number then call your callback when the emps array is full of the data you need.