I want to import the reddit json thread into my mongodb local server, so I can begin analyzing it. Please help me out.
app.js file below
var MongoClient = require('mongodb').MongoClient
, request = require('request');
MongoClient.connect('mongodb://localhost:27017/trasmatter', function(err, db) {
if(err) throw err;
request('https://www.reddit.com/r/Seattle/comments/3e0q5u/why_metro_matters_infographic/.json', function (error, response, body) {
if (!error && response.statusCode == 200) {
var obj = JSON.parse(body);
var stories = obj.data.children.map(function (story) { return story.data; });
db.collection('reddit').insert(stories, function (err, data) {
if(err) throw err;
console.dir(data);
db.close();
});
}
});
});
Console error after running 'node app.js'
app.js:11
var stories = obj.data.children.map(function (story) { return sto
^
TypeError: Cannot read property 'children' of undefined
at Request._callback (/Users/yay/code/pulley/app.js:11:35)
at Request.self.callback (/Users/yay/code/pulley/node_modules/request/request.js:198:22)
at Request.emit (events.js:110:17)
at Request.<anonymous> (/Users/yay/code/pulley/node_modules/request/request.js:1057:14)
at Request.emit (events.js:129:20)
at IncomingMessage.<anonymous> (/Users/yay/code/pulley/node_modules/request/request.js:1003:12)
at IncomingMessage.emit (events.js:129:20)
at _stream_readable.js:908:16
at process._tickCallback (node.js:355:11)
Another script file that works successfully. (I just plugged in
another link and change localhost link)
var MongoClient = require('mongodb').MongoClient
, request = require('request');
MongoClient.connect('mongodb://localhost:27017/course', function(err, db) {
if(err) throw err;
request('http://www.reddit.com/r/technology/.json', function (error, response, body) {
if (!error && response.statusCode == 200) {
var obj = JSON.parse(body);
var stories = obj.data.children.map(function (story) { return story.data; });
db.collection('reddit').insert(stories, function (err, data) {
if(err) throw err;
console.dir(data);
db.close();
});
On your second example, when requesting http://www.reddit.com/r/technology/.json the JSON result is an object:
{"kind":"stuff","data":{}}
But the first request result to an array of objects :
[{"kind":"stuff", "data":{}}, {"kind":"stuff","data":{}}]
In order to get your data, you need a loop or specify which object you want :
var obj = JSON.parse(body);
console.log(obj[0].data); // for example
Credit goes to this guy https://github.com/loganfranken
It looks like https://www.reddit.com/r/Seattle/comments/3e0q5u/why_metro_matters_infographic/.json returns an array as the top-level JSON element, so I think changing this:
obj.data.children
To this:
obj[1].data.children
Should do the trick.
Related
I'm trying to upload an image to Twitter by sending it in a certain Discord channel. This is the code I'm using:
var number = 0;
function downloadJPG(url) { //This function downloads the image in jpg format
request.get(url).on('error', console.error).pipe(fs.createWriteStream('image.jpg'));
}
if (message.attachments.every(attachIsImageJPG)) {
downloadJPG(message.attachments.first().url);
var imageToPostJPG = fs.readFileSync('image.jpg');
client.post('media/upload', {media: imageToPostJPG}, function(err, data, res) {
if (err) console.log(err);
console.log(data);
number = number+1
client.post('statuses/update', {status: 'Success Cop #'+number, media_ids: data.media_id_string}, function(err, params, res) {
if (err) console.log(err);
console.log(params);
});
});
}
The image should be uploaded to Twitter with the status, but only the status gets uploaded, and this is what I see in the console:
Error: HTTP Error: 400 Bad Request
at Request._callback (C:\Users\Silvano\Desktop\Twitter\node_modules\twitter\lib\twitter.js:221:9)
at Request.self.callback (C:\Users\Silvano\Desktop\Twitter\node_modules\request\request.js:185:22)
at Request.emit (events.js:200:13)
at Request.<anonymous> (C:\Users\Silvano\Desktop\Twitter\node_modules\request\request.js:1161:10)
at Request.emit (events.js:200:13)
at IncomingMessage.<anonymous> (C:\Users\Silvano\Desktop\Twitter\node_modules\request\request.js:1083:12)
at Object.onceWrapper (events.js:288:20)
at IncomingMessage.emit (events.js:205:15)
at endReadableNT (_stream_readable.js:1154:12)
at processTicksAndRejections (internal/process/task_queues.js:84:9)
{
request: '/1.1/media/upload.json',
error: 'media type unrecognized.'
}
How can I resolve this error and get the image uploaded too?
I think the problem is that the stream must be finished before readFileSync reads it.
// modified to indicate completion by calling callback
function downloadJPG(url, callback) { //This function downloads the image in jpg format
let stream = request.get(url).on('error', console.error).pipe(fs.createWriteStream('image.jpg'));
stream.on('finish', callback);
}
// modified to do the twtr work in the callback
if (message.attachments.every(attachIsImageJPG)) {
downloadJPG(message.attachments.first().url, function() {
var imageToPostJPG = fs.readFileSync('image.jpg');
client.post('media/upload', {media: imageToPostJPG}, function(err, data, res) {
if (err) console.log(err);
console.log(data);
number = number+1
client.post('statuses/update', {status: 'Success Cop #'+number, media_ids: data.media_id_string}, function(err, params, res) {
if (err) console.log(err);
console.log(params);
});
});
});
}
Side note: I find that after one or two steps, the callback style is too hard to write and read, and that promises are preferable.
My program parses a text document with HTTPS proxies in a string array. It then makes a GET request to ipify.org.
However, my program is throwing: Error: read ECONNRESET at TCP.onStreamRead (internal/stream_base_commons.js:183:27). It terminates my program instead of throwing an error and continuing.
My code:
var fs = require('fs');
var async = require("async");
var request = require("request");
var proxies = fs.readFileSync(__dirname + '\\proxies.txt').toString().split("\n");
console.log(proxies.length + ' proxies loaded!');
async.forEachOf(proxies, proxy => {
request({
'url':'https://api.ipify.org',
'method': "GET",
'proxy': 'http://' + proxy
},function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}
else {
console.log(error.message);
}
})
}, err => {
console.log(err.message);
});
Any help is highly appreciated :)
[Node js ECONNRESET
the error is explained here.I think the url is not being connected for some reason...
I have a set of tests to verify that my Express serves routes properly. For one of my routes launch.js, I receive two different errors, and sometimes the test randomly passes with long (425ms+) response times. Is there a better way to approach this?
launch.js
const authUtils = require('../../lib/authUtils');
const express = require('express');
const VError = require('verror');
const router = express.Router();
router.get('/', (req, res) => {
/**
* Request conformance object
*/
authUtils.getConformance((error, response, body) => {
// If error with HTTP request
if (error) {
throw new VError('Conformance object request failed', error);
// If error from auth server
} else if (response.body.message) {
throw new VError('Conformance object request failed', response.body.message);
// If request for conformance object succeeds
} else {
// Parse conformance object for tokenUri and authUri
authUtils.parseConformanceUris(body, (authUri, tokenUri, parsingError) => {
// Ensure URIs can be parsed from response
if (error) {
throw new VError('Issue while parsing conformance object', parsingError);
} else {
/**
* Data values needed later for /redirect
* #type {{state: string, tokenUri: string}}
*/
const launchData = {
state: authUtils.getUniqueCode(),
tokenUri,
};
// Build URI to request token from auth server
authUtils.buildGetTokenUri(authUri, launchData.state, (getTokenUri) => {
// Store state and tokenUri to session and redirect browser
authUtils.storeLaunchData(req, res, launchData, getTokenUri);
});
}
});
}
});
});
module.exports = router;
index.spec.js
const request = require('supertest');
const app = require('../index');
describe('Express server routes', () => {
describe('GET /launch', () => {
it('responds with HTTP 302', (done) => {
request(app).get('/launch').expect(302, done);
});
});
});
subject.getConformance
/**
* Utility function to request conformance object from auth server
* #param callback
*/
const getConformance = (callback) => {
request({ url: process.env.CONFORMANCE_URI, json: true, method: 'get' }, (error, response, body) => {
if (!error && response.statusCode === 200) {
callback(null, response, body);
} else {
callback(error, response, null);
}
});
};
Error 1
Uncaught TypeError: Cannot read property 'message' of null
at subject.getConformance (test/authUtils.spec.js:28:27)
at Request.request [as _callback] (lib/authUtils.js:7:374)
at Request.self.callback (node_modules/request/request.js:186:22)
at Request. (node_modules/request/request.js:1163:10)
at IncomingMessage. (node_modules/request/request.js:1085:12)
at endReadableNT (_stream_readable.js:1059:12)
at _combinedTickCallback (internal/process/next_tick.js:138:11)
at process._tickCallback (internal/process/next_tick.js:180:9)
Error 2
Uncaught AssertionError: expected 'body' to equal undefined
at subject.getConformance (test/authUtils.spec.js:43:16)
at Request.request [as _callback] (lib/authUtils.js:7:374)
at Request.self.callback (node_modules/request/request.js:186:22)
at Request. (node_modules/request/request.js:1163:10)
at IncomingMessage. (node_modules/request/request.js:1085:12)
at endReadableNT (_stream_readable.js:1059:12)
at _combinedTickCallback (internal/process/next_tick.js:138:11)
at process._tickCallback (internal/process/next_tick.js:180:9)
Assuming that the app object is requiring from a expressjs server, try
.get('/launch')
.expect(302)
.end(function (err, res) {
res.status.should.be.equal(302);
done();
});
If your index.js file is not a server then you need to configure the app object from a valid server. Are you exporting the app from index.js where app is
var express = require("express")
, app = express();
?
I'm using Node v6.7.0 trying to use 'fs' module but there is error as you see below. i Have tried to install it in addition but is not working(even if i add whole path). If i check in the site https://www.npmjs.com/package/fs you can see the message. Any ideas how can use the module?
var filename = process.argv[2];
var version = process.argv[3];
var fs = require('fs');
var prompt = require('C:/Program Files/nodejs/node_modules/prompt');
var p4 = require('C:/Program Files/nodejs/node_modules/p4');
p4.edit(filename, function(err, data) {
if (err) {
console.error(err.message);
}
fs.readFile(filename, 'utf8', function (err, data) {
if (err) {
return console.log(err);
}
var result = data.replace(/string to be replaced/g, version);
fs.writeFile(filename, result, 'utf8', function (err) {
if (err) return console.log(err);
});
});
console.log(data);
prompt.start();
prompt.get('p4 submit -c changelist', function (err, result) {
if(err) {
console.log(err.message);
}
console.log(result);
});
});
fs.js:303
binding.open(pathModule._makeLong(path),
^
TypeError: path must be a string or Buffer
at TypeError (native)
at Object.fs.readFile (fs.js:303:11)
at C:\WorkSpace\http.js:22:9
at C:\Program Files\nodejs\node_modules\p4\p4.js:13:24
at ChildProcess.exithandler (child_process.js:213:5)
at emitTwo (events.js:106:13)
at ChildProcess.emit (events.js:191:7)
at maybeClose (internal/child_process.js:877:16)
at Socket.<anonymous> (internal/child_process.js:334:11)
at emitOne (events.js:96:13)
Process finished with exit code 1
fs is a nodejs core module : here is the fs documentation
I found the answer it should be executed in node command line and var filename = process.argv[2]; must be filled up.
I'm trying to send off an sms (which works in Twilio's API Explorer) but seems to fail under my node installation. I've just done a complete uninstall and reinstalled with no avail.
Error
7 Oct 21:28:37 - [nodemon] starting `node scraper.js`
Free on Xbox One, Xbox 360, PS3, PS4: Tales from the Borderlands (Episode 1)
/Users/rhysedwards/Downloads/insightful/ozbargain/node_modules/twilio/node_modules/q/q.js:126
throw e;
^
TypeError: Cannot read property 'sid' of undefined
at /Users/rhysedwards/Downloads/insightful/ozbargain/scraper.js:39:31
at /Users/rhysedwards/Downloads/insightful/ozbargain/node_modules/twilio/node_modules/q/q.js:1924:17
at flush (/Users/rhysedwards/Downloads/insightful/ozbargain/node_modules/twilio/node_modules/q/q.js:108:17)
at doNTCallback0 (node.js:408:9)
at process._tickCallback (node.js:337:13)
7 Oct 21:28:39 - [nodemon] app crashed - waiting for file changes before starting...
Error with stripped back twilio code;
7 Oct 22:24:44 - [nodemon] starting `node scraper.js`
/Users/rhysedwards/Downloads/insightful/ozbargain/node_modules/twilio/node_modules/q/q.js:126
throw e;
^
TypeError: Cannot read property 'sid' of undefined
at /Users/rhysedwards/Downloads/insightful/ozbargain/scraper.js:12:24
at /Users/rhysedwards/Downloads/insightful/ozbargain/node_modules/twilio/node_modules/q/q.js:1924:17
at flush (/Users/rhysedwards/Downloads/insightful/ozbargain/node_modules/twilio/node_modules/q/q.js:108:17)
at doNTCallback0 (node.js:408:9)
at process._tickCallback (node.js:337:13)
7 Oct 22:24:46 - [nodemon] app crashed - waiting for file changes before starting...
Code
var accountSid = 'AC*******';
var authToken = 'da********';
var fs = require('fs'),
request = require('request'),
cheerio = require('cheerio'),
client = require('twilio')(accountSid, authToken);
url = 'http://www.ozbargain.com.au';
request(url, function(error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
var $el = $("a:contains('Xbox')");
if ($el.length) {
client.messages.create({
to: "61448141065",
from: "+61418739508",
body: "hey",
}, function(err, message) {
console.log(message.sid);
});
console.log($el.text());
} else {
console.log('hey');
}
}
});
The callback of client.messages.create is used as
}, function(err, message) {
console.log(message.sid);
});
When there will be error, the first parameter of the callback err will contain the information related to the error, and the second parameter message will be undefined.
Update the code as follow to handle the erroneous conditions
}, function (err, message) {
if (err) {
// Handle error
// Show appropriate message to user
} else {
// No error
if (message.sid) {
// Use sid here
}
}
console.log(message.sid);
});
You may simply change console.log(message.sid);
to
if(err) {
console.log(err.message);
}
Twilio developer evangelist here.
There are a couple of things wrong on your code on the twilio side of things.
client.messages.create({
to: "61448141065",
from: "+61418739508",
body: "hey",
}, function (err, message) {
console.log(message.sid);
});
on the Toyou're missing a + before the phone number, and after the body, you have an extra comma which you need to remove.
Your final code should look like the following:
var accountSid = 'AC*******';
var authToken = 'da********';
var fs = require('fs'),
request = require('request'),
cheerio = require('cheerio'),
client = require('twilio')(accountSid, authToken);
url = 'http://www.ozbargain.com.au';
request(url, function(error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
var $el = $("a:contains('Xbox')");
if ($el.length) {
client.messages.create({
to: "+61448141065",
from: "+61418739508",
body: "hey"
}, function(err, message) {
console.log(message.sid);
});
console.log($el.text());
} else {
console.log('hey');
}
}
});
I tested it here and it worked fine after changing that.
Hope this helps you.
UPDATE:
Try changing your code to just this:
var accountSid = 'AC*******';
var authToken = 'da********';
var fs = require('fs'),
client = require('twilio')(accountSid, authToken);
client.messages.create({
to: "+61448141065",
from: "+61418739508",
body: "hey"
}, function (err, message) {
console.log(message.sid);
});
It's because your call failed and there is no message object. Replace
console.log(message.sid);
with
if (err) console.log(err.message);
if (message) console.log(message.sid);
Then you will see the error message.