We Keep Coding

Node JS - How to write stream big json data into json array file?

I have difficult to write a json data into json file using stream module.
I learn about this from several blog tutorial, one of them is this page
Let say i am working with big json data on a json file. I think it is not possible to store all json object inside my memory. So i decided to do it using stream module.
Here the codes i have done:
writeStream.js
var Writable = require('stream').Writable,
util = require('util');
var WriteStream = function() {
Writable.call(this, {
objectMode: true
});
};
util.inherits(WriteStream, Writable);
WriteStream.prototype._write = function(chunk, encoding, callback) {
console.log('write : ' + JSON.stringify(chunk));
callback();
};
module.exports = WriteStream;
readStream.js
var data = require('./test_data.json'),
Readable = require('stream').Readable,
util = require('util');
var ReadStream = function() {
Readable.call(this, {
objectMode: true
});
this.data = data;
this.curIndex = 0;
};
util.inherits(ReadStream, Readable);
ReadStream.prototype._read = function() {
if (this.curIndex === this.data.length) {
return this.push(null);
}
var data = this.data[this.curIndex++];
console.log('read : ' + JSON.stringify(data));
this.push(data);
};
module.exports = ReadStream;
Called with this code:
var ReadStream = require('./readStream.js'),
WriteStream = require('./writeStream.js');
var rs = new ReadStream();
var ws = new WriteStream();
rs.pipe(ws);
Problem: I want to write it into different file, how is it possible?
Can you please help me?

If you are looking for a solution to just write the data from your ReadStream into a different file, you can try fs.createWriteStream. It will return you a writeable stream which can be piped directly to your ReadStream.
You will have to make a minor change in your readStream.js. You are currently pushing an object thus making it an object stream while a write stream expects either String or Buffer unless started in the ObjectMode. So you can do one of the following:
Start the write stream in the object mode. More info here.
Push String or Buffer in your read stream as writable stream internally calls writable.write which expects either String or Buffer. More info here.
If we follow the second option as an example, then your readStream.js should look like this:
var data = require('./test_data.json'),
Readable = require('stream').Readable,
util = require('util');
var ReadStream = function() {
Readable.call(this, {
objectMode: true
});
this.data = data;
this.curIndex = 0;
};
util.inherits(ReadStream, Readable);
ReadStream.prototype._read = function() {
if (this.curIndex === this.data.length) {
return this.push(null);
}
var data = this.data[this.curIndex++];
console.log('read : ' + JSON.stringify(data));
this.push(JSON.stringify(data));
};
module.exports = ReadStream;
You can call the above by using the following code
var ReadStream = require('./readStream.js');
const fs = require('fs');
var rs = new ReadStream();
const file = fs.createWriteStream('/path/to/output/file');
rs.pipe(file);
This will write the data from test_data.json to the output file.
Also as a good practice and to reliably detect write errors, add a listener for the 'error' event. For the above code, you can add the following:
file.on('error',function(err){
console.log("err:", err);
});
Hope this helps.

Javascript inherited object overwrites other inherited object

Can someone please explain what the correct way is to have multiple objects inherit from a parent and have their own prototype functions? I'm trying to do this in nodeJS.
I have these files.
ParserA_file
var ParentParser = require('ParentParser_file');
module.exports = ParserA;
ParserA.prototype = Object.create(ParentParser.prototype);
ParserA.prototype.constructor = ParserA;
ParserA.prototype = ParentParser.prototype;
function ParserA(controller, file) {
ParentParser.call(this, controller, file);
this.controller.log('init --- INIT \'parser_A\' parser');
this.date_regex = /([0-9]{1,2})?([A-Z]{3})?([0-9]{2})? ?([0-9]{2}:[0-9]{2})/;
this.date_regex_numeric = /(([0-9]{1,2})([0-9]{2})([0-9]{2}))? ?([0-9]{2}:[0-9]{2})?/;
this.date_format = 'DDMMMYY HH:mm';
}
ParserA.prototype.startParse = function() {
console.log('Starting parse for A');
}
ParserB_file
var ParentParser = require('ParentParser_file');
module.exports = ParserB;
ParserB.prototype = Object.create(ParentParser.prototype);
ParserB.prototype.constructor = ParserB;
ParserB.prototype = ParentParser.prototype;
function ParserB(controller, file) {
ParentParser.call(this, controller, file);
this.controller.log('init --- INIT \'parser_B\' parser');
this.date_regex = /([0-9]{1,2})?([A-Z]{3})?([0-9]{2})? ?([0-9]{2}:[0-9]{2})/;
this.date_regex_numeric = /(([0-9]{1,2})([0-9]{2})([0-9]{2}))? ?([0-9]{2}:[0-9]{2})?/;
this.date_format = 'DDMMMYY HH:mm';
}
ParserB.prototype.startParse = function() {
console.log('Starting parse for B');
}
ParentParser_file
ParentParser = function(controller, file) {
if (!controller) {
throw (new Error('Tried to create a Parser without a controller. Failing now'));
return;
}
if (!file ) {
throw (new Error('Tried to create a Parser without a file. Failing now'));
return;
}
this.controller = null;
this.file = null;
}
module.exports = ParentParser;
Now I require them both in my node app
var ParserA = require('ParserA_file');
var ParserB = require('ParserB_file');
Now, when only one parser is loaded the there is no problem, however, loading them both into my node app and starting parser A
var parser = new ParserA(this, file);
parser.startParse()
returns
init --- INIT 'parser_B' parser'
Now for the question, how come ParserB's function startParse overwrites the startParse from ParserA?

That's because they refer to the same prototype object.
ParserA.prototype = ParentParser.prototype;
...
ParserB.prototype = ParentParser.prototype;
ParserA.prototype === ParserB.prototype; // true
Remove those two lines (which are overwriting the two lines above them anyway) and you'll be good to go.

how to read a file, store data and then write it

I have a text file with a ton of values that I want to convert to meaningful JSON using node.js fs module.
I want to store the first value of every line in an array unless the value is already present.
7000111,-1.31349,36.699959,1004,
7000111,-1.311739,36.698589,1005,
8002311,-1.262245,36.765884,2020,
8002311,-1.261135,36.767544,2021,
So for this case, I'd like to write to a file:
[7000111, 8002311]
Here's what I have so far. It writes [] to the file.
var fs = require('fs');
var through = require('through');
var split = require('split');
var shape_ids = [];
var source = fs.createReadStream('data/shapes.txt');
var target = fs.createWriteStream('./output3.txt');
var tr = through(write, end);
source
.pipe(split())
.pipe(tr)
// Function definitions
function write(line){
var line = line.toString();
var splitted = line.split(',');
// if it's not in array
if (shape_ids.indexOf(splitted[0]) > -1){
shape_ids.push(splitted[0]);
}
}
function end(){
shape_ids = JSON.stringify(shape_ids);
target.write(shape_ids);
console.log('data written');
}
The code is using the split and through modules
How do I store values in the array and write the populated array to the file?
== === ====== =================
Update:
This is what I want to do, but it's in Ruby:
shape_ids = []
File.open("data/shapes.txt").readlines.each do |line|
data = line.split(',')
shape_id = data.first
if !shape_ids.include? shape_id
shape_ids.push(shape_id)
end
end
puts shape_ids # array of unique shape_ids
Can I do this in javascript?

Unless you are super comfortable with the new Stream API in node, use the event-stream module to accomplish this:
var fs = require('fs');
var es = require('event-stream');
function getIds(src, target, callback) {
var uniqueIDs = [];
es.pipeline(
fs.createReadStream(src),
es.split(),
es.map(function (line, done) {
var id = line.split(',').shift();
if (uniqueIDs.indexOf(id) > -1) return done();
uniqueIDs.push(id);
done(null);
}),
es.wait(function (err, text) {
// Here we create our JSON — keep in mind that valid JSON starts
// as an object, not an array
var data = JSON.stringify({ ids: uniqueIDs});
fs.writeFile(target, data, function (err) {
if ('function' == typeof callback) callback(err);
});
})
);
}
getIds('./values.txt', './output.json');
Unfortunately there is no "easy" way to keep this as a pure stream flow so you have to "wait" until the data is done filtering before turning into a JSON string. Hope that helps!

python to node.js confusion

So I have this python code that I'm trying to convert to node.js, but I am not sure how.
import urllib.request, re
def getDef(word):
link = "http://www.merriam-webster.com/dictionary/%s" % word
data = urllib.request.urlopen(link).read().decode()
try:
return re.search("<p>: (.*?)</p><p>", data).group(1)
except:
return "No match"
class newDefinition:
def __init__(self, word):
self.definition = getDef(word);
>>> definition = newDefintion("color")
>>> print(definition.definition)
a quality such as red, blue, green, yellow, etc., that you see when you look at something
In node.js however though it I can seem to return it like in python because of it's callback way of doing things, or at least I can't seem to return it which is why I'm asking how would I do the node.js equivalent or is their no equivalent? Here is what I have so far maybe you can spot what I'm doing wrong and how to fix it
var urllib = require("urllib"); // installed with npm
var getDef = function(word){
var link = "http://www.merriam-webster.com/dictionary/" + word;
var urlData = urllib.request(link, {}, function(err, data, res){
var re = new RegExp("<p>: (.*?)</p><p>");
var results = data.toString();
var match = re.exec(results)[1];
return match; // Expected it to give urlData the definition
});
return urlData;
}
var Definition = function(word){
this.definition = getDef(word);
}
definition = new Definition("color");
console.log(definition.definition); // this won't give the definition but the information of the urllib itself rather.
So in general trying to figure out is how to use asynchronous code so I can return things that I need, but I am not use to this concept either so is there an equivalent to this in python? Also if you can point me to some good documentation on asynchronous code that would be great also.

Since return will actually just exit your function instead of returning a value, you need to use a callback. It would look like this:
var urllib = require("urllib");
var getDef = function(word, callback){
var link = 'http://www.merriam-webster.com/dictionary/' + word;
urllib.request(link, {}, function(err, data, res) {
var re = new RegExp('<p>: (.*?)</p><p>');
var results = data.toString();
var match = re.exec(results)[1];
callback(match);
});
};
Then you would pass a callback while calling the function:
getDef('color', function(definition) {
console.log(definition);
});
Edit: Setting an object's property has the same idea. It might look like this instead:
var Definition = function(word) {
var self = this;
getDef(world, function(definition, callback) {
self.definition = definition;
callback.call(self);
});
};
And would be called like so:
var definition = new Definition('color', function() {
console.log(definition.definition);
});

Here is my two cent worth suggestion.
Never ever use regular expressions to parse HTML (Refer here for more details), instead use the XPath like library to parse the document. You can use libraries like cheerio or phantomjs.
Here is a clean solution.
var request = require('request'),
when = require('when'),
cheerio = require('cheerio');
var URL = 'http://www.merriam-webster.com/dictionary/';
/**
* #param word: Word to search the dictionary
* #returns
* Promise object which resolves to array of
* definitions of the word
*/
var getDef = function(word){
var defer = when.defer();
request(URL + word, function(err, res, body){
if (err || res.statusCode !== 200){
defer.reject();
}
var defs = [];
var $ = cheerio.load(body);
$('.wordclick .headword:first-child p').each(function(i,ele){
var definition = $(ele).text();
defs.push(definition);
});
defer.resolve(defs);
});
return defer.promise;
}
getDef('happy').then(function(words){
console.log(words);
});
Note: Here I am using when (a Promise+ library) instead of the Node's standard CPS style.

How can I use these Node modules to accept HTML through a file or URL and then output JSON as validation of existing HTML elements?

Essentially what I need to do is to take a local grader.js file and then use it at the command line to input HTML, which will then output JSON data to the console to validate the existence of several HTML elements. The usage looks something like this:
./grader.js --checks checks.json --file index.html
./grader.js --checks checks.json --url http://google.com
The Node modules being used are Commander (for working at the command line), Cheerio (for HTML), and Restler (for getting HTML from URL).
The checks.json file is straightforward in that it's simply asking to check for the existence of a few simple HTML elements to find out whether or not they exist on the page:
["h1",
".navigation",
".logo",
".blank",
".about",
".heading",
".subheading",
".pitch",
".video",
".thermometer",
".order",
".social",
".section1",
".section2",
".faq",
".footer"]
The grader.js file is where things get a little more complicated. The following code actually works insofar as it takes the command line arguments and does indicate a true or false value as to whether the HTML elements exist. But it doesn't work properly after adding the URL check at the bottom. There is something wrong with my checkURL function and the way that I implement it using the Commander code at the bottom. Even though the true and false values are correct dependent upon the HTML file/URL I use, I end up spitting out both checks to the console even if I only want to check either the file or the URL, not both. I'm fairly new to this so I'm surprised that it works at all. It may have something to do with the default values, but when I try to make those changes the checkURL function seems to break down. Thanks in advance for your help I really do appreciate it.
#!/usr/bin/env node
var fs = require('fs');
var program = require('commander');
var cheerio = require('cheerio');
var rest = require('restler');
var HTMLFILE_DEFAULT = "index.html";
var CHECKSFILE_DEFAULT = "checks.json";
var URL_DEFAULT = "http://cryptic-spire-7925.herokuapp.com/index.html";
var assertFileExists = function(infile) {
var instr = infile.toString();
if(!fs.existsSync(instr)) {
console.log("%s does not exist. Exiting.", instr);
process.exit(1); // http://nodejs.org/api/process.html#process_process_exit_code
}
return instr;
};
var cheerioHtmlFile = function(htmlfile) {
return cheerio.load(fs.readFileSync(htmlfile));
};
var loadChecks = function(checksfile) {
return JSON.parse(fs.readFileSync(checksfile));
};
var checkHtmlFile = function(htmlfile, checksfile) {
$ = cheerioHtmlFile(htmlfile);
var checks = loadChecks(checksfile).sort();
var out = {};
for(var ii in checks) {
var present = $(checks[ii]).length > 0;
out[checks[ii]] = present;
}
return out;
};
var checkUrl = function(url, checksfile) {
rest.get(url).on('complete', function(data) {
$ = cheerio.load(data);
var checks = loadChecks(checksfile).sort();
var out = {};
for(var ii in checks) {
var present = $(checks[ii]).length > 0;
out[checks[ii]] = present;
}
console.log(out);
});
}
var clone = function(fn) {
// Workaround for commander.js issue.
// http://stackoverflow.com/a/6772648
return fn.bind({});
};
if(require.main == module) {
program
.option('-f, --file <html_file>', 'Path to index.html', clone(assertFileExists), HTMLFILE_DEFAULT)
.option('-u, --url <url>', 'URL to index.html', URL_DEFAULT)
.option('-c, --checks <check_file>', 'Path to checks.json', clone(assertFileExists), CHECKSFILE_DEFAULT)
.parse(process.argv);
var checkJson = checkHtmlFile(program.file, program.checks);
var outJson = JSON.stringify(checkJson, null, 4);
console.log(outJson);
var checkJson2 = checkUrl(program.url, program.checks);
var outJson2 = JSON.stringify(checkJson2, null, 4);
console.log(outJson2);
}
else {
exports.checkHtmlFile = checkHtmlFile;
}

Depending on the arguments call either one of checkHtmlFile() or checkUrl()
Something like:
if (program.url)
checkUrl(program.url, program.checks);
else checkHtmlFile(program.file, program.checks);
Read this for more references: commander.js option parsing
Also, checkJson2 is undefined as checkUrl() isn't returning anything.

Those commander .option lines look wrong to me.
Delete the clone function and revise your option lines as follows:
.option('-f, --file <html_file>', 'Path to index.html', HTMLFILE_DEFAULT)
.option('-u, --url <url>', 'URL to index.html', URL_DEFAULT)
.option('-c, --checks <check_file>', 'Path to checks.json', CHECKSFILE_DEFAULT)
This should solve your commander problem.

Here is the updated checkUrl function after the helpful hints from #David and #ankitsabharwal.
var checkUrl = function(url, checksfile) {
rest.get(url).on('complete', function(data) {
$ = cheerio.load(data);
var checks = loadChecks(checksfile).sort();
var out = {};
for(var ii in checks) {
var present = $(checks[ii]).length > 0;
out[checks[ii]] = present;
}
var outJson = JSON.stringify(out, null, 4);
console.log(outJson);
});
}
And here is the updated Commander code below:
if(require.main == module) {
program
.option('-f, --file <html_file>', 'Path to index.html')
.option('-u, --url <url>', 'URL to index.html')
.option('-c, --checks <check_file>', 'Path to checks.json')
.parse(process.argv);
if (program.url) {
checkUrl(program.url, program.checks);
} else {
checkHtmlFile (program.file, program.checks);
var checkJson = checkHtmlFile(program.file, program.checks);
var outJson = JSON.stringify(checkJson, null, 4);
console.log(outJson);
}
}