I want to pipe content to file. Here's what I've wrote:
var fs = require('fs');
var Readable = require('stream').Readable;
var ws = fs.createWriteStream('xfiles.txt');
var rs = new Readable;
ws.on('finish', function() {
console.log('Written ' + ws.bytesWritten + ' ' + ws.path);
});
rs.push('foo bar baz');
rs.push('foo bar baz');
rs.push('foo bar baz');
rs.push(null);
rs.pipe(ws);
ws.end();
I expected the xfiles.txt to have 3 lines of 'foo bar baz'. But the output tells me that 0 bytes written, and when I check xfiles.txt it's empty.
What do I miss?
EDIT
Sorry, found the answer right after posting this question. The answer in this SO question solved my problem.
Found the answer just after posting this question. This SO question have the answer that also solved my problem.
What I have to do is to call ws.end() after the stream completes its async operation. So I have to put it inside a callback:
ws.on('end', function() {
ws.end();
});
Related
I've been playing with Raspberry Pi and Node for fun. I thought for a simple experiment what if I grabbed some user input to turn an LED on and off.
const readline = require('readline');
const log = console.log;
const five = require('johnny-five');
const raspi = require('raspi-io');
const board = new five.Board({
io: new raspi(),
});
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
board.on('ready', function() {
const led = new five.Led('P1-7');
const recursiveAsyncReadLine = function () {
rl.question('Command: ', function (answer) {
switch(answer) {
case 'on':
log('Got it! Your answer was: "', answer, '"');
led.on();
break;
case 'off':
log('Got it! Your answer was: "', answer, '"');
led.stop().off();
break;
default:
}
recursiveAsyncReadLine();
});
};
recursiveAsyncReadLine();
});
It works however I get 2 strange bugs. In the console output below you'll see it prompts for my input... I enter my input, then it repeats my input in a distorted string of text
(see Example 1). Then after my verification message is output (Got it! Your answer was: " on ") I am met with a ReferenceError: on is not defined (Example 2) even though the
LED lit up perfectly.
Command: on
oonn //Example 1
Got it! Your answer was: " on "
Command:
ReferenceError: on is not defined //Example 2
>> off
ooffff
Got it! Your answer was: " off "
Command:
ReferenceError: off is not defined
>> on
oonn
Got it! Your answer was: " on "
Command:
ReferenceError: on is not defined
>> off
ooffff
Got it! Your answer was: " off "
Command:
ReferenceError: off is not defined
>> on
oonn
Got it! Your answer was: " on "
Command:
ReferenceError: on is not defined
>> off
ooffff
Got it! Your answer was: " off "
Command:
ReferenceError: off is not defined
I am thinking this is not so much a Raspberry Pi/Johnny-five thing and just a plain old JavaScript or Node issue.
Any ideas?
Ok retracing my previous statement looks like your issue is probably
led.stop().off();
In examples for Johny-five they show
led.stop();
led.off();
http://johnny-five.io/examples/led/
Good luck, happy hacking :-)
What happens if you reduce to this (works for me)?
const readline = require('readline');
const log = console.log;
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
const recursiveAsyncReadLine = function () {
rl.question('Command: ', function (answer) {
switch(answer) {
case 'on':
log('Got it! Your answer was: "', answer, '"');
break;
case 'off':
log('Got it! Your answer was: "', answer, '"');
break;
default:
}
recursiveAsyncReadLine();
});
};
recursiveAsyncReadLine();
If that works, that at least tells us it has something to do with the interaction with either raspi-io or johnny-five. Also where does this run, does this code execute on the pi, if so, do you enter the text directly on the pi or is this being entered remotely. If entered remotely what is the platform of the machine you are entering the text on?
Pasted below is the code of my javascript verticle which is deployed successfully. Since I couldn't find a way to debug the js verticle, I printed all the properties of the routingContext for reference. I am trying to get body of the request, however all the functions starting with getBody are returning null. I have pasted the request I am making below the code, with the log printed in the intellij terminal. Am I doing something wrong?
JS Verticle Code
var Router = require("vertx-web-js/router");
var server = vertx.createHttpServer();
var router = Router.router(vertx);
function writeHelp(obj) {
var txt = "";
for (var x in obj){
txt += x + "->" + obj[x] + "\n";
}
return txt;
}
router.post("/provider").handler(function (routingContext) {
var response = routingContext.response();
var json = routingContext.getBodyAsString();
response.putHeader("content-type", "text/plain");
//console.log("Helper\n" + writeHelp(routingContext));
//console.log("Request=" + writeHelp(routingContext.request()));
console.log("body=" + json);
response.end("Success");
});
server.requestHandler(router.accept).listen(8890);
POST Request made through ARC
Log from Intellij
Router: 257219241 accepting request POST http://localhost:8890/provider
Route matches: Route[ path:/provider pattern:null handler:io.vertx.core.Handler$$NashornJavaAdapter#69b539e failureHandler:null order:0 methods:[POST]]#32182409
Calling the handler
body=null
You have to add the BodyHandler to get parsed body:
From the official documentation:
var BodyHandler = require("vertx-web-js/body_handler");
// This body handler will be called for all routes
router.route().handler(BodyHandler.create().handle);
I have a large json file that looks like that:
[
{"name": "item1"},
{"name": "item2"},
{"name": "item3"}
]
I want to stream this file (pretty easy so far), for each line run a asynchronous function (that returns a promise) upon the resolve/reject call edit this line.
The result of the input file could be:
[
{"name": "item1", "response": 200},
{"name": "item2", "response": 404},
{"name": "item3"} // not processed yet
]
I do not wish to create another file, I want to edit on the fly the SAME FILE (if possible!).
Thanks :)
I don't really answer the question, but don't think it can be answered in a satisfactory way anyway, so here are my 2 cents.
I assume that you know how to stream line by line, and run the function, and that the only problem you have is editing the file that you are reading from.
Consequences of inserting
It is not possible to natively insert data into any file (which is what you want to do by changing the JSON live). A file can only grow up at its end.
So inserting 10 bytes of data at the beginning of a 1GB file means that you need to write 1GB to the disk (to move all the data 10 bytes further).
Your filesystem does not understand JSON, and just sees that you are inserting bytes in the middle of a big file so this is going to be very slow.
So, yes it is possible to do.
Write a wrapper over the file API in NodeJS with an insert() method.
Then write some more code to be able to know where to insert bytes into a JSON file without loading the whole file and not producing invalid JSON at the end.
Now I would not recommend it :)
=> Read this question: Is it possible to prepend data to an file without rewriting?
Why do it then?
I assume that want to either
Be able to kill your process at any time, and easily resume work by reading the file again.
Retry partially treated files to fill only the missing bits.
First solution: Use a database
Abstracting the work that needs to be done to live edit files at random places is the sole purpose of existence of databases.
They all exist only to abstract the magic that is behind UPDATE mytable SET name = 'a_longer_name_that_the_name_that_was_there_before' where name = 'short_name'.
Have a look at LevelUP/Down, sqlite, etc...
They will abstract all the magic that needs to be done in your JSON file!
Second solution: Use multiple files
When you stream your file, write two new files!
One that contain current position in the input file and lines that need to be retried
The other one the expected result.
You will also be able to kill your process at any time and restart
According to this answer writing to the same file while reading is not reliable. As a commenter there says, better to write to a temporary file, and then delete the original and rename the temp file over it.
To create a stream of lines you can use byline. Then for each line, apply some operation and pipe it out to the output file.
Something like this:
var fs = require('fs');
var stream = require('stream');
var util = require('util');
var LineStream = require('byline').LineStream;
function Modify(options) {
stream.Transform.call(this, options);
}
util.inherits(Modify, stream.Transform);
Modify.prototype._transform = function(chunk, encoding, done) {
var self = this;
setTimeout(function() {
// your modifications here, note that the exact regex depends on
// your json format and is probably the most brittle part of this
var modifiedChunk = chunk.toString();
if (modifiedChunk.search('response:[^,}]+') === -1) {
modifiedChunk = modifiedChunk
.replace('}', ', response: ' + new Date().getTime() + '}') + '\n';
}
self.push(modifiedChunk);
done();
}, Math.random() * 2000 + 1000); // to simulate an async modification
};
var inPath = './data.json';
var outPath = './out.txt';
fs.createReadStream(inPath)
.pipe(new LineStream())
.pipe(new Modify())
.pipe(fs.createWriteStream(outPath))
.on('close', function() {
// replace input with output
fs.unlink(inPath, function() {
fs.rename(outPath, inPath);
});
});
Note that the above results in only one async operation happening at a time. You could also save the modifications to an array and once all of them are done write the lines from the array to a file, like this:
var fs = require('fs');
var stream = require('stream');
var LineStream = require('byline').LineStream;
var modifiedLines = [];
var modifiedCount = 0;
var inPath = './data.json';
var allModified = new Promise(function(resolve, reject) {
fs.createReadStream(inPath).pipe(new LineStream()).on('data', function(chunk) {
modifiedLines.length++;
var index = modifiedLines.length - 1;
setTimeout(function() {
// your modifications here
var modifiedChunk = chunk.toString();
if (modifiedChunk.search('response:[^,}]+') === -1) {
modifiedChunk = modifiedChunk
.replace('}', ', response: ' + new Date().getTime() + '}');
}
modifiedLines[index] = modifiedChunk;
modifiedCount++;
if (modifiedCount === modifiedLines.length) {
resolve();
}
}, Math.random() * 2000 + 1000);
});
}).then(function() {
fs.writeFile(inPath, modifiedLines.join('\n'));
}).catch(function(reason) {
console.error(reason);
});
If instead of lines you wish to stream chunks of valid json which would be a more robust approach, take a look at JSONStream.
As mentioned in the comment, the file you have is not proper JSON, although is valid in Javascript. In order to generate proper JSON, JSON.stringify() could be used. I think it would make life difficult for others to parse nonstandard JSON as well, therefore I would recommend furnishing a new output file instead of keeping the original one.
However, it is still possible to parse the original file as JSON. This is possible via eval('(' + procline + ')');, however it is not secure to take external data into node.js like this.
const fs = require('fs');
const readline = require('readline');
const fr = fs.createReadStream('file1');
const rl = readline.createInterface({
input: fr
});
rl.on('line', function (line) {
if (line.match(new RegExp("\{name"))) {
var procline = "";
if (line.trim().split('').pop() === ','){
procline = line.trim().substring(0,line.trim().length-1);
}
else{
procline = line.trim();
}
var lineObj = eval('(' + procline + ')');
lineObj.response = 200;
console.log(JSON.stringify(lineObj));
}
});
The output would be like this:
{"name":"item1","response":200}
{"name":"item2","response":200}
{"name":"item3","response":200}
Which is line-delimited JSON (LDJSON) and could be useful for streaming stuff, without the need for leading and trailing [, ], or ,. There is an ldjson-stream package for it as well.
When working in Python I always have this simple utility function which returns the file name and line number from where the function is called:
from inspect import getframeinfo, stack
def d():
""" d stands for Debug. It returns the file name and line number from where this function is called."""
caller = getframeinfo(stack()[1][0])
return "%s:%d -" % (caller.filename, caller.lineno)
So in my code I simply put a couple debug lines like this to see how far we get before some error occurs:
print d()
# Some buggy code here
print d()
# More buggy code here
print d(), 'here is the result of some var: ', someVar
This works really well for me because it really helps debugging quickly.
I'm now looking for the equivalent in a node backend script. I was searching around but I can't find anything useful (maybe I'm looking for the wrong words?).
Does anybody know how I can create a Javascript/nodejs function which outputs the file name and line number from where the function is called? All tips are welcome!
You can create an Error to get where the Error is, and its stack trace. Then you can put that into a function, to get the line where it is.
function thisLine() {
const e = new Error();
const regex = /\((.*):(\d+):(\d+)\)$/
const match = regex.exec(e.stack.split("\n")[2]);
return {
filepath: match[1],
line: match[2],
column: match[3]
};
}
console.log(thisLine());
This works for me in Google Chrome.
And also in node.
Note to #j08691's comment:
Both this and this seem to be using lineNumber, which is not present (as far as I could test) in NodeJS.
Printing line number with custom string
const moment = require('moment');
const log = console.log;
const path = require('path');
function getTime(time) { return moment().format('YYYY-MM-DD HH:mm:ss') };
function line(num = 2) {
const e = new Error();
const regex = /\((.*):(\d+):(\d+)\)$/
const match = regex.exec(e.stack.split("\n")[num]);
const filepath = match[1];
const fileName = path.basename(filepath);
const line = match[2];
const column = match[3];
return {
filepath,
fileName,
line,
column,
str: `${getTime()} - ${fileName}:${line}:${column}`
};
}
log(line().str, "mylog1");
log(line().str, "mylog2");
log(line().str, "mylog3");
OUTPUT
2021-11-22 13:07:15 - test.js:44:5 mylog1
2021-11-22 13:07:15 - test.js:45:5 mylog2
2021-11-22 13:07:15 - test.js:46:5 mylog3
You can use this gulp plugin gulp-log-line . It Logs file and line number without the extra cost of reading the stack.
you just have to install gulp and gulp-log-line using the
npm install gulp --save and npm install gulp-log-line command. after that you need to create and write the below code in gulpfile.js and run
gulp log-line to create a duplicate file in the build folder :-
var gulp = require('gulp');
var logLine = require('gulp-log-line');
gulp.task('line-log', function() {
return gulp.src("file.js", {buffer : true})
//Write here the loggers you use.
.pipe(logLine(['console.log', 'winston.info']))
.pipe(gulp.dest('./build'))
})
gulp.task('default', ['line-log'])
Example
file.js :-
console.log('First log')
var someVariable
console.log('Second log')
Becomes
console.log('file.js:1', 'First log')
var someVariable
console.log('file.js:3', 'Second log')
The only way I've found to get anything relating to line numbers is to trap the window.onerror function, and when there's an error that will get passed the error message, the file URL and the line number:
window.onerror = function(msg, url, line) {
alert(msg + "\n" + url + ":" + line);
};
This works for me on Chrome - I don't know about other browsers.
EDIT when this answer was given in Feb' 15 there was no mention of NodeJS in the question. That was only added in November '17.
I generate some jS code on the front end and i want to write it to a js file by making an api call to a node server and i'm having some trouble with 1) ensuring the code isnt written to file as a literal string and 2) ensuring that multiple lines are preserved.
Here's my code for how ive attempted this:
//client
var bar = "mongolia"; //this value changes hence pulling it into variable
var jsCode = "function(){ \n
baz = foo(" + bar + "); \n
return baz*2;"
var data = {
code: jsCode
}
$http.post('api/code', data).success(function(savedCode){
console.log("successful and the saved code is", savedCode);
});
//server
'use strict';
var jf = require('jsonfile'),
util = require('util');
var codeFile = './code/js/autoGen.js';
exports.addCode = function(req, res) {
var newCode = req.body.code;
//write to file
return jf.writeFile(codeFile, newCode, function(err) {
if(!err) {
return res.json({newCode: newCode});
}
else {
return res.send(err);
}
});
};
When i run this, I get a syntax error in my jsCode definition.
JavaScript syntax for multiline strings is as follows:
var jsCode = "function(){ \
baz = foo(" + bar + "); \
return baz*2;"
I got around both issues by
1) Using coffeescript for multiline strings and the #{} interpolation pattern for including the variable portions
2) Using fs for writing to file (Thanks for the tip #verybadalloc)