We Keep Coding

Using child process output globally

For a project I need to incorporate a backend Python function with Javascript (main code for a chatbot). Using Child processes, it seems to work (when using 'node script.js'). However, I need to access the data from the called python function. Right now, all I am getting is the the output.I tried to store it in the global variable, but it's showing as 'undefined'. Is there a way to actually access the data so I can use it outside the stdout.on?
This is the Javascript code for running the pythonscript:
// Give a path to the QR scanner Python file
const qrScannerPath = "python/qrCodeScanner.py"
const base64Arg = "base64_2.txt"
// Provide the '.exe' python file. If python is available as an 'environment varaible', then simply refer to it as 'python'"
const pythonExe = "python"
// Function to convert a utf-8 array to a string
const utfConverter = function (data) {
return String.fromCharCode.apply(String,(data))
}
// let's us handle python scripts
const spawn = require("child_process").spawn
const scriptExe = spawn(pythonExe, [qrScannerPath, base64Arg])
// If all goes well, the program should execute the Python code
let counterpartyData = {}
scriptExe.stdout.on("data", function (data) {
console.log("getting the Python script data...")
let cp = JSON.parse(utfConverter(data))
counterpartyData = {... cp} //should store the data to the variable
});
console.log(counterpartyData) // shows 'undefinied"
// In case our python script cannot be run, we'll get an error
scriptExe.stderr.on("data", (data) => {
console.error("error : " + data.toString())
});
//logs error message
scriptExe.on('error', (error) => {
console.error('error: ', error.message);
});
// Logs a message saying that our code worked perfectly fine
scriptExe.on("exit", (code) => {
console.log("Process quit with code : " + code)
})
If I run this code with node, the output of 'counterpartyData' is undefined. However, inside the stdout, it actually prints out the data I want.
Furthermore, I get python import errors when running the app on Heroku :(.
Thank you in advance!!
Happy New Year and joyful greetings <3

Spawn child_process with pipe (Node)

I am actually stuck on my code and don't really know what's happening.
I am trying to use child process to listen on a pocsag frequency and send the results to multimon-ng to be able to decode the message. Considering the following code :
const ps = spawn('rtl_fm', ['-s', '22050', '-f', frequency + 'M']);
const grep = spawn('multimon-ng', ['-t', 'raw', '-a', 'POCSAG512', '-a', 'POCSAG1200', '-a', 'POCSAG2400', '-f', 'alpha /dev/stdin']);
ps.stdout.on('data', (data) => {
console.log(data.toString())
grep.stdin.write(data);
});
grep.stdin.on('data', (data) => {
console.log(data.toString())
// DO SPECIFIC STUFF WITH DECODED DATA FROM MULTIMON
});
One error occurs :
Cannot call write after a stream was destroyed
What am I missing ?
Thank !

How to obtain a more specific error message when using node.js spawn?

Right now I have the following code to spawn a python script in node.js:
logger.info('Spawning python process');
let proc1 = spawn('python3',
['experimental/paraphrase/paraphrase_detect.py',
results[0].answer, request.query.response]);
proc1.stdout.on('data', (data) => {
logger.info('Python script returned data');
let retVal;
if (data) {
logger.info(data.toString());
retVal = JSON.parse(data.toString());
}
return response.json({data: retVal});
});
proc1.on('close', (code, test) => {
if (code === 1) {
const msg = 'Python exited with exit code 1 meaning there was a ' +
'problem executing the script.';
logger.error(msg);
logger.error(test);
}
});
proc1.on('error', (data) => {
logger.error('Error with calling python script');
if (data) {
logger.error(data.toString());
}
});
The issue I'm dealing with is that specific error messages aren't being returned to node, such as permissions issues, a misspelling of the script name or a missing python package. The best I've been able to do so far is catch an exit code of 1 which just indicates generically that there was a problem running the script.

Try proc1.stderr.on("data")
More details.
https://nodejs.org/api/child_process.html#child_process_child_process

Node.js - How can I prevent interrupted child processes from surviving?

I have found that some child processes are failing to terminate if the calling script is interrupted.
Specifically, I have a module that uses Ghostscript to perform various actions: extract page images, create a new pdf from a slice, etc. I use the following to execute the command and return a through stream of the child's stdout:
function spawnStream(command, args, storeStdout, cbSuccess) {
storeStdout = storeStdout || false;
const child = spawn(command, args);
const stream = through(data => stream.emit('data', data));
let stdout = '';
child.stdout.on('data', data => {
if (storeStdout === true) stdout += data;
stream.write(data);
});
let stderr = '';
child.stderr.on('data', data => stderr += data);
child.on('close', code => {
stream.emit('end');
if (code > 0) return stream.emit('error', stderr);
if (!!cbSuccess) cbSuccess(stdout);
});
return stream;
}
This is invoked by function such as:
function extractPage(pathname, page) {
const internalRes = 96;
const downScaleFactor = 1;
return spawnStream(PATH_TO_GS, [
'-q',
'-sstdout=%stderr',
'-dBATCH',
'-dNOPAUSE',
'-sDEVICE=pngalpha',
`-r${internalRes}`,
`-dDownScaleFactor=${downScaleFactor}`,
`-dFirstPage=${page}`,
`-dLastPage=${page}`,
'-sOutputFile=%stdout',
pathname
]);
}
which is consumed, for example, like this:
it('given a pdf pathname and page number, returns the image as a stream', () => {
const document = path.resolve(__dirname, 'samples', 'document.pdf');
const test = new Promise((resolve, reject) => {
const imageBlob = extract(document, 1);
imageBlob.on('data', data => {
// do nothing in this test
});
imageBlob.on('end', () => resolve(true));
imageBlob.on('error', err => reject(err));
});
return Promise.all([expect(test).to.eventually.equal(true)]);
});
When this is interrupted, for example if the test times out or an unhandled error occurs, the child process doesn't seem to receive any signal and survives. It's a bit confusing, as no individual operation is particularly complex and yet the process appears to survive indefinitely, using 100% of CPU.
☁ ~ ps aux | grep gs | head -n 5
rwick 5735 100.0 4.2 3162908 699484 s000 R 12:54AM 6:28.13 gs -q -sstdout=%stderr -dBATCH -dNOPAUSE -sDEVICE=pngalpha -r96 -dDownScaleFactor=1 -dFirstPage=3 -dLastPage=3 -sOutputFile=%stdout /Users/rwick/projects/xan-desk/test/samples/document.pdf
rwick 5734 100.0 4.2 3171100 706260 s000 R 12:54AM 6:28.24 gs -q -sstdout=%stderr -dBATCH -dNOPAUSE -sDEVICE=pngalpha -r96 -dDownScaleFactor=1 -dFirstPage=2 -dLastPage=2 -sOutputFile=%stdout /Users/rwick/projects/xan-desk/test/samples/document.pdf
rwick 5733 100.0 4.1 3154808 689000 s000 R 12:54AM 6:28.36 gs -q -sstdout=%stderr -dBATCH -dNOPAUSE -sDEVICE=pngalpha -r96 -dDownScaleFactor=1 -dFirstPage=1 -dLastPage=1 -sOutputFile=%stdout /Users/rwick/projects/xan-desk/test/samples/document.pdf
rwick 5732 100.0 4.2 3157360 696556 s000 R 12:54AM 6:28.29 gs -q -sstdout=%stderr -dBATCH -dNOPAUSE -sDEVICE=pdfwrite -sOutputFile=%stdout /Users/rwick/projects/xan-desk/test/samples/document.pdf /Users/rwick/projects/xan-desk/test/samples/page.pdf
I thought to use a timer to send a kill signal to the child but selecting an arbitrary interval to kill a process seems like it would effectively be trading an known problem for an unknown one and kicking that can down the road.
I would really appreciate any insight into what I'm missing here. Is there a better option to encapsulate child processes so the termination of the parent is more likely to precipitate the child's interrupt?

listen to error event
child.on('error', function(err) {
console.error(err);
// code
try {
// child.kill() or child.disconnect()
} catch (e) {
console.error(e);
}
});

Stream child process output in flowing mode

I have custom command line written using Python which prints its output using "print" statement. I am using it from Node.js by spawning a child process and sending commands to it using child.stdin.write method. Here's source:
var childProcess = require('child_process'),
spawn = childProcess.spawn;
var child = spawn('./custom_cli', ['argument_1', 'argument_2']);
child.stdout.on('data', function (d) {
console.log('out: ' + d);
});
child.stderr.on('data', function (d) {
console.log('err: ' + d);
});
//execute first command after 1sec
setTimeout(function () {
child.stdin.write('some_command' + '\n');
}, 1000);
//execute "quit" command after 2sec
//to terminate the command line
setTimeout(function () {
child.stdin.write('quit' + '\n');
}, 2000);
Now the issue is I am not receiving the output in flowing mode. I want get the output from child process as soon as it's printed but I am receiving the output of all the commands only when child process is terminated (using custom cli's quit command).

You need to flush the output in the child process.
Probably you think this isn't necessary because when testing and letting the output happen on a terminal, then the library flushes itself (e. g. when a line is complete). This is not done when printing goes to a pipe (due to performance reasons).
Flush yourself:
#!/usr/bin/env python
import sys, time
while True:
print "foo"
sys.stdout.flush()
time.sleep(2)

The best way is to use unbuffered mode of python standard output. It will force python to write output to output streams without need to flush yourself.
For example:
var spawn = require('child_process').spawn,
child = spawn('python',['-u', 'myscript.py']); // Or in custom_cli add python -u myscript.py
child.stdout.on('data', function (data) {
console.log('stdout: ' + data);
});
child.stderr.on('data', function (data) {
console.log('stderr: ' + data);
});

In my case in Python I'm using sys.stdin.readline and yielding last line:
def read_stdin():
'''
read standard input
yeld next line
'''
try:
readline = sys.stdin.readline()
while readline:
yield readline
readline = sys.stdin.readline()
except:
# LP: avoid to exit(1) at stdin end
pass
for line in read_stdin():
out = process(line)
ofp.write(out)
sys.stdout.flush()
and when in Node.js
var child = spawn(binpath, args);
// register child process signals
child.stdout.on('data', function (_data) {
var data = Buffer.from(_data, 'utf-8').toString().trim();
console.log(data);
});
child.stderr.on('data', function (data) {
console.warn('pid:%s stderr:%s', child.pid, data);
});
child.stdout.on('exit', function (_) {
console.warn('pid:%s exit', child.pid);
});
child.stdout.on('end', function (_) {
console.warn('pid:%s ended', child.pid);
});
child.on('error', function (error) {
console.error(error);
});
child.on('close', (code, signal) => { // called after `end`
console.warn('pid:%s terminated with code:%d due to receipt of signal:%s with ', child.pid, code, signal);
});
child.on('uncaughtException', function (error) {
console.warn('pid:%s terminated due to receipt of error:%s', child.pid, error);
});