We Keep Coding

Writing and reading data between two nodejs servers using node serial port

I would like to send "Hello world" from one nodejs server to another using node-serialport. I have verified that the radios connecting the two are connected and sending info because they keep displaying buffer information after running my current code.
here is what I have so far.
server1
// Import dependencies
const SerialPort = require("serialport");
const Readline = require("#serialport/parser-readline");
var sf = require('sf');
//SerialPort.list(function (err, results) {
// if (err) {
// throw err;
// }
SerialPort.list().then(ports => {
ports.forEach(function(port) {
console.log(port.path);
console.log(port.pnpId);
console.log(port.manufacturer);
});
});
// Defining the serial port
const port = new SerialPort('COM3',{baudRate: 9600}, function (err) {
if (err) {
return console.log('Port Error: ', err.message)
}
})
port.write('main screen turn on', function(err) {
if (err) {
return console.log('Error on write: ', err.message)
}
console.log('message written')
})
// Read data that is available but keep the stream in "paused mode"
port.on('readable', function () {
console.log('Data:', port.read())
})
// Switches the port into "flowing mode"
port.on('data', function (data) {
console.log('Data:', data)
})
// Pipe the data into another stream (like a parser or standard out)
const lineStream = port.pipe(new Readline())
lineStream.on('data', console.log)
server 2
// Import dependencies
// in Ubuntu need to run command: sudo chmod 666 /dev/ttyS0 to open port for use
const SerialPort = require("serialport");
const Readline = require("#serialport/parser-readline");
var stoploop = true;
// Defining the serial port
const port = new SerialPort('/dev/ttyUSB0', function (err) {
if (err) {
return console.log('Error: ', err.message)
}
})
port.write('chicken butt', function(err) {
if (err) {
return console.log('Error on write: ', err.message)
}
console.log('message written')
})
// port.write("hello?");
// Read data that is available but keep the stream in "paused mode"
port.on('readable', function () {
console.log('Data:', port.read())
})
// Switches the port into "flowing mode"
port.on('data', function (data) {
console.log('Data:', data)
})
// Pipe the data into another stream (like a parser or standard out)
const lineStream = port.pipe(new Readline())
any help or even an example of how to send hello world between the two would be greatly appreciated! please let me know if any more info is needed.
edit : I recently tried doing something like
port.on('data', (data) => {
try {
console.log(data.toString());
} catch (err) {
console.log('Oops');
}
});
this is taking data that used to appear as <buffer # # # # #> and turning it into an odd string like "(
)))) ) ) )))
!)☺)!))) ) )
)(☺!�"

I found the answer myself!
I was using the wrong baudRate, and also needed to stringify the data being sent as a JSON string

Is there a way to get 'live' output lines from a python script spawned by child_process.execFile without flushing stdout every time?

I am trying to get the lines a ('never ending') python script puts into stdout. But currently my code would only log something to the console when the python process exits. Is there a way I can get the 'live' output of the python script line by line?
spawn_child.js:
let execFile = require("child_process").execFile;
var child = execFile("python3", ["PATH_TO_FILE"]);
child.stdout.on("data", data=>{
console.log(data.toString());
});
child.stderr.on("data", data=>{
console.log(data.toString());
});
child.on("exit", code=>{
console.log("Child exited with code "+code);
});
The python file:
from time import sleep
while True:
sleep(3)
print("test")
Edit: It works when using a nodejs script instead of a python script

change python script to
import time
import sys
while True:
time.sleep(1)
print("test")
sys.stdout.flush()
and increase the buffer size of the child process
const child = execFile("python", ["./runner.py"], {
detached: true,
maxBuffer: 10 * 1024 * 1024 * 1024
});
or you can do it without the flushing to stdout with python-shell
const { PythonShell } = require('python-shell');
let pyshell = new PythonShell('runner.py');
pyshell.on('message', function (message) {
console.log(message);
});
pyshell.end(function (err, code, signal) {
if (err) throw err;
console.log('The exit code was: ' + code);
console.log('The exit signal was: ' + signal);
console.log('finished');
});

Use spawn instead of execFile, dont forget options shell and stdio.
const spawn = require("child_process").spawn;
const child = spawn("python3", ["file.py"], {shell: true, stdio: 'inherit'});
child.on('data', function(data) {
console.log(data);
});
child.on('close', function(code) {
console.log('Child process exited with exit code '+code);
});
You can also add cwd option.

Was trying to implement something similar inside a NextJS application and wanted live output from my python script and using python-shell had the same issue that it was only giving me output when the process existed and I ended up using node-pty instead which worked as expected:
import { spawn } from "node-pty"
const pyProcess = spawn("python", ["path/to/python/script"], {
name: 'xterm-color',
cols: 80,
rows: 30,
cwd: process.cwd(),
});
pyProcess.on('data', function (data: { toString: () => any; }) {
console.log(data.toString());
});
pyProcess.on('exit', (code: any) => {
console.log(`child process exited with code ${code}`);
});

NodeJS mongoose - copy entire collection to database in another server

I'm trying to move my database to another server.
I have collection with 410k~ documents, I want to move them partially 100 by 100 to my new server with mongodb database.
There is my code:
var mongoose = require('mongoose');
var itemdataModel = require('./model/ItemData');
mongoose.connection.on('error', function(err) {
console.log('MongoDB Connection Error.' + err);
process.exit(1);
});
mongoose.connection.on('close', function() {
console.log('Connection closed');
process.exit(1);
});
const ins = async (itemz) => {
try {
console.log("Inserting..")
await mongoose.connect('<url to new database>', { useNewUrlParser: true });
await gamedataModel.insertMany(itemz,
async (err, result) => {
if (err) {
console.log("Insert query error" + err)
await mongoose.connection.close()
} else {
console.log("Inserted!");
await mongoose.connection.close()
}
});
} catch (e) {
console.log('insert error ' + e)
}
}
(async () => {
mongoose.connect('<url to old database>', { useNewUrlParser: true });
const wyn = await itemdataModel.find({}).countDocuments()
console.log('Documents count: ' + wyn)
for (let i = 0; i < wyn; i += 100) {
const docs = await itemdataModel.find({status: 'processed'}, '', {'skip': i, 'limit': 100 }).lean().exec()
console.log('Selected ' + docs.length + ' documents to move')
await mongoose.connection.close()
await ins(docs)
}
})();
There is problem with connecting to second database in "ins" function, there is my console output:
Documents count: 411975
Selected 100 docs to move
Connection closed
How to get it working ?
It is required to close first connection before starting another to insert 100 documents, then close it and back to my loop to move another 100 docs?
I do not want to overload the servers by opening 411k/100 connections at once on both of them

I would not use nodejs for this task. Mongodb comes with utilities for backing up and restoring which will almost certainly do it better and more efficiently than you can, especially if you're adding the mongoose overhead.
https://docs.mongodb.com/manual/tutorial/backup-and-restore-tools/

Node child_process and twitch api

I'm trying to create a simple script to run a few node files, these files are just api's for a quote database , Authentication , and a twitch bot.
The Quote and Authentication work just find however they twitch bot does not it says its connected to the server but when it goes to execute any commands it gets a unhandled promise rejection or some such error
However when I run the twitch bot separately in its own terminal session and not from the script it works just fine with no thrown errors.
Why will it work from a separate session that I create but not from a child_process?
note: I have had the child_process start up a new shell and its still the same issue
The process script:
const exec = require('child_process').exec;
const spawn = require('child_process').spawn
// I think twtich bot process has an issue without how much
//data is being passed to stdout or stderr and I need
//to specify how much data is allowed
twitchBot = exec('node ./twitchbot-api/index.js',
function(error, stdout, stderr){
console.log('stdout: ' + stdout);
console.log('stderr: ' + stderr);
if(error !== null){
console.log('exec error: ' + error);
}
})
twitchBot.stdout.on('data', (data) => {
console.log(`twitchBot stdout:\n${data}`);
});
//whenver I recieve an error from quoteDataBase it also displays in the parent process
twitchBot.stderr.on('data', (data) => {
console.error(`twitchBot stderr:\n${data}`);
});
//this throws an error the first time I try a command saying not connected to server , I think I need to do this as exec
// let twitchBot = spawn('node ./twitchbot-api/index.js',{
// stdio: 'inherit',
// shell: true,
// detached: true,
// })
// //twitchBot.unref();
// twitchBot.on('error', (error)=>{
// console.log(`the erorr ${error}`)
// })
//starts up a child exec process for my quotes database
let quoteDataBase = exec('node ./quotes-api/index.js',
function(error, stdout, stderr){
console.log('stdout: ' + stdout);
console.log('stderr: ' + stderr);
if(error !== null){
console.log('exec error: ' + error);
}
})
//whatever the child exec process quoteDatabase pushs to the standard out (console) displays on the parent process
// aka the terminal I run apiStart from
quoteDataBase.stdout.on('data', (data) => {
console.log(`quoteDataBase stdout:\n${data}`);
});
//whenver I recieve an error from quoteDataBase it also displays in the parent process
quoteDataBase.stderr.on('data', (data) => {
console.error(`quoteDataBase stderr:\n${data}`);
});
//starts up a child process for my user Authentication
let AuthenDataBase = exec('node ./authen-api/index.js',
function(error, stdout, stderr){
console.log('stdout: ' + stdout);
console.log('stderr: ' + stderr);
if(error !== null){
console.log('exec error: ' + error);
}
})
And then the twitch bot code:
const tmi = require('tmi.js')
const haikudos = require('haikudos')
require('dotenv').config()
require('es6-promise').polyfill();
require('isomorphic-fetch');
// Valid commands start with:
let commandPrefix = '!'
// Define configuration options:
let opts = {
identity: {
username: process.env.user,
password: process.env.pass
},
channels: [
"dshrops1"
]
}
// These are the commands the bot knows (defined below):
let knownCommands = { echo, haiku, quote }
// Function called when the "echo" command is issued:
function echo (target, context, params) {
// If there's something to echo:
if (params.length) {
// Join the params into a string:
const msg = params.join(' ')
// Send it back to the correct place:
sendMessage(target, context, msg)
} else { // Nothing to echo
console.log(`* Nothing to echo`)
}
}
// Function called when the "haiku" command is issued:
function haiku (target, context) {
// Generate a new haiku:
haikudos((newHaiku) => {
// Split it line-by-line:
newHaiku.split('\n').forEach((h) => {
// Send each line separately:
sendMessage(target, context, h)
})
})
}
async function quote (target, context){
//cant deploy this on AWS yet Untill I deploy my database api as well.
let quote = await fetch('http://localhost:3006/random').then(resp =>resp.text())
sendMessage(target,context,quote)
}
// Helper function to send the correct type of message:
function sendMessage (target, context, message) {
if (context['message-type'] === 'whisper') {
client.whisper(target, message)
} else {
client.say(target, message)
}
}
// Create a client with our options:
let client = new tmi.client(opts)
// Register our event handlers (defined below):
client.on('message', onMessageHandler)
client.on('connected', onConnectedHandler)
client.on('disconnected', onDisconnectedHandler)
// Connect to Twitch:
client.connect()
// Called every time a message comes in:
function onMessageHandler (target, context, msg, self) {
if (self) { return } // Ignore messages from the bot
// This isn't a command since it has no prefix:
if (msg.substr(0, 1) !== commandPrefix) {
console.log(`[${target} (${context['message-type']})] ${context.username}: ${msg}`)
return
}
// Split the message into individual words:
const parse = msg.slice(1).split(' ')
// The command name is the first (0th) one:
const commandName = parse[0]
// The rest (if any) are the parameters:
const params = parse.splice(1)
// If the command is known, let's execute it:
if (commandName in knownCommands) {
// Retrieve the function by its name:
const command = knownCommands[commandName]
// Then call the command with parameters:
command(target, context, params)
console.log(`* Executed ${commandName} command for ${context.username}`)
} else {
console.log(`* Unknown command ${commandName} from ${context.username}`)
}
}
// Called every time the bot connects to Twitch chat:
function onConnectedHandler (addr, port) {
console.log(`* Connected to ${addr}:${port}`)
}
// Called every time the bot disconnects from Twitch:
function onDisconnectedHandler (reason) {
console.log(`Womp womp, disconnected: ${reason}`)
process.exit(1)
}
Sorry About any formatting issues
Despite all the issues that might be wrong with the twitch bot it does work fine when I run it from a separate terminal I am curious as to why it does work from the script and what I can do.
notes: I figured it might have to do with the twitch api being a IRC.

How to exit properly from node.js

The following will read and import many CSV files from disk into MongoDB but NodeJS won't exit after importing all the files if it doesn't go through the resizePhoto() function (Which contains a process.exit after resizing the images).
How can I have it to close properly after importing all files without interrupting? If I add a process.exit .on end it will exit after importing the first file.
var importData = function(fileName) {
// Get file from disk.
var filePath = path.join(folder, fileName);
// Read and import the CSV file.
csv.fromPath(filePath, {
objectMode: true,
headers: keys
})
.on('data', function (data) {
var Obj = new models[fileName](data);
models[fileName].find({}).remove().exec();
Obj.save(function (err, importedObj) {
if (err) {
console.log(fileName, err);
} else if (fileName === 'PHOTOS') {
resizePhoto(importedObj);
}
});
})
.on('end', function() {
console.log(fileName + ': Imported.');
});
};
module.exports = importData;

Use the module async, method parallel (https://github.com/caolan/async#parallel). It can call your tasks (import) in parallel and call the final handler (exit) after all tasks end.
In your case:
1) Somewhere in project
csvImp=require('importData.js');
async.parallel([
function(done){csvImp(name1,done);},
function(done){csvImp(name2,done);},
...
function(done){csvImp(nameN,done);}
],
function(err, results){
process.exit();
});
2) in importData.js
var importData = function(fileName,done) {
...
.on('end', function() {
console.log(fileName + ': Imported.');
done();
});
So, next you need to prepare list of tasks. Something like
names.forEach(function(n){
tasks.push(function(done){csvImp(n,done);});
});
And call async.parallel with tasks.
Good luck)