process npm module in Node and accessing another file that does computations for me. The problem is that when on. message event/callback I am not sure what it is actually but there i am trying to access global variable and it says it is undefined. If somebody have can have a good explained solution.
_addBlock(newBlock)
{
newBlock.previousHash = this._getLatestBlock().hash;
var child =
childProcess.fork('C:\\Users\\Yoana\\WebstormProjects\\ChildProcess\\mining1.js'
);
child.on('message', function(newBlock)
{
// Receive results from child process
console.log('received: ' , newBlock);
this.chain.push(newBlock);
})
// Send child process some work
child.send(newBlock);
}
It says that this.chain.push is undefined. The method _addBlock is part of a class Blockchain and this.chain is globally accessible.
I'm not sure which model are you using i.e. node.js master/worker architecture with cluster native module or child_process native module with message passing etc., by the way despite of sharing globals it is not recommended (how to handle the shared memory? how to handle protected memory?), you can do in this way:
global.GlobalBotFactory = function() {
if (typeof(instance)=="undefined")
instance = new MyClass(options);
return instance;
}
and then you can reference it in other files like
this.instance = GlobalBotFactory(); // the shared factory instance
But this approach, despite it works, could led to several issues like
concurrent variable modification
shared memory corruption
reader/writer problem
etc. so I strongly suggest to follow a node cluster module with master/worker approach and then message passing:
/// node clustering
const cluster = require('cluster');
const numCPUs = require('os').cpus().length;
if (cluster.isMaster) { // master node
var masterConfig=require('./config/masterconfig.json');
// Fork workers.
var maxCPUs = masterConfig.cluster.worker.num;
maxCPUs=(maxCPUs>=numCPUs)?numCPUs:maxCPUs;
for (let i = 0; i < maxCPUs; i++) {
const worker=cluster.fork();
}
var MasterNode=require('./lib/master');
var master= new MasterNode(masterConfig);
master.start()
.then(done=> {
console.log(`Master ${process.pid} running on ${masterConfig.pubsub.node}`);
})
.catch(error=> {
console.error(`Master ${process.pid} error`,error);
});
}
else if (cluster.isWorker) { // worker node
var workerConfig=require('./config/workerconfig.json');
var WorkerNode=require('./lib/worker');
var worker= new WorkerNode(workerConfig);
worker.start()
.then(done=> {
console.log(`Worker ${process.pid} running on ${workerConfig.pubsub.node}`);
})
.catch(error=> {
console.error(`Worker ${process.pid} error`,error);
});
}
For the message passing part take care since you will deal with async forked process, and in node.js there is not guarantee that a message will be delivered, so you need a ack logic or you can use a pubsub approach (Redis will offer this for free, please check here), by the way here you are
for (var i = 0; i < 2; i++) {
var worker = cluster.fork();
// Receive messages from this worker and handle them in the master process.
worker.on('message', function(msg) {
console.log('Master ' + process.pid + ' received message from worker ' + this.pid + '.', msg);
});
// Send a message from the master process to the worker.
worker.send({msgFromMaster: 'This is from master ' + process.pid + ' to worker ' + worker.pid + '.'});
}
this will fork the workers and listen for incoming messages from the master or other workers. But please keep in mind that the delivery logic it's up to you. See here for more info about subprocess.send.
Related
I'm trying to create several child forks (using separate worker js file) and have them communicate with my main js file.
I've tried the following:
for (var i = 1; i <= 2; i++) {
worker = childProcess.fork(path.join(__dirname, workerPath));
workers.push(worker);
worker.send("variable");
worker.on('message', function(message) {
worker.send("variable");
});
}
This works fine with one worker, however, when I use two workers, it seems the message listener is overwritten when creating the second worker. This results in the first worker stopping and only the second worker continuing. For example if I use the following then the workers work fine:
worker1 = childProcess.fork(path.join(__dirname, workerPath));
worker1.send("variable");
worker1.on('message', function(message) {
worker1.send("variable");
});
worker2 = childProcess.fork(path.join(__dirname, workerPath));
worker2.send("variable");
worker2.on('message', function(message) {
worker2.send("variable");
});
Any solutions for this?
Okay so the key is passing the worker number in the worker 'message' because using workers[i-1].send("variable) gives an undefined error:
for (var i = 1; i <= 2; i++) {
workers.push(childProcess.fork(path.join(__dirname, workerPath)));
workers[i-1].send("variable");
workers[i-1].on('message', function(message) {
workers[message-1].send("variable);
});
(note, I'm using Node.js to create a telnet server and handle user input)
I'm having issues using variables/arrays in javascript inside functions in other scripts.
Let me set a simpler example:
var connections = []
is an array in my main.js
Broadcast.js is a function I put in a separate module and attempt to use the broadcast() function in my main.js.
The error I get is stating that connections is undefined. How do get Broadcast.js able to see the connections array in the main.js
~~
For example in my main.js I set an array to handle clients connecting to a server
//point to Telnet library
const Telnet = require('ranvier-telnet');
const logger = require('./logger.js')
var outspeak = []
var connections = []
var clients = []
let server = new Telnet.TelnetServer(rawSocket => {
let telnetSocket = new Telnet.TelnetSocket();
//attaching socket
telnetSocket.attach(rawSocket);
//setting telnet options
telnetSocket.telnetCommand(Telnet.Sequences.WILL, Telnet.Options.OPT_EOR);
//giving clients a name
telnetSocket.name = rawSocket.remoteAddress + ":" + rawSocket.remotePort
//pushing client names to array
clients.push(telnetSocket.name);
//pushing client connections to an array
connections.push(rawSocket);
console.log(`${telnetSocket.name} has connected`)
logger(`${telnetSocket.name} has connected`)
broadcast(telnetSocket.name + " connected.")
telnetSocket.on('data', function (data) {
//broadcast (telnetSocket.name + ">" + data, telnetSocket);
}
function broadcast (message, sender) {
connections.forEach(function (connection) {
//don't want to send it to sender
if (connection === sender) return;
connection.write(`${message} \n`);
});
}
Now inside my main script, I could called that array/push to that array, read from that array, as long as I type out the function inside the main.js file.
And it can easily use the broadcast function.
Now I want to make it more advance and make reduce my lines on my main.js
but once I separate the broadcast function into it's own module.
use strict'
//broadcast function
function broadcast (message, sender) {
connections.forEach(function (connection) {
//don't want to send it to sender
if (connection === sender) return;
connection.write(`${message} \n`);
});
}
module.exports = broadcast
I get a connection undefined error any time I try to invoke that broadcast function. It's like my global variable/array can't be seen by broadcast.js function.
this is how I'm invoking it
// handle input
telnetSocket.on('data', function (data) {
broadcast (telnetSocket.name + ">" + data, telnetSocket);
});
And yes, const broadcast = require('./broadcast.js'); as been added to the file at the top.
Here's the broken code complete:
'use strict'
//point to Telnet library
const Telnet = require('ranvier-telnet');
const logger = require('./logger.js');
const broadcast = require('./broadcast.js');
var connections = []
var clients = []
//had to call message as global variable
//Asan's timestamp functionm
//telnetstuff
console.log("Starting...");
let server = new Telnet.TelnetServer(rawSocket => {
let telnetSocket = new Telnet.TelnetSocket();
//attaching socket
telnetSocket.attach(rawSocket);
//setting telnet options
telnetSocket.telnetCommand(Telnet.Sequences.WILL, Telnet.Options.OPT_EOR);
//giving clients a name
telnetSocket.name = rawSocket.remoteAddress + ":" + rawSocket.remotePort
//pushing client names to array
clients.push(telnetSocket.name);
//pushing client connections to an array
connections.push(rawSocket);
console.log(`${telnetSocket.name} has connected`)
logger(`${telnetSocket.name} has connected`)
broadcast(telnetSocket.name + " connected.")
// handle input
telnetSocket.on('data', function (data) {
broadcast (telnetSocket.name + ">" + data, telnetSocket);
});
//removing client/connection from array
rawSocket.on('end', function () {
clients.splice(clients.indexOf(telnetSocket), 1);
connections.splice(connections.indexOf(rawSocket), 1);
broadcast(telnetSocket.name + " has left.\n");
logger(telnetSocket.name + " has left.");
console.log(telnetSocket.name + " has left.");
});
}).netServer
server.listen(4000);
console.log('ServerRunning...');
logger('>Server started.');
What I'm missing here? Also I apologize in advance this is my first question ever asked and I've gone through as much I could today to even figure out how to ask my question, maybe I'm not using correct lingo/terms? any help is appreciative.
refactor\broadcast.js:5
connections.forEach(function (connection) {
^
ReferenceError: connections is not defined
In nodejs, when you declare a variable not inside any function definitions, it is scoped to the file only. (This is different from browser javascript.) If you want something to be accessible from outside, you need to export it:
module.exports.connections = connections;
Then import it into the other file:
const connections = require(myFile);
This will work as long as you don't try to set the value of the variable in either file, but if you do that they'll end up pointing to separate objects. But mutating it, calling methods on it, etc should work fine.
I am using service workers to intercept requests for me and provide the responses to the fetch requests by communicating with a Web worker (also created from the same parent page).
I have used message channels for direct communication between the worker and service worker. Here is a simple POC I have written:
var otherPort, parentPort;
var dummyObj;
var DummyHandler = function()
{
this.onmessage = null;
var selfRef = this;
this.callHandler = function(arg)
{
if (typeof selfRef.onmessage === "function")
{
selfRef.onmessage(arg);
}
else
{
console.error("Message Handler not set");
}
};
};
function msgFromW(evt)
{
console.log(evt.data);
dummyObj.callHandler(evt);
}
self.addEventListener("message", function(evt) {
var data = evt.data;
if(data.msg === "connect")
{
otherPort = evt.ports[1];
otherPort.onmessage = msgFromW;
parentPort = evt.ports[0];
parentPort.postMessage({"msg": "connect"});
}
});
self.addEventListener("fetch", function(event)
{
var url = event.request.url;
var urlObj = new URL(url);
if(!isToBeIntercepted(url))
{
return fetch(event.request);
}
url = decodeURI(url);
var key = processURL(url).toLowerCase();
console.log("Fetch For: " + key);
event.respondWith(new Promise(function(resolve, reject){
dummyObj = new DummyHandler();
dummyObj.onmessage = function(e)
{
if(e.data.error)
{
reject(e.data.error);
}
else
{
var content = e.data.data;
var blob = new Blob([content]);
resolve(new Response(blob));
}
};
otherPort.postMessage({"msg": "content", param: key});
}));
});
Roles of the ports:
otherPort: Communication with worker
parentPort: Communication with parent page
In the worker, I have a database say this:
var dataBase = {
"file1.txt": "This is File1",
"file2.txt": "This is File2"
};
The worker just serves the correct data according to the key sent by the service worker. In reality these will be very large files.
The problem I am facing with this is the following:
Since I am using a global dummyObj, the older dummyObj and hence the older onmessage is lost and only the latest resource is responded with the received data.
In fact, file2 gets This is File1, because the latest dummyObj is for file2.txt but the worker first sends data for file1.txt.
I tried by creating an iframe directly and all the requests inside it are intercepted:
<html>
<head></head>
<body><iframe src="tointercept/file1.txt" ></iframe><iframe src="tointercept/file2.txt"></iframe>
</body>
</html>
Here is what I get as output:
One approach could be to write all the files that could be fetched into IndexedDB in the worker before creating the iframe. Then in the Service Worker fetch those from indexed DB. But I don't want to save all the resources in IDB. So this approach is not what I want.
Does anybody know a way to accomplish what I am trying to do in some other way? Or is there a fix to what I am doing.
Please Help!
UPDATE
I have got this to work by queuing the dummyObjs in a global queue instead of having a global object. And on receiving the response from the worker in msgFromW I pop an element from the queue and call its callHandler function.
But I am not sure if this is a reliable solution. As it assumes that everything will occur in order. Is this assumption correct?
I'd recommend wrapping your message passing between the service worker and the web worker in promises, and then pass a promise that resolves with the data from the web worker to fetchEvent.respondWith().
The promise-worker library can automate this promise-wrapping for you, or you could do it by hand, using this example as a guide.
If you were using promise-worker, your code would look something like:
var promiseWorker = new PromiseWorker(/* your web worker */);
self.addEventListener('fetch', function(fetchEvent) {
if (/* some optional check to see if you want to handle this event */) {
fetchEvent.respondWith(promiseWorker.postMessage(/* file name */));
}
});
Using the cluster module in node I can allow for a http server to run on multiple cores, increasing concurrency, which is great. But if my app contains other tasks which I do not want to run multiple times, such as a scheduled event. How can I do this?
My basic code using the cluster module is like so...
var cluster = require('cluster');
if(cluster.isMaster){
var numWorkers = require('os').cpus().length;
for(var i = 0; i < numWorkers; i++) {
cluster.fork();
}
//restart dead workers
cluster.on('exit', function(worker, code, signal) {
console.log('Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal);
console.log('Starting a new worker');
cluster.fork();
});
}else{
var http = require('http');
var server = http.createServer(function(req, res) {
res.write(process.pid.toString());
res.end();
});
server.listen(5000);
console.log("Server is listening");
}
Now if I have a task that I want to run every hour like.
setInterval(function(){
doSomething();
}, 60*60*1000);
Where would I put it? I have read that the master should only manage workers, to avoid any risk of the master throwing an error and crashing. Is it possible to give individual events to specific workers maybe?
I fork workers in my Node.js application via the Cluster module and pass a custom ID to the environment of all my workers. That works well for far.
However, I do not know how I can access this id in my master when an 'online' or 'exit' event is emitted.
The documentation is not very helpful. Could you please point me the right way?
var cluster = require('cluster');
if (cluster.isMaster) {
//MASTER
function fork() {
var worker_env = {worker_id:'my_custom_id'};
cluster.fork(worker_env);
}
cluster.on('online', function(worker) {
console.log(worker.process.env.worker_id); // undefined
//
// How can I access my custom worker id here?
//
});
cluster.on('exit', function(worker, code, signal) {
//
// And here...?
//
fork();
});
} else {
// WORKER
console.log(process.env.worker_id); // my_custom_id
}
theres no way, the worker process env is not exposed to the master.
One aproach can be a map of our cluster (a object containig the needed info).
Something like these:
var cluster = require('cluster');
if (true === cluster.isMaster) {
//CODE EXECUTED BY MASTER
var cluster_map = {}; // Here we store the workers info in a object
var restart_Limit = 10; // max global worker restart (10)
function fork_worker(myWorkerId) {
// these makes worker_id available in the worker
var worker = cluster.fork({
worker_id: myWorkerId
});
// max restarts limit (global)
if (worker.id >= restart_Limit) {
console.log('Restart limit reached, bye!');
process.kill();
}
// here we add the key "myWorkerId" to the cluster map
cluster_map[worker.id] = myWorkerId;
// WORKER AUTO-KILL
setTimeout(function() {
console.log('stoping...' + myWorkerId);
worker.kill();
}, 3000);
}
cluster.on('online', function(worker) {
var online_proc = cluster_map[worker.id];
console.log('worker online: ' + online_proc + '\n Restarts: ' + worker.id);
});
cluster.on('exit', function(worker, code, signal) {
var exited_proc = cluster_map[worker.id];
// delete the process from the cluster map
delete cluster_map[worker.id];
console.log("worker offline: " + exited_proc);
// WORKER AUTO-RESTART
setTimeout(function() {
console.log('Restarting... ' + exited_proc);
fork_worker(exited_proc);
}, 3000);
});
// start the magic ( 3 workers )
(function() {
fork_worker('id_1');
fork_worker('id_2');
fork_worker('id_3');
})();
} else {
//CODE EXECUTED BY EACH WORKER (process env is present here).
console.log('hi from the worker, process.env: ' + process.env.worker_id);
// all the hard work for the workers here.
}