I currently have an operation in a module that is blocking, so I'm looking at making this into a child process that I fork instead.
If I want to do that, then I of course need to modify the architecture of my module. The module requires that a dependency is injected by calling the module as a function, passing in the dependency, like so:
var dependency = { name: "Bob" }
require('worker')(dependency)
Then in my worker module:
module.exports = function (dependency) {
// Outputs { name: "Bob" }
console.log(dependency)
}
How can I turn this example into a child process being forked?
When using .fork() you are spinning up a completely separate process, so you are not able to pass around references between the parent and child processes (and are limited to messaging after the process has been created).
An approach not requiring messaging is to pass arguments (in an array) when you fork the process. Although I believe you'll have to stick with simple string/number values (but it looks like this might be enough for you from the code). Eg.:
At top level:
var name = 'bob'
var args = [name];
var childProcess = require('child_process').fork(__dirname + '/worker', args);
In the worker process:
var name = process.argv[2]; //AFIAK elements 0 and 1 are already populated with env info
Update
If you really want to go the messaging route (which I'd hesitate to recommend if you already need to send messages), then you could differentiate between the types of messages something like this (there may be more elegant ways):
At top level:
var childProcess = require('child_process').fork(__dirname + '/worker');
childProcess.send({msgtype:'dependencies', content:dependencies});
//Then to send 'normal' message:
childProcess.send({msgtype:'myothermessagetype', content:'some content'}
In worker process:
process.on('message', function(msg){
if(msg.mtype == 'dependencies') {
var dependencies = msg.content;
//Do something with dependencies
} else if(msg.mtype == 'myothermessagetype') {
var normalmessage = msg.content;
//Do something in response to normal message.
}
});
a.js
var fork = require ("child_process").fork;
var child;
var init = false;
var m = module.exports = {};
m.init = function (o){
if (init) return;
init = true;
child = fork (__dirname + "/child");
child.send ({ init: o });
};
m.print = function (o){
if (!init) return;
child.send ({ msg: o });
};
m.uninit = function (){
if (!init) return;
child.on ("exit", function (){
init = false;
});
child.kill ();
};
child.js
var dependency;
var print = function (o){
console.log (o + dependency.name);
};
process.on ("message", function (o){
if (o.init){
dependency = o.init;
}else{
print (o.msg);
}
});
b.js
var a = require ("./a");
a.init ({ name: "asd" });
a.print ("hi, ");
setTimeout (function (){
a.uninit ();
}, 1000);
Prints: hi, asd
In the main module:
var dependency = {message: 'hello there'};
var args = [JSON.stringify(dependency)];
var child = require('child_process').fork('worker', args);
child.send('sayhello');
child.send('exit');
And in the child process module (worker.js):
var dependency = JSON.parse(process.argv[2]);
process.on('message', function(m){
if(m == 'sayhello') console.log(dependency.message);
else if(m == 'exit') process.exit();
});
Related
Little info, i have an arp.js file which takes a subnet address "192.168.2" and gets all strings returned from arp -a and stores in an array.
I can't figure out why my arpList function is returning an undefined value in my index.js file.
All the console.logs are returning the correct values in the arp.js page when called from the index.js, but the ipObj is coming up undefined. Even the console.log before i return of ipObj works.
Any help would be greatly appreciated.
var { spawn } = require('child_process');
const arpLs = spawn('arp', ['-a']);
var bufferData;
module.exports = {
arpList: function (subnet) {
arpLs.stdout.on('data', data => {
bufferData += data
})
arpLs.stderr.on('data', data => {
console.log('error: ' + data);
});
arpLs.on('exit', function (code) {
if (code != 0) {
console.log("Error exiting"); //if error occurs
}
console.log("exit start 1"); // checking internal processes at stages
var dataArray = bufferData.split(' ');
var ipArray = [];
for (i = 0; i < dataArray.length; i++) {
if (dataArray[i].includes(subnet)) {
ipArray.push(dataArray[i]);
console.log("loop working");
}
}
var ipObj = { "lanIps": ipArray };
console.log("Object is there: "+ipObj)
return ipObj; // this obj should be returned to the index.js call using
})
},
sayMyName: function () {
return "Hello";
}
}
//arpList(ipSubnet);
//INDEX.js
//the index page looks like this
//var arp = require('./arp.js);
//var ipSubnet = "192.168.2";
//var lanIps = arp.arpList(ipSubnet);
//console.log(lanIps);
I ended up adding a callback function to arpList - function (subnet, callback)
Then instead of returning the value pass it into the callback
Then on the index.js side instead of
var lanIps = arp.arpList(value)
i used
arp.arpList(value, function(res){lanIps = res}
return ipObj; // this obj should be returned to the index.js call using
It won't be returned. The reference say nothing about return value. Node-style callbacks rarely work like that because they are potentially asynchronous and returned value cannot be taken into account.
This a special case of this well-known problem. The process is asynchronous and is finished after arp.arpList(ipSubnet) call, there's nothing to assign to lanIps. This is a use case for promises. There are already third-party promisified counterparts like child-process-promise.
The problem can be also solved by moving to synchronous API. child_process functions have synchronous counterparts, including spawnSync.
Is it possible to fork a child_process from an electron render process? I found some posts across the net, but there were no hint how helps me to get my code working.
I created a module, that fork child processes. This code works, when I run this with cmd and under node. But when I try to integrate it in my electron app, I can not communicate with the child.send() method.
// create fork
const fork = require('child_process').fork;
const fs = require('fs');
const img_path = [
'path/to/an/image1.jpg',
'path/to/an/image2.jpg',
'path/to/an/image3.jpg'
];
const cp = [];
const temp_path = img_path.map((item) => item);
createAndResize(2);
function createAndResize(num) {
return childResize(createChildProcess(num));
}
function createChildProcess(num) {
if(num <= 0) {
return cp;
} else {
let cf = fork('./child.js');
cp.push(cf);
num -= 1;
return createChildProcess(num);
}
}
function childResize(list) {
if(list.length <=0) {
return true;
} else {
// child_process is created
let child = list.shift();
child.on('message', function (data) {
if (!temp_path.length) {
process.kill(data);
} else {
child.send(temp_path.shift());
}
});
child.send(temp_path.shift());
setTimeout(function() {
childResize(list);
}, 1000);
}
}
//child.js
process.on('message', function(msg) {
console.log(msg); //this is never reached
};
EDIT: based on the comment below, I fork child processes on the main process. The comunication seems to work with few exceptions. But first my new code:
// myView.js
const { remote } = require('electron');
const mainProcess = remote.require('./main.js');
const { forkChildfromMain } = mainProcess;
forkChildfromMain();
// main.js
const fork = require('child_process').fork;
let cp = [];
function forkChildfromMain() {
createAndResize(4);
}
function createAndResize(num) {
return childResize(createChildProcess(num));
}
function createChildProcess(num) {
if(num <= 0) {
return cp;
} else {
let cf = fork('./resize.js');
cp.push(cf);
num -= 1;
return createChildProcess(num);
}
}
function childResize(list) {
if(list.length <=0) {
return true;
} else {
let child = list.shift();
child.on('message', function (msg) {
// logs 'Hello World' to the cmd console
console.log(msg);
});
child.send('Hello World');
setTimeout(function() {
childResize(list);
}, 1000);
}
}
exports.forkChildfromMain = forkChildfromMain;
// child.js
process.on('message', function(msg) {
// this console statement get never loged
// I think, I must integrate an icpModule
console.log(msg);
//process send msg back to main.js
process.send(msg);
})
OUTDATED: The main problem now is, that I think electron 'spawn' new child processes and do not fork.
Because, when I look at my task manager I see only one instance from electron. When I run the code in a node env, I see there were fork multiple node instances.
The reason why I prefer to fork my child processes in multiple node instances is, that I want to make many image manipulation. So when I fork childs, then every child has it own node instance with memory and so on. I think that would be more performant then when I only have one instance who shared the memory and resources to all of the childs.
The second unexpected behavior is, that the console.log statement in the child is not printed to my cmd console. But this is the smaller ones :)
EDIT: After I analyse my task manager a little more in depth, I saw, that electron spawn multiple child processes like it should.
Electron's renderer process is not the right place for forking child processes, you should think about moving this to the main process.
Nonetheless, it should work the way you describe. If you'd make a minimal example available somewhere I could take a closer look.
Given Node.js boasts of asynchronous event driven model,
I was expecting, I should be able to write any Nodejs function,
e.g as simple as going through a loop, e.g IamLooper() below,
which might or might not involve file I/O and then pass that looping function to a mother nodeJs function e.g Invoke(),to which I also pass another call back functiont e.g happyend() below.
My expectation was after IamLooper is finished ,happyend () will be invoked by the NodeJs supplied function .
e.g :
==>
gdata =[];
function IamLooper() {
var pi = Array;
for (var ii = 0 ; ii <4 ; ii ++)
{
pi[ii] = 13* ii;;
gdata.push(ii);
}
console.log("looper done -tell the callback") ;
}
function happyend() { console.log("looper says done");}
I want to invoke IamLooper() and supply the happyend at time of invocation.
i.e. I am looking for a ready made node function e.g Invoke, which can be called like this:
Invoke(IamLooper(), happyend());
if(gdata.length > 0) {console.log("looping has started");}
In essence Invoke should do the same for any two functions I supply to it so that we have just a working template of a callback execution strategy.
Also the Invoke being executed async, my program progresses beyond Invoke before it finishes.
Is my expectation is misguided ? Can any one give me some guidance here.
If you are looking for a preexisting way of easily doing callbacks in node, you should use event emitters (https://nodejs.org/api/events.html):
var EventEmitter = require('events').EventEmitter;
var eventExample = new EventEmitter;
//You can create event listeners:
eventExample.on('anEvent', function(someData){
//Do something with someData
});
//To trigger an event listener you must emit:
eventExample.emit('anEvent', someData);
With your code, it'd look something like this:
var EventEmitter = require('events').EventEmitter;
var looper = new EventEmitter;
looper.on('invoke', function(data){
var callFunction = data.callFunction;
var finishFunction = data.finishFunction;
var callParameters = data.callParameters;
var finishParameters = data.finishParameters;
if(callParameters == null){
callFunction({callbackPara: finishParameters, callbackFunction: finishFunction});
}
else{
callFunction(callParameters, {callbackParameters: finishParameters, callbackFunction: finishFunction});
}
});
looper.on('finish', function(data){
var finishFunction = data.callbackFunction;
var parameters = data.callbackParameters;
if(parameters == null){
finishFunction();
}
else{
finishFunction(parameters);
}
});
gdata =[];
function IamLooper(g, callback){
var pi = Array;
for (var ii = 0 ; ii <4 ; ii ++){
pi[ii] = 13* ii;;
g.push(ii);
}
looper.emit('finish', callback);
}
function happyend() { console.log("looper says done");}
And then call it like:
looper.emit('invoke', {callFunction: IamLooper, finishFunction: happyend, callParameters: gdata, finishParameters: null});
You can also always do normal callbacks:
gdata =[];
function IamLooper(g, callback){
var pi = Array;
for (var ii = 0 ; ii <4 ; ii ++){
pi[ii] = 13* ii;;
g.push(ii);
}
callback();
}
IamLooper(gdata, function(){ console.log("looper says done");}
I'm just learning node.js and wanted to write a simple test program that copied a file from a source folder to a destination folder. I piped a fs.ReadStream to a fs.WriteStream and that worked perfectly. I next tried to use non-flowing mode but the following program fails 99% of the time on larger files (anything over 1MB.) I'm assuming that given certain timing the event queue becomes empty and so exits. Should the following program work?
var sourcePath = "./source/test.txt";
var destinationPath = "./destination/test.txt";
// Display number of times 'readable' callback fired
var callbackCount = 0;
process.on('exit', function() {
console.log('Readable callback fired %d times', callbackCount);
})
var fs = require('fs');
var sourceStream = fs.createReadStream(sourcePath);
var destinationStream = fs.createWriteStream(destinationPath);
copyStream(sourceStream, destinationStream);
function copyStream(src, dst) {
var drained = true;
// read chunk of data when ready
src.on('readable', function () {
++callbackCount;
if (!drained) {
dst.once('drain', function () {
writeToDestination();
});
} else {
writeToDestination();
}
function writeToDestination() {
var chunk = src.read();
if (chunk !== null) {
drained = dst.write(chunk);
}
}
});
src.on('end', function () {
dst.end();
});
}
NOTE: If I remove the drain related code the program always works but the node.js documentation indicates that you should wait on a drain event if the write function returns false.
So should the above program work as is? If it shouldn't how should I reorganize it to work with both readable and drain events?
It looks like you're most of the way there; there are just a couple of things you need to change.
When writing to dst you need to keep reading from src until either you get a null chunk, or dst.write() returns false.
Instead of listening for all readable events on src, you should only be listening for those events when it's ok to write to dst and you currently have nothing to write.
Something like this:
function copyStream(src, dst) {
function writeToDestination() {
var chunk = src.read(),
drained = true;
// write until dst is saturated or there's no more data available in src
while (drained && (chunk !== null)) {
if (drained = dst.write(chunk)) {
chunk = src.read();
}
}
if (!drained) {
// if dst is saturated, wait for it to drain and then write again
dst.once('drain', function() {
writeToDestination();
});
} else {
// if we ran out of data in src, wait for more and then write again
src.once('readable', function() {
++callbackCount;
writeToDestination();
});
}
}
// trigger the initial write when data is available in src
src.once('readable', function() {
++callbackCount;
writeToDestination();
});
src.on('end', function () {
dst.end();
});
}
Here is the error I'm getting when trying to test a basic Socket.io and Express set up (per the example on the socket.io website):
/Users/scottcorgan/Projects/sevenly/campaigns/node_modules/socket.io/lib/manager.js:659
var socket = this.namespaces[i].socket(data.id, true);
^
TypeError: Object function extend(another) {
var properties = Object.keys(another);
var object = this;
properties.forEach(function (property) {
object[property] = another[property];
});
return object;
} has no method 'socket'
at Manager.handleClient (/Users/scottcorgan/Projects/sevenly/campaigns/node_modules/socket.io/lib/manager.js:659:41)
at Manager.handleUpgrade (/Users/scottcorgan/Projects/sevenly/campaigns/node_modules/socket.io/lib/manager.js:588:8)
at HTTPServer.<anonymous> (/Users/scottcorgan/Projects/sevenly/campaigns/node_modules/socket.io/lib/manager.js:119:10)
at HTTPServer.emit (events.js:88:20)
at Socket.<anonymous> (http.js:1390:14)
at TCP.onread (net.js:334:27)
Appreciate any help I can get, please :)
This problem stems from the fact that you or a library you use are adding functions to Object.prototype.
Thus this code:
Object.prototype.foo = function() {};
Object.prototype.bar = function() {};
var myObj = { x: 1 };
for (var i in myObj) {
console.log(i)
}
will print: x, foo, bar (not necessarily in that order) and not just x as you expect.
In your case this happens in manager.js:
// initialize the socket for all namespaces
for (var i in this.namespaces) {
var socket = this.namespaces[i].socket(data.id, true);
// echo back connect packet and fire connection event
if (i === '') {
this.namespaces[i].handlePacket(data.id, { type: 'connect' });
}
}
This code doesn't expect to encounter the declared: extend key, as you can see from the stack trace of the error:
TypeError: Object function extend(another) {
var properties = Object.keys(another);
var object = this;
properties.forEach(function (property) {
object[property] = another[property];
});
return object;
} has no method 'socket'
the program is actually tried to invoke socket() on the extend function.
See Bob's rant here about adding functions to Object.prototype.
As for the solution, you can either add a conditional statement inside manager.js like so:
// initialize the socket for all namespaces
for (var i in this.namespaces) {
if ('extend' == i) continue; // ADDED
var socket = this.namespaces[i].socket(data.id, true);
// echo back connect packet and fire connection event
if (i === '') {
this.namespaces[i].handlePacket(data.id, { type: 'connect' });
}
}
or you can remove the Object.prototype.extend = function(...) {} declaration, which is my personal preference.
Your this.namespaces[i].socket(data.id, true); does not exist. Do something like console.log(typeof this.namespaces[i].socket(data.id, true)); You'll probably get an undefined
I bet one of the elements of your namespaces array is missing.