Node.js sending an object with function definitions to worker thread - multithreading

So I am working on a project in Node.js and I want to open up some extra threads to handle the processing load more efficiently. But I am using classes with function definitions with them and when I try to send those objects to the worker thread, the functions defined in the object disappear and I am only left with the other fields in the object. Is there a way to send the worker an object and preserve the functions so they can be called within the worker?
var cluster = require('cluster');
if(cluster.isMaster){
Monster = function(species){
this.attack = function(){
console.log('CHOMP');
};
this.name = species;
};
var vamp = new Monster('vampire'),
worker = cluster.fork();
worker.send({'monster' : vamp});
}
else{
process.on('message', function(msg) {
console.log(msg.monster); //this logs "{ name: 'vampire' }"
msg.monster.attack(); //TypeError: Object #<Object> has no method 'attack'
});
}

No, there is no way to pass functions between threads. You can pass only JS plain objects (data only) and handle it with functions defined in current thread (like create new object with received data).

Charlie, I realize you asked this question a year ago, but I was wanting to do something very similar and I came across your question which you didn't mark an answer to yet. I thought I would take a "stab" at it and show you what I have done with your code. This different way of organizing code is for me a very acceptable workaround in my node.js work. I am pretty sure this gives you a way to accomplish what you want, even though you can't do it in the manner you wanted.
Declare your "class" outside the cluster code, like this:
var cluster = require('cluster');
var Monster = function(species){
this.attack = function(){
console.log('CHOMP!');
};
this.die = function() {
console.log("Oh, what did I eat? I don't feel so good....\r\n");
process.exit(0);
};
this.scare = function() {
console.log("BOO! I am a " + this.name + "!");
};
this.name = species;
};
if(cluster.isMaster){
worker = cluster.fork();
worker.send({'species' : 'Vampire'});
}
else{
process.on('message', function(msg) {
if(typeof msg.species !== "undefined") {
myMonster = new Monster(msg.species);
myMonster.scare();
myMonster.attack();
myMonster.die();
}
});
}
Give that a whirl and see if this is an answer you can accept!

Ok, stumbled upon this answer, and I found it strange that no one brought this up, but it might be a more modern feature than the question:
eval
let str = "() => { console.log('test') }"
let func = eval(str)
func()
Think it's obvious what's going on here, you can parse any string to javascript, and you can send strings to workers, so you can build and object with functions:
let obj = { a: "() => { ... }" }
and send the object over. (JSON.stringify(obj) first, and than you will have to parse the object first, and than all the substrings seperately)

Related

Passing a return from one function to another function that already has set parameters?

Edit: I know JS is asynchronous, I have looked over the How to Return thread. The issue I'm having is that going from "foo" examples to something specific = I'm not quite sure where to re-format this.
Also here is some context: https://github.com/sharkwheels/beanballs/blob/master/bean-to-osc-two.js
I have a question about returns in node. It might be a dumb question, but here goes. I have a function that connects to a socket, and gets OSC messages from processing:
var sock = dgram.createSocket("udp4", function(msg, rinfo) {
try {
// get at all that info being sent out from Processing.
//console.log(osc.fromBuffer(msg));
var getMsg = osc.fromBuffer(msg);
var isMsg = getMsg.args[0].value;
var isName = getMsg.args[1].value;
var isAdd = getMsg.address;
var isType = getMsg.oscType;
// make an array out of it
var isAll = [];
isAll.push(isName);
isAll.push(isMsg);
isAll.push(isAdd);
isAll.push(isType);
// return the array
console.log(isAll);
return isAll;
} catch (error) {
console.log(error);
}
});
Below I have the start of another function, to write some of that array to a BLE device. It needs name and characteristics from a different function. How do I get the below function to use isAll AND two existing parameters?
var writeToChars = function (name, characteristics) { // this is passing values from the BLE setup function
// i need to get isAll to here.
// eventually this will write some values from isAll into a scratch bank.
}
Thanks.
async call in this case be written something like this. state can be maintained in the variables in closure if required. In this particular case - you can do without any state (isAll) as well.
var isAll;
var soc = dgram.createSocket('udp4', oncreatesocket);
function oncreatesocket(msg, rinfo)
{
isAll = parseMessage(msg);
writeData(isAll);
}
function parseMessage(msg) {
...
// code to parse msg and return isAll
}
function writeData() {}
if the writeData is small enough function. It can be inside oncreatesocket without impacting the readability of the code.
Alright. So I figured out what to do, at least in this scenario. I'm sure there is a better way to do this, but for now, this works.
I'm mapping an existing global array of peripherals into the write function, while passing the OSC message to it as a parameter. This solved my issue of "how do I get two pieces of information to the same place". It figures out which peripheral is which and writes a different value to each scratch bank of each peripheral accordingly. Leaving here for future reference.
var writeToBean = function(passThrough){
var passThrough = passThrough;
console.log("in Write to bean: ", passThrough);
_.map(beanArray, function(n){
if(n.advertisement.localName === passThrough.name){
//var name = n.advertisement.localName;
n.discoverSomeServicesAndCharacteristics(['a495ff20c5b14b44b5121370f02d74de'], [scratchThr], function(error, services, characteristics){
var service = services[0];
var characteristic = characteristics[0];
var toSend = passThrough.msg;
console.log("service", service);
console.log("characteristic", characteristic);
if (toSend != null) {
characteristic.write(new Buffer([toSend]), false, function(error) {
if (error) { console.log(error); }
console.log("wrote " + toSend + " to scratch bank 3");
});
}
// not sure how to make the program resume, it stops here. No error, just stops processing.
});
}
});
}

Blocking node from reading simultaneously from db

I have a set of values in a mongoDB I need to only be read ones. So when I have read them I delete that line from the DB. But since node is async if I do getValue() twice in quick succession I get the same value, since the DB has not had time to delete the old one. Does anyone know a good way to fix this problem. I can think of a couple but nothing good.
I don’t have my code here so just wrote a quick example to show my problem.
Var getValue = function() {
ReadfromDB(function(data){
deleteRecord(); // show that what we read has been updated
});
}
You could try something like this. The reading is placed into a promise that's done when the deleting is done as well. This can stack so if you do getValue() 20 times it still should execute in the correct order.
var getValue = function(){
var currentPromise;
var read = function(){
return ReadfromDB().then(function(data){
deleteRecord(); // show that what we read has been updated
});
}
return function() {
if(currentPromise){
currentPromise = currentPromise.then(read)
}else{
currentPromise = read();
}
}
}
What you need to do outside this code is make sure ReadfromDB returns a promise object.
Sounds like a good use case for a closure!
var getValue = (function() {
var called = false;
return function() {
if (!called) {
called = true;
ReadFromDB(function(data) {
deleteRecord();
});
}
};
}());
You could also use once to do exactly the same thing.
var once = require('once');
var getValue = once(function() {
ReadFromDB(function(data) {
deleteRecord();
});
});

What's going on with Meteor and Fibers/bindEnvironment()?

I am having difficulty using Fibers/Meteor.bindEnvironment(). I tried to have code updating and inserting to a collection if the collection starts empty. This is all supposed to be running server-side on startup.
function insertRecords() {
console.log("inserting...");
var client = Knox.createClient({
key: apikey,
secret: secret,
bucket: 'profile-testing'
});
console.log("created client");
client.list({ prefix: 'projects' }, function(err, data) {
if (err) {
console.log("Error in insertRecords");
}
for (var i = 0; i < data.Contents.length; i++) {
console.log(data.Contents[i].Key);
if (data.Contents[i].Key.split('/').pop() == "") {
Projects.insert({ name: data.Contents[i].Key, contents: [] });
} else if (data.Contents[i].Key.split('.').pop() == "jpg") {
Projects.update( { name: data.Contents[i].Key.substr(0,
data.Contents[i].Key.lastIndexOf('.')) },
{ $push: {contents: data.Contents[i].Key}} );
} else {
console.log(data.Contents[i].Key.split('.').pop());
}
}
});
}
if (Meteor.isServer) {
Meteor.startup(function () {
if (Projects.find().count() === 0) {
boundInsert = Meteor.bindEnvironment(insertRecords, function(err) {
if (err) {
console.log("error binding?");
console.log(err);
}
});
boundInsert();
}
});
}
My first time writing this, I got errors that I needed to wrap my callbacks in a Fiber() block, then on discussion on IRC someone recommending trying Meteor.bindEnvironment() instead, since that should be putting it in a Fiber. That didn't work (the only output I saw was inserting..., meaning that bindEnvironment() didn't throw an error, but it also doesn't run any of the code inside of the block). Then I got to this. My error now is: Error: Meteor code must always run within a Fiber. Try wrapping callbacks that you pass to non-Meteor libraries with Meteor.bindEnvironment.
I am new to Node and don't completely understand the concept of Fibers. My understanding is that they're analogous to threads in C/C++/every language with threading, but I don't understand what the implications extending to my server-side code are/why my code is throwing an error when trying to insert to a collection. Can anyone explain this to me?
Thank you.
You're using bindEnvironment slightly incorrectly. Because where its being used is already in a fiber and the callback that comes off the Knox client isn't in a fiber anymore.
There are two use cases of bindEnvironment (that i can think of, there could be more!):
You have a global variable that has to be altered but you don't want it to affect other user's sessions
You are managing a callback using a third party api/npm module (which looks to be the case)
Meteor.bindEnvironment creates a new Fiber and copies the current Fiber's variables and environment to the new Fiber. The point you need this is when you use your nom module's method callback.
Luckily there is an alternative that takes care of the callback waiting for you and binds the callback in a fiber called Meteor.wrapAsync.
So you could do this:
Your startup function already has a fiber and no callback so you don't need bindEnvironment here.
Meteor.startup(function () {
if (Projects.find().count() === 0) {
insertRecords();
}
});
And your insert records function (using wrapAsync) so you don't need a callback
function insertRecords() {
console.log("inserting...");
var client = Knox.createClient({
key: apikey,
secret: secret,
bucket: 'profile-testing'
});
client.listSync = Meteor.wrapAsync(client.list.bind(client));
console.log("created client");
try {
var data = client.listSync({ prefix: 'projects' });
}
catch(e) {
console.log(e);
}
if(!data) return;
for (var i = 1; i < data.Contents.length; i++) {
console.log(data.Contents[i].Key);
if (data.Contents[i].Key.split('/').pop() == "") {
Projects.insert({ name: data.Contents[i].Key, contents: [] });
} else if (data.Contents[i].Key.split('.').pop() == "jpg") {
Projects.update( { name: data.Contents[i].Key.substr(0,
data.Contents[i].Key.lastIndexOf('.')) },
{ $push: {contents: data.Contents[i].Key}} );
} else {
console.log(data.Contents[i].Key.split('.').pop());
}
}
});
A couple of things to keep in mind. Fibers aren't like threads. There is only a single thread in NodeJS.
Fibers are more like events that can run at the same time but without blocking each other if there is a waiting type scenario (e.g downloading a file from the internet).
So you can have synchronous code and not block the other user's events. They take turns to run but still run in a single thread. So this is how Meteor has synchronous code on the server side, that can wait for stuff, yet other user's won't be blocked by this and can do stuff because their code runs in a different fiber.
Chris Mather has a couple of good articles on this on http://eventedmind.com
What does Meteor.wrapAsync do?
Meteor.wrapAsync takes in the method you give it as the first parameter and runs it in the current fiber.
It also attaches a callback to it (it assumes the method takes a last param that has a callback where the first param is an error and the second the result such as function(err,result).
The callback is bound with Meteor.bindEnvironment and blocks the current Fiber until the callback is fired. As soon as the callback fires it returns the result or throws the err.
So it's very handy for converting asynchronous code into synchronous code since you can use the result of the method on the next line instead of using a callback and nesting deeper functions. It also takes care of the bindEnvironment for you so you don't have to worry about losing your fiber's scope.
Update Meteor._wrapAsync is now Meteor.wrapAsync and documented.

Node JS : Requesting for code snippet explanation

I am just curious to know what is the purpose of,
function MyStream() {
events.EventEmitter.call(this);
}
in the following block of code taken from Nodejs.org Docs section,
var util = require("util");
var events = require("events");
function MyStream() {
events.EventEmitter.call(this);
}
util.inherits(MyStream, events.EventEmitter);
MyStream.prototype.write = function(data) {
this.emit("data", data);
}
var stream = new MyStream();
console.log(stream instanceof events.EventEmitter); // true
console.log(MyStream.super_ === events.EventEmitter); // true
stream.on("data", function(data) {
console.log('Received data: "' + data + '"');
})
stream.write("It works!"); // Received data: "It works!"
Please explain.
This isn't the first time this question has been asked: https://groups.google.com/forum/#!topic/nodejs/ab_Xih1L5R8/discussion
Quote from Bradley Meck:
Using EventEmitter.call on an object will do the setup of instance methods / properties (not inherited) of an EventEmitter. It is similar in purpose to super(...) in Java or base(...) in C#, but it is not implicit in Javascript. Because of this, we must manually call it ourselves. As for the talk about util.inherits, this will make the MyStream function inherit from another prototyped function so that instanceof works (side note: javascript only allows single inheritance). Due to how the new operator works, if we have the this variable in a function set to an instanceof EventEmitter, and call EventEmitter.call it look for all intents and purposes as if EventEmitter's constructor is being called on our MyStream object.

http.Clientrequest.abort() ends program

I'm having some issues using Node.js as a http client against an existing long polling server. I'm using 'http' and 'events' as requires.
I've created a wrapper object that contains the logic for handling the http.clientrequest. Here's a simplified version of the code. It works exactly as expected. When I call EndMe it aborts the request as anticipated.
var http = require('http');
var events = require('events');
function lpTest(urlHost,urlPath){
this.options = {
host: urlHost,
port: 80,
path: urlPath,
method: 'GET'
};
var req = {};
events.EventEmitter.call(this);
}
lpTest.super_ = events.EventEmitter;
lpTest.prototype = Object.create(events.EventEmitter.prototype, {
constructor: {
value: lpTest,
enumerable: false
}
});
lpTest.prototype.getData = function getData(){
this.req = http.request(this.options, function(res){
var httpData = "";
res.on('data', function(chunk){
httpData += chunk;
});
res.on('end', function(){
this.emit('res_complete', httpData);
}
};
}
lpTest.prototype.EndMe = function EndMe(){
this.req.abort();
}
module.exports = lpTest;
Now I want to create a bunch of these objects and use them to long poll a bunch of URL's. So I create an object to contain them all, generate each object individually, initiate it, then store it in my containing object. This works a treat, all of the stored long-polling objects fire events and return the data as expected.
var lpObject = require('./lpTest.js');
var objWatchers = {};
function DoSomething(hostURL, hostPath){
var tempLP = new lpObject(hostURL,hostPath);
tempLP.on('res_complete', function(httpData){
console.log(httpData);
this.getData();
});
objWatchers[hosturl + hostPath] = tempLP;
}
DoSomething('firsturl.com','firstpath');
DoSomething('secondurl.com','secondpath);
objWatchers['firsturl.com' + 'firstpath'].getData();
objWatchers['secondurl.com' + 'secondpath'].getData();
Now here's where it fails... I want to be able to stop a long-polling object while leaving the rest going. So naturally I try adding:
objWatchers['firsturl.com' + 'firstpath'].EndMe();
But this causes the entire node execution to cease and return me to the command line. All of the remaining long-polling objects, that are happily doing what they're supposed to do, suddenly stop.
Any ideas?
Could it have something to do with the fact that you are only calling getData() when the data is being returned?
Fixed code:
function DoSomething(hostURL, hostPath){
var tempLP = new lpObject(hostURL,hostPath);
tempLP.on('res_complete', function(httpData){
console.log(httpData);
});
tempLP.getData();
objWatchers[hosturl + hostPath] = tempLP;
}
I have seemingly solved this, although I'm note entirely happy with how it works:
var timeout = setTimeout(function(){
objWatchers['firsturl.com' + 'firstpath'].EndMe();
}, 100);
By calling the closing function on the object after a delay I seem to be able to preserve the program execution. Not exactly ideal, but I'll take it! If anyone can offer a better method please feel free to let me know :)

Resources