I have a worker class that has lots of different utility methods, and emits error messages when there's a problem anywhere in the execution. Due to NodeJS magic, error messages are special, and if nothing is listening to them, they're turned into thrown Errors, so I currently do:
var myWorkerFunction = function(input) {
var w = myFactory();
try {
w.dothis();
w.dothat(input);
w.hokeypokey();
return w.finalize();
} catch(e) {
return false;
}
}
What I wonder though, is it possible to avoid the try/catch block entirely? NodeJS's documentation seems to indicate it's best to avoid not re-throwing a caught exception (I'm doing no checks here to see that it really was my worker's logic that threw the exception and not a critical fault from Node).
So I'd like to do something like:
var myWorkerFunction = function(input) {
var w = myFactory();
w.on('error', function() {
// How to tell caller of myWorkerFunction() I failed,
// and stop the rest of the myWorkerFunction function? (return false)
});
w.dothis();
w.dothat(input);
w.hokeypokey();
return w.finalize();
}
But how to trigger that "return" for myWorkerFunction inside that event listener function? I could have myWorkerFunction emit an "error" message too, but that just kicks the can to the next layer, and doesn't stop the execution of the worker script (i.e. if dothis() fails, don't keep going and call dothat(input)). Is there a programming pattern for situations like this?
EDIT: The one solution I could come up with is something like:
var myWorkerFunction = function(input) {
var w = myFactory();
var hasFailed = false;
w.on('error', function() {
hasFailed = true;
});
w.dothis();
if (hasFailed) return fa;se
w.dothat(input);
if (hasFailed) return false;
w.hokeypokey();
if (hasFailed) return false;
var out = w.finalize();
if (hasFailed) return false;
return out;
}
Which is not very elegant having to constantly check if we've failed or not before every line of code.
A couple thoughts on this. First you might want to take a look at Caolan McMahon's fine async nodejs library because it feels like you're reinventing flow control that would be handled quite well with 'async.waterfall'.
If you don't feel it's a good fit you - following the pattern you've adopted you could opt to post events on completion of each step (see the 'done' events in the sample below). This would allow you to listen for those events as indication that no error has occurred and that it makes sense to move onto the next step. Let 'finalize' take an optional error to allow your worker to differentiate normal vs abnormal completion. This is not tested but here's what something like that might look like:
var myWorkerFunction = function(input) {
var w = myFactory();
w.on('error', function(error) {
// Handle error, log whatever - and finalize with an error?
w.finalize(error);
});
w.on('doneWithThis', function() {
w.dothat(input);
});
w.on('doneWithThat', function() {
w.hokeypokey();
});
w.on('doneWithHokeypokey', function() {
w.finalize();
});
w.dothis(); //starts the ball rolling.
}
Related
So I'm working with websockets to process data from website's API. For every new event I also send some http requests back to the website in order to obtain more data. Up untill now everything has worked fine, but now that I started using async requests to speed it up a bit things got a bit different. My code used to process one event and then move on to the next one (these events come in extremely quick - around 10 per second) but now it just seems to ignore the async (non blocking) part and move on to the next event and that way it just skips over half of the code. Note that the code works fine outside the Pusher. I'm using the 'pusher-client' module. My code looks like this:
var Request = require("request");
var requestSync = require('sync-request');
var Pusher = require('pusher-client');
var events_channel = pusher.subscribe('inventory_changes');
events_channel1.bind('listed', function(data)
{
var var2;
//Async request (to speed up the code)
function myFunction(callback){
request("url", function(error, response, body) {
if (!error && response.statusCode == 200)
{
result = JSON.stringify(JSON.parse(body));
return callback(null, result);
}
else
{
return callback(error, null);
}
});
}
myFunction(function(err, data){
if(!err)
{
var2 = data
return(data);
}
else
{
return(err);
}
});
//The part of the code below waits for the callback and the executes some code
var var1 = var2;
check();
function check()
{
if(var2 === var1)
{
setTimeout(check, 10);
return;
}
var1 = var2;
//A CHUNK OF CODE EXECUTES HERE (connected to the data from the callback)
}
});
In conclusion the code works, but not inside the pusher due to the pusher skipping the asynchronous request. How would I make the pusher wait for my async request to finish, before processing the next event (I have no idea)? If you happen to know, please let me know :)
You need to implement a queue to handle events one after another. I'm curious how it worked before, even without Pusher you'd have to implement some queue mechanism for it.
const eventsQueue = []
events_channel1.bind('listed', function(data) {
eventsQueue.push(data)
handleNewEvent()
})
let processingEvent = false
function handleNewEvent() {
if (processingEvent) return // do nothing if already processing an event
processingEvent = true
const eventData = eventsQueue.shift() // pick the first element from array
if (!eventData) return // all events are handled at the moment
... // handle event data here
processingEvent = false
handleNewEvent() // handle next event
}
Also, you should call clearTimeout method to clear your timeout when you don;t need it anymore.
And it's better to use promises or async/await instead of callbacks. Your code will be much easier to read and maintain.
I'm trying to implement and existing solution in node.js, specifically, using express.js framework. Now, the existing solution works as follows:
server exposes a GET service that clients can connect to
when a client calls the GET service, the client number increments (a global variable) and then the number of clients is checked;
if there are not at least 3 clients connected, the service is in endless loop, waiting for other clients to connect
if (or rather, when) the rest of the two clients connect, the service sends respond to everyone that enough clients are connected (a 'true' value).
So what basically happens is, the client connects and the connection is active (in a loop) until enough clients connect, then and only then there is a response (to all clients at the same time).
Now I'm not expert in these architectures, but from what I think, this is not a correct or good solution. My initial thought was: this must be solved with sockets. However, since the existing solution works like that (it's not written in node.js), I tried to emulate such behaviour:
var number = (function(){
var count = 0;
return {
increase: function() {
count++;
},
get: function(){
return count;
}
};
})();
app.get('/test', function(req, res){
number.increase();
while (number.get() < 3) {
//hold it here, until enough clients connect
}
res.json(number.get());
});
Now while I think that this is not a correct solution, I have a couple of questions:
Is there any alternative to solving this issue, besides using sockets?
Why does this "logic" work in C#, but not in express.js? The code above hangs, no other request is processed.
I know node.js is single-threaded, but what if we have a more conventional service that responds immediately, and there are 20 requests all at the same time?
I would probably use an event emitter for this:
var EventEmitter = require('events').EventEmitter;
var emitter = new EventEmitter();
app.get('/', function(req, res) {
// Increase the number
number.increase();
// Get the current value
var current = number.get();
// If it's less than 3, wait for the event emitter to trigger.
if (current < 3) {
return emitter.once('got3', function() {
return res.json(number.get());
});
}
// If it's exactly 3, emit the event so we wake up other listeners.
if (current === 3) {
emitter.emit('got3');
}
// Fall through.
return res.json(current);
});
I would like to stress that #Plato is correct in stating that browsers may timeout when a response takes too much time to complete.
EDIT: as an aside, some explanation on the return emitter.once(...).
The code above can be rewritten like so:
if (current < 3) {
emitter.once('got3', function() {
res.json(number.get());
});
} else if (current === 3) {
emitter.emit('got3');
res.json(number.get());
} else {
res.json(number.get());
}
But instead of using those if/else statements, I return from the request handler after creating the event listener. Since request handlers are asynchronous, their return value is discarded, so you can return anything (or nothing). As an alternative, I could also have used this:
if (current < 3) {
emitter.once(...);
return;
}
if (current === 3) {
...etc...
Also, even though you return from the request handler function, the event listener is still referencing the res variable, so the request handler scope is maintained by Node until res.json() in the event listener callback is called.
Your http approach should work
You are blocking the event loop so node refuses to do any other work while it is in the while loop
You're really close, you just need to check every now and then instead of constantly. I do this below with process.nextTick() but setTimeout() would also work:
var number = (function(){
var count = 0;
return {
increase: function() {
count++;
},
get: function(){
return count;
}
};
})();
function waitFor3(callback){
var n = number.get();
if(n < 3){
setImmediate(function(){
waitFor3(callback)
})
} else {
callback(n)
}
}
function bump(){
number.increase();
console.log('waiting');
waitFor3(function(){
console.log('done');
})
}
setInterval(bump, 2000);
/*
app.get('/test', function(req, res){
number.increase();
waitFor3(function(){
res.json(number.get());
})
});
*/
Sorry if the question title is a tad ambiguous, but I'm not entirely sure how to word it.
I'm writing an NPM module that talks to a json-rpc api - this is the current setup.
// The module
function MyModule(config) {
// do some connection stuff here
connected = true
}
MyModule.prototype.sendCommand = function() {
if(connected) {
// do command
} else {
// output an error
}
}
module.exports = MyModule;
// The script interacting with the module
var MyModule = require('./MyModule');
var config = {
// config stuff
};
var mod = new MyModule(config);
var mod.sendCommand;
The command won't send, as at this point it hasn't connected, I assume this is due to NodeJS' asynchronous, non-blocking architecture and that I perhaps need to use promises to wait for a response from the API, where would I implement this? Do I do it in my module or do I do it in the script interacting with the module?
You will need to use either a callback or promises or something like that to indicate when the connection is complete so you can then use the connection in further code that is started via that callback.
Though it is generally not considered the best practice to do asynchronous stuff in a constructor, it can be done:
function MyModule(config, completionCallback) {
// do some connection stuff here
connected = true
completionCallback(this);
}
var mod = new MyModule(config, function(mod) {
// object has finished connecting
// further code can run here that uses the connection
mod.sendCommand(...);
});
A more common design pattern is to not put the connecting in the constructor, but to add a method just for that:
function MyModule(config) {
}
MyModule.prototype.connect = function(fn) {
// code here that does the connection and calls
// fn callback when connected
}
var mod = new MyModule(config);
mod.connect(function() {
// object has finished connecting
// further code can run here that uses the connection
mod.sendCommand(...);
});
don't use promises, use node's programming model where you don't "call functions" but you "call functions with a result handler for dealing with the data once it's actually available":
MyModule.prototype.sendCommand = function(handler) {
if(connected) {
// run stuff, obtain results, send that on:
handler(false, result);
} else {
// output an error, although really we should
// just try to connect if we're not, and say
// there's an error only when it actually fails.
handler(new Error("ohonoes"));
}
}
and then you call the function as
var MyModule = require('./MyModule');
var mod = ...
mod.sendCommand(function(err, result) {
// we'll eventually get here, at which point:
if (err) { return console.error(err); }
run();
more();
code();
withResult(result);
});
I am having difficulty using Fibers/Meteor.bindEnvironment(). I tried to have code updating and inserting to a collection if the collection starts empty. This is all supposed to be running server-side on startup.
function insertRecords() {
console.log("inserting...");
var client = Knox.createClient({
key: apikey,
secret: secret,
bucket: 'profile-testing'
});
console.log("created client");
client.list({ prefix: 'projects' }, function(err, data) {
if (err) {
console.log("Error in insertRecords");
}
for (var i = 0; i < data.Contents.length; i++) {
console.log(data.Contents[i].Key);
if (data.Contents[i].Key.split('/').pop() == "") {
Projects.insert({ name: data.Contents[i].Key, contents: [] });
} else if (data.Contents[i].Key.split('.').pop() == "jpg") {
Projects.update( { name: data.Contents[i].Key.substr(0,
data.Contents[i].Key.lastIndexOf('.')) },
{ $push: {contents: data.Contents[i].Key}} );
} else {
console.log(data.Contents[i].Key.split('.').pop());
}
}
});
}
if (Meteor.isServer) {
Meteor.startup(function () {
if (Projects.find().count() === 0) {
boundInsert = Meteor.bindEnvironment(insertRecords, function(err) {
if (err) {
console.log("error binding?");
console.log(err);
}
});
boundInsert();
}
});
}
My first time writing this, I got errors that I needed to wrap my callbacks in a Fiber() block, then on discussion on IRC someone recommending trying Meteor.bindEnvironment() instead, since that should be putting it in a Fiber. That didn't work (the only output I saw was inserting..., meaning that bindEnvironment() didn't throw an error, but it also doesn't run any of the code inside of the block). Then I got to this. My error now is: Error: Meteor code must always run within a Fiber. Try wrapping callbacks that you pass to non-Meteor libraries with Meteor.bindEnvironment.
I am new to Node and don't completely understand the concept of Fibers. My understanding is that they're analogous to threads in C/C++/every language with threading, but I don't understand what the implications extending to my server-side code are/why my code is throwing an error when trying to insert to a collection. Can anyone explain this to me?
Thank you.
You're using bindEnvironment slightly incorrectly. Because where its being used is already in a fiber and the callback that comes off the Knox client isn't in a fiber anymore.
There are two use cases of bindEnvironment (that i can think of, there could be more!):
You have a global variable that has to be altered but you don't want it to affect other user's sessions
You are managing a callback using a third party api/npm module (which looks to be the case)
Meteor.bindEnvironment creates a new Fiber and copies the current Fiber's variables and environment to the new Fiber. The point you need this is when you use your nom module's method callback.
Luckily there is an alternative that takes care of the callback waiting for you and binds the callback in a fiber called Meteor.wrapAsync.
So you could do this:
Your startup function already has a fiber and no callback so you don't need bindEnvironment here.
Meteor.startup(function () {
if (Projects.find().count() === 0) {
insertRecords();
}
});
And your insert records function (using wrapAsync) so you don't need a callback
function insertRecords() {
console.log("inserting...");
var client = Knox.createClient({
key: apikey,
secret: secret,
bucket: 'profile-testing'
});
client.listSync = Meteor.wrapAsync(client.list.bind(client));
console.log("created client");
try {
var data = client.listSync({ prefix: 'projects' });
}
catch(e) {
console.log(e);
}
if(!data) return;
for (var i = 1; i < data.Contents.length; i++) {
console.log(data.Contents[i].Key);
if (data.Contents[i].Key.split('/').pop() == "") {
Projects.insert({ name: data.Contents[i].Key, contents: [] });
} else if (data.Contents[i].Key.split('.').pop() == "jpg") {
Projects.update( { name: data.Contents[i].Key.substr(0,
data.Contents[i].Key.lastIndexOf('.')) },
{ $push: {contents: data.Contents[i].Key}} );
} else {
console.log(data.Contents[i].Key.split('.').pop());
}
}
});
A couple of things to keep in mind. Fibers aren't like threads. There is only a single thread in NodeJS.
Fibers are more like events that can run at the same time but without blocking each other if there is a waiting type scenario (e.g downloading a file from the internet).
So you can have synchronous code and not block the other user's events. They take turns to run but still run in a single thread. So this is how Meteor has synchronous code on the server side, that can wait for stuff, yet other user's won't be blocked by this and can do stuff because their code runs in a different fiber.
Chris Mather has a couple of good articles on this on http://eventedmind.com
What does Meteor.wrapAsync do?
Meteor.wrapAsync takes in the method you give it as the first parameter and runs it in the current fiber.
It also attaches a callback to it (it assumes the method takes a last param that has a callback where the first param is an error and the second the result such as function(err,result).
The callback is bound with Meteor.bindEnvironment and blocks the current Fiber until the callback is fired. As soon as the callback fires it returns the result or throws the err.
So it's very handy for converting asynchronous code into synchronous code since you can use the result of the method on the next line instead of using a callback and nesting deeper functions. It also takes care of the bindEnvironment for you so you don't have to worry about losing your fiber's scope.
Update Meteor._wrapAsync is now Meteor.wrapAsync and documented.
Rather than copy and pasting my code onto here, I have uploaded it to github. The RequireJS module does have a dependency on jquery.signalr and in tern has a dependency on jquery but also have a dependency on the javascript held in /signalr/hubs. There is a bit of config to do with Require.Config.
Basically what is happening is on the first time you load the page the connection is made to the hubs within signalr and the "server side" code is executed and does the desired thing. When you refresh the page it does not. All client side code is called, so for example:
var myViewModel = new MyViewMode();
myViewModel.init();
and within your init method you have
var connection = $.connection.myHub;
this.init = function() {
connection.server.myMethod();
}
this would then go off to
public MyHub : Hub
{
public void MyMethod()
{
Client.Request.populateSomeInformation() // I think it's request but I'm doing this from memory!
}
}
and then call
connection.client.populateSomeInformation = function () { .. )
but doesn't call this :(
It looks like a connection has been made (using the good old console.log() to see what it outputs) and indeed debugging the project it executes the code within the hub but there is no response made back to the javascript.
So wonderful people of the internet, where am I going wrong? Do I need to check the state of $.connection.hub.start(); before attempting to start it again?
Time for beer :)
I believe it should be
connection.client.populateSomeInformation = function () { .. )
(not connection.server)
http://www.asp.net/signalr/overview/hubs-api/hubs-api-guide-javascript-client#callclient
(observations on the code you have on github right now)
var isLoaded = false;
// ... some code that doesn't change isLoaded ...
if (isLoaded == false) {
scrollIntervalId = window.setInterval(function () {
signalRLoaded();
}, 30);
}
I think isLoaded will always be false at this point. Not sure what you intended this to accomplish.
var connection = $.connection.hub.start();
I don't think you're supposed to open the connection before defining any client functions. I don't see any client functions being defined here, so maybe you're doing that somewhere else? I don't know if it really matters other than if the server attempts to call a client function that hasn't yet been defined...
function SignalRReady(callback) {
if (isLoaded) {
callback(connection);
} else {
readyCalls = callback;
}
return SignalRReady;
}
SignalRReady.version = "1.0.0";
SignalRReady.load = function(name, request, onLoad, config) {
if (config.isBuild) {
onLoad();
} else {
SignalRReady(onLoad);
}
};
return SignalRReady;
I'm confused by this bit of code, probably because I don't see how it's being used. Is this an attempt at a kind of singleton? I see that SignalRReady is the "class" being returned for the module. You're not really returning an object, you're returning a constructor which implies that you're instantiating it in other places, something like
define(['SignalRReady'], function(sigR)
{
var srr = new sigR();
});
But then you have that load function defined that calls the constructor and makes this look all weird. How are you using this?
Anyways, I'm thinking you might be hitting some kind of race condition where the client function may not always be available at the time the server is trying to call it.
(additional comments/code 2013-09-06)
Your connection object is actually a jQuery promise ( http://api.jquery.com/category/deferred-object/ ).
If you're unfamiliar with promises, think of them generically as a queue of callbacks to be executed later. In this case, when connected, all the callbacks will be executed (in the order they were added). If a callback is added after being connected, it will get executed immediately. This is how your code is working now. You add the callback to the .done queue after the connection is made and is executed immediately.
If you insist on creating the connection object yourself, then you do not need to use the stateChanged event. You just add the callback to the .done queue:
define(function()
{
function signalRReady(callback)
{
if (window.connection == undefined) {
window.connection = $.connection.hub.start();
}
window.connection.done(callback);
}
signalRReady.version = "1.0.0";
return signalRReady;
});
However, I believe it's not a good idea to initiate the connection yourself. Because your module isn't a complete wrapper around SignalR such that people would only use your module to do SignalR stuff, you are not guaranteed (and cannot expect) other code will not initiate the connection. Especially if someone is adding your module to an existing codebase.
Your module is simply adding a new event, so keep it simple. Take the callback and execute it yourself when appropriate:
define(function()
{
function signalRReady(callback)
{
$.connection.hub.stateChanged(function (state)
{
if(state.newState === $.signalR.connectionState.connected)
{
callback();
}
});
}
signalRReady.version = "1.0.0";
return signalRReady;
});
Nowadays, promises are pretty popular. You might want to implement a promise-based module like:
define(function()
{
var deferred = $.Deferred();
$.connection.hub.stateChanged(function (state)
{
if(state.newState === $.signalR.connectionState.connected)
{
// executes all callbacks attached by the "ready" function below
deferred.resolve();
}
});
return {
ready: function(callback)
{
deferred.done(callback);
},
version: "1.0.0"
};
});
If callbacks are attached after the connection has been made, they are executed immediately.
Also, notice this example module's init function returns an object instead of a function. Since RequireJS will pass the same instance around to any module that requires it, state is maintained - we can use local variables instead of global.