Nodejs debug errors in production - node.js

I have a nodejs script running in production
I unlikely (once in thousand times) get errors like this:
TypeError: value is out of bounds
at checkInt (buffer.js:1009:11)
at Buffer.writeUInt16LE (buffer.js:1067:5)
at Object.foo.bar (/fake/path/name.js:123:1);
at Object.foo.bar2 (/fake/path/name2.js:123:1);
at Object.foo.bar3 (/fake/path/name3.js:123:1);
Causing the production server to crash...
Great I have a stacktrace! But I want to know what it's current data is for each call or all the data?
What are some great tools or code to use to for error logging(with it's current data) on production code?

I highly recommend in using either Winston or Bunyan. The selection of npm package is a decision of your application.
You can benchmark the available npm packages by going through the stats in the available in npm pages. The stats are basically the following.
downloads in the last day
downloads in the last week
downloads in the last month
open issues and open pull requests.
Having higher number of downloads recently will indicate that there is a great support for the module you are using in the long run. So that is important.
Both Winstan and Bunyan being best logging npm packages in the market, the main difference is that, Winstan is really awesome and flexible for normal logging purposes. Of course Winstan offers a great deal of logging capabilities. But how ever, to make use of these capabilities, some effort need to put on compared to Bunyan.
Bunyan on the other hand is specially support the fact of "analysing logs". So basically Bunyan is for logs processing. So if you want to analyse your logs, log files it is highly recommend to use Bunyan. Tweaking logs with Bunyan is fairly easy vis-a-vis to Winstan.
I did a thorough comparison between Bunyan and Winstan. Please do check the link below to view how the Winstan and Bunyan can use depending on the scope, use-case and necessity of the logging to the Node application.
link : https://docs.google.com/document/d/1pD9PLyxlcHVxxOvserNLO9tAz-QA_Co-xo6cWLhLghc/edit?usp=sharing
Also in the Production environment, make sure to use the logging levels wisely. The mostly used logging levels in production environment are:
error
info
debug

Yoou can use Winston or Pino
With winston you can load many modules for log where you want and maybe store logs online. I never use pino, but i have read good things about it.
Set env variables for choose where output your log, for example you wont to show output on stdout only if you are in develop and store online only if the app is in production.

A good way to handle the asynchronous functions in node.js is by using the decofun debug tool.
The main feature of it is to parse the code and the names of the anonymous functions according to their context.
You can deanonymise any anonymous function by running it with deco filename.js
A simple example would be as mentioned in the documentation
function gravy() {
return function returnedᅠfromᅠgravyᅠㅣlineᅠ2 () {
return {
prop: function asᅠpropertyᅠpropᅠㅣlineᅠ4 () {
setTimeout(function passedᅠintoᅠsetTimeoutᅠㅣlineᅠ5 () {
console.trace('Getting a trace...');
}, 10)
}
}
}
}
Trace: Getting a trace...
at passedᅠintoᅠsetTimeoutᅠㅣlineᅠ5 [as _onTimeout] (/home/ubuntu/workspace/node_modules/decofun/examples/loadable/index.js:6:22)
at Timer.listOnTimeout (timers.js:92:15)
Since it comes with embedded cute-stack library which normalises the path to the current directory
By applying the command deco examples/loadable --cute table
the output would be shown as
The best thing i like about it is that it transforms the functions based upon their calls to the original one as seen in the example from this
function one (a, cb) {
}
one('blah', function () {
})
function two () {
return function () { }
}
function three () {
return {
shoe: function () {}
}
}
function four () {
return function () {
return function () {
}
}
}
function five () {
return function () {
return function () {
return function () {
foo('blue', function () {
})
}
}
}
}
var six = function () {
}
var seven = function (err, cb) {
return function () {
cb(function () {
})
}
}
var o = {};
o.eight = function (cb) { }
o.eight(function () { })
o.eight.nine = function () {}
o.eight.nine(function () { })
var o2;
o2 = function () { }
;(function () {}())
!function () { }()
function toodeep () {
return function () {
return function () {
return function () {
return function () {
return function () {
return function () {
return function () {
return function () {
return function () {
return function () {
}
}
}
}
}
}
}
}
}
}
}
into this
function one (a, cb) {
}
one('blah', function passedᅠintoᅠoneᅠㅣlineᅠ6 () {
})
function two () {
return function returnedᅠfromᅠtwoᅠㅣlineᅠ11 () { }
}
function three () {
return {
shoe: function asᅠpropertyᅠshoeᅠㅣlineᅠ17 () {}
}
}
function four () {
return function returnedᅠfromᅠfourᅠㅣlineᅠ22 () {
return function returnedᅠfromᅠᐸᅠreturnedᅠfromᅠfourᅠᐳᅠㅣlineᅠ23 () {
}
}
}
function five () {
return function returnedᅠfromᅠfiveᅠㅣlineᅠ30 () {
return function returnedᅠfromᅠᐸᅠreturnedᅠfromᅠfiveᅠᐳᅠㅣlineᅠ31 () {
return function returnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠfiveᅠᐳᅠᐳᅠㅣlineᅠ32 () {
foo('blue', function passedᅠintoᅠfooᅠㅣlineᅠ33 () {
})
}
}
}
}
var six = function asᅠvarᅠsixᅠㅣlineᅠ42 () {
}
var seven = function asᅠvarᅠsevenᅠㅣlineᅠ47 (err, cb) {
return function returnedᅠfromᅠᐸᅠasᅠvarᅠsevenᅠᐳᅠㅣlineᅠ49 () {
cb(function passedᅠintoᅠcbᅠㅣlineᅠ50 () {
})
}
}
var o = {};
o.eight = function asᅠpropertyᅠeightᅠㅣlineᅠ58 (cb) { }
o.eight(function passedᅠintoᅠoːeightᅠㅣlineᅠ61 () { })
o.eight.nine = function asᅠpropertyᅠnineᅠㅣlineᅠ63 () {}
o.eight.nine(function passedᅠintoᅠeightːnineᅠㅣlineᅠ64 () { })
var o2;
o2 = function asᅠvarᅠo2ᅠㅣlineᅠ68 () { }
;(function IIFEᅠㅣlineᅠ71 () {}())
!function IIFEᅠㅣlineᅠ73 () { }()
function toodeep () {
return function returnedᅠfromᅠtoodeepᅠㅣlineᅠ78 () {
return function returnedᅠfromᅠᐸᅠreturnedᅠfromᅠtoodeepᅠᐳᅠㅣlineᅠ79 () {
return function returnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠtoodeepᅠᐳᅠᐳᅠㅣlineᅠ80 () {
return function returnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠtoodeepᅠᐳᅠᐳᅠᐳᅠㅣlineᅠ82 () {
return function returnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠtoodeepᅠᐳᅠᐳᅠᐳᅠᐳᅠㅣlineᅠ83 () {
return function returnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠtoodeepᅠᐳᅠᐳᅠᐳᅠᐳᅠᐳᅠㅣlineᅠ84 () {
return function returnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠᐸᅠreturnedᅠfromᅠtoodeepᅠᐳᅠᐳᅠᐳᅠᐳᅠᐳᅠᐳᅠㅣlineᅠ86 () {
return function () {
return function () {
return function () {
}
}
}
}
}
}
}
}
}
}
}
Hope this might help a bit! Cheers!

I am using pm2 which is a process manager for node.js and also rollbar error reporting. I think you should define some metrics for the part of your code which this error comes from.

For any uncaughtException the server will stop in order to make the server keep on running even when there is an uncaught exception what i have done is created a separate collection for storing error, save error once an uncaught exception occurs and returns.
Collection
var ErrorSchema = new mongoose.Schema({
err_Message:{type:String},
err_Stack:{type:String},
date:{type:Date}
});
Controller
process.on('uncaughtException', function (err) {
console.log(err);
console.error((new Date).toUTCString() + ' uncaughtException:', err.message);
console.error(err.stack);
var newError = new Error;
newError.err_Message = err.message;
newError.err_Stack = err.stack;
newError.date = moment();
newError.save(function(saveErr,errData){
if(!saveErr)
console.log('New Error is saved');
else
console.log('Error in saving error');
});
//process.exit(1)
});
The above methods stores the uncaught exception in the Error collection and the process/server does not stops.
Hope this helps.

Related

Awaiting for all the callbacks to return before HttpsCallableResult completes

I am using Firebase Functions with Unity. The main function returns before the database functions finish. I am still new to Node.js and I am still trying to get my head around all the Async Callback stuff.
I have tried CallAsync, ContinueWith, and Coroutines, but the function always continues after the first return (I use Task.isCompleted() to check for that).
My Node.js functions are something like this:
exports.my_fn = functions.https.onCall((data, context) => {
dbsessions.child(id).once("value").then(function(snapshot) {
if (snapshot.val()) {
Create();
} else {
Move(session);
}});
});
function Move(session) {
if (session["error"]) {
return return_stuff;
} else {
if (some_condition) {
dbsessions.child(id).set(sson, function(set_error) {
if (set_error) {
return return_stuff;
} else {
return return_stuff;
}
});
} else {
dbaimoves.child(stt).child(dif).once("value").then(function(snapshot) {
if (snapshot.val()) {
return return_stuff;
} else {
if (!first) {
dbsessions.child(id).set(sson, function(set_error) {
if (set_error) {
return return_stuff;
} else {
return return_stuff;
}
});
} else {
return return_stuff;
}
}
}, function(errorObject) {
if (errorObject) {
return return_stuff;
}
});
}}}
var Create = function(data, callback) {
dbdifficulty.child(data).once("value").then(function(snapshot) {
if (snapshot.val()) {
return callback();
} else {
dbsessions.child(data.id).set(data, function(set_error) {
if (set_error) {
return callback();
} else {
return callback();
}});
}});
}
(I skipped unnecessary data to keep the question simple). It is basically nested returns and database operations, callbacks and functions call each other.
My C# Unity code is something like this:
private async Task<string> AddMessageAsync(string text)
{
// Create the arguments of the callable function.
var data = new Dictionary<string, string>();
data["s"] = text;
data["d"] = "0";
var function = func.GetHttpsCallable("my_fn");
var Tfn = function.CallAsync(data);
var TRes = await Tfn;
if (Tfn.IsCompleted)
{
JFunc result = JsonUtility.FromJson<JFunc>(TRes.Data.ToString());
Debug.Log("error:" + result.error);
return result.move;
}
return "error";
}
The codes above resemble my actual code, which calls the function from Unity, the function runs on Firebase and returns shortly (before it goes into Create() or Move()), Unity receives the result (null). A few seconds later the function finishes successfully on Firebase, but Unity does not receive anything about that (or maybe it does, but I can't handle it properly).
I need to know:
how to make the main function return what the other functions return, and
how to make C# wait and keep listening to the returned values, instead of thinking the task has completed after the first return. It would be even better if I can only return only when the result is ready.
To make the Cloud Functions code return a value, make sure that each function returns a value or a promise. Promises "bubble up" meaning that the value you return from the most-nested code will be return to the top-level, as long as you have a return on each level.
So in your code from a quick scan, you need:
exports.my_fn = functions.https.onCall((data, context) => {
return dbsessions.child(id).once("value").then(function(snapshot) {
if (snapshot.val()) {
return Create();
} else {
return Move(session);
}});
});
var Create = function(data, callback) {
return dbdifficulty.child(data).once("value").then(function(snapshot) {
if (snapshot.val()) {
return callback();
} else {
return dbsessions.child(data.id).set(data, function(set_error) {
if (set_error) {
return callback();
} else {
return callback();
}});
}});
}
I've only instrumented the top-level my_fn and Create here, to show what to do. You'll have to do the same for Move yourself.

jest mocking functions without passing as a callback

I have some code like:
module.exports = {
idCheck: function(errors) {
errors.some( (error) => {
if (error.parentSchema.regexp === '/^((?!\\bMyId\\b).)*$/i') {
this._recordError('IDCHECK');
}
});
}
};
I am trying to test it using jest with this:
const IDCheck = require(
'./IDCheck'
);
let errors = [
{
parentSchema: {
regexp: '/^((?!\\bMyId\\b).)*$/i'
}
}
];
describe('IDCheck', () => {
afterEach(() => {
jest.restoreAllMocks();
});
it('calls _recordError with IDCHECK', () => {
jest.spyOn(this, '_recordError');
IDCheck.idCheck(errors);
});
});
however, when running jest, I get
Cannot spy the _recordError property because it is not a function; undefined given instead
Is there a way of mocking, testing for _recordError() having been called, or not called and with the correct parameter, without passing _recordError through as a parameter?
A few things about this line: jest.spyOn(this, '_recordError');
this has to be IDCheck because there is no this in scope since you are using arrow functions that inherit this if previously set (which it isn't). You can console.log(this) right above the line to prove that point.
'_recordError' is not a method of IDCheck. spyOn checks the target's methods, not methods called within it. Now if _recordError is a method of IDCheck, then you should be ok.
Finally, you basically have to return the data you want in order to verify it. There's no real way to check what was passed unless you return it.
Here's a solution I came up with that does not include some fixes you'd have to implement to fix the potential workflow flaws.
const IDCheck = {
idCheck: function(errors) {
return errors.map(error => {
if (error.parentSchema.regexp === '/^((?!\\bMyId\\b).)*$/i') {
return this._recordError('IDCHECK')
}
})
},
_recordError: function(data) {
return data
}
}
let errors = [
{
parentSchema: {
regexp: '/^((?!\\bMyId\\b).)*$/i'
}
}
];
describe('IDCheck', () => {
afterEach(() => {
jest.restoreAllMocks();
});
it('calls _recordError with IDCHECK', () => {
const spy = jest.spyOn(IDCheck, '_recordError')
const check = IDCheck.idCheck(errors).includes('IDCHECK')
expect(spy).toHaveBeenCalled()
expect(check).toBe(true)
});
});

How to attach parameters to a nodejs event emmiter

I've developed a way to make two separate services to comunicate using a pub/sub channel using Redis, this is the main part of the code:
var Intercom = (function () {
var _event = new events.EventEmitter();
listener.subscribe("intercom");
listener.on("message", function(channel, message) {
try {
var data = JSON.parse(message);
_event.emit(data.controller, data.payload);
}
catch (e) {}
});
return {
on: function (evt, callback) {
_event.on(evt, callback);
},
emit: function (controller, payload) {
try {
sender.publish("intercom", JSON.stringify({ controller: controller, payload: payload}));
}
catch (e) {}
}
}
})();
Im using it on the main app just by: intercom.on('hook', hookController.main);
As you can see, if the query is "hook" the main() function of hookController is called. Its a very ExpressJs like approach.
The hookController does a very simple thing:
exports.main = function(req) {
console.log(req);
}
It is not very clear to my how the parameter "req" is getting passed to main(), and probably because of my lack of understanding about it, I cant figure it out how to pass another parameter to main from the main app, something like:
var foo = 'bar';
intercom.on('hook', hookController.main, foo);
I found a way to do it;
var foo = 'bar';
intercom.on('hook', function(req) { hookController.main(req, foo) });
Its a little bit ugly.

How to create parameterized and reusable gulp tasks

Is there a way to make this generic to the point where I can have one copy of it and pass the config item, and file list into it rather than duplicating it for every file/config combination?
I'd love to have something like
gulp.task('foo_test', function (cb) {
run_tests(files.foo_list, config.fooCoverage);
cb();
}
Note on potential oddities in the code
I'm using lazypipe and gulp-load-plugins full file here
// test the server functions and collect coverage data
gulp.task('api_test', function (cb) {
gulp.src(files.api_files)
.pipe(istanbulPre())
.on('end', function () {
gulp.src(files.api_test_files)
.pipe(mochaTask())
.pipe(istanbulAPI())
.on('end', cb);
});
});
var istanbulAPI = lazypipe()
.pipe(plugins.istanbul.writeReports, config.apiCoverage);
config = {
apiCoverage: {
reporters: ['json'],
reportOpts: {
json: {
dir: 'coverage',
file: 'coverage-api.json'
}
}
},
Gulp is just JavaScript.
You can write plain old regular functions, just as you normally would:
function run_tests(srcFiles, srcTestFiles, coverageConfig, cb) {
var istanbul = lazypipe()
.pipe(plugins.istanbul.writeReports, coverageConfig);
gulp.src(srcFiles)
.pipe(istanbulPre())
.on('end', function () {
gulp.src(srcTestFiles)
.pipe(mochaTask())
.pipe(istanbul())
.on('end', cb);
});
}
gulp.task('unit_test', function (cb) {
run_tests(files.lib_files, files.unit_test_files, config.unitCoverage, cb);
});
gulp.task('api_test', function (cb) {
run_tests(files.api_files, files.api_test_files, config.apiCoverage, cb);
});
Note that the callback cb is just another parameter that is passed to the run_tests function. If it was called immediately after calling run_tests that would signal task completion to gulp before the asynchronous code in run_tests has actually finished.
Lazypipe was a solution (and there are other alternatives) but since Gulp 4 these do not seem to work anymore. Gulp 4 does not pass the stream to pipe functions. Yet the gulp.src(...) function returns a stream.
Also a nice feature of gulp 4 is that functions with Promises can also be a task.
So in the end I came up with this solution that worked for me. With my gulpfile.js looking something like this:
const {
src,
dest,
series,
parallel
} = require('gulp');
// other gulp packages omitted in this example...
const myReusableJsParser = (sources, destination) => {
return src(sources)
.pipe(stripComments({...}))
.pipe(obfuscator({compact:true}))
.pipe(...) //etc
.pipe(dest(destination));
};
const parseScriptsX = () => {
return myReusableJsParser('./js/x/**/*.js', './dist/js/x/');
}
const parseScriptsY = () => {
return myReusableJsParser('./js/y/**/*.js', './dist/js/y/');
}
// more
...
const cleanUp = () => new Promise((resolve, reject) => {
try {
deleteFiles('./dist/').then(resolve).catch(reject);
} catch(err) {
reject(err);
}
});
// export
module.exports = {
default: series(cleanUp, paralell(parseScriptsX, parseScriptsY), paralell(...)),
...,
clean: cleanUp
};

How to call function on shared service - SailsJS

I have service:
module.exports = {
functionA: function () {
sails.log('Hello!');
},
functionB: function () {
functionA();
}
}
then I got an error: "countPoint is not defined"
as title, I want to call another function on shared function, but I dont now how to do. Can anyone help me?
Sorry for my bad English~
FunctionA does not "exist" yet. Remember that it is a property from a JSON, not a function per se. Using quotes should help you understand:
module.exports = {
"functionA": function () {
sails.log('Hello!');
},
"functionB": function () {
functionA(); // this is actually module.exports.functionA()
}
}
Put your functions outside module.exports and reference them without ():
module.exports = {
"functionA": functionA,
"functionB": functionA
}
function functionA() {
sails.log('Hello!');
}
#nodeman: tks for your response. But I can not call function functionA() outside of module. Then I found a solution:
file Demo.js:
module.exports = {
functionA: function () {
sails.log('Hello!');
},
functionB: function () {
Demo.functionA();
}
}
by call function with file name prefix, I called function in same module.

Resources