Nodejs crashes when database connection fails - node.js

While my database server is not available and any function of my node-express rest service like hiExpress is called, Nodejs crashes the node server and console of node reports
sql server connection closed
I do not want this to happen because either it should go to err function or at least it must be cautht by catch block. What could i do to avoid the crash of nodejs when database server is not available I am using following code which is absolutely fine as long as database server is available.
var sqlServer = require('seriate');
app.get('/hiExpress',function(req, res)
{
var sr = {error:'',message:''};
var sql= 'select * from table1 where id=? and name=?';
var params = {id: 5, name:'sami'};
exeDB(res,sr,sql, params);//sent only 4 parameters (not 6)
});
function exeDB(res, sr, sql, params, callback, multiple) {
try {
var obj = {};
for (p in params) {
if (params.hasOwnProperty(p)) {
obj[p] = {
type: sqlServer.VARCHAR,
val: params[p]
};
}
};
var exeOptions = {
query: sql,
params: obj
};
if (multiple) {
exeOptions.multiple = true;
}
sqlServer.execute(sqlServerConfigObject, exeOptions).then(function (results) {
sr.data = results;
if (callback)
callback(sr);
else
res.json(sr); //produces result when success
}, function (err) {
//sr.message = sql;
console.log(11);
sr.error = err.message;
res.json(sr);
});
}
catch (ex) {
console.log(21);
sr.error = ex.message;
res.json(sr);
}
}
Why I preferred to use seriate
I had not been much comfortable with node-SQL, especially when when it came to
multiple queries option even not using a transaction. It facilitates easy go to parameterized queries.

You can use transaction without seriate but with async like below
async.series([
function(callback) {db.run('begin transaction', callback)},
function(callback) {db.run( ..., callback)},
function(callback) {db.run( ..., callback)},
function(callback) {db.run( ..., callback)},
function(callback) {db.run('commit transaction', callback)},
], function(err, results){
if (err) {
db.run('rollback transaction');
return console.log(err);
}
// if some queries return rows then results[query-no] contains them
})
The code is very dirty. Pass req and res params to db-layer is not a good idea.
Try change exeDB. I'm not sure, but probably you don't set error catcher to promise
function exeDB(res, sr, sql, params, callback, multiple) {
// It will execute with no error, no doubt
var obj = {};
for (p in params) {
if (params.hasOwnProperty(p)) {
obj[p] = {
type: sqlServer.VARCHAR,
val: params[p]
};
}
};
var exeOptions = {
query: sql,
params: obj
};
if (multiple) {
exeOptions.multiple = true;
}
// Potential problem is here.
// Catch is useless because code below is asynchronous.
sqlServer.execute(sqlServerConfigObject, exeOptions).then(function (results) {
sr.data = results;
if (callback)
callback(sr);
else
res.json(sr); //produces result when success
}).error(function(err){ // !!! You must provide on-error
console.log(err);
};
}

Related

Azure Redis Cache cache.set() Not operable within Promise.then?

This one has had me spinning my tires for about 2 days now, I'm ready to reach out for help :)
I have a google firebase functions app, running as a middle-ware to an angular SPA. Hoping to avoid some of the pay-by-use cost of Azure SQL, I wanted to implement a caching option for the most common queries.
I thought I knew redis, I've worked with it before. There's a simple enough example on the repo: https://www.npmjs.com/package//redis
Everything works fine, if it is top level.
But the way my application is built, i need the ability to set a cache value from within the .then of a Promise, and when I try to do that, all operation just stops, with no identifiable error logging, or even response from redis. Even in Azure insights, i'm not getting much feed-back, only that the 'set' operation isn't being counted in metrics.
So, just to clarify, this works:
// "cache", the object, set globally
export const testCache = functions.https.onRequest(
async (req: any, res) => {
await cache.connect();
var redisKey = 'testing_global';
var result = await cache.get(redisKey);
await cache.set(redisKey, 'testing new class')
console.log("\nDone");
cache.disconnect();
res.send('done');
})
But, this does not:
import { createClient } from 'redis';
const cache = createClient({
url: "rediss://" + process.env.REDIS_HOST_NAME + ":6380",
password: process.env.REDIS_KEY,
});
export const getValues = functions.https.onRequest(
(req: any, response) => {
cors(req, response, async () => {
response.set('Access-Control-Allow-Origin', origin);
var searchText = req.body['search'];
var offset = req.body['offset'];
var fetch = req.body['fetch'];
var x = req.body['x'];
var y = req.body['y'];
var districts = req.body['districtFilter'];
var sort = req.body['sort'];
if (offset) {
if (!/^\d+$/.test(offset))
throw new Error('bad number');
} else {
offset = 0;
}
if (fetch) {
if (!/^\d+$/.test(fetch))
throw new Error('bad number');
} else {
fetch = 200;
}
if (x) {
if (!/^-?\d+$/.test(x))
throw new Error('bad number');
} else {
x = null;
}
if (y) {
if (!/^-?\d+$/.test(y))
throw new Error('bad number');
} else {
y = null;
}
if (districts && districts.length > 0) {
districts = sanitizeStringArray(districts);
} else {
districts = null;
}
if (sort) {
switch (sort) {
case "scoreAsc":
case "scoreDesc":
case "priceAsc":
case "priceDesc":
break;
default:
sort = '';
}
}
var redisKey = `get_values_${searchText}_${offset}_${fetch}_${x}_${y}_${districts}_${sort}`;
var cacheResult: any = null;
await cache.connect();
// because I want to end up in the 'else', for testing
cacheResult = await cache.getFromCache('someOtherKey');
if (null !== cacheResult) {
response.send({
"status": "success",
"totalCount": cacheResult.totalCount,
"data": cacheResult.result
});
cache.disconnect();
} else {
var connection = new Connection(sqlConfig);
var totalCount: number = 0;
connection.on('connect', function (err: any) {
// If no error, then good to proceed.
console.log("Connected");
var sql = `EXEC SomeSPC;`;
const sqlRequest = new Request(sql, function (err: any) {
if (err) {
console.log(err);
}
});
const countRequest = new Request(
`EXEC SomeOtherSPC;`
, function (err: any) {
if (err) {
console.log(err);
}
}
)
sqlRequest.connection = connection;
countRequest.connection = connection;
var result: any[] = [];
sqlRequest.on('row', function (columns: any[]) {
var rowResult: any = {};
columns.forEach(function (column: any) {
rowResult[column['metadata']['colName']] = column['value'];
});
result.push(rowResult);
});
sqlRequest.on("requestCompleted", function (rowCount: any, more: any) {
console.log(rowCount + ' rows returned');
connection.execSql(countRequest);
countRequest.on('row', function (columns: any[]) {
totalCount = columns[0]['value'];
});
countRequest.on('requestCompleted', async function (rowCount: any, more: any) {
connection.close();
cacheResult = {
totalCount: totalCount
, result: result
};
// ******************************************************************
// Does Not Work, Just Fails, Without Much to Go On
await cache.set(redisKey, 'in Promise')
response.send({
"status": "success",
"totalCount": totalCount,
"data": result
});
})
});
connection.execSql(sqlRequest);
});
connection.on('infoMessage', infoError);
connection.on('errorMessage', infoError);
connection.on('end', end);
connection.on('debug', debug);
connection.connect();
console.log("Reading rows from the Table...");
}
})
}
)
There's **update- no longer ** a fair amount of psuedo-code here, so please **update- No Need to ** disregard any inconsistent lines. I went ahead and put in the full function, including all the fluff, since trimming the fat seems to make things difficult for others to understand what is being asked.
The sql stuff all works, if i take out the cache.set() everything is fine, but that line, in the result of the Promise, just fails, and I can't figure out why.
I've tried using cache locally and globally, extracting the cache operations to a function, and then to a separate class, and in all cases, i'm getting the same result.
Is there a known reason this wouldn't work?
As far as I can understand, given you didn't provide a reproducible example, your code is binding to the requestCompleted event for the second request only after it runs execSql(): I would suggest moving the binding block before that, otherwise the event may be skipped.

nodejs get sqlite3 query result using promise or wait

This is my first personal project in Nodejs. I'm trying to get in live soon.
I have a Nodejs server that uses sqlite3. There are only 3000 rows with word, transform and a precalculated value each in a column of the table, which is already populated.
I need to just lookup the word in the DB to be sure it is valid.
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.get("SELECT * FROM tab WHERE w = ?", word, function(err, row) {
if(err) { console.log("Lookup:",word,", Error => ",err); return false; }
return true;
});
The problem is that the caller of this code has a lot of context and need the operation to wait. So, I tried this
function dbLookup(db, w) {
return function(cb) {
var rows = [];
db.exec('SELECT w FROM tab WHERE w = "'+w+'"')
.on('row', function(r) {
rows.push(r)
})
.on('result', function() {
cb(rows);
});
}
async.each([word], function(w) {
dbLookup(this.db, w);
}, function(err) {
if(err) {console.log("...ERROR..."); return false; }
else {console.log("...SUCCESS..."); return true; }
});
This doesn't solve the wait issue as the callback can fire at its own pace.
I read that promise using something like bluebird can solve my problem
but now I'm not able to get the value/result of the query out:
I've been pulling my hair for so long. Please help me either get the async working or get the result back from the promise approach.
var async = require('async');
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
function check(word, callback) {
db.get("SELECT count(1) cnt FROM tab WHERE w = ?", word, callback)
}
async.map(words, check, function(err, results) {
if (err)
return console.log('Query error')
var all_checked = results.filter(function(r) {
return r.cnt > 0
});
...
});
Or
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.all("SELECT distinct w FROM tab", function(err, rows) {
var all_checked = words.filter(function (w) {
return rows.indexOf(w) != -1;
})
...
})

Node.js + socket.io + MySQL correction of syntax

Considering that my server.js looks almost like this. Just send you the relevant part. I did not receive anything from the query, I do have data in the database, and "sendNotification" is triggered by the jQuery function in the client. Everything works and since var notis = []; returns an empty value and is what is shows as response. I know I have to debug SQL and that's what I'm going to do but anyway want to be sure of this other things. So my questions are:
1) Is a right syntax for node.js, considering this async behavior? (which I still don't understand )
2) The query always should be inside of the "io.sockets.on('connection')" part?
connection = mysql.createConnection({
host: 'localhost',
user: '',
password: "",
database: 'table' //put your database name
}),
...
connection.connect(function(err) {
// connected! (unless `err` is set)
console.log(err);
});
…
var sqlquery = function(uID,vs){
var notis = [];
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
return notis.push(data);
});
};
io.sockets.on('connection', function(socket) {
...
socket.on("sendNotification", function(data) {
var roomBName = data.room_name.replace("room-",""),
found = [];
var roomSelected = _.find(rooms, function (room) { return room.id == roomBName });
for (var person in people) {
for (var i = 0, numAttending = roomSelected.peopleAttending.length; i < numAttending; i++) {
if (people[person].name == roomSelected.peopleAttending[i]) {
found.push(person);
}
}
}
for (var i = 0, numFound = found.length; i < numFound; i++) {
**result = sqlquery(9,2);**
io.to(found[i]).emit('notification', result);
};
});
Your sqlquery() function will not accomplish anything useful. Because connection.query() is asynchronous, that means it provides the response sometime LATER after sqlquery() has already finished.
The only way in node.js to use an async result is to actually use it in the callback that provides it. You don't just stuff it into some other variable and expect the result to be there for you in other code. Instead, you use it inside that callback or you call some other function from the callback and pass it the data.
Here's one way, you could change your sqlquery() function:
var sqlquery = function(uID, vs, callback){
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
callback(null, data);
});
// need to add error handling here if the query returns an error
// by calling callback(err)
};
Then, you could use the sqlquery function like this:
found.forEach(function(person, index) {
sqlquery(..., function(err, result) {
if (err) {
// handle an error here
} else {
io.to(person).emit('notification', result);
}
});
});
And, it looks like you probably have similar async issues in other places too like in connection.connect().
In addition to #jfriend00, this could be done with new ES6 feature Promise :
var sqlquery = function(uID, vs){
return new Promise(function(resolve, reject){
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
resolve(data);
});
});
};
Now you can use it like :
found.forEach(function(person, index) {
sqlquery(...)
.then(function(result){
io.to(person).emit('notification', result);
});
});

module.export function not returning results

I'm having some problems returning results from a module function.
Below are two files that I'm working with.
When I call the exported function it returns nothings.
Any suggestions/fixes as to why? Does it have to do with callbacks?
models/index.js
module.exports = exports = function(library) {
modCodes.findOne({name: library}, {modcode:1}, function(err, mc) {
if (err) throw new Error(err);
var db = mongoose.createConnection('mongodb://localhost:27017/' + mc.modcode + '?safe=true');
var models = {
Books: db.model('books', require('./schemas/books'))
}
return models;
});
};
books.js
var Models = require('../models');
console.log(Models("myLibrary")); //return nothing
The reason you're getting no results is that you're trying to return a function value synchronously from an asynchronous callback. Instead of providing a function value, the return statement will instead stop the function, as return; would normally do. This is why you must use a callback for asynchronous operations:
module.exports = exports = function(library, callback) {
modCodes.findOne({name: library}, {modcode: 1}, function (err, mc) {
if (err) throw new Error(err);
var db = mongoose.createConnection('mongodb://localhost:27017/' + mc.modcode + '?safe=true');
var models = {
Books: db.model('books', require('./schemas/books'))
}
callback(models);
});
};
And this is how you would be able to use it:
var Models = require('../models');
Models('myLibrary', function(models) {
console.log(models);
});
i solve similar problem in different way. I am not sure whether it is right way.
in main node js I am using a model named product. I am passing product and res to misc.js. Following is part of my server.js file
var misc = require('./misc');
app.get('/groupbyCategory', function(req,res,next)
{
var res2;
misc.addX(product,res);
})
IN misc.js doing group by function and will return that value to straight way to angular controller. it is not necessary to return the result to server.js and from server.js to return angular controller. So i feel waiting and other call back seems unnecessary.
Inside misc.js i keep following code.
exports.addX = function(product,res) {
product.aggregate([
{ $group: {
_id: {category: "$category"},
count: { $sum: 1 }
}}
], function (err, result) {
if (err) {
console.log(err);
return err;
}
else
{
//return result;
console.log(result);
res.send(result);
}
});
};

Iterating over a mongodb cursor serially (waiting for callbacks before moving to next document)

Using mongoskin, I can do a query like this, which will return a cursor:
myCollection.find({}, function(err, resultCursor) {
resultCursor.each(function(err, result) {
}
}
However, I'd like to call some async functions for each document, and only move on to the next item on the cursor after this has called back (similar to the eachSeries structure in the async.js module). E.g:
myCollection.find({}, function(err, resultCursor) {
resultCursor.each(function(err, result) {
externalAsyncFunction(result, function(err) {
//externalAsyncFunction completed - now want to move to next doc
});
}
}
How could I do this?
Thanks
UPDATE:
I don't wan't to use toArray() as this is a large batch operation, and the results might not fit in memory in one go.
A more modern approach that uses async/await:
const cursor = db.collection("foo").find({});
while(await cursor.hasNext()) {
const doc = await cursor.next();
// process doc here
}
Notes:
This may be even more simple to do when async iterators arrive.
You'll probably want to add try/catch for error checking.
The containing function should be async or the code should be wrapped in (async function() { ... })() since it uses await.
If you want, add await new Promise(resolve => setTimeout(resolve, 1000)); (pause for 1 second) at the end of the while loop to show that it does process docs one after the other.
If you don't want to load all of the results into memory using toArray, you can iterate using the cursor with something like the following.
myCollection.find({}, function(err, resultCursor) {
function processItem(err, item) {
if(item === null) {
return; // All done!
}
externalAsyncFunction(item, function(err) {
resultCursor.nextObject(processItem);
});
}
resultCursor.nextObject(processItem);
}
since node.js v10.3 you can use async iterator
const cursor = db.collection('foo').find({});
for await (const doc of cursor) {
// do your thing
// you can even use `await myAsyncOperation()` here
}
Jake Archibald wrote a great blog post about async iterators, that I came to know after reading #user993683's answer.
This works with large dataset by using setImmediate:
var cursor = collection.find({filter...}).cursor();
cursor.nextObject(function fn(err, item) {
if (err || !item) return;
setImmediate(fnAction, item, arg1, arg2, function() {
cursor.nextObject(fn);
});
});
function fnAction(item, arg1, arg2, callback) {
// Here you can do whatever you want to do with your item.
return callback();
}
If someone is looking for a Promise way of doing this (as opposed to using callbacks of nextObject), here it is. I am using Node v4.2.2 and mongo driver v2.1.7. This is kind of an asyncSeries version of Cursor.forEach():
function forEachSeries(cursor, iterator) {
return new Promise(function(resolve, reject) {
var count = 0;
function processDoc(doc) {
if (doc != null) {
count++;
return iterator(doc).then(function() {
return cursor.next().then(processDoc);
});
} else {
resolve(count);
}
}
cursor.next().then(processDoc);
});
}
To use this, pass the cursor and an iterator that operates on each document asynchronously (like you would for Cursor.forEach). The iterator needs to return a promise, like most mongodb native driver functions do.
Say, you want to update all documents in the collection test. This is how you would do it:
var theDb;
MongoClient.connect(dbUrl).then(function(db) {
theDb = db; // save it, we'll need to close the connection when done.
var cur = db.collection('test').find();
return forEachSeries(cur, function(doc) { // this is the iterator
return db.collection('test').updateOne(
{_id: doc._id},
{$set: {updated: true}} // or whatever else you need to change
);
// updateOne returns a promise, if not supplied a callback. Just return it.
});
})
.then(function(count) {
console.log("All Done. Processed", count, "records");
theDb.close();
})
You can do something like this using the async lib. The key point here is to check if the current doc is null. If it is, it means you are finished.
async.series([
function (cb) {
cursor.each(function (err, doc) {
if (err) {
cb(err);
} else if (doc === null) {
cb();
} else {
console.log(doc);
array.push(doc);
}
});
}
], function (err) {
callback(err, array);
});
You could use a Future:
myCollection.find({}, function(err, resultCursor) {
resultCursor.count(Meteor.bindEnvironment(function(err,count){
for(var i=0;i<count;i++)
{
var itemFuture=new Future();
resultCursor.nextObject(function(err,item)){
itemFuture.result(item);
}
var item=itemFuture.wait();
//do what you want with the item,
//and continue with the loop if so
}
}));
});
You can get the result in an Array and iterate using a recursive function, something like this.
myCollection.find({}).toArray(function (err, items) {
var count = items.length;
var fn = function () {
externalAsyncFuntion(items[count], function () {
count -= 1;
if (count) fn();
})
}
fn();
});
Edit:
This is only applicable for small datasets, for larger one's you should use cursors as mentioned in other answers.
A more modern approach that uses for await:
const cursor = db.collection("foo").find({});
for await(const doc of cursor) {
// process doc here with await
await processDoc(doc);
}
You could use simple setTimeOut's. This is an example in typescript running on nodejs (I am using promises via the 'when' module but it can be done without them as well):
import mongodb = require("mongodb");
var dbServer = new mongodb.Server('localhost', 27017, {auto_reconnect: true}, {});
var db = new mongodb.Db('myDb', dbServer);
var util = require('util');
var when = require('when'); //npm install when
var dbDefer = when.defer();
db.open(function() {
console.log('db opened...');
dbDefer.resolve(db);
});
dbDefer.promise.then(function(db : mongodb.Db){
db.collection('myCollection', function (error, dataCol){
if(error) {
console.error(error); return;
}
var doneReading = when.defer();
var processOneRecordAsync = function(record) : When.Promise{
var result = when.defer();
setTimeout (function() {
//simulate a variable-length operation
console.log(util.inspect(record));
result.resolve('record processed');
}, Math.random()*5);
return result.promise;
}
var runCursor = function (cursor : MongoCursor){
cursor.next(function(error : any, record : any){
if (error){
console.log('an error occurred: ' + error);
return;
}
if (record){
processOneRecordAsync(record).then(function(r){
setTimeout(function() {runCursor(cursor)}, 1);
});
}
else{
//cursor up
doneReading.resolve('done reading data.');
}
});
}
dataCol.find({}, function(error, cursor : MongoCursor){
if (!error)
{
setTimeout(function() {runCursor(cursor)}, 1);
}
});
doneReading.promise.then(function(message : string){
//message='done reading data'
console.log(message);
});
});
});

Resources