async.parallel not executing last cllback function when using with lambda functions - node.js

I am writing a lambda function in node js. I have main js file (index.js) which based on input event calls a methods in 2nd js file (loader.ls).
The issue is that after the method execution completes in loader.js, the callback in index.js is not getting executed. Below is the code
Code in index.js
var yates = require("./yatesLoader");
module.exports.handler = function(event, context) {
if(event.app === undefined || event.data === undefined){
context.fail("invalid request");
}
if(event.app === 'YatesDataLoader'){
var result = yates.sendNotification(event.data, function(err, resp){
if(err){
console.log("Error : " + JSON.stringify(err));
context.done(null, "error occured");
}
console.log("response : " + JSON.stringify(resp));
context.done(null, resp); // SUCCESS with message
});
}
};
Code in loader.js
var config = require("./config");
var optDynamo = config.DynamoDb;
var optSlack = config.Slack;
var async = require("async");
var req = require("request");
var AWS = require("aws-sdk");
AWS.config.update({
region: optDynamo.region
});
var DynamoDb = new AWS.DynamoDB();
var sendNotification = function(data, callback){
async.parallel([
sendSlackNotification.bind(null, data, cb),
saveDataToDynamoDb.bind(null, data, cb)
], function(err, results){
if (err) {
console.error("Error JSON:", JSON.stringify(err, null, 2));
return callback(err, null);
} else {
console.log("Success:", JSON.stringify(results, null, 2));
return callback(null, results);
}
});
};
var cb = function(err, resp){
if(err){
console.log("Error");
}else {
console.log("success");
}
};
var saveDataToDynamoDb = function(data, cb){
var params = {
"TableName" : optDynamo.table,
"Item" : {
"TimeStamp" : {"S" : new Date().toString() },
"ErrorMessage" : {"S" : data }
}
};
console.log("adding new data to DynamoDb");
DynamoDb.putItem(params, function(err, data){
if (err) {
console.error("Unable to add item. Error JSON:", JSON.stringify(err, null, 2));
return cb(err, null);
} else {
console.log("Added item:", JSON.stringify(data, null, 2));
return cb(null, data);
}
});
};
var sendSlackNotification = function(data, cb){
var options = {
method : 'post',
body : {"text" : data},
json : true,
url : optSlack.url
};
console.log("sending msg to slack");
req(options, function(err, resp){
if(err){
console.error("Unable to send message to Slack. Error JSON:", JSON.stringify(err, null, 2));
return cb(err, null);
} else {
console.log("Message sent to Slack:", JSON.stringify(resp, null, 2));
return cb(null, resp);
}
})
};
module.exports = {sendNotification : sendNotification};
Can someone help in understanding what is wrong here.

Its because of the way you are using .bind(). It is changing the arguments that get passed in to saveDataToDynamoDb() and sendSlackNotification(). So what they see is actually:
var saveDataToDynamoDb = function(data, cb, next) { ....
where next is the callback that the async library is expecting you to call. So you could call next after cb.
var saveDataToDynamoDb = function(data, cb, next){
...
DynamoDb.putItem(params, function(err, data){
if (err) {
cb(err, null);
return next(err, null);
} else {
cb(err, null);
return next(null, data);
}
});
};

Related

AWS Lambda function timing out after successful completion

I have a lambda function that does work successfully. I am able to create a file on S3 and another FTP server. The issue is even after successfull completion it time outs. It just doesn't finish executing. I could see from the logs it takes around 2 seconds and the timeout specified is 5 seconds. How do I make my function stops as soon as it is done? Here is the complete code:
"use strict";
var config = require('./config/dev');
var sql = require("mssql");
var AWS = require('aws-sdk');
var PromiseFtp = require('promise-ftp');
var fs = require('fs');
const request = require('request');
exports.handler = (event, context, callback) => {
GetDataFromServer(event, context, callback);
};
function GetDataFromServer(event, context, callback) {
console.log("Fetching data from database...");
var keys = [], outputText = '';
sql.connect(config.db, err => {
if (err) {
console.log("Error while connecting database :- " + err);
return callback(true, 'Error in fetching records from database...');
}
else {
new sql.Request()
.input('ClientId', sql.Int, 469)
.execute('ForesightDailyDataPull', (err, result) => {
if (!err) {
//Create Text here and assign to "outputText"
CreateFileOnS3(outputText, event, context, callback);
}
else {
console.log('Error in fetching records from database...');
return callback(true, 'Error in fetching records from database...');
}
})
}
});
sql.on('error', err => {
console.log('Error in fetching records from database...');
return callback(true, 'Error in fetching records from database...');
})
}
function CreateFileOnS3(fileData, event, context, callback) {
const fileName = generateFileName();
console.log('Sending file to S3...');
const s3 = new AWS.S3(config.awsCredentials);
const params = {
Bucket: config.app.s3Bucket,
Key: fileName,
Body: fileData
};
s3.upload(params, function (s3Err, data) {
if (s3Err) {
console.log('There was an error creating file on S3');
return callback(true, 'There was an error creating file on S3');
}
else {
console.log(`File uploaded successfully at ${data.Location}`);
CreatefileOnFTP(fileData, fileName, event, context, callback);
}
});
}
function CreatefileOnFTP(fileData, fileName, event, context, callback) {
console.log('Sending file to FTP...');
var ftpObject = {
"fileData": fileData,
"fileName": fileName,
"ftpURL": config.ftpDetails.ftpProtocol + "://" + config.ftpDetails.host,
"ftpUserName": config.ftpDetails.user,
"ftpPassword": config.ftpDetails.password
};
request({
url: config.ftpUploadURL,
method: "POST",
json: true,
body: ftpObject
}, function (error, response, body) {
if (!error) {
console.log('File sent successfully to FTP server.');
return callback(null, 'File sent successfully to FTP...');
}
else {
console.log('An error occurred while sending file to FTP.');
return callback(true, 'Error in sending file to FTP...');
}
});
}
function generateFileName() {
var _d = new Date(),
y = _d.getFullYear(),
m = _d.getMonth() + 1,
d = _d.getDate();
return y + '-' + (m < 10 ? '0' + m : m) + '-' + (d < 10 ? '0' + d : d) + '.txt';
}
You should close the open sql connection after the function completes by calling sql.close() after the action resolves.
function GetDataFromServer(event, context, callback) {
console.log("Fetching data from database...");
var keys = [], outputText = '';
sql.connect(config.db, err => {
if (err) {
console.log("Error while connecting database :- " + err);
return callback(true, 'Error in fetching records from database...');
}
else {
new sql.Request()
.input('ClientId', sql.Int, 469)
.execute('ForesightDailyDataPull', (err, result) => {
if (!err) {
//Create Text here and assign to "outputText"
CreateFileOnS3(outputText, event, context, callback);
sql.close() //HERE
}
else {
console.log('Error in fetching records from database...');
sql.close() //HERE
return callback(true, 'Error in fetching records from database...');
}
})
}
});
sql.on('error', err => {
console.log('Error in fetching records from database...');
sql.close() //HERE
return callback(true, 'Error in fetching records from database...');
})
}

Node js mongoerror 'write ECONNRESET'

When creating new document in node js gives MongoError of write ECONNRESET.
Following is the error in console when creating the new document:
error: Error adding new agreementTemplate ! Error :
{"name":"MongoError","message":"write ECONNRESET"}
following is the controller function creating the error.
function addSubChapter(req, res) {
logger.debug('Processing request for creating a new agreement.');
console.log(req.body);
async.waterfall([
function (callback) {
agreementService.getAgreementTemplate( callback);
},
function (data, callback) {
agreementTmplService.AgreementTemplateSetDefaultFalse(function(err, result){
callback(null, data);
});
},
function (data, callback) {
var agreementTemplate =data.result[0];
var chapter ={
name: req.body.name
}
agreementTemplate = agreementTemplate.toObject(); // swap for a plain javascript object instance
delete agreementTemplate["_id"];
agreementTemplate.createdBy= req.body.user;
agreementTemplate.created= new Date();
//agreementTemplate.Chapters.push(chapter);
var chapters = agreementTemplate.Chapters;
for(var i=0;i<chapters.length; i++){
if(chapters[i].name == req.body.chapter){
var subChap ={
name: req.body.name
}
chapters[i].subChapters.push(subChap);
}
}
console.log('----------------------');
console.log(agreementTemplate);
agreementService.createAgreementTemplate(agreementTemplate, callback);
},
function (data, callback) {
agreementTmplService.addSubChapter(req.body, callback);
}
], function (err, result) {
utils.processResponse(err, result, res);
});
}
When creating the agreement template it causes the error, here is the service function:
function createAgreementTemplate(data, callback) {
serviceHelper.createModel(AgreementTemplate, 'agreementTemplate', data, callback);
}
and the createModel function is as follows.
function createModel(model, modelName, modelData, callback) {
logger.debug('Creating new Model : %s', modelName);
var modelObj = model(modelData);
modelObj.save(function (err, newModelObj) {
if (err) {
logger.error('Error adding new %s ! Error : %s', modelName, JSON.stringify(err));
callback(utils.handleMongodbError(err), null);
} else {
logger.info('New Model of type : %s created with id : %s', modelName, JSON.stringify(newModelObj._id));
var result = {};
result.status = httpStatus.OK;
result.result = newModelObj._id;
callback(null, result);
}
});
}

async.eachSeries runs only once with async.waterfall inside for each iteration

I am new to async library. I have used async.eachSeries and async.waterfall for each iteration. I see, the async.waterfall runs only once.
Here is my code :
var fs = require('fs'),
async = require('async'),
Client = require('node-rest-client').Client;
// REST API Call and output in jsonOutput.results
console.log(jsonOutput.results.length); // jsonOutput.results has 124 records.
async.eachSeries(jsonOutput.results, function(account, callback) {
var dataObject = {};
dataObject.updatetime = new Date();
var setAccountInfoURL = ""; // Data Update REST API Request
async.waterfall([
function setAccountInfo(updateCallback) {
// client.get(setAccountInfoURL, function (data, response) {
// var jsonOutput = JSON.parse(data.toString('utf8'));
updateCallback(null, "output", account)
// });
},
function saveAccountInfo(jsonOutput, account, updateCallback) {
var debuglog = JSON.stringify(account) + "\n" + jsonOutput;
fs.appendFile("debuginfo.json", debuglog + "\n", function (err) {
if(err) {
console.log(err);
}
console.log("JSON saved to " + "debuginfo.json");
updateCallback(null);
});
}
],function asyncComplete(err) {
if (err) {
console.warn('Error setting account info.', err);
}
console.log('async completed');
});
}, function(err){
if (err) {
console.log('error in loop');
}
console.log('loop completed');
});
Output:
124
JSON saved to debuginfo.json
async completed
Any help is really appreciated.
I found my mistake. I missed calling the callback after each iteration just after async is completed.
var fs = require('fs'),
async = require('async'),
Client = require('node-rest-client').Client;
// REST API Call and output in jsonOutput.results
console.log(jsonOutput.results.length); // jsonOutput.results has 124 records.
async.eachSeries(jsonOutput.results, function(account, callback) {
var dataObject = {};
dataObject.updatetime = new Date();
var setAccountInfoURL = ""; // Data Update REST API Request
async.waterfall([
function setAccountInfo(updateCallback) {
// client.get(setAccountInfoURL, function (data, response) {
// var jsonOutput = JSON.parse(data.toString('utf8'));
updateCallback(null, "output", account)
// });
},
function saveAccountInfo(jsonOutput, account, updateCallback) {
var debuglog = JSON.stringify(account) + "\n" + jsonOutput;
fs.appendFile("debuginfo.json", debuglog + "\n", function (err) {
if(err) {
console.log(err);
}
console.log("JSON saved to " + "debuginfo.json");
updateCallback(null);
});
}
],function asyncComplete(err) {
if (err) {
console.warn('Error setting account info.', err);
}
console.log('async completed');
callback(null); // this is the change.
});
}, function(err){
if (err) {
console.log('error in loop');
}
console.log('loop completed');
});

async.each confusion in NodeJS

I'm taking my first steps in NodeJS, and I'm having an issue with the async-module. I had the following code which works fine:
var http = require('http');
var fs = require('fs');
var async = require('async');
function load_albums(callback) {
fs.readdir("albums", function(err, content) {
console.log(content);
if(err) {
callback(err);
return;
}
var directories = [];
(function iterator(index) {
if(index == content.length) {
callback(null, directories);
return;
}
fs.stat("albums/" + content[index], function(err, stats) {
if (err) {
callback(err);
}
if(stats.isDirectory()) {
directories.push(content[index]);
}
console.log(index);
iterator(index + 1);
});
})(0);
});
}
function handle_request(request, response) {
load_albums(function(err, albums) {
if (err) {
response.writeHead(503, {"Content-Type": "application/json"});
response.end(JSON.stringify(err) + "\n");
return;
}
var out = { error: null,
data: { albums: albums}};
response.writeHead(200, { "Content-Type" : "application/json" });
response.end(JSON.stringify(out) + "\n");
});
}
var s = http.createServer(handle_request);
s.listen(8080);
This works fine, and gives the expected output:
{"error":null,"data":{"albums":["testdir1","testdir2"]}}
However, I intended to replace the iterator with the async.each function.
I ended up with this:
function load_albums(callback) {
fs.readdir("albums", function(err, content) {
console.log(content);
if(err) {
callback(err);
return;
}
var directories = [];
async.each(content, function(item, callback2) {
fs.stat("albums/" + item, function(err,stats) {
if(stats.isDirectory()) {
directories.push(item);
}
callback2();
});
});
callback(null, directories);
});
}
However, this doesn't seem to work, as "albums" seems to be empty now:
{"error":null,"data":{"albums":[]}}
What am I missing here? I guess it has something to do with calling the fs.stats() function, but I'm unsure about what I'm doing wrong.
async.each() takes three arguments. You are not passing the last one which is the one that tells you when it is done. You also haven't implemented error handling on fs.stat(). You can change to this:
function load_albums(callback) {
fs.readdir("albums", function(err, content) {
console.log(content);
if(err) {
callback(err);
return;
}
var directories = [];
async.each(content, function(item, callback2) {
fs.stat("albums/" + item, function(err,stats) {
if (!err && stats.isDirectory()) {
directories.push(item);
}
callback2(err);
});
}, function(err) {
callback(err, directories);
});
});
}
As answered by #jfriend00 final callback is third parameter of asyn.each. Currently this callback is running without waiting for async.each to complete.
Also you're serving albums for all the request. They should be served on a particular resource URL like /albums or /albums/
I have made these modifications to the code, now it loads albums on http://localhost:8080/albums otherwise it returns 'No Content.
var http = require('http');
var fs = require('fs');
var async = require('async');
function load_albums(loadCompleteCallback) {
fs.readdir("albums", function(err, content) {
console.log(content);
if(err) {
callback(err);
return;
}
var directories = [];
async.each(content, function(item, doneCallback) {
fs.stat("albums/" + item, function(err,stats) {
if(stats.isDirectory()) {
directories.push(item);
}
return doneCallback(err);
});
}
, function (err) {
loadCompleteCallback(err, directories);
});
});
}
function handle_request(request, response) {
console.log("requested path: " + request.url);
if(request.url.match(/^\/albums[\/]?$/) ) {
load_albums(function(err, albums) {
if (err) {
response.writeHead(503, {"Content-Type": "application/json"});
response.end(JSON.stringify(err) + "\n");
return;
}
var out = { error: null,
data: { albums: albums}};
response.writeHead(200, { "Content-Type" : "application/json" });
response.end(JSON.stringify(out) + "\n");
});
} else {
response.writeHead(200, { "Content-Type" : "application/json" });
response.end("No Content\n");
}
}
var s = http.createServer(handle_request);
s.listen(8080);
console.log("server running at : http://localhost:" + 8080);

wrap couchbase access function

below is my couchbase nodejs code
kdatabase.js
var couchbase = require('couchbase');
var db = new couchbase.Connection({
host: "http://127.0.0.1:8091",
bucket: "default",
},
function(err) {
if (err) throw err;
db.get('id1', function(err, result) {
if (err) throw err;
console.log(result.value);
process.exit(0);
});
});
it works
but I hope to wrap it to object that can be easily to operate
module.exports = function(app) {
return new KDatabase(app);
};
var KDatabase = function(app) {
this.app = app;
};
//couchbase
KDatabase.prototype.query = function(userName) {
var couchbase = require('couchbase');
var db = new couchbase.Connection({
host: "http://127.0.0.1:8091",
bucket: "default",
},
function(err) {
if (err) throw err;
console.log(userName + '!!!!--');
db.get(userName, function(err, result) {
if (err) throw err;
var o = result.value;
console.log(o['password'] + '***--');
return o['password'];
});
});
};
then I call
var db = require('kdatabase.js')();
var s = db.query(msg.username, function(err) {
if (err) {
console.log('aaa');
}
console.log('bbb');
return;
});
the lines
console.log(userName + '!!!!--');
console.log(o['password'] + '***--');
display correctly
but
console.log('aaa');
console.log('bbb');
are never executed
Your query method does not take a callback argument, so you never call it.
KDatabase.prototype.query = function(userName, cb) {
/* snip */
console.log(o['password'] + '***--');
cb(err, result);

Resources