I am using aws-sdk for javascript.
The code below works fine when using in a stand-alone program
//program.js
const AWS = require('aws-sdk');
const firehose = new AWS.Firehose({
accessKeyId: "XXX",
secretAccessKey: "YY"
});
const params = {
DeliveryStreamName: 'demo1',
Record: {
Data: new Buffer("Hello World")
}
};
firehose.putRecord(params, function (err, data){
if (err) {
console.log(err);
return;
}
console.log(data); // successful response
});
Again, the above code works fine as a stand alone file. Data gets pushed into firehose and then further down to Redshift.
so if i execute
node program.js
I am able to see my data in Redshift. Yay!!
=============================
However, what i really want to achieve is to push data to firehose when a certain route gets hit in my express application. So I take the exact same code as above and stick it in my route
// router.js
const AWS = require('aws-sdk');
const firehose = new AWS.Firehose({
accessKeyId: "XXX",
secretAccessKey: "YY"
});
router
.get('/v1/locations/:id?', (req, res) => {
const params = {
DeliveryStreamName: 'demo1',
Record: {
Data: new Buffer("Hello World")
}
};
firehose.putRecord(params, function (err, data){
if (err) {
console.log(err);
return;
}
console.log(data);
});
// do the work that needs to be done for this route and send a response
res.send("some data");
});
The minute firehose.putRecord is executed .. it crashes my program with the following error:
```
TypeError: doneCallback.cal is not a function
at Request.callListeners (/api-project/node_modules/aws-sdk/lib/sequential_executor.js:115:18)
at callNextListener (/api-project/node_modules/aws-sdk/lib/sequential_executor.js:95:12)
at /api-project/node_modules/aws-sdk/lib/event_listeners.js:74:9
at finish (/api-project/node_modules/aws-sdk/lib/config.js:315:7)
at /api-project/node_modules/aws-sdk/lib/config.js:333:9
at Credentials.get (/api-project/node_modules/aws-sdk/lib/credentials.js:126:7)
at getAsyncCredentials (/api-project/node_modules/aws-sdk/lib/config.js:327:24)
at Config.getCredentials (/api-project/node_modules/aws-sdk/lib/config.js:347:9)
at Request.VALIDATE_CREDENTIALS (/api-project/node_modules/aws-sdk/lib/event_listeners.js:69:26)
at Request.callListeners (/api-project/node_modules/aws-sdk/lib/sequential_executor.js:101:18)
I can't understand why this code crashes my express program. is this is bug in the aws-sdk library or am i doing something wrong ?
You should be sending the express response inside your success callback.
firehose.putRecord(params, function (err, data) {
if (err) {
console.log(err);
return;
}
console.log(data);
res.send("some data");
}
);
FYI, your res.send(data) will effectively exit the program and send data. However, your putRecord callback is the time when your exit should occur. In node, things do not happen in a sequence from top to bottom of the code, but instead it executes in order of the callback events. so the execution flow for your code would be like this:
file executes
some operation is performed
callback for operation occurs
then if theres additional code outside of operate, it will continue, otherwise the code will exit in that callback. Hence, put the res.send in your putRecord callback.
Related
I have this code in node.js (nodejs10.x) running in AWS Lambda.
module.exports.register = (event, context, callback) => {
// Body es un json por lo que hay que deserializarlo
let body = JSON.parse(event.body);
let rules = {
'name': 'required|min:3',
'family_name': 'required|min:3',
'email': 'required|email',
'curp': 'required|size:18',
'modules': 'required',
'password': 'required'
};
let validation = new validator(body, rules);
// If errors this validation exits using the callback
if(validation.fails()){
console.log(validation.errors.all())
const response = {
statusCode: 422,
body: JSON.stringify(validation.errors.all())
};
callback(null, response);
}
// just for testing
const isModulesValid = false;
if(!isModulesValid){
console.log('Modules validation failed. ')
const response = {
statusCode: 422,
body: JSON.stringify({'modules': 'Invalid modules string. '})
};
callback(null, response);
// However this is not working
}
// and even if there is an error this code is executed
console.log('XXXX');
I am testing it locally with a code like this.
// Callback
let callback = function(err, result) {
if (err)
console.log(err);
if (result)
console.log(result);
// Terminate execution once done
process.exit(0);
}
// lambda.generateToken(event, context, callback);
lambda.register(event, context, callback);
Locally if isModulesValid = false the code exits and the console.log('XXXX') is not executed. However when running it in AWS Lambda, even if the validation fails the code continues to run and the console.log() is executed.
I cannot figure out whats going on. Help please?
Locally you are using the callback which has process.exit(0); which is making the process finish and hence the next line is not executing. Callback doesn't mean that the code afterwards wouldn't be executed. the code flow continues after that as well. It all depends on what you have in your callback.
This piece of code solved the problem:
if(!isModulesValid){
console.log('Modules validation failed. ')
const response = {
statusCode: 422,
body: JSON.stringify({'modules': 'Invalid modules string. '})
};
// Return callback explcitly
return callback(null, response);
}
Apparently it has to do with how AWS Lambda handles the task queue. I found a good explanation here: https://blog.danillouz.dev/aws-lambda-and-the-nodejs-event-loop/
so on my first time learning AWS stuff (it is a beast), I'm trying to create e-mail templates, I have this lambda function:
// Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({ region: "us-east-1" });
exports.handler = async (event, context, callback) => {
// Create createTemplate params
var params = {
Template: {
TemplateName: "notification" /* required */,
HtmlPart: "HTML_CONTENT",
SubjectPart: "SUBJECT_LINE",
TextPart: "sending emails with aws lambda"
}
};
// Create the promise and SES service object
const templatePromise = new AWS.SES({ apiVersion: "2010-12-01" })
.createTemplate(params)
.promise();
// Handle promise's fulfilled/rejected states
templatePromise
.then((data) => {
console.log(data);
callback(null, JSON.stringify(data) );
// also tried callback(null, data);
}, (err) => {
console.error(err, err.stack);
callback(JSON.stringify(err) );
});
as far as I am understanding, this function should return me a template? an object, anything? when I use the lambda test functionality I always got null in the request response
does anyone know what I am doing wrong here?
edit: and It is not creating the e-mail template, I check the SES Panel - email templates and it is empty
edit2: if I try to return a string eg: callback(null, "some success message"); it does return the string, so my guess is something wrong with the SES, but this function is exactly what we have in the AWS docs, so I assume it should just work..
Try not to resolve the Promise and change your code to just returning it as-is:
return await templatePromise;
which should present you some more detail of what is really going wrong in your code - it might be some hidden access issue - so you might need to adjust the role your lambda function is using. createTemplate on the other side should not return much in case of successful execution but just create the template.
Also try to follow the following try/catch pattern when using async (as described here in more detail: https://aws.amazon.com/de/blogs/compute/node-js-8-10-runtime-now-available-in-aws-lambda/)
exports.handler = async (event) => {
try {
data = await lambda.getAccountSettings().promise();
}
catch (err) {
console.log(err);
return err;
}
return data;
};
I have created a Lambda function in AWS that export logs from Cloudfront to Elasticsearch.
From the AWS console, I still have a warining in front of Invocation error, though the metrics show there is none for more than 24hours.
A typical workflow of logs looks like
START RequestId: 302f0b95-7856-11e8-9486-55b3f10e7d4e Version: $LATEST
Request complete
END RequestId: 302f0b95-7856-11e8-9486-55b3f10e7d4e
REPORT RequestId: 302f0b95-7856-11e8-9486-55b3f10e7d4e Duration: 794.93 ms Billed Duration: 800 ms Memory Size: 128 MB Max Memory Used: 52 MB
There is no error in the logs, and the only thing I guess could trigger this invocation error is that sometimes two request starts at the same time
09:01:47
START RequestId: 63cd00e1-7856-11e8-8f96-1f900def8e65 Version: $LATEST
09:01:47
START RequestId: 63e1e7f3-7856-11e8-97e6-3792244f6ab0 Version: $LATEST
Except from this, I don't understand where this error comes from.
Do I miss something? Or do I have to wait more than 24hours before the satus change? May be there is a way to pinpoint the error with AWS console/API that I did not find about?
Would be happy to hear what you think about this.
Edit: In case you'd like to take a look at the code itself.
var aws = require('aws-sdk');
var zlib = require('zlib');
var async = require('async');
const CloudFrontParser = require('cloudfront-log-parser');
var elasticsearch = require('elasticsearch');
var s3 = new aws.S3();
var client = new elasticsearch.Client({
host: process.env.ES_HOST,
log: 'trace',
keepAlive: false
});
exports.handler = function(event, context, callback) {
var srcBucket = event.Records[0].s3.bucket.name;
var srcKey = event.Records[0].s3.object.key;
async.waterfall([
function fetchLogFromS3(next){
console.log('Fetching compressed log from S3...');
s3.getObject({
Bucket: srcBucket,
Key: srcKey
},
next);
},
function uncompressLog(response, next){
console.log("Uncompressing log...");
zlib.gunzip(response.Body, next);
},
function publishNotifications(jsonBuffer, next) {
console.log('Filtering log...');
var json = jsonBuffer.toString();
var records;
CloudFrontParser.parse(json, { format: 'web' }, function (err, accesses) {
if(err){
console.log(err);
} else {
records = accesses;
}
});
var bulk = [];
records.forEach(function(record) {
bulk.push({"index": {}})
bulk.push(record);
});
client.bulk({
index: process.env.ES_INDEXPREFIX,
type: 'log',
body: bulk
}, function(err, resp, status) {
if(err) {
console.log('Error: ', err);
}
console.log(resp);
next();
});
console.log('CloudFront parsed:', records);
}
], function (err) {
if (err) {
console.error('Failed to send data: ', err);
} else {
console.log('Successfully send data.');
}
callback(null, 'success');
});
};
You need to explicitly return information back to the caller.
Here's the related documentation:
The Node.js runtimes v6.10 and v8.10 support the optional callback
parameter. You can use it to explicitly return information back to the
caller. The general syntax is:
callback(Error error, Object result); Where:
error – is an optional parameter that you can use to provide results
of the failed Lambda function execution. When a Lambda function
succeeds, you can pass null as the first parameter.
result – is an optional parameter that you can use to provide the
result of a successful function execution. The result provided must be
JSON.stringify compatible. If an error is provided, this parameter is
ignored.
If you don't use callback in your code, AWS Lambda will call it
implicitly and the return value is null.
When the callback is called (explicitly or implicitly), AWS Lambda
continues the Lambda function invocation until the event loop is
empty.
The following are example callbacks:
callback(); // Indicates success but no information returned to the caller.
callback(null); // Indicates success but no informatio returned to the caller.
callback(null, "success"); // Indicates success with information returned to the caller.
callback(error); // Indicates error with error information returned to the caller
https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html
I have a simple node Express app that has a service that makesa call to a node server. The node server makes a call to an AWS web service. The AWS simply lists any S3 buckets it's found and is an asynchronous call. The problem is I don't seem to be able to get the server code to "wait" for the AWS call to return with the JSON data and the function returns undefined.
I've read many, many articles on the web about this including promises, wait-for's etc. but I think I'm not understanding the way these work fully!
This is my first exposer to node and I would be grateful if somebody could point me in the right direction?
Here's some snippets of my code...apologies if it's a bit rough but I've chopped and changed things many times over!
Node Express;
var Httpreq = new XMLHttpRequest(); // a new request
Httpreq.open("GET","http://localhost:3000/listbuckets",false);
Httpreq.send(null);
console.log(Httpreq.responseText);
return Httpreq.responseText;
Node Server
app.get('/listbuckets', function (req, res) {
var bucketData = MyFunction(res,req);
console.log("bucketData: " + bucketData);
});
function MyFunction(res, req) {
var mydata;
var params = {};
res.send('Here are some more buckets!');
var request = s3.listBuckets();
// register a callback event handler
request.on('success', function(response) {
// log the successful data response
console.log(response.data);
mydata = response.data;
});
// send the request
request.
on('success', function(response) {
console.log("Success!");
}).
on('error', function(response) {
console.log("Error!");
}).
on('complete', function() {
console.log("Always!");
}).
send();
return mydata;
}
Use the latest Fetch API (https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) to make HTTP calls. It has built-in support with Promise.
fetch('http://localhost:3000/listbuckets').then(response => {
// do something with the response here
}).catch(error => {
// Error :(
})
I eventually got this working with;
const request = require('request');
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
parseString(body, function (err, result) {
console.log(JSON.stringify(result));
});
// from within the callback, write data to response, essentially returning it.
res.send(body);
}
else {
// console.log(JSON.stringify(response));
}
})
Currently i am using putObject to upload the large file to AWS s3 with REST api call.
var params ={
Bucket:'lambdacushbu',
Key:req.files.image.name,
Body:req.files.image.data
}
s3.putObject(params,function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else {
console.timeEnd('Uploadtime');
console.log("uploaded",data);
res.json({
'status':'Uploaded',
'url':data.Location
});
} // successful response
});
But its looks like asynchronous i want the above in synchronous mode also a timeout is occurred but the file is being uploaded to the AWS s3.
So how can i increase the timeout value?? tried with connect-timeout package
app.use(timeout('600000'));
But it dosen't worked
Try using upload function instead of putObject. That should solve your timeout problem.
Here is a documentation for that function: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property
Synchronous call will definitely lower your app's performance. Can you provide me more details about your problem so we can find an async solution?
EDIT:
Here is how you should return response in your controller:
router.post('/your-route',
//additional middlewares
function(req, res, next) {
var params = {
Bucket:'lambdacushbu',
Key:req.files.image.name,
Body:req.files.image.data
}
s3.upload(params,function(err, data) {
if (err) { res.json(err); }
else {
res.json({
'status':'Uploaded',
'url':data.Location
}
});
}
);
And make sure you don't call res.json() or res.send() anywhere else in this route