NodeJs npm requestretry package Not Work on AWS Lambda Nodejs8.10 - node.js

Here is my nodejs8.10 test code:
var request = require('requestretry');
const testRequest = async() =>{
request({
url: 'https://jsonplaceholder.typicode.com/posts/1',
json: true,
maxAttempts: 5, // (default) try 5 times
retryDelay: 5000, // (default) wait for 5s before trying again
retryStrategy: request.RetryStrategies.HTTPOrNetworkError
}, function(err, response, body){
if (response) {
console.log('The number of request attempts: ' + response.attempts);
console.log(body);
}else{
console.log('error: ', err);
}
});
}
testRequest(); // for local implementation
exports.handler = async (event) => {
let x = await testRequest();
const response = {
statusCode: 200,
body: JSON.stringify('Hello from Test Lambda!'),
};
return response;
};
vs. nodejs6.10 test code:
var request = require('requestretry');
function testRequest () {
request({
url: 'https://jsonplaceholder.typicode.com/posts/1',
json: true,
// The below parameters are specific to request-retry
maxAttempts: 5, // (default) try 5 times
retryDelay: 5000, // (default) wait for 5s before trying again
retryStrategy: request.RetryStrategies.HTTPOrNetworkError // (default) retry on 5xx or network errors
}, function(err, response, body){
// this callback will only be called when the request succeeded or after maxAttempts or on error
if (response) {
console.log(body);
}
});
}
testRequest();
exports.handler = (event, context, callback) => {
testRequest();
const response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
callback(null, response);
};
It won't display the log that related to requestretry package on lambda Nodejs 8.10 and no any error messages. However, it can get the response and logs by running with command node <filename> in the local machine after npm install with the necessary packages like 'requestretry' and its required packages. If the above file converted into Nodejs6.10 format and it won't have the issue on both aws lambda and local machine.
I need to find out the causing before I convert the codes from nodejs6.10 (callback format)to nodejs8.10 (async format).

Related

AWS API Gateway with Lambda HTTP GET Request (Node.js) 502 Bad Gateway

I'm new to AWS lambda functions and NodeJS. I'm trying to create an API Gateway call to a Lambda function that calls an external API and return some JSON data. It took me a while but I was finally able to get something to work based on this post:
AWS Lambda HTTP POST Request (Node.js)
The problem was the API Gateway kept erroring with a 502 Bad Gateway; which turns out to be that the JSON response was malformed. In the post I referenced above everyone seem to have success with just returning the JSON as-is, but I had to follow the instructions here to fix my issue:
https://aws.amazon.com/premiumsupport/knowledge-center/malformed-502-api-gateway/
My question is: if you look at the last 10 lines of my code that finally worked I had to reformat my response, as well as use a callback in a async function. I am new to nodeJS and Lambda but it looks wrong to me, even though it works. The post I referenced seem to have much more elegant code, and I hope someone can tell me what I am doing wrong.
const https = require('https');
var responseBody = {"Message": "If you see this then the API call did not work"};
const doGetRequest = () => {
return new Promise((resolve, reject) => {
const options = {
host: 'my.host.com',
path: '/api/v1/path?and=some&parameters=here',
method: 'GET',
headers: {
'Authorization': 'Bearer token for testing',
'X-Request-Id': '12345',
'Content-Type': 'application/json'
}
};
var body='';
//create the request object with the callback with the result
const req = https.request(options, (res) => {
res.on('data', function (chunk) {
body += chunk;
});
res.on('end', function () {
console.log("Result", body.toString());
responseBody = body;
});
resolve(JSON.stringify(res.statusCode));
});
// handle the possible errors
req.on('error', (e) => {
reject(e.message);
});
//finish the request
req.end();
});
};
exports.handler = async (event, context, callback) => {
await doGetRequest();
var response = {
"statusCode": 200,
"headers": {
"my_header": "my_value"
},
"body": JSON.stringify(responseBody),
"isBase64Encoded": false
};
callback(null, response);
};
I see couple of things.
We need to get the values from method doGetRequest and use the response, we can do that by await response = doGetRequest() or doGetRequest.then(), since we ant to capture errors as well, i went with second method.
We also need to resolve or reject the actual response from within promise.
I tested with a different api(with url of this question). Here is the updated code.
const https = require('https');
var responseBody = {"Message": "If you see this then the API call did not work"};
const doGetRequest = () => {
return new Promise((resolve, reject) => {
const options = {
host: 'stackoverflow.com',
path: '/questions/66376601/aws-api-gateway-with-lambda-http-get-request-node-js-502-bad-gateway',
method: 'GET'
};
var body='';
//create the request object with the callback with the result
const req = https.request(options, (res) => {
res.on('data', function (chunk) {
body += chunk;
});
res.on('end', function () {
console.log("Result", body.toString());
resolve(body);
});
});
// handle the possible errors
req.on('error', (e) => {
reject(e.message);
});
//finish the request
req.end();
});
};
exports.handler = (event, context, callback) => {
console.log('event',event, 'context',context);
doGetRequest().then(result => {
var response = {
"statusCode": 200,
"headers": {
"my_header": "my_value"
},
"body": JSON.stringify(result),
"isBase64Encoded": false
};
callback(null, response);
}).catch(error=> {
callback(error);
})
};

Reason why lambda function calling AWS glue using Node.JS does not console.log?

I'm trying to kick off a AWS glue job using a lambda function, using node.js. I can test the lambda function fine, but nothing appear to happen after the script has run its course. I've added a few console.log lines, but during the SDK method call to start the AWS glue job, none of the console.log lines log anything (I am checking the output on the lambda code configuration page, and on CloudWatch). Am I missing something here? I tested the below using the in-browser "TEST" button.
var AWS = require('aws-sdk');
AWS.config.update({region: 'us-east-2'});
var glue = new AWS.Glue();
exports.handler = async (event) => {
console.log("Hello!")
var params = {
JobName: 'ETL-store-inventory',
};
//Invoke job run
glue.startJobRun(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
console.log("Done")
const response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
return response;
};
I get the following from the console:
Response:
{
"statusCode": 200,
"body": "\"Hello from Lambda!\""
}
Request ID:
"e205ec08-dce1-4710-b944-f490544b1486"
Function Logs:
START RequestId: e205ec08-dce1-4710-b944-f490544b1486 Version: $LATEST
2019-05-03T17:17:55.427Z e205ec08-dce1-4710-b944-f490544b1486 Hello!
2019-05-03T17:17:55.525Z e205ec08-dce1-4710-b944-f490544b1486 Done
END RequestId: e205ec08-dce1-4710-b944-f490544b1486
REPORT RequestId: e205ec08-dce1-4710-b944-f490544b1486 Duration: 324.11 ms
Billed Duration: 400 ms Memory Size: 128 MB Max Memory Used: 68 MB
Your function is returning and closing before the callback from your glue job returns. You can move the return inside the callback to make the function complete once the callback returns
var AWS = require('aws-sdk'); AWS.config.update({region: 'us-east-2'});
var glue = new AWS.Glue();
exports.handler = async (event) => {
console.log("Hello!")
var params = {
JobName: 'ETL-store-inventory',
};
//Invoke job run
return glue.startJobRun(params, function(err, data) {
if (err) {
console.log(err, err.stack); // an error occurred
const response = {
statusCode: 200,
body: JSON.stringify('An error occurred!'),
};
return response
} else {
console.log(data); // successful response
console.log("Done")
const response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
return response;
}
});

How to use Nodejs "request" work aysn in AWS Lambda?

When I'm trying to use "request" to access external API and get back the response in AWS Lambda, I don't know how to properly put my "return" code to return the response.
NodeJs 8.10
var request = require('request');
module.exports.sendcode = async (event) => {
let options = {
url: 'https://api.netease.im/sms/sendcode.action?' + content,
method: 'POST'
};
return await request(options, function (error, response, body) {
console.log(body);
return {
statusCode: 200,
body: JSON.stringify({
message: body,
input: event,
}),
};
});
};
When I run this code in serverless framework, I got a null response, there is nothing in the body, actually it should have at least the "input" attribute.
But console.log already logs the actual response from API.
It looks like my "return" code is not executed at all.
(If I remove async and await, then the program hangs until timeout)
Can anyone help how to modify this code to make it work?
Request does not use promises, so your await keyword is doing nothing. If you want to use promises with request, you need to find a library that supports promises, as noted here: https://www.npmjs.com/package/request#promises--asyncawait A popular option is https://github.com/request/request-promise-native
However, it is a simple matter to simply wrap a request in a promise, so that you don't need to use another dependency
var request = require('request');
const requestHandler = (options) => new Promise((resolve, reject) => {
request(options, (error, response, body) => {
if (error) {
console.error(error);
reject(error);
} else {
console.log(response, body);
resolve({ response, body });
}
});
});
module.exports.sendcode = async (event) => {
let options = {
url: 'https://api.netease.im/sms/sendcode.action?' + content,
method: 'POST'
};
const { response, body } = await requestHandler(options);
return {
statusCode: 200,
body: JSON.stringify({
message: body,
input: event,
})
}
};
Instead of writing your own promise wrapper, maybe request-promise pacakge with promise wrapper on top of request by the same authors, would be of interest to you.
const rp = require('request-promise')
module.exports.sendcode = async (event) => {
let options = {
url: 'https://api.netease.im/sms/sendcode.action?' + content,
method: 'POST'
};
const body = await rp(options)
console.log(body);
return {
statusCode: 200,
body: JSON.stringify({
message: body,
input: event,
}),
};
};
If you want full response you've to simply set resolveWithFullResponse to true
let options = {
url: 'https://api.netease.im/sms/sendcode.action?' + content,
method: 'POST',
resolveWithFullResponse: true
};
// now response contains full respose
const response = await rp(options)
// response.body
// response.statusCode
Use Async/Await with promises. You cannot use Async-Await syntax with callback. Either you must write a promise wrapper for your 'request'callback or you can use request-promise module which is already available.
I have already provided answer for similar problem which you can refer here.
Undefined value after returning an array of values from a MySQL query in a different file

Node.js serverless offline hangs after 1st request

first time when i make a request its working fine, but when i make the request again without restarting the server, its going time out and returning some promise error
following is the code and error
module.exports.getOne = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
db.once('open', () => {
Client.findOne({name:{ $regex : new RegExp(event.pathParameters.name, "i") }})
.then(client => callback(null, {
statusCode: 200,
body: JSON.stringify(client)
}))
.catch(err => callback(null, {
statusCode: err.statusCode || 500,
headers: { 'Content-Type': 'text/plain' },
body: 'Could not fetch the note.'
}))
.finally(() => {
// Close db connection or node event loop won't exit , and lambda will timeout
db.close();
});
});
};
Error:
(node:3273) UnhandledPromiseRejectionWarning: Error: Cannot stop server while in stopping state
The issue is that you aren't completing the call by calling callback so it thinks that your getOne() function isn't finished.
Take a look at the sample code that serverless provides when you setup a new serverless app with their template
module.exports.hello = (event, context, callback) => {
const response = {
statusCode: 200,
body: JSON.stringify({
message: 'Go Serverless v1.0! Your function executed successfully!',
input: event,
}),
};
callback(null, response);
};

Issues with uploading to S3 from node stream using request

Using Node.js, I am trying to upload a large file (700MB ~ 1GB), that i get as a response to a POST request (using request), to my S3 bucket.
Using the aws-sdk for Javascript iv'e tried 2 approaches but each had a different issue:
Approach 1 - Trying to invoke the s3.upload() function on the response event:
const sendRequest = (data) => {
try {
return new Promise((resolve, reject) => {
AWS.config.loadFromPath('./config/awsConfig.json');
let s3 = new AWS.S3({ params:{ Bucket:'myBucket', Key:'path/to/file.csv' } });
request({
method: 'POST',
uri: 'https://www.example.com/apiEndpoint',
headers: {
host: 'example.com',
'content-type': 'application/json'
},
body: JSON.stringify(data)
}).on('response', (response) => { // 1st approach
if (200 == response.statusCode) {
s3.upload({
Body: response,
ACL: 'public-read',
CacheControl: "5184000"
}, (err, data) => { //2 months
console.log(err, data);
});
}
}).on('error', (error) => {
reject();
}).on('end', () => {
resolve();
});
});
} catch (error) {
throw new Error('Unable to get and upload file');
}
}
Result: s3.upload() is called once. The file is created in the bucket but has no data in it (zero bytes).
Approach 2 - Trying to invoke the s3.upload() function on the data event:
const sendRequest = (data) => {
try {
return new Promise((resolve, reject) => {
AWS.config.loadFromPath('./config/awsConfig.json');
let s3 = new AWS.S3({ params:{ Bucket:'myBucket', Key:'path/to/file.csv' } });
request({
method: 'POST',
uri: 'https://www.example.com/apiEndpoint',
headers: {
host: 'example.com',
'content-type': 'application/json'
},
body: JSON.stringify(data)
}).on('data', (data) => { // 2nd approach
s3.upload({
Body: data,
ACL: 'public-read',
CacheControl: "5184000"
}, (err, data) => { //2 months
console.log(err, data);
});
}).on('error', (error) => {
reject();
}).on('end', () => {
resolve();
});
});
} catch (error) {
throw new Error('Unable to get and upload file');
}
}
Result: s3.upload() is called every time there is a data event. The file is created in the bucket but each time the event is emitted, the new data overwrites the old one. In the end there is only the last data that was emitted (7kb ~ 10kb).
Also, after resolve() is called, s3.upload() is still being called multiple times.
Notes:
1) The function returns a Promise because my entire process is synchronous.
2) Both approaches are taken from the answers to Stream response from nodejs request to s3 and from Piping from request.js to s3.upload results in a zero byte file
3) A 3rd approach is to stream to a local file on my server and only then upload to s3. I would very much like to avoid that.
Any ideas on how to get it to work?

Resources