I have one AWS lambda function, which is in node.js
var pg = require("pg");
exports.handler = (event, context, callback) => {
var client = new pg.Client({
user: "41231qd123",
password: "lkjlkasjdlkasldkjas",
database: "12312312asdasd",
port: 5432,
host: "kdoiedjeifk.compute-1.amazonaws.com",
ssl: true
});
client.connect();
console.log('Connected to PostgreSQL database');
client.query("SELECT products.* from products where location_id =
"+event.location_id+" AND company_id = "+event.company_id+" order
by products.name ASC;", (err, res) => {
if (err) {
callback(err);
}
var jsonString = JSON.stringify(res.rows);
var jsonObj = JSON.parse(jsonString);
console.log(jsonString);
client.end();
context.succeed(jsonObj);
});
};
When i test this function in aws lambda "Test", it was working perfectly, but after i added the api-gateway and test it ther i am getting this message 'Process exited before completing request', is there any specific reason with the code or configuration?
First, check your logs for a lambda error.
If there is no lambda crash
If the function is taking longer than 29 seconds then the API Gateway integration can time out. Check the lambda logs to see if there is a crash there, and if not check the time taken.
If there is a lambda crash
Your lambda logs in cloudwatch will give a better reason why. "Process exited before completing request" is usually the boilerplate AWS Lambda error for a limits error. You likely either:
Ran out of memory in the lambda VM
Ran out of disk space in the lambda VM
Timed out.
Check the REPORT line of your logs, they should look like this:
REPORT RequestId: asdf-wertvgh-12345-fdghdfgh Duration: 12345.13 ms Billed Duration: 12400 ms Memory Size: 128 MB Max Memory Used: 64 MB
If that isn't the error, your logs should give you some idea of what happened. Posting a traceback might help.
Related
I have an AWS Lambda function setup using NodeJS, which makes a call to a postgres database using the pg-promise library to retrieve data - which then sends HTTPS GET requests etc. but none of the rest is important.
I was using the 'pg' library originally but ran into connection closing & async issues which is why I switched to pg-promise (Which DID fix my other problem!). With the regular pg library I was getting expected latency of <1000ms
I have boiled down and redacted the code to just a simple query, and I am getting these response times from the last 3 test runs: 11790.78 ms, 11232.22 ms, 12002.04 ms. Every single time it is over 10000ms...
EDIT: Fixed code
const pgp = require('pg-promise')();
const https = require('https');
const xmlParser = require('xml2js').Parser();
const client = pgp({
database: process.env.DATABASE,
host: process.env.HOST,
port: process.env.PORT,
user: process.env.USERNAME,
password: process.env.PASSWORD
});
exports.handler = function(event, context, callback) {
client.one("SELECT period FROM pay WHERE company='XXX' ORDER BY moddate DESC LIMIT 1;")
.then(function(data) {
callback(null, {
"statusCode": 200,
"headers": {
'Content-Type' : 'application/json'
},
"body": data.period
});
})
.catch(function(error) {
console.error(error);
});
};
As stated, I was having no problems with latency when using the 'pg' library, so I know there is no problem with the lambda-RDS postgres connection.
Does anyone have any idea why this is?
Thanks,
In the end I figured out what the problem was... leaving pg-promise to automatically shut down the connection pool is what was causing the latency.
Chaining a
.finally(pgp.end);
after the .catch gave me a 200ms response time.
Thanks everyone
I'm trying to verify that I'm able to connect to my RDS database, but nothing is logged in the connection promise / callback. I was able to connect from my local machine, but not on AWS. I'm out of ideas on how else I can debug this as I've done everything I can find online.
I've verified that the RDS instance and Lambda function in the same VPC, VPC security group, and subnets as suggested by this question. For that VPC security group, I've added 0.0.0.0/0 and ::/0, and the inbound rules can be seen below:
The RDS instance is set to be publicly accessible, and setting to not publicly accessible doesn't make a difference. Below is the output I get from running the lambda function.
START RequestId: 9567a1be-d8d1-4b61-b9c4-4dd06ff36a4b Version: $LATEST
2021-07-21T23:52:47.115Z 9567a1be-d8d1-4b61-b9c4-4dd06ff36a4b INFO Lambda invoked
END RequestId: 9567a1be-d8d1-4b61-b9c4-4dd06ff36a4b
REPORT RequestId: 9567a1be-d8d1-4b61-b9c4-4dd06ff36a4b Duration: 52.71 ms Billed Duration: 53 ms Memory Size: 128 MB Max Memory Used: 71 MB Init Duration: 193.40 ms
I'm using the pg code I got from the node-postgres documentation. I went through the Amazon tutorial for connecting lambda function to rds, giving it a role with AWSLambdaVPCAccessExecutionRole (I didn't use the CLI as they have, I used the GUI on the website). I also read that the console object inside promises don't always return, so I've wrapped every promise in a try catch block and still nothing is returned.
const {Client, Pool} = require('pg')
const pool = new Pool({
user: 'myusername',
password: 'mypassword',
host: 'blahblah.somestuff.us-east-2.rds.amazonaws.com',
port: 5432,
database: 'postgres'
})
pool.on('error', (err, client) => {
console.error('Unexpected error on idle client', err)
process.exit(-1)
})
exports.handler = function(event, context) {
console.log('Lambda invoked') // this logs just fine
try {
var client = pool.connect((err, client, done) => {
if (err) throw err
console.log('connected')
try {
client.query('SELECT * FROM users WHERE id = $1', [1], (err, res) => {
done()
console.log('query has run')
if (err) {
throw err;
// console.log(err.stack)
} else {
console.log(res.rows[0])
}
})
} catch(err) {
throw err
}
})
}
catch(err) {
console.warn(err)
}
};
Node-postgres 6.4.2, PostgreSQL 12.6R1
The proper configuration of the Security Groups would be:
A Security Group on the Lambda function (Lambda-SG) with default settings of Allow All Outbound
A Security Group on the Amazon RDS database (DB-SG) with an Inbound rule that permits traffic from Lambda-SG on port 5432 (for PostgreSQL)
That is, DB-SG specifically references Lambda-SG as permissible for inbound traffic. This is much cleaner than putting resources "in the same Security Group", which is an incorrect model because Security Groups apply to each resource individually.
I was trying to make a post query the db (RDS) using handler.async.
However, I ran into the following issues.
Half of the time, the lambda function completes but the query is not successfully sent to RDS. The other half of the time, it will be completely send to lambda. Tried adding a setTimeout function to increase lambda execution time by 3 secs and the query will be sent all the time.
Also the log will shows the error:
INFO Error: Cannot enqueue Query after fatal error.
The following are my code:
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '***',
user : '***',
password : '***',
database : '***'
});
exports.handler = async (event) => {
const sql = `INSERT INTO forms VALUES(777,2,3,4,5,6,7,8,9,10,11);`;
const query = (x) => {
return new Promise ((resolve,reject)=>{
resolve(connection.query(x, function (error, results, fields) {
console.log(error)
console.log(results)
console.log(fields)
}))})}
await query(sql)
}
With the timeout function,
var mysql = require('mysql');
var connection = mysql.createConnection({
host : '***',
user : '***',
password : '***',
database : '***'
});
exports.handler = async (event) => {
const sql = `INSERT INTO forms VALUES(777,2,3,4,5,6,7,8,9,10,11);`;
const query = (x) => {
return new Promise ((resolve,reject)=>{
resolve(connection.query(x, function (error, results, fields) {
console.log(error)
console.log(results)
console.log(fields)
}))})}
await query(sql)
await wait(3000)
}
const wait = (x) => {
return new Promise ((resolve,reject)=>{
setTimeout(()=>{resolve(console.log("delay"))}, x);
})
}
The first value is a primary key. A constant 777 is sent to check, if error shows duplicate primary key, it will mean that the query is successfully sent. If there is no error, it means that the query is unsuccessfully sent although lambda finishes.
execution result succeeded but shows:
START RequestId: e541fe4b-6927-4fbb-90b4-750f77e5f460 Version: $LATEST
2019-12-19T01:54:45.212Z e541fe4b-6927-4fbb-90b4-750f77e5f460 INFO Error: **Cannot enqueue Query after fatal error**.
at Protocol._validateEnqueue (/var/task/node_modules/mysql/lib/protocol/Protocol.js:212:16)
at Protocol._enqueue (/var/task/node_modules/mysql/lib/protocol/Protocol.js:138:13)
at Connection.query (/var/task/node_modules/mysql/lib/Connection.js:201:25)
at /var/task/index.js:14:24
at new Promise (<anonymous>)
at query (/var/task/index.js:13:10)
at Runtime.exports.handler (/var/task/index.js:20:7)
at Runtime.handleOnce (/var/runtime/Runtime.js:66:25) {
code: 'PROTOCOL_ENQUEUE_AFTER_FATAL_ERROR',
fatal: false
}2019-12-19T01:54:45.213Z e541fe4b-6927-4fbb-90b4-750f77e5f460 INFO undefined2019-12-19T01:54:45.213Z e541fe4b-6927-4fbb-90b4-750f77e5f460 INFO undefined2019-12-19T01:54:45.262Z e541fe4b-6927-4fbb-90b4-750f77e5f460 INFO delayEND RequestId: e541fe4b-6927-4fbb-90b4-750f77e5f460
REPORT RequestId: e541fe4b-6927-4fbb-90b4-750f77e5f460 Duration: 51.09 ms Billed Duration: 100 ms Memory Size: 128 MB Max Memory Used: 80 MB
May you please advise and also tell me which is the best way to execute it??
Managing RDBMS connections in any environment is not a trivial task. Lambda adds a layer of complexity here. You need to understand the distinction between warm and cold restarts, what it means for resources created outside of your handler function, when connection pools are appropriate, and when and how to release connections.
Persistent connections to a database are not particularly suitable in a microservices, FaaS environment like Lambda. That's one reason that Aurora Serverless supports an HTTP Data API (and hopefully other DB engines will too at some point).
Read How To: Manage RDS Connections from AWS Lambda Serverless Functions.
Also be aware of the new Amazon RDS Proxy with AWS Lambda.
In your particular case, the most obvious concern is that you are repeatedly creating DB connections but never releasing them (unless that is a built-in feature of the mysql package's query function that I'm not aware of).
You can increase the lambda timeout upto 15 minutes. But if you are calling the lambda through api gateway, the timeout is 29 seconds.
here is the code working for me.
const mysql = require('mysql');
const con = mysql.createConnection({
host: process.env.RDS_HOSTNAME,
user: process.env.RDS_USERNAME,
password: process.env.RDS_PASSWORD,
port: process.env.RDS_PORT,
connectionLimit: 10,
multipleStatements: true,// Prevent nested sql statements
debug: true
// ,database:'testdb1'
});
exports.handler = async (event) => {
try {
const data = await new Promise((resolve, reject) => {
con.connect(function (err) {
if (err) {
reject(err);
}
const sql = `INSERT INTO forms VALUES(777,2,3,4,5,6,7,8,9,10,11);`;
con.query(sql, function (err, result) {
if (err) {
console.log("Error->" + err);
reject(err);
}
resolve(result);
});
})
});
return {
statusCode: 200,
body: JSON.stringify(data)
}
} catch (err) {
return {
statusCode: 400,
body: err.message
}
}
};
reference: aws lambda with rds mysql DDL command not working
I have an AWS RDS which is publicly accessible and I want to connect to that RDS using AWS Lambda. I am using API Gateway to create a Rest API named "/hello", which needs to return the RDS connection status.
The Lambda code is given below.
var mysql = require('mysql');
var ApiBuilder = require('claudia-api-builder'),
api = new ApiBuilder();
var con = mysql.createConnection({
host: "host",
user: "user",
password: "password"
});
module.exports = api;
api.get('/hello', function () {
//any code written here, works perfectly, when this is called
con.connect(function(err) { //this part doesn't work
if (err) return err;
return "connected";
});
});
The database is publicly accessible, even from my local machine. The Lambda function also has the required permissions with AWS RDS.
The response I get on the browser is as follows
{}
I checked the cloud watch logs, and it is given below.
08:16:11
START RequestId: bf804be1-4797-11e8-8b3f-5b09118631a6 Version: $LATEST
08:16:11
END RequestId: bf804be1-4797-11e8-8b3f-5b09118631a6
08:16:11
REPORT RequestId: bf804be1-4797-11e8-8b3f-5b09118631a6 Duration: 34.28 ms Billed Duration: 100 ms Memory Size: 128 MB Max Memory Used: 26 MB
The Lambda function doesn't go past con.connect(...) where there is no error being shown anywhere (not even on RDS logs). The API Gateway works, because any code before con.connect(...) works perfectly. What could be the issue here?
con.connect(function(err) {
if (err) return err;
return "connected";
});
This is an asynchronous call. Before connect callback function is called lambda will complete and exit.
Change this to
con.connect(function(err) {
if (err) context.done(err);
context.done(null, "connected");
});
As #ConfusedCoder explained, this is caused due to con.connect(...) being an asynchronous call. Using the context in Lambda is not an option here, as Lambda is being invoked using API Gateway, and that there is no exports.myhandler being used in the scenario. Therefore the context object cannot be used to control the flow of Lamda.
exports.myHandler = function(event, context) {
...
}
I tried playing with promises, async await and other techniques until I found a node package named sync-mysql. This makes synchronous queries to a mysql database, where AWS Lambda does not proceed without executing the database call. The updated code is given below, using 'sync-mysql'.
var ApiBuilder = require('claudia-api-builder'), api = new ApiBuilder();
var MySql = require('sync-mysql');
var connection = new MySql({
host: "host",
user: "user",
password: "password"
});
module.exports = api;
api.get('/hello', function(request) {
return connection.query('SELECT * FROM DB.DummyTable');
});
I wrote some basic js to just list the files of a FTP but I get:
"Process exited before completing request"
Is that because Lambda can't interact with FTP?
I'm using jsftp btw.
Here's my setup:
I use Serverless to create the project
For my lambda, I used nodejs and I'm using JSFTP to deal with the ftp stuff.
My code:
// Require Serverless ENV vars
var ServerlessHelpers = require('serverless-helpers-js').loadEnv();
// Require Logic
var lib = require('../lib');
// Lambda Handler
module.exports.handler = function (event, context) {
lib.respond(event, function (error, response) {
return context.done(error, response);
});
};
My ftp lambda code:
var JSFtp = require("jsftp");
module.exports.respond = function (event, cb) {
var ftp = new JSFtp({
host: "host",
user: "user",
password: "password"
});
ftp.auth(ftp.user, ftp.password, function(err, res) {
if (err) console.log(err);
else console.log(res);
ftp.ls(".", function (err, res) {
var results = [];
res.forEach(function (file) {
results.push(file.name);
});
ftp.raw.quit();
return cb(null, results.length);
})
});
};
I added some console.log() all over the place and it seems like it choked once it tried to ftp.auth.
The output I see in cloud watch:
START RequestId: __ID__ Version: $LATEST
END RequestId: __ID__
REPORT RequestId: __ID__ Duration: 526.46 ms Billed Duration: 600 ms Memory Size: 1024 MB Max Memory Used: 33 MB
Process exited before completing request
So it looks like it just choked somewhere...
in short, ftp will not work with lambda since they use ephemeral ports.
sftp will work nicely with lambda. i tested using java code via jsch with no issues; tho i cant see how it wouldnt work with any js sftp lib.
It is possible tested just now.
Make sure ur timeout is set to be long enough and you are calling context.succeed() on process termination
function __main__(event, context) {
var JSFtp = require("jsftp");
var ftp = new JSFtp({
host: "speedtest.tele2.net",
port: 21, // defaults to 21
});
ftp.ls(".", function(err, res) {
var results = []; res.forEach(function(file) {
results.push(file.name);
});
context.succeed(results);
});
};
By default, Lambda functions only have 3 seconds to complete. If it takes longer than that, you'll get the error you are seeing.
You can adjust the timeout to anything up to 5 minutes. To change it using the aws CLI, run:
aws lambda update-function-configuration --function-name my-lambda-function --timeout 300