I wrote some basic js to just list the files of a FTP but I get:
"Process exited before completing request"
Is that because Lambda can't interact with FTP?
I'm using jsftp btw.
Here's my setup:
I use Serverless to create the project
For my lambda, I used nodejs and I'm using JSFTP to deal with the ftp stuff.
My code:
// Require Serverless ENV vars
var ServerlessHelpers = require('serverless-helpers-js').loadEnv();
// Require Logic
var lib = require('../lib');
// Lambda Handler
module.exports.handler = function (event, context) {
lib.respond(event, function (error, response) {
return context.done(error, response);
});
};
My ftp lambda code:
var JSFtp = require("jsftp");
module.exports.respond = function (event, cb) {
var ftp = new JSFtp({
host: "host",
user: "user",
password: "password"
});
ftp.auth(ftp.user, ftp.password, function(err, res) {
if (err) console.log(err);
else console.log(res);
ftp.ls(".", function (err, res) {
var results = [];
res.forEach(function (file) {
results.push(file.name);
});
ftp.raw.quit();
return cb(null, results.length);
})
});
};
I added some console.log() all over the place and it seems like it choked once it tried to ftp.auth.
The output I see in cloud watch:
START RequestId: __ID__ Version: $LATEST
END RequestId: __ID__
REPORT RequestId: __ID__ Duration: 526.46 ms Billed Duration: 600 ms Memory Size: 1024 MB Max Memory Used: 33 MB
Process exited before completing request
So it looks like it just choked somewhere...
in short, ftp will not work with lambda since they use ephemeral ports.
sftp will work nicely with lambda. i tested using java code via jsch with no issues; tho i cant see how it wouldnt work with any js sftp lib.
It is possible tested just now.
Make sure ur timeout is set to be long enough and you are calling context.succeed() on process termination
function __main__(event, context) {
var JSFtp = require("jsftp");
var ftp = new JSFtp({
host: "speedtest.tele2.net",
port: 21, // defaults to 21
});
ftp.ls(".", function(err, res) {
var results = []; res.forEach(function(file) {
results.push(file.name);
});
context.succeed(results);
});
};
By default, Lambda functions only have 3 seconds to complete. If it takes longer than that, you'll get the error you are seeing.
You can adjust the timeout to anything up to 5 minutes. To change it using the aws CLI, run:
aws lambda update-function-configuration --function-name my-lambda-function --timeout 300
Related
I have an index.js written in node.js 8.10 with the necessary node modules uploaded and the pem file in an aws lambda function. The lambda function needs to ssh to an ec2 instance and run a python script (creates another file inside the directory) inside it.
On running a test, I am getting 200 success but I don't see a new file (intended output of script). I am using simple-ssh to get to run the ec2 script.
'use strict';
console.log('Loading lambda function');
exports.handler = function(event, context, callback) {
let bag_size = event.bag_size === undefined ? 10 : event.bag_size;
var SSH = require('simple-ssh');
var fs = require('fs');
var ssh = new SSH({
host: '##############',
user: 'ubuntu',
key: fs.readFileSync('key.pem'),
passphrase: '##########'
//pass: 'password'
});
var pythonCommand = 'python lambda_test.py ' + bag_size;
ssh.exec('cd /home/ubuntu/***/***/***').exec('ls -al', {
out: function(stdout) {
console.log('ls -al got:');
console.log(stdout);
console.log('now launching command');
console.log(pythonCommand);
}
}).exec('' + pythonCommand, {
out: console.log.bind(console),
exit: function(code, stdout, stderr) {
console.log('operation exited with code: ' + code);
console.log('STDOUT from EC2:\n' + stdout);
console.log('STDERR from EC2:\n' + stderr);
context.succeed('Success!');
}
}).start();
var response = {
statusCode: 200,
headers: {
"Access-Control-Allow-Credentials" : true, // Required for cookies, authorization headers with HTTPS
"Access-Control-Allow-Origin":"*",
"Access-Control-Allow-Methods":"POST,GET,OPTIONS",
"Access-Control-Allow-Headers":"Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token",
"Content-Type":"application/json"
},
body: JSON.stringify({ "message": "Success" })
};
// Return response to the caller
callback(null, response);
};
Log Output from CloudWatch:
START RequestId: 247cd************************480b Version: $LATEST
END RequestId: 247cd************************480b
REPORT RequestId: 247247cd************************5b480b0b Duration: 10962.61 ms Billed Duration: 11000 ms Memory Size: 128 MB Max Memory Used: 49 MB
I am not sure where I am going wrong. Please help!
Looks like you call the handler's callback before the exec function is finished since it continues to run asynchronously, which causes the lambda to terminate.
Make sure to call the callback only once it's done
(you can do that by passing a callback parameter to the start function).
Check https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html.
I want to a file from a given url, in a aws lambda function.
I wrote this code:
exports.handler = (event, context, callback) => {
var http = require('http');
var url= "https://mail.google.com/mail/u/0/?ui=2&ik=806f533220&attid=0.1&permmsgid=msg-a:r-8750932957918989452&th=168b03149469bc1f&view=att&disp=safe&realattid=f_jro0gbqh0"
//var client = http.createClient(80, url);
var request = http.request({
port: 80,
host: url
});
request.on('response', function( res ) {
res.on('data', function( data ) {
console.log(data);
});
});
request.end();
const result = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
callback(null, result);
};
but I get an error saying:
"Response: {
"errorMessage": "RequestId: 52baec5e-60bc-47ea-911e-8e6cb1d2f1da Process exited before completing request"
}"
since I only need the first 2 bytes' I thought maybe I should read them, and not the whole file.
any ideas?
thanks a lot!
Did you increase your lambda execution Timeout limit? When you first created, lambda comes setted with only 3 seconds by default. You can change that under Basic settings. Change Timeout to 2 or 3 minutes to allow your lambda to finish execution. Also check if your memory is enough. You may need to increase it a little bit. I have mine with 256 MB.
When you test your lambda, pay attention at the Duration and Memory Size values. Lambda will print that in the last line of the log output. So if you set your lambda execution Timeout to 5 minutes and it only takes 2 minutes or if your lambda Memory Size is to close to your Memory you might want to increase it so your lambda do not fail execution due to a memory problem.
I usually do this with fetch or request. You can make it with request like this:
exports.handler = (event, context, callback) => {
var request = require('request');
var url= "https://mail.google.com/mail/u/0/?ui=2&ik=806f533220&attid=0.1&permmsgid=msg-a:r-8750932957918989452&th=168b03149469bc1f&view=att&disp=safe&realattid=f_jro0gbqh0"
request(url, { json: true, timeout: 1000 }, (err, response, body) => {
if (err) {
console.log(err);
callback(err, null);
} else {
console.log(body);
callback(null, "Hello from Lambda");
}
});
};
Just run npm install request to get module request and you are good to go.
It is always a good idea to start with your local development environment and when everything is ready, you just zip your files and then upload them to lambda. This way you know everything is just fine with your code and you can focus on the lambda configuration details. This way it much easier to test and you do not consume any lambda resource.
This is how to upload your file to lambda:
I have an AWS RDS which is publicly accessible and I want to connect to that RDS using AWS Lambda. I am using API Gateway to create a Rest API named "/hello", which needs to return the RDS connection status.
The Lambda code is given below.
var mysql = require('mysql');
var ApiBuilder = require('claudia-api-builder'),
api = new ApiBuilder();
var con = mysql.createConnection({
host: "host",
user: "user",
password: "password"
});
module.exports = api;
api.get('/hello', function () {
//any code written here, works perfectly, when this is called
con.connect(function(err) { //this part doesn't work
if (err) return err;
return "connected";
});
});
The database is publicly accessible, even from my local machine. The Lambda function also has the required permissions with AWS RDS.
The response I get on the browser is as follows
{}
I checked the cloud watch logs, and it is given below.
08:16:11
START RequestId: bf804be1-4797-11e8-8b3f-5b09118631a6 Version: $LATEST
08:16:11
END RequestId: bf804be1-4797-11e8-8b3f-5b09118631a6
08:16:11
REPORT RequestId: bf804be1-4797-11e8-8b3f-5b09118631a6 Duration: 34.28 ms Billed Duration: 100 ms Memory Size: 128 MB Max Memory Used: 26 MB
The Lambda function doesn't go past con.connect(...) where there is no error being shown anywhere (not even on RDS logs). The API Gateway works, because any code before con.connect(...) works perfectly. What could be the issue here?
con.connect(function(err) {
if (err) return err;
return "connected";
});
This is an asynchronous call. Before connect callback function is called lambda will complete and exit.
Change this to
con.connect(function(err) {
if (err) context.done(err);
context.done(null, "connected");
});
As #ConfusedCoder explained, this is caused due to con.connect(...) being an asynchronous call. Using the context in Lambda is not an option here, as Lambda is being invoked using API Gateway, and that there is no exports.myhandler being used in the scenario. Therefore the context object cannot be used to control the flow of Lamda.
exports.myHandler = function(event, context) {
...
}
I tried playing with promises, async await and other techniques until I found a node package named sync-mysql. This makes synchronous queries to a mysql database, where AWS Lambda does not proceed without executing the database call. The updated code is given below, using 'sync-mysql'.
var ApiBuilder = require('claudia-api-builder'), api = new ApiBuilder();
var MySql = require('sync-mysql');
var connection = new MySql({
host: "host",
user: "user",
password: "password"
});
module.exports = api;
api.get('/hello', function(request) {
return connection.query('SELECT * FROM DB.DummyTable');
});
I have one AWS lambda function, which is in node.js
var pg = require("pg");
exports.handler = (event, context, callback) => {
var client = new pg.Client({
user: "41231qd123",
password: "lkjlkasjdlkasldkjas",
database: "12312312asdasd",
port: 5432,
host: "kdoiedjeifk.compute-1.amazonaws.com",
ssl: true
});
client.connect();
console.log('Connected to PostgreSQL database');
client.query("SELECT products.* from products where location_id =
"+event.location_id+" AND company_id = "+event.company_id+" order
by products.name ASC;", (err, res) => {
if (err) {
callback(err);
}
var jsonString = JSON.stringify(res.rows);
var jsonObj = JSON.parse(jsonString);
console.log(jsonString);
client.end();
context.succeed(jsonObj);
});
};
When i test this function in aws lambda "Test", it was working perfectly, but after i added the api-gateway and test it ther i am getting this message 'Process exited before completing request', is there any specific reason with the code or configuration?
First, check your logs for a lambda error.
If there is no lambda crash
If the function is taking longer than 29 seconds then the API Gateway integration can time out. Check the lambda logs to see if there is a crash there, and if not check the time taken.
If there is a lambda crash
Your lambda logs in cloudwatch will give a better reason why. "Process exited before completing request" is usually the boilerplate AWS Lambda error for a limits error. You likely either:
Ran out of memory in the lambda VM
Ran out of disk space in the lambda VM
Timed out.
Check the REPORT line of your logs, they should look like this:
REPORT RequestId: asdf-wertvgh-12345-fdghdfgh Duration: 12345.13 ms Billed Duration: 12400 ms Memory Size: 128 MB Max Memory Used: 64 MB
If that isn't the error, your logs should give you some idea of what happened. Posting a traceback might help.
I'm try to create a task on AWS Lambda that create PDF file from PhantomJS then upload it AWS S3 later.
Now, I try to run it on Lambda but it's always Timeout.
My Lambda has 128mb of ram. The runtime is node.js 4.4.3.
This is the error I got from Lambda
"errorMessage": "2017-03-01T08:05:56.255Z dfd4cfe8-fe55-11e6-bf24-e7edf412e037 Task timed out after 10.00 seconds"
Also these are the log output
REPORT RequestId: dfd4cfe8-fe55-11e6-bf24-e7edf412e037 Duration: 10000.08 ms Billed Duration: 10000 ms Memory Size: 128 MB Max Memory Used: 29 MB
2017-03-01T08:05:56.255Z dfd4cfe8-fe55-11e6-bf24-e7edf412e037 Task timed out after 10.00 seconds
This is my code.
Index.js
var childProcess = require('child_process');
var path = require('path');
exports.handler = function(event, context, callback) {
// Set the path as described here: https://aws.amazon.com/blogs/compute/running-executables-in-aws-lambda/
process.env['PATH'] = process.env['PATH'] + ':' + process.env['LAMBDA_TASK_ROOT'];
// Set the path to the phantomjs binary
var phantomPath = path.join(__dirname, 'phantomjs_linux-x86_64');
// Arguments for the phantom script
var processArgs = [
path.join(__dirname, 'phantom-script.js'),
event.url
];
// Launc the child process
childProcess.execFile(phantomPath, processArgs, function(error, stdout, stderr) {
if (error) {
context.fail(error);
return;
}
if (stderr) {
context.fail(error);
return;
}
context.succeed(stdout);
});
}
phantom-script.js
var system = require('system');
var args = system.args;
// Example of how to get arguments passed from node script
// args[0] would be this file's name: phantom-script.js
const url = "https://google.com";
system.stdout.write('hello from phantom!');
console.log("task start, target url = " + url);
console.time("execute time");
phantom.create().then(function(ph) {
console.time("create Page");
ph.createPage().then(function(page) {
console.timeEnd("create Page");
console.time("open website");
page.open(url).then(function(status) {
console.timeEnd("open website");
console.time("render as pdf");
page.render('google.pdf').then(function() {
console.timeEnd("render as pdf");
console.log('Page Rendered');
ph.exit();
// send to s3
console.timeEnd("execute time");
});
});
});
});
// Send some info node's childProcess' stdout
system.stdout.write('hello from phantom!')
phantom.exit();
I try to do my work following this answer but it's not working.
I didn't get any of log from my phantom-script.js like it is not trigger but my task always timeout.
After I spend a lot of time on it. I found the package name Phantomjs-Prebuilt that you can install it via npm. you have to do npm install on amazon linux instance or docker amazon linux that has node version 4.x (lambda is using node version 4.3). Otherwise it will not work on lambda.
Then, I updated my code like this.
Index.js
var phantomjs = require('phantomjs-prebuilt')
exports.handler = function(event, context, callback) {
var sourceUrl = "https://example.com"
var program = phantomjs.exec('phantom-script.js', sourceUrl)
program.stdout.pipe(process.stdout)
program.stderr.pipe(process.stderr)
program.on('exit', code => {
// do something here after you phantomjs finish.
return
})
}
phantom-script.js
var system = require('system')
var args = system.args
// Example of how to get arguments passed from node script
// args[0] would be this file's name: phantom-script.js
var url = args[1] // received sourceUrl value
// Send some info node's childProcess' stdout
system.stdout.write('phantomJS running\r\n')
var webpage = require('webpage').create()
webpage.paperSize = {
format: 'A4',
orientation: 'landscape'
}
webpage.open(url, function(status) {
system.stdout.write('start open page\r\n')
webpage.render('/tmp/web.pdf', {
format: 'pdf',
quality: '100'
})
system.stdout.write('finish render page\r\n')
phantom.exit()
})
On lambda the place you can write a file is /tmp folder that why i saved the file there.
I'm running this via lambda with 192mb of ram. It's work really fine. I can create a screenshot of webpage that has 500 images with this setting. The most important thing is make sure your lambda is able to connect internet.
FYI, I realize that when phantom-script.js (the file i wrote phantom script in.) has an error your lambda will freeze until it timeout. That's why I always got this response from lambda Task timed out after 10.00 seconds.