I'm try to create a task on AWS Lambda that create PDF file from PhantomJS then upload it AWS S3 later.
Now, I try to run it on Lambda but it's always Timeout.
My Lambda has 128mb of ram. The runtime is node.js 4.4.3.
This is the error I got from Lambda
"errorMessage": "2017-03-01T08:05:56.255Z dfd4cfe8-fe55-11e6-bf24-e7edf412e037 Task timed out after 10.00 seconds"
Also these are the log output
REPORT RequestId: dfd4cfe8-fe55-11e6-bf24-e7edf412e037 Duration: 10000.08 ms Billed Duration: 10000 ms Memory Size: 128 MB Max Memory Used: 29 MB
2017-03-01T08:05:56.255Z dfd4cfe8-fe55-11e6-bf24-e7edf412e037 Task timed out after 10.00 seconds
This is my code.
Index.js
var childProcess = require('child_process');
var path = require('path');
exports.handler = function(event, context, callback) {
// Set the path as described here: https://aws.amazon.com/blogs/compute/running-executables-in-aws-lambda/
process.env['PATH'] = process.env['PATH'] + ':' + process.env['LAMBDA_TASK_ROOT'];
// Set the path to the phantomjs binary
var phantomPath = path.join(__dirname, 'phantomjs_linux-x86_64');
// Arguments for the phantom script
var processArgs = [
path.join(__dirname, 'phantom-script.js'),
event.url
];
// Launc the child process
childProcess.execFile(phantomPath, processArgs, function(error, stdout, stderr) {
if (error) {
context.fail(error);
return;
}
if (stderr) {
context.fail(error);
return;
}
context.succeed(stdout);
});
}
phantom-script.js
var system = require('system');
var args = system.args;
// Example of how to get arguments passed from node script
// args[0] would be this file's name: phantom-script.js
const url = "https://google.com";
system.stdout.write('hello from phantom!');
console.log("task start, target url = " + url);
console.time("execute time");
phantom.create().then(function(ph) {
console.time("create Page");
ph.createPage().then(function(page) {
console.timeEnd("create Page");
console.time("open website");
page.open(url).then(function(status) {
console.timeEnd("open website");
console.time("render as pdf");
page.render('google.pdf').then(function() {
console.timeEnd("render as pdf");
console.log('Page Rendered');
ph.exit();
// send to s3
console.timeEnd("execute time");
});
});
});
});
// Send some info node's childProcess' stdout
system.stdout.write('hello from phantom!')
phantom.exit();
I try to do my work following this answer but it's not working.
I didn't get any of log from my phantom-script.js like it is not trigger but my task always timeout.
After I spend a lot of time on it. I found the package name Phantomjs-Prebuilt that you can install it via npm. you have to do npm install on amazon linux instance or docker amazon linux that has node version 4.x (lambda is using node version 4.3). Otherwise it will not work on lambda.
Then, I updated my code like this.
Index.js
var phantomjs = require('phantomjs-prebuilt')
exports.handler = function(event, context, callback) {
var sourceUrl = "https://example.com"
var program = phantomjs.exec('phantom-script.js', sourceUrl)
program.stdout.pipe(process.stdout)
program.stderr.pipe(process.stderr)
program.on('exit', code => {
// do something here after you phantomjs finish.
return
})
}
phantom-script.js
var system = require('system')
var args = system.args
// Example of how to get arguments passed from node script
// args[0] would be this file's name: phantom-script.js
var url = args[1] // received sourceUrl value
// Send some info node's childProcess' stdout
system.stdout.write('phantomJS running\r\n')
var webpage = require('webpage').create()
webpage.paperSize = {
format: 'A4',
orientation: 'landscape'
}
webpage.open(url, function(status) {
system.stdout.write('start open page\r\n')
webpage.render('/tmp/web.pdf', {
format: 'pdf',
quality: '100'
})
system.stdout.write('finish render page\r\n')
phantom.exit()
})
On lambda the place you can write a file is /tmp folder that why i saved the file there.
I'm running this via lambda with 192mb of ram. It's work really fine. I can create a screenshot of webpage that has 500 images with this setting. The most important thing is make sure your lambda is able to connect internet.
FYI, I realize that when phantom-script.js (the file i wrote phantom script in.) has an error your lambda will freeze until it timeout. That's why I always got this response from lambda Task timed out after 10.00 seconds.
Related
I have an index.js written in node.js 8.10 with the necessary node modules uploaded and the pem file in an aws lambda function. The lambda function needs to ssh to an ec2 instance and run a python script (creates another file inside the directory) inside it.
On running a test, I am getting 200 success but I don't see a new file (intended output of script). I am using simple-ssh to get to run the ec2 script.
'use strict';
console.log('Loading lambda function');
exports.handler = function(event, context, callback) {
let bag_size = event.bag_size === undefined ? 10 : event.bag_size;
var SSH = require('simple-ssh');
var fs = require('fs');
var ssh = new SSH({
host: '##############',
user: 'ubuntu',
key: fs.readFileSync('key.pem'),
passphrase: '##########'
//pass: 'password'
});
var pythonCommand = 'python lambda_test.py ' + bag_size;
ssh.exec('cd /home/ubuntu/***/***/***').exec('ls -al', {
out: function(stdout) {
console.log('ls -al got:');
console.log(stdout);
console.log('now launching command');
console.log(pythonCommand);
}
}).exec('' + pythonCommand, {
out: console.log.bind(console),
exit: function(code, stdout, stderr) {
console.log('operation exited with code: ' + code);
console.log('STDOUT from EC2:\n' + stdout);
console.log('STDERR from EC2:\n' + stderr);
context.succeed('Success!');
}
}).start();
var response = {
statusCode: 200,
headers: {
"Access-Control-Allow-Credentials" : true, // Required for cookies, authorization headers with HTTPS
"Access-Control-Allow-Origin":"*",
"Access-Control-Allow-Methods":"POST,GET,OPTIONS",
"Access-Control-Allow-Headers":"Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token",
"Content-Type":"application/json"
},
body: JSON.stringify({ "message": "Success" })
};
// Return response to the caller
callback(null, response);
};
Log Output from CloudWatch:
START RequestId: 247cd************************480b Version: $LATEST
END RequestId: 247cd************************480b
REPORT RequestId: 247247cd************************5b480b0b Duration: 10962.61 ms Billed Duration: 11000 ms Memory Size: 128 MB Max Memory Used: 49 MB
I am not sure where I am going wrong. Please help!
Looks like you call the handler's callback before the exec function is finished since it continues to run asynchronously, which causes the lambda to terminate.
Make sure to call the callback only once it's done
(you can do that by passing a callback parameter to the start function).
Check https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html.
I want to a file from a given url, in a aws lambda function.
I wrote this code:
exports.handler = (event, context, callback) => {
var http = require('http');
var url= "https://mail.google.com/mail/u/0/?ui=2&ik=806f533220&attid=0.1&permmsgid=msg-a:r-8750932957918989452&th=168b03149469bc1f&view=att&disp=safe&realattid=f_jro0gbqh0"
//var client = http.createClient(80, url);
var request = http.request({
port: 80,
host: url
});
request.on('response', function( res ) {
res.on('data', function( data ) {
console.log(data);
});
});
request.end();
const result = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
callback(null, result);
};
but I get an error saying:
"Response: {
"errorMessage": "RequestId: 52baec5e-60bc-47ea-911e-8e6cb1d2f1da Process exited before completing request"
}"
since I only need the first 2 bytes' I thought maybe I should read them, and not the whole file.
any ideas?
thanks a lot!
Did you increase your lambda execution Timeout limit? When you first created, lambda comes setted with only 3 seconds by default. You can change that under Basic settings. Change Timeout to 2 or 3 minutes to allow your lambda to finish execution. Also check if your memory is enough. You may need to increase it a little bit. I have mine with 256 MB.
When you test your lambda, pay attention at the Duration and Memory Size values. Lambda will print that in the last line of the log output. So if you set your lambda execution Timeout to 5 minutes and it only takes 2 minutes or if your lambda Memory Size is to close to your Memory you might want to increase it so your lambda do not fail execution due to a memory problem.
I usually do this with fetch or request. You can make it with request like this:
exports.handler = (event, context, callback) => {
var request = require('request');
var url= "https://mail.google.com/mail/u/0/?ui=2&ik=806f533220&attid=0.1&permmsgid=msg-a:r-8750932957918989452&th=168b03149469bc1f&view=att&disp=safe&realattid=f_jro0gbqh0"
request(url, { json: true, timeout: 1000 }, (err, response, body) => {
if (err) {
console.log(err);
callback(err, null);
} else {
console.log(body);
callback(null, "Hello from Lambda");
}
});
};
Just run npm install request to get module request and you are good to go.
It is always a good idea to start with your local development environment and when everything is ready, you just zip your files and then upload them to lambda. This way you know everything is just fine with your code and you can focus on the lambda configuration details. This way it much easier to test and you do not consume any lambda resource.
This is how to upload your file to lambda:
I'm trying to capture the output from an external program launched from an AWS Lambda written in NodeJS. Full example code below. Any test event would do since it's not really used.
exports.handler = async (event) => {
console.log ("Entering lambda" + JSON.stringify(event))
var spawn = require('child_process').spawnSync;
child_process = spawn ('aws', [' --version'], {
stdio: 'inherit',
stderr: 'inherit',
shell: true
})
console.log ("done");
const response = {
statusCode: 200,
body: JSON.stringify('done'),
};
return response;
};
When I run it, I get the following as output (I removed the test event details for brevity, since it's irrelevant).
What I don't see is the results of the aws --version command that I expected (I'm using it to test the correct invocation of the AWS CLI, but any Linux command would do). The code does execute synchronously because if I replace the invocation with child_process = spawn ('sleep', ['1'], {, lambda's execution time grows to 1117.85 ms, so the one-second sleep happens. But there's nothing captured in the execution logs.
START RequestId: 0c1287e2-d2ee-4436-a577-bc8ec3608120 Version: $LATEST
2019-01-16T19:12:45.130Z 0c1287e2-d2ee-4436-a577-bc8ec3608120 Entering lambda {...}
2019-01-16T19:12:45.143Z 0c1287e2-d2ee-4436-a577-bc8ec3608120 done
END RequestId: 0c1287e2-d2ee-4436-a577-bc8ec3608120
REPORT RequestId: 0c1287e2-d2ee-4436-a577-bc8ec3608120 Duration: 13.29 ms Billed Duration: 100 ms Memory Size: 128 MB Max Memory Used: 20 MB
Am I doing something wrong? Or is there any other way to capture the output (status code, stdio, stderr) for a Lambda written in NodeJS?
This works for me (node.js 8.10 runtime)
exports.handler = async (event) => {
const spawnSync = require('child_process').spawnSync;
const process = spawnSync('echo', ['hello', 'world'], {
stdio: 'pipe',
stderr: 'pipe'
});
console.log(process.status);
console.log(process.stdout.toString());
};
When trying to run with aws, it throws a ENOENT error. In other words, the command is not available. As mentioned in the question comments by #jarmod, I also believe the awscli is not available in the Lambda container.
What is available is the SDK, so that you can require('aws-sdk'); without bundling it to your Lambda deployment package.
aws-cli is a python package and is not installed in a Lambda environment.
To verify my statement, you can type shell commands here http://www.lambdashell.com/ and check what is installed by default in your execution environment or check the official documentation.
Your code above returns no output, because trying to execute aws returns ENOENT, meaning the file is not available.
If you want to run aws inside your lambda function, you can follow these instructions : Call aws-cli from AWS Lambda
I would however question why do you want to do this ? The AWS-SDK is included in the local runtime environment and you can call any AWS API directly from your code, without the need to spawn a process and deal with stdin/stdout. I would strongly advise to not spawn aws cli from your code but rather use the SDK instead.
But, if you do want to run a process from Lambda and capture stdout and stderr, here is how I am doing it.
'use strict';
const childProcess = require('child_process');
/*
* Handle the chile process and returns a Promise
* that resoved when process finishes executing
*
* The Promise resolves an exit_code
*/
function handleProcess(process) {
return new Promise((resolve, reject) => {
process.stdout.on('data', (data) => {
console.log(`stdout: ${data}`);
console.log('stdout');
});
process.stderr.on('data', (data) => {
console.log(`stderr: ${data}`);
});
process.on('close', (code) => {
console.log(`child process exited with code ${code}`);
if (code === 0) {
resolve(code);
} else {
reject(code);
}
});
});
}
exports.handler = (event, context, callback) => {
// console.log(JSON.stringify(process.env, null, 2));
// console.log(JSON.stringify(event, null, 2));
return handleProcess(childProcess.spawn('ls', ['--version']))
.then((exit_code) => {
console.log(`exit_code = ${exit_code}`)
let response = {
statusCode: (0 == exit_code) ? 200 : 500,
body: exit_code
};
callback(null, response);
})
.catch((error) => {
console.error(error);
let response = {
statusCode: 500,
body: error
};
callback(null, response);
});
}
const winston = require('winston');
var S3StreamLogger = require('s3-streamlogger').S3StreamLogger;
var s3_stream = new S3StreamLogger({
bucket: "bucket_name",
access_key_id: "access_key_id",
secret_access_key: "secret_access_key"
});
var logger = new (winston.Logger)({
transports: [
new (winston.transports.File)({
stream: s3_stream
})
]
});
exports.handler = function (event, context) {
winston.info('Hello Winston!');
console.log("This is logging demo");
logger.info('Logging is in progress');
console.log("Consol Starting Lambda function");
var responseCode = 200;
var responseBody = {
message: "Hello !",
input: event
};
var response = {
statusCode: responseCode,
headers: {
"x-custom-header": "my custom header value"
},
body: JSON.stringify(responseBody)
};
console.log("response: " + JSON.stringify(response));
context.succeed(response);
};
On executing above code(successfully) on aws lambda. I am getting cloudwatch logs but there is nothing generated inside my S3 bucket.
I was hoping for a log file to be created over there but nothing created.
Am i lacking something in the code ?
My aim is to generate log file in amazon S3 bucket so that i can download the file for analysis.
s3-streamlogger buffers content before uploading it to s3. By default it will buffer for 20 seconds, or 10kB of content (whichever is less) – so if your lambda function doesn't execute for at least 20 second, or write 10KB of logs nothing will be written before your function is suspended.
You could either specify smaller thresholds for the buffering via the upload_every and buffer_size options (see https://github.com/coggle/s3-streamlogger#upload_every), or manually flush the file after each request by calling flushFile on the stream.
I wrote some basic js to just list the files of a FTP but I get:
"Process exited before completing request"
Is that because Lambda can't interact with FTP?
I'm using jsftp btw.
Here's my setup:
I use Serverless to create the project
For my lambda, I used nodejs and I'm using JSFTP to deal with the ftp stuff.
My code:
// Require Serverless ENV vars
var ServerlessHelpers = require('serverless-helpers-js').loadEnv();
// Require Logic
var lib = require('../lib');
// Lambda Handler
module.exports.handler = function (event, context) {
lib.respond(event, function (error, response) {
return context.done(error, response);
});
};
My ftp lambda code:
var JSFtp = require("jsftp");
module.exports.respond = function (event, cb) {
var ftp = new JSFtp({
host: "host",
user: "user",
password: "password"
});
ftp.auth(ftp.user, ftp.password, function(err, res) {
if (err) console.log(err);
else console.log(res);
ftp.ls(".", function (err, res) {
var results = [];
res.forEach(function (file) {
results.push(file.name);
});
ftp.raw.quit();
return cb(null, results.length);
})
});
};
I added some console.log() all over the place and it seems like it choked once it tried to ftp.auth.
The output I see in cloud watch:
START RequestId: __ID__ Version: $LATEST
END RequestId: __ID__
REPORT RequestId: __ID__ Duration: 526.46 ms Billed Duration: 600 ms Memory Size: 1024 MB Max Memory Used: 33 MB
Process exited before completing request
So it looks like it just choked somewhere...
in short, ftp will not work with lambda since they use ephemeral ports.
sftp will work nicely with lambda. i tested using java code via jsch with no issues; tho i cant see how it wouldnt work with any js sftp lib.
It is possible tested just now.
Make sure ur timeout is set to be long enough and you are calling context.succeed() on process termination
function __main__(event, context) {
var JSFtp = require("jsftp");
var ftp = new JSFtp({
host: "speedtest.tele2.net",
port: 21, // defaults to 21
});
ftp.ls(".", function(err, res) {
var results = []; res.forEach(function(file) {
results.push(file.name);
});
context.succeed(results);
});
};
By default, Lambda functions only have 3 seconds to complete. If it takes longer than that, you'll get the error you are seeing.
You can adjust the timeout to anything up to 5 minutes. To change it using the aws CLI, run:
aws lambda update-function-configuration --function-name my-lambda-function --timeout 300