nodejs - aws lambda random timeout when trying to access S3.getBucketLocation - node.js

For the following code, I was asked to check, which runs on AWS Lambda and performs the simple task of uploading files to S3, timesout randomly when run manually from the aws console,but at the same time when run on a schedule, will always timeout at the first attempt and then run successfully the second time(it is run immediately after failing the first time).The code always seems to hang after calling s3.getBucketLocation, never reaching the getBucketLocation's callback(on the first attempt!).
All VPC's and NAT are set-up correctly, such that, when I remove s3.getBucketLocation from the code and call s3Upload.upload(params, cb); directly, the code runs without timeouts.
I would really like to understand why the first function fails alternatively when trying to retrieve the buckets region from AWS.
"use strict";
var urlHelper = require('url');
var fs = require('fs');
var AWS = require('aws-sdk');
var exp = {};
function saveToFile(path, content, cb) {
fs.writeFile(path, content, cb);
}
function saveToS3(bucket, key, content, cb) {
var s3 = new AWS.S3({apiVersion: '2006-03-01'});
var params = {
Bucket: bucket
};
s3.getBucketLocation(params, function(err, data) {
console.log('THIS IS NOT REACHED');
if (err) {
return cb(err);
}
var s3Upload = new AWS.S3({
apiVersion: '2006-03-01',
region: data.LocationConstraint
});
var params = {
Bucket: bucket,
Key: key,
Body: content,
ACL: 'public-read',
ContentType: 'text/xml',
CacheControl: 'max-age=60'
};
s3Upload.upload(params, cb);
});
}
exp.upload = function(path, content, cb) {
var url = urlHelper.parse(path);
switch(url.protocol) {
case 'file:':
case null:
saveToFile(path, content, cb);
break;
case 's3:':
saveToS3(url.host, url.pathname.substring(1), content, cb);
break;
default:
console.log('No matching protocol found for upload!');
cb(null, null);
}
};
module.exports = exp;

Related

Cannot upload to AWS S3 inside my Lambda function

I have the following lambda function. It received an XML, looks through it, finds a base64 pdf file and tries to upload it to S3.
index.js
const AWS = require('aws-sdk');
const xml2js = require('xml2js');
const pdfUpload = require('./upload_pdf');
const s3 = new AWS.S3();
exports.handler = async (event, context, callback) => {
let attachment;
xml2js.parseString(event.body, function(err, result) {
attachment =
result.Attachment[0].Data[0];
if (attachment) {
pdfUpload(attachment);
}
});
return {
statusCode: 200
}
};
upload_pdf.js
/**
*
* #param {string} base64 Data
* #return {string} Image url
*/
const pdfUpload = async (base64) => {
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const base64Data = new Buffer.from(base64, 'base64');
// With this setup, each time your user uploads an image, will be overwritten.
// To prevent this, use a different Key each time.
// This won't be needed if they're uploading their avatar, hence the filename, userAvatar.js.
const params = {
Bucket: 'mu-bucket',
Key: `123.pdf`,
Body: base64Data,
ACL: 'public-read',
ContentEncoding: 'base64',
ContentType: `application/pdf`
}
let location = '';
let key = '';
try {
const { Location, Key } = await s3.upload(params).promise();
location = Location;
key = Key;
} catch (error) {
// console.log(error)
}
console.log(location, key);
return location;
}
module.exports = pdfUpload;
No matter what I do, the file does not get uploaded. I have checked the permissions, and the lambda has access to the bucket. Running the lambda I'm not receiving any errors either. Can anybody see what might be wrong here?
First, as an advice, I think you should put more logs to see at which steps the function is stuck / failing
The second thing you can try is to put await
await pdfUpload(attachment);

AWS textract methods in node js are not getting invoked

I want to extract text from image using node js so created a lambda in aws. Please find the below code snippet. Issue is that the textract method detectDocumentText is not getting invoked.
As far as permission I had given s3 full access and textract full access to the lambda. Am I missing anything?
var AWS = require("aws-sdk");
var base64 = require("base-64");
var fs = require("fs");
exports.handler = async (event, context, callback) => {
// Input for textract can be byte array or S3 object
AWS.config.region = "us-east-1";
//AWS.config.update({ region: 'us-east-1' });
var textract = new AWS.Textract({ apiVersion: "2018-06-27" });
//var textract = new AWS.Textract();
console.log(textract);
var params = {
Document: {
/* required */
//'Bytes': imageBase64
S3Object: {
Bucket: "717577",
Name: "Picture2.png"
}
}
};
textract.detectDocumentText(params, function(err, data) {
if (err) {
console.log(err); // an error occurred
} else {
console.log(data); // successful response
callback(null, data);
}
});
};
As well as I don't see any error logs in cloudwatch logs.
The problem is that you have marked your method as async which means that you are returning a promise. In your case you are not returning a promise so for lambda there is no way to complete the execution of the method. You have two choices here
Remove async
Or more recommended way is to convert your callback style to use promise. aws-sdk support .promise method on all methods so you could leverage that. The code will look like this
var AWS = require("aws-sdk");
var base64 = require("base-64");
var fs = require("fs");
exports.handler = async (event, context) => {
// Input for textract can be byte array or S3 object
AWS.config.region = "us-east-1";
//AWS.config.update({ region: 'us-east-1' });
var textract = new AWS.Textract({ apiVersion: "2018-06-27" });
//var textract = new AWS.Textract();
console.log(textract);
var params = {
Document: {
/* required */
//'Bytes': imageBase64
S3Object: {
Bucket: "717577",
Name: "Picture2.png"
}
}
};
const data = await textract.detectDocumentText(params).promise();
return data;
};
Hope this helps.

Lambda runs but no console output

I am trying to build a lambda function in AWS - when I go and test the function, it says it completes successfully but the expected results are not showing up (ie transcribe job or file in S3). I put code to look for errors but nothing is showing in console either
Have the files uploaded as a zip in what I believe is the right way - the root of the zip file has node modules and index.js
Also can see it firing in xray
exports.handler = async (event, context) => {
var Parser = require('rss-parser');
var request = require('request');
var AWS = require('aws-sdk');
var bucketName = 'transcribebucketkm';
var bucketNameOut = 'transcribebucketkm-out';
var parser = new Parser();
var s3 = new AWS.S3();
var transcribeservice = new AWS.TranscribeService();
AWS.config.update({region:'us-east-1'});
var datetime = new Date
var datetime = datetime.toISOString().slice(0,10) + ".mp3";
var mediafileuri= "http://s3.amazonaws.com/transcribebucketkm/" + datetime
parser.parseURL('https://bridgetown.podbean.com/feed.xml', function (err, feed) {
console.log(err)
request({ method: 'GET', url: feed.items[0].enclosure.url, encoding: null},function (err, response, body) {
console.log(err)
console.log(body)
s3.upload({ Bucket: bucketName, Key: datetime, Body: body}, function(err, data) {
console.log(data)
transcribeservice.startTranscriptionJob({LanguageCode: "en-US", Media:{MediaFileUri: mediafileuri}, MediaFormat: "mp3", TranscriptionJobName: datetime, OutputBucketName: bucketNameOut});
});
});
});
}
Console does show only null - making me believe no errors

Stubbing S3 uploads in Node.js

How would I go about stubbing S3 uploads in Node.js?
For insight, I'm using Mocha for tests and Sinon for stubbing, but I'm open to changing anything. I have a file that exports a function that performs the upload. It looks like this:
var AWS = require('aws-sdk');
var s3 = new AWS.S3({ params: { Bucket: process.env.S3_BUCKET }});
var params = { Key: key, Body: body };
s3.upload(params, function (error, data) {
// Handle upload or error
});
If I try to stub AWS.S3 or AWS.S3.prototype, nothing changes. I assume this is because my tests have required aws-sdk themselves and have their own copy of each function.
My test looks like this:
describe('POST /files', function () {
var url = baseURL + '/files';
it('uploads the file to s3', function (done) {
var fs = require('fs');
var formData = {
video: fs.createReadStream(process.cwd() + '/test/support/video.mp4')
};
var params = {url: url, formData: formData};
request.post(params, function (error, response, body) {
expect(response.statusCode).to.eq(200);
expect(response.body).to.eq('Uploaded');
done();
});
});
});
This test works fine, but it does not stub the upload to S3, so the upload actually goes through :X.
There are several options to mock S3 in Node.
Some modules specific to S3:
https://www.npmjs.com/package/mock-s3
https://www.npmjs.com/package/mock-aws-s3
Some modules for general AWS mocking:
https://www.npmjs.com/package/aws-sdk-mock
https://www.npmjs.com/package/mock-aws-sinon
https://www.npmjs.com/package/mock-aws
And you can even start a simple server that responds to some of the S3 API calls:
https://www.npmjs.com/package/s3rver
The last one can easily be used in other languages and runtimes, not only in Node.
You can stub with Sinon.js as follows if you'd like:
Expose the AWS.S3 instance:
var AWS = require('aws-sdk');
var s3 = new AWS.S3({ params: { Bucket: process.env.S3_BUCKET }});
var params = { Key: key, Body: body };
exports.s3.upload(params, function (error, data) {
});
//Expose S3 instance
exports.s3 = s3;
Stub the same instance like so:
var sinon = require('sinon');
//Import module you are going to test
var UploadService = require('./uploadService');
describe('POST /files', function () {
before(function() {
//Stub before, and specify what data you'd like in the callback.
this.uploadS3Stub = sinon.stub(uploadService.s3, 'upload').callsArgWith(1, null, { data: 'Desired response' });
});
after(function() {
//Restore after
this.uploadS3Stub.restore();
});
var url = baseURL + '/files';
it('uploads the file to s3', function (done) {
var fs = require('fs');
var formData = {
video: fs.createReadStream(process.cwd() + '/test/support/video.mp4')
};
var params = {url: url, formData: formData};
var self = this;
request.post(params, function (error, response, body) {
expect(response.statusCode).to.eq(200);
expect(response.body).to.eq('Uploaded');
//You can also check whether the stub was called :)
expect(self.uploadS3Stub.calledOnce).to.eql(true);
done();
});
});
});
s3UploadStub = AWS.S3.prototype.upload = sandbox.stub();
it("should upload successfully", async function() {
s3UploadStub.yields(null, data);
let response = await FileUtil.uploadFile(fileReference,data);
expect(s3UploadStub.calledOnce).to.be.true;
expect(response.statusCode).to.equal(200);
expect(response.body).to.equal(fileReference);
});
You need to create stubs using sinon and replace the variables in the code you're testing with the stubs. There are a few modules to do this, try Mockery or Rewire. For example with Rewire you load the module you're testing using rewire instead of require then use __set__ to set the variables.
var rewire = require('rewire');
var moduleUnderTest = rewire('myModule');
moduleUnderTest.__set__('s3', stubbedS3);

nodejs aws s3 replace files

Im trying to upload a folder from a local directory to an AWS S3 Bucket.
I have the following code.
var s3 = require('s3');
var awsS3Client = new AWS.S3({
accessKeyId: 'XXXXXXX',
secretAccessKey: 'XXXXXXX'
});
var options = {
s3Client: awsS3Client
};
var client = s3.createClient(options);
var params = {
localDir: "./zips",
deleteRemoved: true, // default false, whether to remove s3 objects
// that have no corresponding local file.
s3Params: {
Bucket: "node-files",
Prefix: "test/unzip/"
},
};
var uploader = client.uploadDir(params);
uploader.on('error', function (err) {
console.error("unable to sync:", err.stack);
});
uploader.on('progress', function () {
console.log("progress", uploader.progressAmount, uploader.progressTotal);
});
uploader.on('end', function () {
console.log("done uploading");
});
All works fine when uploading for the first time, the directory and all of its files are in tact and in the bucket.
However when i try a second time, the buffer just gets stuck and times out.
Im assuming i either need to set some kind of option to overwrite the existing files?

Resources