Stubbing S3 uploads in Node.js - node.js

How would I go about stubbing S3 uploads in Node.js?
For insight, I'm using Mocha for tests and Sinon for stubbing, but I'm open to changing anything. I have a file that exports a function that performs the upload. It looks like this:
var AWS = require('aws-sdk');
var s3 = new AWS.S3({ params: { Bucket: process.env.S3_BUCKET }});
var params = { Key: key, Body: body };
s3.upload(params, function (error, data) {
// Handle upload or error
});
If I try to stub AWS.S3 or AWS.S3.prototype, nothing changes. I assume this is because my tests have required aws-sdk themselves and have their own copy of each function.
My test looks like this:
describe('POST /files', function () {
var url = baseURL + '/files';
it('uploads the file to s3', function (done) {
var fs = require('fs');
var formData = {
video: fs.createReadStream(process.cwd() + '/test/support/video.mp4')
};
var params = {url: url, formData: formData};
request.post(params, function (error, response, body) {
expect(response.statusCode).to.eq(200);
expect(response.body).to.eq('Uploaded');
done();
});
});
});
This test works fine, but it does not stub the upload to S3, so the upload actually goes through :X.

There are several options to mock S3 in Node.
Some modules specific to S3:
https://www.npmjs.com/package/mock-s3
https://www.npmjs.com/package/mock-aws-s3
Some modules for general AWS mocking:
https://www.npmjs.com/package/aws-sdk-mock
https://www.npmjs.com/package/mock-aws-sinon
https://www.npmjs.com/package/mock-aws
And you can even start a simple server that responds to some of the S3 API calls:
https://www.npmjs.com/package/s3rver
The last one can easily be used in other languages and runtimes, not only in Node.

You can stub with Sinon.js as follows if you'd like:
Expose the AWS.S3 instance:
var AWS = require('aws-sdk');
var s3 = new AWS.S3({ params: { Bucket: process.env.S3_BUCKET }});
var params = { Key: key, Body: body };
exports.s3.upload(params, function (error, data) {
});
//Expose S3 instance
exports.s3 = s3;
Stub the same instance like so:
var sinon = require('sinon');
//Import module you are going to test
var UploadService = require('./uploadService');
describe('POST /files', function () {
before(function() {
//Stub before, and specify what data you'd like in the callback.
this.uploadS3Stub = sinon.stub(uploadService.s3, 'upload').callsArgWith(1, null, { data: 'Desired response' });
});
after(function() {
//Restore after
this.uploadS3Stub.restore();
});
var url = baseURL + '/files';
it('uploads the file to s3', function (done) {
var fs = require('fs');
var formData = {
video: fs.createReadStream(process.cwd() + '/test/support/video.mp4')
};
var params = {url: url, formData: formData};
var self = this;
request.post(params, function (error, response, body) {
expect(response.statusCode).to.eq(200);
expect(response.body).to.eq('Uploaded');
//You can also check whether the stub was called :)
expect(self.uploadS3Stub.calledOnce).to.eql(true);
done();
});
});
});

s3UploadStub = AWS.S3.prototype.upload = sandbox.stub();
it("should upload successfully", async function() {
s3UploadStub.yields(null, data);
let response = await FileUtil.uploadFile(fileReference,data);
expect(s3UploadStub.calledOnce).to.be.true;
expect(response.statusCode).to.equal(200);
expect(response.body).to.equal(fileReference);
});

You need to create stubs using sinon and replace the variables in the code you're testing with the stubs. There are a few modules to do this, try Mockery or Rewire. For example with Rewire you load the module you're testing using rewire instead of require then use __set__ to set the variables.
var rewire = require('rewire');
var moduleUnderTest = rewire('myModule');
moduleUnderTest.__set__('s3', stubbedS3);

Related

node.js sinon replace method

I'm new to sinon and can't achieve the results I want.
I'm trying to stub AWS S3 API getObject to return a test-provided object.
My production code has:
let s3 = new AWS.S3({ apiVersion: '2006-03-01' });
let params = {
Bucket: aws_bucket,
Key: path
};
s3.getObject(params, function(err, data) {
...
});
My test code has:
describe('GET /image', function() {
beforeEach(function() {
let stub = sinon.createStubInstance(AWS.S3);
stub.getObject.callsFake(() => { console.log('stubbed'); });
});
The AWS S3 class instance is fully stubbed when I run the test, which is great, but it is not calling my fake.
What am I missing?
I found a working approach.
Step 1 Wrap AWS.S3 instance in another module.
// s3.js
var AWS = require('aws-sdk');
var s3 = new AWS.S3({ apiVersion: '2006-03-01' });
module.exports = s3;
Step 2 Change production code to use this instance instead of making its own.
// image.js
var s3 = require('./s3');
// ... other code ...
s3.getObject(...);
Step 3 Stub what needs to be stubbed.
// image-test.js
var s3 = require('./s3');
var getObjectStub;
describe('GET /image', function() {
beforeEach(function() {
getObjectStub = sinon.stub(s3, 'getObject').callsFake(() => { console.log('stubbed'); });
});
afterEach(() => {
getObjectStub.restore();
});
// test code continues
});

AWS textract methods in node js are not getting invoked

I want to extract text from image using node js so created a lambda in aws. Please find the below code snippet. Issue is that the textract method detectDocumentText is not getting invoked.
As far as permission I had given s3 full access and textract full access to the lambda. Am I missing anything?
var AWS = require("aws-sdk");
var base64 = require("base-64");
var fs = require("fs");
exports.handler = async (event, context, callback) => {
// Input for textract can be byte array or S3 object
AWS.config.region = "us-east-1";
//AWS.config.update({ region: 'us-east-1' });
var textract = new AWS.Textract({ apiVersion: "2018-06-27" });
//var textract = new AWS.Textract();
console.log(textract);
var params = {
Document: {
/* required */
//'Bytes': imageBase64
S3Object: {
Bucket: "717577",
Name: "Picture2.png"
}
}
};
textract.detectDocumentText(params, function(err, data) {
if (err) {
console.log(err); // an error occurred
} else {
console.log(data); // successful response
callback(null, data);
}
});
};
As well as I don't see any error logs in cloudwatch logs.
The problem is that you have marked your method as async which means that you are returning a promise. In your case you are not returning a promise so for lambda there is no way to complete the execution of the method. You have two choices here
Remove async
Or more recommended way is to convert your callback style to use promise. aws-sdk support .promise method on all methods so you could leverage that. The code will look like this
var AWS = require("aws-sdk");
var base64 = require("base-64");
var fs = require("fs");
exports.handler = async (event, context) => {
// Input for textract can be byte array or S3 object
AWS.config.region = "us-east-1";
//AWS.config.update({ region: 'us-east-1' });
var textract = new AWS.Textract({ apiVersion: "2018-06-27" });
//var textract = new AWS.Textract();
console.log(textract);
var params = {
Document: {
/* required */
//'Bytes': imageBase64
S3Object: {
Bucket: "717577",
Name: "Picture2.png"
}
}
};
const data = await textract.detectDocumentText(params).promise();
return data;
};
Hope this helps.

Lambda runs but no console output

I am trying to build a lambda function in AWS - when I go and test the function, it says it completes successfully but the expected results are not showing up (ie transcribe job or file in S3). I put code to look for errors but nothing is showing in console either
Have the files uploaded as a zip in what I believe is the right way - the root of the zip file has node modules and index.js
Also can see it firing in xray
exports.handler = async (event, context) => {
var Parser = require('rss-parser');
var request = require('request');
var AWS = require('aws-sdk');
var bucketName = 'transcribebucketkm';
var bucketNameOut = 'transcribebucketkm-out';
var parser = new Parser();
var s3 = new AWS.S3();
var transcribeservice = new AWS.TranscribeService();
AWS.config.update({region:'us-east-1'});
var datetime = new Date
var datetime = datetime.toISOString().slice(0,10) + ".mp3";
var mediafileuri= "http://s3.amazonaws.com/transcribebucketkm/" + datetime
parser.parseURL('https://bridgetown.podbean.com/feed.xml', function (err, feed) {
console.log(err)
request({ method: 'GET', url: feed.items[0].enclosure.url, encoding: null},function (err, response, body) {
console.log(err)
console.log(body)
s3.upload({ Bucket: bucketName, Key: datetime, Body: body}, function(err, data) {
console.log(data)
transcribeservice.startTranscriptionJob({LanguageCode: "en-US", Media:{MediaFileUri: mediafileuri}, MediaFormat: "mp3", TranscriptionJobName: datetime, OutputBucketName: bucketNameOut});
});
});
});
}
Console does show only null - making me believe no errors

nodejs - aws lambda random timeout when trying to access S3.getBucketLocation

For the following code, I was asked to check, which runs on AWS Lambda and performs the simple task of uploading files to S3, timesout randomly when run manually from the aws console,but at the same time when run on a schedule, will always timeout at the first attempt and then run successfully the second time(it is run immediately after failing the first time).The code always seems to hang after calling s3.getBucketLocation, never reaching the getBucketLocation's callback(on the first attempt!).
All VPC's and NAT are set-up correctly, such that, when I remove s3.getBucketLocation from the code and call s3Upload.upload(params, cb); directly, the code runs without timeouts.
I would really like to understand why the first function fails alternatively when trying to retrieve the buckets region from AWS.
"use strict";
var urlHelper = require('url');
var fs = require('fs');
var AWS = require('aws-sdk');
var exp = {};
function saveToFile(path, content, cb) {
fs.writeFile(path, content, cb);
}
function saveToS3(bucket, key, content, cb) {
var s3 = new AWS.S3({apiVersion: '2006-03-01'});
var params = {
Bucket: bucket
};
s3.getBucketLocation(params, function(err, data) {
console.log('THIS IS NOT REACHED');
if (err) {
return cb(err);
}
var s3Upload = new AWS.S3({
apiVersion: '2006-03-01',
region: data.LocationConstraint
});
var params = {
Bucket: bucket,
Key: key,
Body: content,
ACL: 'public-read',
ContentType: 'text/xml',
CacheControl: 'max-age=60'
};
s3Upload.upload(params, cb);
});
}
exp.upload = function(path, content, cb) {
var url = urlHelper.parse(path);
switch(url.protocol) {
case 'file:':
case null:
saveToFile(path, content, cb);
break;
case 's3:':
saveToS3(url.host, url.pathname.substring(1), content, cb);
break;
default:
console.log('No matching protocol found for upload!');
cb(null, null);
}
};
module.exports = exp;

How do I expect a function to be called with specific args with sinon, mocha, and chai in nodejs?

I have been having a problem trying to make sure Q.ninvoke is called with the args I am passing in. I am new to testing with Sinon, Mocha and Chai. I have been trying everything I have found online for 2 days now and I still cant get my test pass. What am I doing wrong?
This is my code under test.
var cuid = require('cuid');
var fs = require('fs');
var Q = require('q');
var AWS = require('aws-sdk');
var S3 = new AWS.S3();
module.exports = {
initialize: initialize
};
function initialize(options) {
return Q.nfcall(fs.readFile, options.path).then(function (file) {
var fileParams = {
Bucket: options.bucket,
Key: options.name,
Body: file,
ContentType: options.contentType
};
return Q.ninvoke(S3, 'upload', fileParams).then(function(data){
return data.Location;
});
});
}
Here is my test.
describe.only('when a file is read successfully', function() {
var spy;
beforeEach(function() {
spy = chai.spy.on(Q, 'ninvoke');
sinon.stub(Q, 'nfcall').withArgs(fs.readFile, fileParams.path).returns(Q.resolve(file));
});
it('Q.ninvoke should be called with args', function() {
UploadCommand.initialize(fileParams)
expect(spy).to.have.been.called.with(S3, 'upload', params);
});
});
This is the error I am getting.
1) UploadCommand .initialize when a file is read successfully
Q.ninvoke should be called with args:
AssertionError: expected { Spy } to have been called with [ Array(3) ]
try this:
var cuid = require('cuid');
var fs = require('fs');
var Q = require('q');
var AWS = require('aws-sdk');
var S3 = new AWS.S3();
module.exports = {
initialize: initialize
};
function initialize(options) {
return Q.nfcall(fs.readFile, options.path).then(function (file) {
var fileParams = {
Bucket: options.bucket,
Key: options.name,
Body: file,
ContentType: options.contentType
};
return Q.ninvoke(S3, 'upload', fileParams);
});
}
note in particular that you should return a promise from your initialize function. then in the test:
describe.only('when a file is read successfully', function() {
var spy;
beforeEach(function() {
spy = chai.spy.on(Q, 'ninvoke');
sinon.stub(Q, 'nfcall').withArgs(fs.readFile,fileParams.path).returns(Q.resolve(file));
});
it('Q.ninvoke should be called with args', function(done) {
UploadCommand.initialize(fileParams).then(function(data) {
expect(spy).to.have.been.called.with(S3, 'upload', params);
done();
});
});
});
a couple of other things to note, in your main application code, you will also want to chain your initialize function to a 'then' function, and in the body of that then function is where the rest of your application code should go. also, the 'done' callback is the way you tell mocha that it is an asynchronous test.
Mike I was able to get it working finally thanks to you. I really appreciate it! Here is the final test.
describe.only('when a file is read successfully', function() {
beforeEach(function() {
sinon.stub(Q, 'nfcall').withArgs(fs.readFile, fileParams.path).returns(Q.resolve(file));
sinon.stub(Q, 'ninvoke').withArgs(S3, 'upload', params).returns(Q.resolve('url'));
chai.spy.on(Q, 'ninvoke')
});
it('Q.ninvoke should be called with args', function(done) {
UploadCommand.initialize(fileParams).then(function(data) {
expect(Q.ninvoke).to.have.been.called.with(S3, 'upload', params);
done();
});
});
});
You can use sinon to stub as shown below
let yourFunctionStub
yourFunctionStub= sinon.stub(yourClass, "yourFunction");
yourFunctionStub.withArgs(arg1, arg2, arg3).returns(resultYouWant);
if this a promise you can return
....returns(Promise.resolve(resultYouWant));
if for argument you are not clear you can you
sinon.match.any
Hope this helps you. :)

Resources