Nodejs - testing AWS with Mocha - node.js

Question

I'm having trouble writing the tests for the following nodejs code which uses AWS and graphicsmagick. I have tried to search for examples also on how to write tests for async's waterfall method but without any definite results.
// dependencies
var async = require('async');
var AWS = require('aws-sdk');
var gm = require('gm').subClass({ imageMagick: true });
var util = require('util');
// get reference to S3 client
var s3 = new AWS.S3();
exports.AwsHandler = function(event, context) {
// Read options from the event.
console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
var srcBucket = event.Records[0].s3.bucket.name;
var srcKey = event.Records[0].s3.object.key;
var dstnKey = srcKey;
// Infer the image type.
var typeMatch = srcKey.match(/\.([^.]*)$/);
if (!typeMatch) {
console.error('unable to infer image type for key ' + srcKey);
return;
}
var imageType = typeMatch[1];
if (imageType != "jpg" && imageType != "png") {
console.log('skipping non-image ' + srcKey);
return;
}
//Download the image from S3, transform, and upload to same S3 bucket but different folders.
async.waterfall([
function download(next) {
// Download the image from S3 into a buffer.
s3.getObject({
Bucket: srcBucket,
Key: srcKey
},
next);
},
function transformSave(response, next) {
var _buffer = null;
for (var i = 0; i<len; i++) {
// Transform the image buffer in memory.
gm(response.Body, srcKey)
.resize(_sizesArray[i].width)
.toBuffer(imageType, function(err, buffer) {
if (err) {
next(err);
} else {
console.log(buffer);
_buffer = buffer;
}
});
// put newly resized image into respective folder
s3.putObject({
Bucket: srcBucket,
Key: "dst/" + _sizesArray[i].destinationPath + "/" + dstnKey,
Body: _buffer,
ContentType: response.ContentType
}, next);
}
},
], function (err) {
if (err) {
console.error(
'---->Unable to resize ' + srcBucket + '/' + srcKey +
' and upload to ' + srcBucket + '/dst' +
' due to an error: ' + err
);
} else {
console.log(
'---->Successfully resized ' + srcBucket +
' and uploaded to ' + srcBucket + "/dst"
);
}
context.done();
}
);
};
My tests for this module so far:
require('blanket')({
pattern: function (filename) {
return !/node_modules/.test(filename);
}
});
// in terminal, type the following command to get code coverage: mocha -R html-cov > coverage.html
var chai = require('chai');
var sinonChai = require("sinon-chai");
var expect = chai.expect;
var sinon = require('sinon');
chai.use(sinonChai);
var sync = require("async");
var proxyquire = require('proxyquire');
describe('Image Resizing module', function () {
var gmSubclassStub = sinon.stub();
var getObjectStub = sinon.stub();
var putObjectSpy = sinon.spy();
var testedModule = proxyquire('../index', {
'gm': {subClass: sinon.stub().returns(gmSubclassStub)},
'AWS': {
"s3": {
getObject: sinon.stub().returns(getObjectStub),
putObject: putObjectSpy
}
}
});
describe('AwsHandler', function () {
var event = {
"Records": [
{
"s3": {
"bucket": {
"name": "testbucket"
},
"object": {
"key": "test.jpg"
}
}
}
]
};
it("should call gm write with correct files", function () {
// Arrange
// Spies are the methods you expect were actually called
var buffer800Spy = sinon.spy();
var buffer500Spy = sinon.spy();
var buffer200Spy = sinon.spy();
var buffer45Spy = sinon.spy();
// This is a stub that will return the correct spy for each iteration of the for loop
var resizeStub = sinon.stub();
resizeStub.withArgs(800).returns({toBuffer: buffer800Spy});
resizeStub.withArgs(500).returns({toBuffer: buffer500Spy});
resizeStub.withArgs(200).returns({toBuffer: buffer200Spy});
resizeStub.withArgs(45).returns({toBuffer: buffer45Spy});
// Stub is used when you just want to simulate a returned value
var nameStub = sinon.stub().yields({"name": "testbucket"});
var keyStub = sinon.stub().yields({"key": "test.jpg"});
gmSubclassStub.withArgs(event).returns({resize:resizeStub});
getObjectStub.withArgs(event).yields({name: nameStub}, {key: keyStub});
// Act - this calls the tested method
testedModule.AwsHandler(event);
// Assert
});
});
});

Answer 1

It's hard to respond this kind of question here; the question is not very specific and it's not an open question which can be replied with opinions, thoughts, etc.
Hence, I've created an similar implementation which solve the async.waterfall issue and provide a test which test the AwsHandler with 100% coverage.
The code is in this gist, because it's more handy and readable to be there than here.
I've also written a blog post related with this implementation

Answer 2

There are a few things that need to be changed:
You want to test the operation of the unit, without testing the implementation. That's why you should ignore the async in your tests (as you did).
It is just a way of implementing the method, the inner workings of the unit.
What you should test is that in given conditions, the unit gives the end result expected, in this case it's calling s3.putObject.
So you should stub everything that is external (gm and aws), and spy on the s3.putObject method, because that is the expected end result.
In your stubs you used "yield", which calls the callback function, but only if it is the first parameter.
If it's not, like in our case, you need to use "callsArgWith(index,...)" with the index of the parameter which is the callback.
The proxyquire has to have the injected modules with exactly the same name as in the code that requires them - changed 'AWS' to 'aws-sdk'
A way of checking if the stubs were injected correctly is in the debugger, put a watch on
"s3" variable, and check that it is "function proxy()" and not "function()". You can also print it to console if you're not using a debugger.
Your module is calling next in the for loop, which causes the waterfall to split into a tree with 36 calls to done(!).
Maybe you should use a different async model like map reduce. I fixed it by adding a silly condition, but that's not good code.
As a side note, you can see that the test is becoming awfully complicated.
This can be an indication that the tested code could use some separation of concerns.
For example, moving the gm operations, and the s3 operations to two separate modules can help separate things, and also make it easier to test.
Changes in the module itself, to prevent calling next 4*4 times:
function transform(response, next) {
for (var i = 0; i<len; i++) {
// Transform the image buffer in memory.
gm(response.Body, srcKey)
.resize(_sizesArray[i].width)
.toBuffer(imageType, function(err, buffer) {
if (err) {
next(err);
} else {
next(null, response.ContentType, buffer, i);
}
});
}
},
function upload(contentType, data, i, next) {
// Stream the transformed image to a different folder.
s3.putObject({
Bucket: srcBucket,
Key: "dst/" + _sizesArray[i].destinationPath + "/" + dstnKey,
Body: data,
ContentType: contentType
},
function(err) {
if (i==3) next(err);
});
}
And the test:
describe.only('Image Resizing module', function () {
var gmSubclassStub = sinon.stub();
var s3Stub = {};
var proxyquire = require('proxyquire');
var testedModule = proxyquire('../index', {
'gm': {subClass: sinon.stub().returns(gmSubclassStub)},
'aws-sdk': {"S3": sinon.stub().returns(s3Stub)}
});
describe('AwsHandler', function () {
var event = {};
// The done callback is used for async testing
it("should call gm write with correct files", function (done) {
// Arrange
var resizeStub = sinon.stub();
var buffer800Spy = sinon.stub().withArgs("jpg").callsArgWith(1, null, "800 buffer");
var buffer500Spy = sinon.stub().withArgs("jpg").callsArgWith(1, null, "500 buffer");
var buffer200Spy = sinon.stub().withArgs("jpg").callsArgWith(1, null, "200 buffer");
var buffer45Spy = sinon.stub().withArgs("jpg").callsArgWith(1, null, "45 buffer");
resizeStub.withArgs(800).returns({toBuffer: buffer800Spy});
resizeStub.withArgs(500).returns({toBuffer: buffer500Spy});
resizeStub.withArgs(200).returns({toBuffer: buffer200Spy});
resizeStub.withArgs(45).returns({toBuffer: buffer45Spy});
gmSubclassStub.withArgs("response body", "test.jpg").returns({resize: resizeStub});
s3Stub.getObject = sinon.stub()
.withArgs({name: "testbucket", key: "test.jpg"})
.callsArgWith(1, null, {
Body: "response body",
ContentType: "response content type"
});
var putObjectMock = sinon.mock();
s3Stub.putObject = putObjectMock;
putObjectMock.callsArgWith(1, null, {}); // return behaviour of the mock
putObjectMock.exactly(4); // sets expectation that it is called 4 times
// Act - this calls the tested method
testedModule.AwsHandler(event, {
done: function () {
// Assertions need to be inside callback because it is async
assert.deepEqual(putObjectMock.getCall(0).args[0], {
Bucket: "testbucket",
Key: "dst/large/test.jpg",
Body: "800 buffer",
ContentType: "response content type"
});
assert.deepEqual(putObjectMock.getCall(1).args[0], {
Bucket: "testbucket",
Key: "dst/medium/test.jpg",
Body: "500 buffer",
ContentType: "response content type"
});
assert.deepEqual(putObjectMock.getCall(2).args[0], {
Bucket: "testbucket",
Key: "dst/small/test.jpg",
Body: "200 buffer",
ContentType: "response content type"
});
assert.deepEqual(putObjectMock.getCall(3).args[0], {
Bucket: "testbucket",
Key: "dst/thumbnail/test.jpg",
Body: "45 buffer",
ContentType: "response content type"
});
// This ends the async test
done();
}
});
});
});
});