AWS S3 performance using Node.js SDK - node.js

Question

I am trying to pushing the upload(and download) performance of my program to its limits.
I am getting about 1000Mbps when uploading 256MB files using aws's command line interface.
But I get stuck at about 600Mbps upload with the following program
if (process.argv.length < 7) {
console.log ("usage: " + process.argv [0] + " " + process.argv[1] + " <config> <region> <bucket> <key> <file>")
return -1
}
var config = process.argv[2]
var region = process.argv[3]
var bucketName = process.argv[4]
var key = process.argv[5]
var file = process.argv[6]
var multipartMap = { Parts: [] }
var uploadStartTime // = new Date()
var partSize = 1024 * 1024 * 8 // at least 5MB, specified by amazon
var partNum
var multipartParams = {
Bucket: bucketName,
Key: key,
ContentType: "binary",
StorageClass: "REDUCED_REDUNDANCY",
}
var part = 0
var maxRetry = 3
var fs = require ('fs')
var aws = require ('aws-sdk')
function upload (bucket, multipart, partParams, trial) {
var trial = trial || 1;
bucket.uploadPart (partParams, function (err, data) {
if (err) {
console.log ("failed: ", err)
if (trial < maxRetry) {
console.log ("retrying part: ", partParams.PartNumber)
upload (bucket, multipart, partParams, trial + 1)
} else {
console.log ("failed: ", err, " unable to upload part: ", partParams.PartNumber)
}
return;
}
multipartMap.Parts[this.request.params.PartNumber - 1] = {
ETag: data.ETag,
PartNumber: Number (this.request.params.PartNumber)
}
if (--partNum > 0) return;
var doneParams = {
Bucket: bucketName,
Key: key,
MultipartUpload: multipartMap,
UploadId: multipart.UploadId
}
console.log ("success")
bucket.completeMultipartUpload (doneParams, function (err, data){
if (err) {
console.log("An error occurred while completing the multipart upload");
console.log(err);
} else {
var delta = (new Date() - uploadStartTime) / 1000;
console.log('Completed upload in', delta, 'seconds');
console.log('Final upload data:', data);
}
})
})
}
var kickoffTime = new Date ()
aws.config.loadFromPath (config)
aws.config.region = region
var bucket = new aws.S3 ({params: {Bucket: bucketName}})
console.log ("filename: ", file)
buffer = fs.readFileSync (file)
partNum = Math.ceil (buffer.length / partSize) // number of parts
var totalPart = partNum
uploadStartTime = new Date ()
bucket.createMultipartUpload (multipartParams, function (err, multipart) {
if (err) {
console.log ("cannot create multipart upload: ", err)
return -1
}
for (var i = 0; i < buffer.length; i += partSize) {
++part
var end = Math.min (i + partSize, buffer.length)
var body = buffer.slice (i, end)
var partParams = {
Body: body,
Bucket: bucketName,
Key: key,
PartNumber: String (part),
UploadId: multipart.UploadId,
ContentLength: end - i
}
upload (bucket, multipart, partParams);
}
})
var kickoffTimeDelta = (new Date () - kickoffTime) / 1000
console.log ("Kickoff time: ", kickoffTimeDelta)
This program will not work for empty files, but please ignore this case. The above program is coded with reference to this.
As for downloading, the speed also stuck at about 600Mbps, here is the code
if (process.argv.length < 7) {
console.log ("usage: " + process.argv [0] + " " + process.argv1 + " ")
return -1
}
var config = process.argv[2]
var region = process.argv[3]
var bucketName = process.argv[4]
var key = process.argv[5]
var file = process.argv[6]
var fs = require ('fs')
var aws = require ('aws-sdk')
fs.readFile (config, "utf8", function (err, configFile) {
if (err) {
console.log ("Config file cannot be read: ", err)
return -1
}
aws.config = JSON.parse (configFile)
aws.config.region = region
var bucket = new aws.S3 ({params: {Bucket: bucketName}})
bucket.createBucket (function () {
var data = {Key: key}
bucket.getObject (data, function (err, fileData) {
if (err) {
console.log ("Error downloading data: ", err)
} else {
fs.writeFile (file, fileData.Body, function (err) {
if (err) {
console.log ("Error writing data: ", err)
} else {
console.log ("Successfully downloaded!")
}
})
}
})
})
})
I am new to node.js and aws sdk, is there anything missing to achieve better throughtput?
Thanks

Answer 1

Hmm...had a clarifying question but don't have the reputation to post as such.
How many requests per second are you seeing on both ends? If you're regularly hitting S3 with more than 100 requests per second, you'll get better performance by randomizing the start of your key name.
See this article for an explanation and some suggestions:
http://docs.aws.amazon.com/AmazonS3/latest/dev/request-rate-perf-considerations.html
Basically if you have a bunch of files with a key (subdirectory) that starts with the same characters, you can overwhelm the index partition...so for high-volume read/write activities, random keynames speed up the performance.