stream response from nodejs request to s3 - node.js

Question

How do you use request to download contents of a file and directly stream it up to s3 using the aws-sdk for node?
The code below gives me Object #<Request> has no method 'read' which makes it seem like request does not return a readable stream...
var req = require('request');
var s3 = new AWS.S3({params: {Bucket: myBucket, Key: s3Key}});
var imageStream = req.get(url)
.on('response', function (response) {
if (200 == response.statusCode) {
//imageStream should be read()able by now right?
s3.upload({Body: imageStream, ACL: "public-read", CacheControl: 5184000}, function (err, data) { //2 months
console.log(err,data);
});
}
});
});
Per the aws-sdk docs Body needs to be a ReadableStream object.
What am I doing wrong here?
This can be pulled off using the s3-upload-stream module, however I'd prefer to limit my dependencies.

Answer 1

Since I had the same problem as #JoshSantangelo (zero byte files on S3) with request#2.60.0 and aws-sdk#2.1.43, let me add an alternative solution using Node's own http module (caveat: simplified code from a real life project and not tested separately):
var http = require('http');
function copyToS3(url, key, callback) {
http.get(url, function onResponse(res) {
if (res.statusCode >= 300) {
return callback(new Error('error ' + res.statusCode + ' retrieving ' + url));
}
s3.upload({Key: key, Body: res}, callback);
})
.on('error', function onError(err) {
return callback(err);
});
}
As far as I can tell, the problem is that request does not fully support the current Node streams API, while aws-sdk depends on it.
References:
request issue about the readable event not working right
generic issue for "new streams" support in request
usage of the readable event in aws-sdk

Answer 2

You want to use the response object if you're manually listening for the response stream:
var req = require('request');
var s3 = new AWS.S3({params: {Bucket: myBucket, Key: s3Key}});
var imageStream = req.get(url)
.on('response', function (response) {
if (200 == response.statusCode) {
s3.upload({Body: response, ACL: "public-read", CacheControl: 5184000}, function (err, data) { //2 months
console.log(err,data);
});
}
});
});

Answer 3

As Request has been deprecated, here's a solution utilizing Axios
const AWS = require('aws-sdk');
const axios = require('axios');
const downloadAndUpload = async function(url, fileName) {
const res = await axios({ url, method: 'GET', responseType: 'stream' });
const s3 = new AWS.S3(); //Assumes AWS credentials in env vars or AWS config file
const params = {
Bucket: IMAGE_BUCKET,
Key: fileName,
Body: res.data,
ContentType: res.headers['content-type'],
};
return s3.upload(params).promise();
}
Note, that the current version of the AWS SDK doesn't throw an exception if the AWS credentials are wrong or missing - the promise simply never resolves.