I'm using Nodejs to try and push an image to an S3 instance with the aws-sdk. Currently, it reads from a file on the client and then saves it on the server (I'm using a meteor framework.) I'd like to push it to the S3 server instead of saving it on the meteor server. When I tried to migrate it over, the images seem to gain about 30% when they are on S3. If I try and download them off of S3 the image is no longer viewable either, so it looks like it has changed encoding or something.
Here is the code to load the file on the client side:
saveFile = function( blob, name, path, type, callback ) {
var fileReader = new FileReader();
var method;
var encoding = 'binary';
var type = type || 'binary';
switch( type ) {
case 'text':
method = 'readAsText';
encoding = 'utf8';
break;
case 'binary':
method = 'readAsBinaryString';
encoding = 'binary';
break;
default:
method = 'readAsBinaryString';
encoding = 'binary';
break;
}
// Call the save function on the server after the file has been read.
fileReader.onload = function( file ) {
console.log( "File loaded..." );
Meteor.call( 'saveFile', file.srcElement.result, name, path, encoding, callback );
}
// Read the file
fileReader[ method ]( blob );
}
On the server side:
saveFile: function( file, name, path, encoding ) {
s3.createBucket({Bucket: bucketName}, function() {
var params = {Bucket: bucketName, Key: keyName, ContentType: 'binary', ContentEncoding: 'utf8', Body: file};
s3.putObject(params, function(err, data) {
if (err)
console.log(err)
else
console.log("Successfully uploaded data to " + bucketName + "/" + keyName);
});
});
I figured out the solution, it was to encapsulate the 'file' object in a
new Buffer()
Simple, but oh so difficult to find!!
Related
I have implemented a s3 uploader per these instructions https://aws.amazon.com/blogs/compute/uploading-to-amazon-s3-directly-from-a-web-or-mobile-application/
This is the Lambda function code
AWS.config.update({ region: process.env.AWS_REGION })
const s3 = new AWS.S3()
const URL_EXPIRATION_SECONDS = 300
// Main Lambda entry point
exports.handler = async (event) => {
return await getUploadURL(event)
}
const getUploadURL = async function(event) {
const randomID = parseInt(Math.random() * 10000000)
const Key = `${randomID}.jpg`
// Get signed URL from S3
const s3Params = {
Bucket: process.env.UploadBucket,
Key,
Expires: URL_EXPIRATION_SECONDS,
Currently the filename (key) is generated using a random ID.
I would like to change that to use the original filename of the uploaded file.
I tried a couple approaches such as using the the fs.readfile() to get the filename but have not had any luck.
There is a webpage with a form that works in conjunction with the Lambda to upload the file to s3.
How do I get the filename?
If you want to save the file with the original filename, you have to pass that filename as part of the key you use to request the signed url. You don't show how you're getting the file to upload, but if it is part of a web site, you get this from the client.
On the client side you have the user identify the file to upload and pass that to your code that calls getUploadURL(). Maybe in your code it is part of event? Then you send the signed URL back to the client and then the client can send the file to the signed URL.
Therefore to upload a file, your client has to send two requests to your server -- one to get the URL and one to upload the file.
You do mention that you're using fs.readFile() If you're able to get the file with this call, then you already have the file name. All you have to do is pass the same name to getUploadURL() as an additional parameter or as part of event. You may have to parse the filename first or within getUploadURL() if it includes a path to someplace other than your current working directory.
The code above looks like it may be a Lambda that's getting called with some event. If that event is a trigger of some sort that you can include a file name, then you can look pull it from that variable. For example:
const getUploadURL = async function(event) {
const randomID = parseInt(Math.random() * 10000000)
const Key = `${event.fileNameFromTrigger}`
// Get signed URL from S3
const s3Params = {
Bucket: process.env.UploadBucket,
Key,
Expires: URL_EXPIRATION_SECONDS.
...
}
If the file name includes the extension, then you don't need to append that as you were with the random name.
I modified the Lambda
changed this
const randomID = parseInt(Math.random() * 10000000)
const Key = `${randomID}.jpg`
to this
const Key = event.queryStringParameters.filename
And this the frontend code with my endpoint redacted. Note the query ?filename= appended to the endpoint and how I used this.filename = file.name
<script>
const MAX_IMAGE_SIZE = 1000000
/* ENTER YOUR ENDPOINT HERE */
const API_ENDPOINT = '{api-endpoint}/uploads?filename=' // e.g. https://ab1234ab123.execute-api.us-east-1.amazonaws.com/uploads
new Vue({
el: "#app",
data: {
image: '',
uploadURL: '',
filename: ''
},
methods: {
onFileChange (e) {
let files = e.target.files || e.dataTransfer.files
//let filename = files[0].name
if (!files.length) return
this.createImage(files[0])
},
createImage (file) {
// var image = new Image()
let reader = new FileReader()
reader.onload = (e) => {
//console.log(file.name)
console.log('length: ', e.target.result.includes('data:image/jpeg'))
if (!e.target.result.includes('data:image/jpeg')) {
return alert('Wrong file type - JPG only.')
}
if (e.target.result.length > MAX_IMAGE_SIZE) {
return alert('Image is loo large.')
}
this.image = e.target.result
this.filename = file.name
}
reader.readAsDataURL(file)
},
removeImage: function (e) {
console.log('Remove clicked')
this.image = ''
this.filename = ''
},
uploadImage: async function (e) {
console.log('Upload clicked')
// Get the presigned URL
const response = await axios({
method: 'GET',
url: API_ENDPOINT + this.filename
})
console.log('Response: ', response)
console.log('Uploading: ', this.image)
let binary = atob(this.image.split(',')[1])
let array = []
for (var i = 0; i < binary.length; i++) {
array.push(binary.charCodeAt(i))
}
let blobData = new Blob([new Uint8Array(array)], {type: 'image/jpeg'})
console.log('Uploading to: ', response.uploadURL)
const result = await fetch(response.uploadURL, {
method: 'PUT',
body: blobData
})
console.log('Result: ', result)
// Final URL for the user doesn't need the query string params
this.uploadURL = response.uploadURL.split('?')[0]
}
}
})
</script>
I am having issue uploading mp4 files to my s3 bucket in my react/meteor project. It works for the other type of files (mp3, images) but not for video. I don't get any error but when I try to read the file that was uploaded it doesn't work.
here is my client code:
import React from "react";
import { Meteor } from "meteor/meteor";
import PropTypes from "prop-types";
import { types } from "../../../utils/constants/types";
const FileUpload = ({ fileType, type, typeId, subtype, setFileName }) => {
const handleUpload = event => {
event.preventDefault();
const file = event.target.files[0];
const fileExtension = file.type;
var reader = new FileReader();
reader.onload = function () {
Meteor.call(
"uploadFile",
fileExtension,
reader.result,
type,
typeId,
(err, result) => {
if (err) {
console.log(err);
} else {
setFileName(result);
}
}
);
};
reader.readAsDataURL(file);
};
return (
<div>
<input name="Uploader" onChange={handleUpload} type="file" />
</div>
);
};
and there is my meteor method on the server side:
Meteor.methods({
uploadFile: async function (fileType, data, type, typeId) {
let extension;
let contentType = fileType;
if (fileType.includes("jpeg") || fileType.includes("jpg")) {
extension = "jpg";
} else if (fileType.includes("png")) {
extension = "png";
} else if (fileType.includes("mp4")) {
extension = "mp4";
} else if (fileType.includes("audio/mpeg")) {
contentType = "video/mp4";
extension = "mp3";
} else if (fileType.includes("pdf")) {
extension = "pdf";
} else {
throw new Meteor.Error("format-error", "Only authorized format");
}
const random = Random.id();
const key = `random.${extension}`;
const buf =
extension !== "mp4"
? Buffer.from(data.replace(/^data:image\/\w+;base64,/, ""), "base64")
: data;
const config = {
Bucket: bucketName,
Key: key,
Body: buf,
ContentType: contentType,
ACL: "public-read",
};
if (extension !== "mp4") {
config.ContentEncoding = "base64";
}
const uploadResult = await s3.upload(config).promise();
return uploadResult.Location;
},
});
I think it may come from the reader not managing properly video files but I'm a bit lost there. Any input would be appreciated. Thanks.
I wanted to ask, since I see your code is pretty complicated. Why do you upload files via your server?
I think it is more efficient to push files from the client straight to S3. Please have a look at this package that I am maintaining if you are interested to see the concept (https://github.com/activitree/s3up-meta). It is a re-write of an older hard-tested package from years back. Uploads are signed by the Meteor server but uploaded by the client. You don't want to keep sockets/fibers busy with file uploads while also a lot of data is coming in for each upload (data about the status of upload). You would want to have that all on the user/client side.
Another thing to note is the importance of setting the AbortIncompleteMultipartUpload in your bucket/Management/Lifecycle. This is where your S3 size may grow without control.
Finally, when I upload video, I just upload the file (received from the input type file). It looks like this:
video: File
lastModified: 1573722339000
lastModifiedDate: Thu Nov 14 2019 13:05:39 GMT+0400 (Gulf Standard Time) {}
name: "IMG_7847.MOV"
size: 20719894
type: "video/quicktime"
webkitRelativePath: ""
I do use other helpers to determine the file (and limit the time length of the permitted uploads) and file type.
You could monitor the errors on your server side as you upload or/and activate CloudWatch on S3 side to see where things are failing.
I am trying to download multiple pdf files from external sources to my nodejs server (in Heroku) temporarily and upload it to AWS S3 bucket.
I have tried multiple methods all of which works fine in my local machine but not in Heroku Dyno NodeJS Server. I am unable to even create folder in Heroku. I guess due to limited permission.
In Node
1) using var download = require('download-file') (using this currently in below code)
2) axios
3) res.download()
Download Files Code
const downloadFiles = async (unique_files) => {
for (let index = 0; index < unique_files.length; index++) {
let file_ext = unique_files[index].substr(unique_files[index].length - 4);
if(file_ext == ".pdf") {
await downloadzz(unique_files[index])
}
}
}
function downloadzz(link) {
download(link, function(err){
if (err) throw err
console.log("DOWNLOAD Complete");
});
}
Upload Files Code
const uploadFiles = async (unique_files) => {
for (let index = 0; index < unique_files.length; index++) {
let file_ext = unique_files[index].substr(unique_files[index].length - 4);
if(file_ext == ".pdf") {
await uploadzz(unique_files[index])
}
}
}
function uploadzz(link) {
fs.readFile(require('path').resolve(__dirname+'/../external-pdfs/', link.slice(link.lastIndexOf('/') + 1)), function (err, data) {
params = {Bucket: pdfBucket, Key: link.slice(link.lastIndexOf('/') + 1), Body: data, ACL: "public-read" };
s3.putObject(params, function(err, data) {
if (err) {
console.log("Failed Upload", err);
} else {
console.log("Successfully uploaded data to bucket", data);
}
});
});
}
I don't get any error but no folder seem to exist with a name external-pdfs on heroku server.
I am open for better solutions: for example, directly uploading file from external url to s3...
How can I in read file from a external url and directly upload to AWS S3 bucket?
You can use axios. Setting the responseType as stream, you can get the file data and pass it as the body. Here it's an example code to get the pdf from a URL and uploading its info directly to S3:
const AWS = require('aws-sdk');
const axios = require('axios');
AWS.config.loadFromPath('./config.json');
const s3 = new AWS.S3({apiVersion: '2006-03-01'});
const URL = "<YOUR_URL>";
const uploadPdfToS3 = async () => {
try{
const {data, headers} = await axios.get(URL, {responseType: 'stream'});
// Create params for putObject call
const objectParams = {
Bucket: "<YOUR_BUCKET>",
Key: "<YOUR_KEY>",
ContentLength: headers['content-length'],
Body: data
};
// Create object upload promise
await s3.putObject(objectParams).promise();
} catch(err){
console.log("ERROR --->" + err)
}
}
In Angular, we can use FileSaver library to save the pdf file from library.
Find the below sample code to do this way.
enter image description here
I want to concatenate the files uploaded on Amazon S3 server.
How can I do this.
Concatenation on local machine i can do using following code.
var fs = require('fs'),
files = fs.readdirSync('./files'),
clips = [],
stream,
currentfile,
dhh = fs.createWriteStream('./concatfile.mp3');
files.forEach(function (file) {
clips.push(file.substring(0, 6));
});
function main() {
if (!clips.length) {
dhh.end("Done");
return;
}
currentfile = './files/' + clips.shift() + '.mp3';
stream = fs.createReadStream(currentfile);
stream.pipe(dhh, {end: false});
stream.on("end", function() {
main();
});
}
main();
You can achieve what you want by breaking it into two steps:
Manipulating files on s3
Since s3 is a remote file storage, you can't run code on s3 server to do the operation locally (as #Andrey mentioned).
what you will need to do in your code is to fetch each input file, process them locally and upload the results back to s3. checkout the code examples from amazon:
var s3 = new AWS.S3();
var params = {Bucket: 'myBucket', Key: 'mp3-input1.mp3'};
var file = require('fs').createWriteStream('/path/to/input.mp3');
s3.getObject(params).createReadStream().pipe(file);
at this stage you'll run your concatenation code, and upload the results back:
var fs = require('fs');
var zlib = require('zlib');
var body = fs.createReadStream('bigfile.mp3').pipe(zlib.createGzip());
var s3obj = new AWS.S3({params: {Bucket: 'myBucket', Key: 'myKey'}});
s3obj.upload({Body: body}).
on('httpUploadProgress', function(evt) { console.log(evt); }).
send(function(err, data) { console.log(err, data) });
Merging two (or more) mp3 files
Since MP3 file include a header that specifies some information like bitrate, simply concatenating them together might introduce playback issues.
See: https://stackoverflow.com/a/5364985/1265980
what you want to use a tool to that. you can have one approach of saving your input mp3 files in tmp folder, and executing an external program like to change the bitrate, contcatenate files and fix the header.
alternatively you can use an library that allows you to use ffmpeg within node.js.
in their code example shown, you can see how their merge two files together within the node api.
ffmpeg('/path/to/part1.avi')
.input('/path/to/part2.avi')
.input('/path/to/part2.avi')
.on('error', function(err) {
console.log('An error occurred: ' + err.message);
})
.on('end', function() {
console.log('Merging finished !');
})
.mergeToFile('/path/to/merged.avi', '/path/to/tempDir');
Here's my quick take on the problem of downloading and processing S3 objects. My example is focused mostly on getting the data local and then processing it once it's all downloaded. I suggest you use one of the ffmpeg approaches mentioned above.
var RSVP = require('rsvp');
var s3 = new AWS.S3();
var bucket = '<your bucket name>';
var getFile = function(key, filePath) {
return new RSVP.Promise(function(resolve, reject) {
var file = require('fs').createWriteStream(filePath);
if(!file) {
reject('unable to open file');
}
s3.getObject({
Bucket: bucket,
Key: key
}).on('httpData', function(chunk) {
file.write(chunk);
}).on('httpDone', function() {
file.end();
resolve(filePath);
});
});
};
var tempFiles = ['<local temp filename 1>', '<local temp filename 2>'];
var keys = ['<s3 object key 1>', '<s3 object key 2>'];
var promises = [];
for(var i = 0; i < keys.length; ++i) {
var promise = getFile(keys[i], tempFiles[i]);
promises.push(promise);
}
RSVP.all(promises).then(function(data) {
//do something with your files
}).catch(function(error) {
//handle errors
});
I'm using ImageMagick to make thumbnails of my photos on S3.
This is the flow:
Get the image from S3.
Make thumbnail from the data.Body.
Put the thumbnail on S3.
Here is the make thumbnail function:
function makeThumbnail(image) {
var defer = q.defer();
im.resize({
srcData: image.Body,
width: 256
}, function (err, stdout) {
if (err) {
defer.reject(err);
} else {
image.Key.replace(/^images/, "thumbs");
image.Body = new Buffer(stdout, 'binary');
defer.resolve(image);
}
});
return defer.promise;
}
The image object is of the form that S3 SDK expects to get it:
var image = {
Bucket: 'bucket name',
Key: 'object key',
Body: 'body',
ContentType: 'image/png'
};
Yet, when putting the thumbnail on S3, the MIME is set to application/octet-stream for some reason. When downloading the thumbnail it opens like any other picture, yet the browsers do not treat it like it's an image and that is a problem for me.
What causes this issue? and how can I solve it?
Thanks
The docs for node-imagemagick.resize() says the default format is jpg. Maybe your output is not a png like you think.
We use the im-resize package and that contains input and output parameters, where we typically convert .jpeg images uploaded from a web form into .png output file. (We read from the file system after conversion, not stream buffers, just fyi).
Our code is something like this:
var resize = require('im-resize');
var metadata = require('im-metadata');
var AWS = require('aws-sdk');
metadata(data.params.file.path, {}, function(error, metadataResults) {
//error checking
var image = {
//set from metadataResults
};
var output = {
versions: [{
//set as desired
}]
};
resize(image, output, function(error, versions) {
var filename = 'images/' + uuid.v4() + '.png'; //this was enough for us
var fileStream = fs.createReadStream(versions[0].path);
fileStream.on('error', function(err) {
//handle error
});
fileStream.on('open', function() {
var s3 = new AWS.S3();
s3.putObject({
Bucket: 'bucketname',
Key: filename,
Body: fileStream
}, function(err) {
//handle errors
//know the file has been saved to S3
//close up all the squiggly braces and return HTTP response or whatever