I'm letting users upload multiple images directly to Amazon-S3 using Multer-S3 and then displaying those images on the front end via a loop. All works perfectly.
However when the images are uploaded via mobile (image taken on an iPhone or Android) the orientation is correct on mobile but does NOT have correct orientation on desktops. Major problem.
This is due to the images EXIF data I believe.
Seems like ImageMagick or Kraken JS https://kraken.io/docs/storage-s3 might be a way to solve it but for the life of me I cannot figure out how to implement either with the way I'm uploading and showing images shown below.
How would I change my code below to auto-orient the images? Note: It must work for multiple images.
Thanks for any help!
Heres's how I'm letting users upload multiple images at a time directly to Amazon-S3:
aws.config.update({
secretAccessKey: 'AccessKey',
accessKeyId: 'KeyID',
region: 'us-east-2'
});
var s3 = new aws.S3();
var storage = multerS3({
limits : { files: 25 },
s3: s3,
bucket: 'files',
key: function (req, file, cb) {
var fileExtension = file.originalname.split(".")[1];
var path = "uploads/" + req.user._id + Date.now() + "." + fileExtension;
cb(null, path);
},
})
var upload = multer({storage: storage}).any("images", 25);
router.post("/", middleware.isLoggedIn, function(req, res, next){
upload(req,res,function(err) {
if(err) {
console.log(err);
res.redirect('/')
}
Listings.findById(req.params.id, function(err, foundListings){
var allimages = []
if(typeof req.files !== "undefined") {
for(var i = 0; i < req.files.length; i++) {
allimages.push(req.files[i].key);
}
}
var currentimages = allimages;
var newListings = {currentimages:currentimages}
//Removed the other Model aspects
Listings.create(newListings, function(err, newlyCreated){
if(err){
console.log(err);
} else {
res.redirect("/listings");
}
});
});
How I'm displaying the images on the front end. Listings.currentimages is an array containing all image links.
app.locals.awspath = "https://s3.us-east-2.amazonaws.com/myfiles/";
// awspath is the file path to my Amazon-S3 path
<div id='allimages'>
<% for(var i = 0; i < listings.currentimages.length; i++ ) { %>
<div class='smallerImages'>
<% var url2 = awspath + listings.currentimages[i] %>
<img class="small" src="<%= url2 %>">
</div>
<% } %>
</div>
The problem is that iOS sets the image's EXIF metadata which causes this behavior. You can use a library that can read the EXIF metadata and rotate the image for you.
jpeg-autorotate (https://github.com/johansatge/jpeg-autorotate) is a very simple lib and has very nice documentation (you should check it out).
Example
var jo = require('jpeg-autorotate');
var fs = require('fs');
// var options = {quality: 85};
var options = {};
var path = '/tmp/Portrait_8.jpg'; // You can use a Buffer, too
jo.rotate(path, options, function(error, buffer, orientation) {
if (error) {
console.log('An error occurred when rotating the file: ' + error.message);
return;
}
console.log('Orientation was: ' + orientation);
// upload the buffer to s3, save to disk or more ...
fs.writeFile("/tmp/output.jpg", buffer, function(err) {
if(err) {
return console.log(err);
}
console.log("The file was saved!");
});
});
You can find some sample images with different EXIF rotation metadata from here
Converted as an AWS Lambda Function
// Name this file index.js and zip it + the node_modules then upload to AWS Lambda
console.log('Loading function');
var aws = require('aws-sdk');
var s3 = new aws.S3({apiVersion: '2006-03-01'});
var jo = require('jpeg-autorotate');
// Rotate an image given a buffer
var autorotateImage = function(data, callback) {
jo.rotate(data, {}, function(error, buffer, orientation) {
if (error) {
console.log('An error occurred when rotating the file: ' + error.message);
callback(error, null);
} else {
console.log('Orientation was: ' + orientation);
callback(null, buffer);
}
});
};
// AWS Lambda runs this on every new file upload to s3
exports.handler = function(event, context, callback) {
console.log('Received event:', JSON.stringify(event, null, 2));
// Get the object from the event and show its content type
var bucket = event.Records[0].s3.bucket.name;
var key = event.Records[0].s3.object.key;
s3.getObject({Bucket: bucket, Key: key}, function(err, data) {
if (err) {
console.log("Error getting object " + key + " from bucket " + bucket +
". Make sure they exist and your bucket is in the same region as this function.");
callback("Error getting file: " + err, null);
} else {
// log the content type, should be an image
console.log('CONTENT TYPE:', data.ContentType);
// rotate the image
autorotateImage(data.Body, function(error, image) {
if (error) {
callback("Error rotating image: " + error, null);
}
const params = {
Bucket: bucket,
Key: 'rotated/' + key,
Body: image
};
// Upload new image, careful not to upload it in a path that will trigger the function again!
s3.putObject(params, function (err, data) {
if (error) {
callback("Error uploading rotated image: " + error, null);
} else {
console.log("Successfully uploaded image on S3", data);
// call AWS Lambda's callback, function was successful!!!
callback(null, data);
}
});
});
}
});
};
Notes This function upload the rotated images to the same bucket but you can easily change that. If you are just starting with AWS Lambda, I'd suggest you learn more about it (https://www.youtube.com/watch?v=eOBq__h4OJ4, https://www.youtube.com/watch?v=PEatXsXIkLc)
Make sure you've the right permissions (read and write), correct function trigger, correct "Handler" when creating the function! Make sure to checkout the function logs in CloudWatch too, makes debugging a lot easier. If it starts timing out, increase the function timeout and increase it's memory.
Related
I am trying to download multiple pdf files from external sources to my nodejs server (in Heroku) temporarily and upload it to AWS S3 bucket.
I have tried multiple methods all of which works fine in my local machine but not in Heroku Dyno NodeJS Server. I am unable to even create folder in Heroku. I guess due to limited permission.
In Node
1) using var download = require('download-file') (using this currently in below code)
2) axios
3) res.download()
Download Files Code
const downloadFiles = async (unique_files) => {
for (let index = 0; index < unique_files.length; index++) {
let file_ext = unique_files[index].substr(unique_files[index].length - 4);
if(file_ext == ".pdf") {
await downloadzz(unique_files[index])
}
}
}
function downloadzz(link) {
download(link, function(err){
if (err) throw err
console.log("DOWNLOAD Complete");
});
}
Upload Files Code
const uploadFiles = async (unique_files) => {
for (let index = 0; index < unique_files.length; index++) {
let file_ext = unique_files[index].substr(unique_files[index].length - 4);
if(file_ext == ".pdf") {
await uploadzz(unique_files[index])
}
}
}
function uploadzz(link) {
fs.readFile(require('path').resolve(__dirname+'/../external-pdfs/', link.slice(link.lastIndexOf('/') + 1)), function (err, data) {
params = {Bucket: pdfBucket, Key: link.slice(link.lastIndexOf('/') + 1), Body: data, ACL: "public-read" };
s3.putObject(params, function(err, data) {
if (err) {
console.log("Failed Upload", err);
} else {
console.log("Successfully uploaded data to bucket", data);
}
});
});
}
I don't get any error but no folder seem to exist with a name external-pdfs on heroku server.
I am open for better solutions: for example, directly uploading file from external url to s3...
How can I in read file from a external url and directly upload to AWS S3 bucket?
You can use axios. Setting the responseType as stream, you can get the file data and pass it as the body. Here it's an example code to get the pdf from a URL and uploading its info directly to S3:
const AWS = require('aws-sdk');
const axios = require('axios');
AWS.config.loadFromPath('./config.json');
const s3 = new AWS.S3({apiVersion: '2006-03-01'});
const URL = "<YOUR_URL>";
const uploadPdfToS3 = async () => {
try{
const {data, headers} = await axios.get(URL, {responseType: 'stream'});
// Create params for putObject call
const objectParams = {
Bucket: "<YOUR_BUCKET>",
Key: "<YOUR_KEY>",
ContentLength: headers['content-length'],
Body: data
};
// Create object upload promise
await s3.putObject(objectParams).promise();
} catch(err){
console.log("ERROR --->" + err)
}
}
In Angular, we can use FileSaver library to save the pdf file from library.
Find the below sample code to do this way.
enter image description here
I'm currently working on a project using AWS S3, Rekognition and Lambda. I'm writing in NodeJS and created a working solution to what I want to achieve. The workflow in short is: an image of a face is loaded onto a S3 bucket, then the 'searchFacesByImage' API is called to see if that face has been indexed to the Master collection in the past. If it is a new face, the result will be false, and the 'indexFaces' API is called to index that face to the Master collection. Once that is done, I write the output to 3 separate JSON files that is in the same S3 bucket, called: 'metadata.json', 'indexing.json', 'rekognition.json'.
The 'metadata.json' file only contains the ExternalImageID (that I create myself), the date and time of indexing, the filename that was indexed, and a count that counts how many times that face has been indexed in the past.
The 'indexing.json' file contains the same ExternalImageID, the same data and time of indexing, and the response from the 'searchFacesByImage' API.
The 'rekognition.json' file contains the same ExternalImageID and date and time, as well as the response from the 'indexFaces' API.
The problem comes in that when I load on image at a time, the 3 JSON files will start to populate accordingly, but as soon as I load more than a few (I've tested it with 7) images at the same time, all 7 images will run through the workflow and the response data is written out to each file according to the Cloudwatch logs, but when I actually go to view the JSON files, not all the response data is there for all 7 images. Sometimes the data of 5 images are in the JSON, other times its 4 images. The data doesn't have to be in any specific order, it must just be there. I've also tested it where I uploaded 18 images at once and only the response of 10 images was in the JSON.
I believe the problem comes in that I'm calling the 'getObject' API on the JSON files, then I append the response data to those files, and then I'm calling the 'putObject' API on those JSON files to put them back into the S3 bucket, but while the first image is going through this process, the next image wants to do the same, but there is no file to use the 'getObject' on, because it is busy with the previous image, so then it just skips over the image, although the Cloudwatch logs said I has been added to the files.
I have no idea how to work around this. I believe the answer lies in Asynchronous JavaScript (which I don't know that much of so I have no idea where to begin)
My apologies for the long post. Here is my code below:
const AWS = require('aws-sdk');
const s3 = new AWS.S3({apiVersion: "2006-03-01"});
const rekognition = new AWS.Rekognition();
//const docClient = new AWS.DynamoDB.DocumentClient();
const uuidv4 = require('uuid/v4');
let bucket, key;
let dataSaveDate = new Date();
console.log('Loading function');
//-----------------------------------Exports Function---------------------------
exports.handler = function(event, context) {
bucket = event.Records[0].s3.bucket.name;
key = event.Records[0].s3.object.key;
console.log(bucket);
console.log(key);
searchingFacesByImage(bucket, key);
};
//---------------------------------------------------------------------------
// Search for a face in an input image
function searchingFacesByImage(bucket, key) {
let params = {
CollectionId: "allFaces",
FaceMatchThreshold: 95,
Image: {
S3Object: {
Bucket: bucket,
Name: key
}
},
MaxFaces: 5
};
const searchingFace = rekognition.searchFacesByImage(params, function(err, searchdata) {
if (err) {
console.log(err, err.stack); // an error occurred
} else {
// console.log(JSON.stringify(searchdata, null, '\t'));
// if data.FaceMatches > 0 : There that face in the image exists in the collection
if (searchdata.FaceMatches.length > 0) {
console.log("Face is a match");
} else {
console.log("Face is not a match");
let mapping_id = createRandomId();
console.log(`Created mapping_id: ${mapping_id}`);
console.log("Start indexing face to 'allFaces'");
indexToAllFaces(mapping_id, searchdata, bucket, key);
}
}
});
return searchingFace;
}
//---------------------------------------------------------------------------
// If face is not a match in 'allFaces', index face to 'allFaces' using mapping_id
function indexToAllFaces(mapping_id, searchData, bucket, key) {
let params = {
CollectionId: "allFaces",
DetectionAttributes: ['ALL'],
ExternalImageId: mapping_id,
Image: {
S3Object: {
Bucket: bucket,
Name: key
}
}
};
const indexFace = rekognition.indexFaces(params, function(err, data) {
if (err) {
console.log(err, err.stack); // an error occurred
} else {
console.log("INDEXING TO 'allFaces'");
//console.log(JSON.stringify(data, null, '\t'));
logAllData(mapping_id, bucket, key, searchData, data);
}
});
return indexFace;
}
//---------------------------------------------------------------------------
// Counting how many times a face has been indexed and logging ALL data in a single log
function logAllData(mapping_id, bucket, key, searchData, data) {
let params = {
CollectionId: mapping_id,
MaxResults: 20
};
const faceDetails = rekognition.listFaces(params, function(err, facedata) {
if (err) {
console.log(err, err.stack); // an error occurred
} else {
//console.log(JSON.stringify(facedata, null, '\t'));
metadata(mapping_id, bucket, key, facedata);
indexing(mapping_id, bucket, searchData);
rekognitionData(mapping_id, bucket, data);
}
});
return faceDetails;
}
//-----------------------------------------------------------------------------
function metadata(mapping_id, bucket, key, faceData) {
let body = [
{
"mapping_id": mapping_id,
"time": dataSaveDate,
"image_name": key,
"indexing_count": faceData.Faces.length - 1
}
];
//console.log(JSON.stringify(body, null, '\t'));
logData("metadata.json", bucket, body);
}
//------------------------------------------------------------------------------
function indexing(mapping_id, bucket, searchData) {
let body = [
{
"mapping_id": mapping_id,
"time": dataSaveDate,
"IndexingData": searchData
}
];
logData("indexing.json", bucket, body);
}
//------------------------------------------------------------------------------
function rekognitionData(mapping_id, bucket, data) {
let body = [
{
"mapping_id": mapping_id,
"time": dataSaveDate,
"rekognition": data
}
];
logData("rekognition.json", bucket, body);
}
//------------------------------------------------------------------------------
// Function to log all data to JSON files
function logData(jsonFileName, bucket, body) {
let params = {
Bucket: bucket,
Key: jsonFileName
};
const readFile = s3.getObject(params, function(err, filedata) {
if (err) {
console.log(err, err.stack); // an error occurred
} else {
console.log(`READING ${jsonFileName} CONTENTS`);
// Read data from 'jsonFileName'
let raw_content = filedata.Body.toString();
let content = JSON.parse(raw_content);
// Add new data to 'jsonFileName'
content.push(...body);
// Put new data back into jsonFileName
s3.putObject(
{
Bucket: bucket,
Key: jsonFileName,
Body: JSON.stringify(content, null, '\t'),
ContentType: "application/json"
},
function(err, res) {
if (err) {
console.log(err);
} else {
console.log(`DATA SAVED TO ${jsonFileName}`);
}
}
);
}
});
return readFile;
}
//----------------------------------SCRIPT ENDS---------------------------------
When a Node.js Lambda reaches the end of the main thread, it ends all other threads.
To make sure that the lambda does not prematurely terminate those threads, wait until that Promise is complete by using await.
The functions s3.getObject and s3.putObject can be made into a Promise like this:
await s3.getObject(params).promise()
await s3.putObject(params).promise()
I just configured an Amazon Cloud Front distribution, and connected it to an existent Amazon S3 bucket. Having said that, this bucket holds 10GB to 40GB video files, and I want to use CF to make the upload process faster (also the download, but the upload is more important in this case).
Finally, I created a simple JS script to push a file to this bucket connected to CF:
var AWS_ACCESS_KEY_ID = "111",
AWS_SECRET_ACCESS_KEY = "222",
S3_BUCKET = "videos.company.com",
S3_KEY = "remote_filename.mkv",
fs = require('fs'),
AWS = require('aws-sdk');
AWS.config.update({accessKeyId: AWS_ACCESS_KEY_ID, secretAccessKey: AWS_SECRET_ACCESS_KEY});
var startDate = new Date().toUTCString();
var fileStream = fs.createReadStream(__dirname + '/test.mkv');
fileStream.on('error', function (err) {
if (err) { throw err; }
});
fileStream.on('open', function () {
var s3 = new AWS.S3();
s3.putObject({
Bucket: S3_BUCKET,
Key: S3_KEY,
Body: fileStream
}, function(err) {
if (err) {
console.log(err, err.stack);
} else {
console.log('Uploaded file to ' + S3_KEY + ' at ' + new Date().toUTCString());
console.log("Start time: " + startDate + ", end time: " + new Date().toUTCString());
}
}).on('httpUploadProgress', function(progress) {
console.log('Upload', progress.loaded, 'of' , progress.total, 'bytes' + " --> " + ((progress.loaded*100)/progress.total).toFixed(2) + "%");
});
});
After running the script, I did not notice any speed changes (upload with and without CF distribution)... actually the upload time for both cases are exactly the same. However, after reading some forums, I saw people talking about doing a POST request using the CF url.
How can I upload a file from my server to CF directly?
Thanks!
I want to concatenate the files uploaded on Amazon S3 server.
How can I do this.
Concatenation on local machine i can do using following code.
var fs = require('fs'),
files = fs.readdirSync('./files'),
clips = [],
stream,
currentfile,
dhh = fs.createWriteStream('./concatfile.mp3');
files.forEach(function (file) {
clips.push(file.substring(0, 6));
});
function main() {
if (!clips.length) {
dhh.end("Done");
return;
}
currentfile = './files/' + clips.shift() + '.mp3';
stream = fs.createReadStream(currentfile);
stream.pipe(dhh, {end: false});
stream.on("end", function() {
main();
});
}
main();
You can achieve what you want by breaking it into two steps:
Manipulating files on s3
Since s3 is a remote file storage, you can't run code on s3 server to do the operation locally (as #Andrey mentioned).
what you will need to do in your code is to fetch each input file, process them locally and upload the results back to s3. checkout the code examples from amazon:
var s3 = new AWS.S3();
var params = {Bucket: 'myBucket', Key: 'mp3-input1.mp3'};
var file = require('fs').createWriteStream('/path/to/input.mp3');
s3.getObject(params).createReadStream().pipe(file);
at this stage you'll run your concatenation code, and upload the results back:
var fs = require('fs');
var zlib = require('zlib');
var body = fs.createReadStream('bigfile.mp3').pipe(zlib.createGzip());
var s3obj = new AWS.S3({params: {Bucket: 'myBucket', Key: 'myKey'}});
s3obj.upload({Body: body}).
on('httpUploadProgress', function(evt) { console.log(evt); }).
send(function(err, data) { console.log(err, data) });
Merging two (or more) mp3 files
Since MP3 file include a header that specifies some information like bitrate, simply concatenating them together might introduce playback issues.
See: https://stackoverflow.com/a/5364985/1265980
what you want to use a tool to that. you can have one approach of saving your input mp3 files in tmp folder, and executing an external program like to change the bitrate, contcatenate files and fix the header.
alternatively you can use an library that allows you to use ffmpeg within node.js.
in their code example shown, you can see how their merge two files together within the node api.
ffmpeg('/path/to/part1.avi')
.input('/path/to/part2.avi')
.input('/path/to/part2.avi')
.on('error', function(err) {
console.log('An error occurred: ' + err.message);
})
.on('end', function() {
console.log('Merging finished !');
})
.mergeToFile('/path/to/merged.avi', '/path/to/tempDir');
Here's my quick take on the problem of downloading and processing S3 objects. My example is focused mostly on getting the data local and then processing it once it's all downloaded. I suggest you use one of the ffmpeg approaches mentioned above.
var RSVP = require('rsvp');
var s3 = new AWS.S3();
var bucket = '<your bucket name>';
var getFile = function(key, filePath) {
return new RSVP.Promise(function(resolve, reject) {
var file = require('fs').createWriteStream(filePath);
if(!file) {
reject('unable to open file');
}
s3.getObject({
Bucket: bucket,
Key: key
}).on('httpData', function(chunk) {
file.write(chunk);
}).on('httpDone', function() {
file.end();
resolve(filePath);
});
});
};
var tempFiles = ['<local temp filename 1>', '<local temp filename 2>'];
var keys = ['<s3 object key 1>', '<s3 object key 2>'];
var promises = [];
for(var i = 0; i < keys.length; ++i) {
var promise = getFile(keys[i], tempFiles[i]);
promises.push(promise);
}
RSVP.all(promises).then(function(data) {
//do something with your files
}).catch(function(error) {
//handle errors
});
I'm using ImageMagick to make thumbnails of my photos on S3.
This is the flow:
Get the image from S3.
Make thumbnail from the data.Body.
Put the thumbnail on S3.
Here is the make thumbnail function:
function makeThumbnail(image) {
var defer = q.defer();
im.resize({
srcData: image.Body,
width: 256
}, function (err, stdout) {
if (err) {
defer.reject(err);
} else {
image.Key.replace(/^images/, "thumbs");
image.Body = new Buffer(stdout, 'binary');
defer.resolve(image);
}
});
return defer.promise;
}
The image object is of the form that S3 SDK expects to get it:
var image = {
Bucket: 'bucket name',
Key: 'object key',
Body: 'body',
ContentType: 'image/png'
};
Yet, when putting the thumbnail on S3, the MIME is set to application/octet-stream for some reason. When downloading the thumbnail it opens like any other picture, yet the browsers do not treat it like it's an image and that is a problem for me.
What causes this issue? and how can I solve it?
Thanks
The docs for node-imagemagick.resize() says the default format is jpg. Maybe your output is not a png like you think.
We use the im-resize package and that contains input and output parameters, where we typically convert .jpeg images uploaded from a web form into .png output file. (We read from the file system after conversion, not stream buffers, just fyi).
Our code is something like this:
var resize = require('im-resize');
var metadata = require('im-metadata');
var AWS = require('aws-sdk');
metadata(data.params.file.path, {}, function(error, metadataResults) {
//error checking
var image = {
//set from metadataResults
};
var output = {
versions: [{
//set as desired
}]
};
resize(image, output, function(error, versions) {
var filename = 'images/' + uuid.v4() + '.png'; //this was enough for us
var fileStream = fs.createReadStream(versions[0].path);
fileStream.on('error', function(err) {
//handle error
});
fileStream.on('open', function() {
var s3 = new AWS.S3();
s3.putObject({
Bucket: 'bucketname',
Key: filename,
Body: fileStream
}, function(err) {
//handle errors
//know the file has been saved to S3
//close up all the squiggly braces and return HTTP response or whatever