I've built functionality to simultaneously save a record in my MongoDB database and upload image(s) to Amazon S3. That's all working OK. However, in this process, I'm also wanting to save the ID of the data from my MongoDB as the key in my Amazon S3 upload.
I've tried playing around with 'req.params.id' and so on, but it's just coming back as undefined.
Is there a really obvious way of doing this? Hoping I'm missing something straightforward.
Code:
const upload = multer({
storage: multerS3({
s3: s3,
bucket: 'example',
metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
},
contentType: multerS3.AUTO_CONTENT_TYPE,
acl: 'public-read',
key: function (req, file, cb) {
cb(null, `${req.body.id}`
// Date.now().toString()
);
}
})
});
module.exports = (req, res) => {
HouseListing.create(req.body, (error, houselisting) => {
if (error) {
const validationErrors = Object.keys(error.errors).map(key => error.errors[key].message);
req.flash('validationErrors', validationErrors);
return res.redirect("/houseListing");
}
console.log(req.files);
console.log(req.body.mainImages);
console.log(req.body.id);
res.redirect("/");
});
};
The bottom two console logs of my Database function are returning 'undefined'. As is the upload function.
Any tips? :) Please let me know if I've missed any info out!
Related
I'm uploading images from a form to my S3 bucket. When the form is submitted, it creates a new mongoDB object. I'm using key: function (req, file, cb) {cb(null, Date.now().toString())},
to give the uploads unique names. My issue is, when I'm saving the new mongoDB object, I'm not sure how to reference the unique S3 key.
I admit this might be a dumb question, but I'm not very experienced with mongoDB/mongoose and this is my first time ever working with S3.
I would greatly appreciate any advice or recommendations
// productController.js
// if errors on form
else {
let product = new Product({
business: req.body.businessName,
productName: req.body.productName,
category: req.body.productCategory,
price: req.body.productPrice,
description: req.body.productDescription,
website: req.body.website,
imageURL: "https://mybucket.s3-us-west-2.amazonaws.com/" + '???'
});
product.save(function (err) {
if (err) {
console.log(err)
return next(err);
}
res.redirect("/list-product");
});
}
to upload my images im using this:
// uploadController.js
const aws = require("aws-sdk");
const multer = require("multer");
const multerS3 = require("multer-s3");
require("dotenv").config();
aws.config.update({
secretAccessKey: process.env.SECRETACCESSKEY,
accessKeyId: process.env.ACCESSKEYID,
region: "us-west-2",
});
const s3 = new aws.S3();
const fileFilter = (req, file, cb) => {
if (file.mimetype === "image/jpeg" || file.mimetype === "image/png") {
cb(null, true);
} else {
cb(new Error("Invalid format, only JPG and PNG"), false);
}
};
const upload = multer({
fileFilter: fileFilter,
storage: multerS3({
s3: s3,
bucket: "mybucket",
acl: "public-read",
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
console.log(file);
cb(null, Date.now().toString());
},
}),
});
module.exports = upload;
and my routes:
// routes.js
let upload = require('../controllers/uploadController')
const singleUpload = upload.single("image");
// post list product page
router.post('/list-product', singleUpload, listController.list__post)
I'm not a pro on this but i can think in one solution:
I see you handle the request first in the upload, so there you can save on the request the value of the Key, and not use directly the Date.now() on the name of the image when you create the file
So you should do:
req.imageName = new Date.now().toString()
And in the name change that creation of the name with req.imageName
Then in the next controller handling the request, you have access to the name on the object "req" provided to the function.
NOTE: new Date.now() IS NOT a good unique key if you have concurrent request for different clients, you have a high probability of getting the same number if you have too much requests at the same time.
NOTE 2: I can think on uuid for better unique keys, i don't know too much about others librarys that can solve that problem.
Everything I've looked at regarding s3 and multer s3 looks something like this
const upload = multer({
storage: multerS3({
s3: s3,
bucket: 'bucket',
acl: 'public-read',
cacheControl: 'max-age=31536000',
contentType: multerS3.AUTO_CONTENT_TYPE,
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
const key = `pictures/${Date.now().toString()}`;
cb(null, key);
},
}),
});
and then the controller action looks like this
app.post('/profileImage', upload.single('profileImage'), async function (req, res) {
const { _id } = req.params;
const user = await User.findOneAndUpdate({ _id }, { image: req.file.location }, { new: true });
res.json(user);
});
My question is can I send back form data and other data in one call and upload my images to s3. I want to be able to structure my data something like this
const formData = new FormData();
formData.append("picture", this.file);
const data = {
title: this.toDoTitle,
details: this.toDoDetails,
formData,
};
axios
.post("/profileImage", data)
.then(() => {
})
.catch(err => {
console.log(err);
});
and then in my controller be able to save that form data image in s3 but also interact with the other data. I've been able to send back data in the req.params but I want to know if you can send it back in the body as well with the image and be able to upload image and get other data
I am trying to upload an image to a s3 bucket using multerS3. It does save the file to the s3 bucket but not in an image format.
here is my code.
storage: multerS3({
s3: S3,
bucket: 'slsupload',
acl: 'public-read',
metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
},
key: function (req, file, cb) {
cb(null, Date.now().toString()+".jpg")
}
})
});
const singleUpload = upload.single('file');
app.post('/test-upload', (req, res) => {
singleUpload(req, res, function(err, some) {
if (err) {
return res.status(422).send({errors: [{title: 'Image Upload Error', detail: err.message}] });
}
return res.json({'imageUrl': req.file.location});
});
});```
You can try setting the contentType buy hand or let multerS3 discover the contentType automatically:
var upload = multer({
storage: multerS3({
s3: s3,
bucket: 'some-bucket',
contentType: multerS3.AUTO_CONTENT_TYPE,
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
})
I am not able to upload large files using multer-s3. It is not giving me any error as well. It just doesn't upload the file, doesn't even enters the callback and gets timeout. Any way to handle uploading large files to s3 Bucket?
I am using it like this:
var uploadSingle = upload.single('uploadFile');
router.post('/uploadVideo',function(req,res,next){
uploadSingle(req,res,function(err){
// doesn't come here if the file is large
if(err){
//Error Response , Error while uploading Module PDF;
}
else{
//handling file upload
// success response
}
});
}
I had the same issue and after researching this page I found that I need to add contentLength as one of the parameters. Its value is for the length in bytes.
const s3 = new AWS.S3({
accessKeyId: process.env.S3_ACCESS_KEY_ID,
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY
});
var upload = multer({
storage: multerS3({
s3: s3,
bucket: 'myBucket',
contentType: multerS3.AUTO_CONTENT_TYPE,
contentLength: 500000000,
metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
},
key: function (req, file, cb) {
cb(null, file.originalname);
}
})
});
router.post('/uploadToS3', upload.array('photos', 30), function(req, res, next) {
res.send({"message": 'Successfully uploaded ' + req.files.length + ' files!'});
})
I'm currently implementing a file/image upload service for my users. I want to transform these images (resize/optimize) before uploading to my s3 bucket.
What I'm currently doing: Using a multipart form on my frontend (I think the actual implementation doesn't matter here..) and the multer and multer-s3 packages on my backend.
Here my implementation stripped down to the important parts.
// SETUP
var multer = require('multer');
var s3 = require('multer-s3');
var storage = s3({
dirname: 'user/uploads',
bucket: auth.aws.s3.bucket,
secretAccessKey: auth.aws.s3.secretAccessKey,
accessKeyId: auth.aws.s3.accessKeyId,
region: auth.aws.s3.region,
filename: function (req, file, cb) {
cb(null, Date.now());
}
});
var upload = multer({storage: storage}).single('img');
// ROUTE
module.exports = Router()
.post('/', function (req, res, next) {
upload(req, res, function (err) {
if (err) {
return res.status(401).json({err: '...'});
}
return res.json({err:null,url: '..'});
});
});
What I want to do: transform the image before uploading it. I'm not sure if I need to use multer/busboy here or I can just do it with NodeJS (thus I've tagged NodeJS and express as well).
So my question is: where can I intercept the upload and transform it before uploading it to my S3 bucket?
Not sure if you're still looking for an answer to this, but I had the same problem. I decided to extend the multer-s3 package.
I've opened a pull request to the original repository, but for now, you can use my fork.
Here's an example of how to use the extended version:
var upload = multer({
storage: multerS3({
s3: s3,
bucket: 'some-bucket',
shouldTransform: function (req, file, cb) {
cb(null, /^image/i.test(file.mimetype))
},
transforms: [{
id: 'original',
key: function (req, file, cb) {
cb(null, 'image-original.jpg')
},
transform: function (req, file, cb) {
cb(null, sharp().jpg())
}
}, {
id: 'thumbnail',
key: function (req, file, cb) {
cb(null, 'image-thumbnail.jpg')
},
transform: function (req, file, cb) {
cb(null, sharp().resize(100, 100).jpg())
}
}]
})
})
EDIT: My fork is also now available via npm under the name multer-s3-transform.
I've tried using #ItsGreg's fork, but couldn't get it to work. I managed to get this behaviour working by using multer-s3 standard configuration, and inside my file upload endpoint, i.e.,
app.post('/files/upload', upload.single('file'), (req, res) => {...})
I am retrieving the file using request, and passing the Buffer to sharp. The following works (and assumes you are using ~/.aws/credentials):
let request = require('request').defaults({ encoding: null });
let dataURI = `https://s3.amazonaws.com/${process.env.AWS_S3_BUCKET}/${image.defaultUrl}`;
request.get(dataURI, function (error, response, body) {
if (! error && response.statusCode === 200) {
let buffer = new Buffer(body);
const sizes = ['thumbnail', 'medium', 'large'];
sizes.forEach(size => {
sharp(buffer)
.resize(image.sizes[size])
.toBuffer()
.then(data => {
// Upload the resized image Buffer to AWS S3.
let params = {
Body: data,
Bucket: process.env.AWS_S3_BUCKET,
Key: `${image.filePath}${image.names[size]}`,
ServerSideEncryption: "AES256",
};
s3.putObject(params, (err, data) => {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
})
})
}
});