Uploading image to amazon s3 using multer-s3 nodejs - node.js

I am trying to upload an image to amazon s3 using multer-s3, but I am getting this error:
TypeError: Expected opts.s3 to be object
node_modules/multer-s3/index.js:69:20
This is my server code:
var upload = multer({
storage: s3({
dirname: '/',
bucket: 'bucket',
secretAccessKey: 'key',
accessKeyId: 'key',
region: 'us-west-2',
filename: function (req, file, cb) {
cb(null, file.originalname);
}
})
});
app.post('/upload', upload.array('file'), function (req, res, next) {
res.send("Uploaded!");
});
Why I am getting this error?

[Update Mar 2022] It works perfectly fine till-date and now also shows the uploaded file public URL as well.
Complete and working Node Cheat | Upload to s3 using multer-s3 available.
Code:
var express = require('express'),
aws = require('aws-sdk'),
bodyParser = require('body-parser'),
multer = require('multer'),
multerS3 = require('multer-s3');
aws.config.update({
secretAccessKey: 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX',
accessKeyId: 'XXXXXXXXXXXXXXX',
region: 'us-east-1'
});
var app = express(),
s3 = new aws.S3();
app.use(bodyParser.json());
var upload = multer({
storage: multerS3({
s3: s3,
acl: 'public-read',
bucket: 'bucket-name',
key: function (req, file, cb) {
console.log(file);
cb(null, file.originalname); //use Date.now() for unique file keys
}
})
});
//open in browser to see upload form
app.get('/', function (req, res) {
res.sendFile(__dirname + '/index.html');//index.html is inside node-cheat
});
//use by upload form
app.post('/upload', upload.array('upl', 25), function (req, res, next) {
res.send({
message: "Uploaded!",
urls: req.files.map(function(file) {
return {url: file.location, name: file.key, type: file.mimetype, size: file.size};
})
});
});
app.listen(3000, function () {
console.log('Example app listening on port 3000!');
});
For complete repo:
Clone node-cheat express_multer_s3, run node app followed by npm install express body-parser aws-sdk multer multer-s3.
Happy Helping!

#V31 has answered very well still I want to add my 2 cents.
I believe in keeping one responsibility into one file, for better code organization and debugging purpose.
I have created a file for uploading upload.js.
require('dotenv').config();
const AWS = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const s3Config = new AWS.S3({
accessKeyId: process.env.AWS_IAM_USER_KEY,
secretAccessKey: process.env.AWS_IAM_USER_SECRET,
Bucket: process.env.AWS_BUCKET_NAME
});
const fileFilter = (req, file, cb) => {
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png') {
cb(null, true)
} else {
cb(null, false)
}
}
// this is just to test locally if multer is working fine.
const storage = multer.diskStorage({
destination: (req, res, cb) => {
cb(null, 'src/api/media/profiles')
},
filename: (req, file, cb) => {
cb(null, new Date().toISOString() + '-' + file.originalname)
}
})
const multerS3Config = multerS3({
s3: s3Config,
bucket: process.env.AWS_BUCKET_NAME,
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
console.log(file)
cb(null, new Date().toISOString() + '-' + file.originalname)
}
});
const upload = multer({
storage: multerS3Config,
fileFilter: fileFilter,
limits: {
fileSize: 1024 * 1024 * 5 // we are allowing only 5 MB files
}
})
exports.profileImage = upload;
Which is imported inside my routes routes.js
const express = require('express');
const ProfileController = require('../profile/controller');
const { profileImage } = require('../utils/upload.js');
const routes = (app) => {
const apiRoutes = express.Router();
apiRoutes.use('/profile', profileRoutes);
profileRoutes.post('/',profileImage.single('profileImage'), ProfileController.saveProfile);
app.use('/api', apiRoutes);
}
module.exports = routes
Postman screen shot for post body

I just want to add my cents,
There are many comments in all answers like how to get public URL after uploading and S3 response object and lets see implementation and cases,
// INITIALIZE NPMS
var AWS = require('aws-sdk'),
multer = require('multer'),
multerS3 = require('multer-s3'),
path = require('path');
// CONFIGURATION OF S3
AWS.config.update({
secretAccessKey: '***********************************',
accessKeyId: '****************',
region: 'us-east-1'
});
// CREATE OBJECT FOR S3
const S3 = new AWS.S3();
// CREATE MULTER FUNCTION FOR UPLOAD
var upload = multer({
// CREATE MULTER-S3 FUNCTION FOR STORAGE
storage: multerS3({
s3: S3,
acl: 'public-read',
// bucket - WE CAN PASS SUB FOLDER NAME ALSO LIKE 'bucket-name/sub-folder1'
bucket: 'bucket-name',
// META DATA FOR PUTTING FIELD NAME
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
// SET / MODIFY ORIGINAL FILE NAME
key: function (req, file, cb) {
cb(null, file.originalname); //set unique file name if you wise using Date.toISOString()
// EXAMPLE 1
// cb(null, Date.now() + '-' + file.originalname);
// EXAMPLE 2
// cb(null, new Date().toISOString() + '-' + file.originalname);
}
}),
// SET DEFAULT FILE SIZE UPLOAD LIMIT
limits: { fileSize: 1024 * 1024 * 50 }, // 50MB
// FILTER OPTIONS LIKE VALIDATING FILE EXTENSION
fileFilter: function(req, file, cb) {
const filetypes = /jpeg|jpg|png/;
const extname = filetypes.test(path.extname(file.originalname).toLowerCase());
const mimetype = filetypes.test(file.mimetype);
if (mimetype && extname) {
return cb(null, true);
} else {
cb("Error: Allow images only of extensions jpeg|jpg|png !");
}
}
});
There are three cases, if we want to retrieve files res object from S3 after upload:
Case 1: When we are using .single(fieldname) method it will return file object in req.file
app.post('/upload', upload.single('file'), function (req, res, next) {
console.log('Uploaded!');
res.send(req.file);
});
Case 2: When we are using .array(fieldname[, maxCount]) method it will return file object in req.files
app.post('/upload', upload.array('file', 1), function (req, res, next) {
console.log('Uploaded!');
res.send(req.files);
});
Case 3: When we are using .fields(fields) method it will return file object in req.files
app.post('/upload', upload.fields([
{ name: 'avatar', maxCount: 1 },
{ name: 'gallery', maxCount: 8 }
]), function (req, res, next) {
console.log('Uploaded!');
res.send(req.files);
});

s3 needs to be an object to be passed. According to the docs, the object needs to be like this:
var upload = multer({
storage: multerS3({
s3: s3,
bucket: 'some-bucket',
metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
},
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
})
MulterS3 Docs

/*** Using Multer To Upload Image image is uploading */
const fileStorage = multer.diskStorage({
destination: function(req, file, cb) {
cb(null, "./public/uploads");
},
filename: function(req, file, cb) {
cb(null, file.originalname);
}
});
/** AWS catalog */
aws.config.update({
secretAccessKey: process.env.SECRET_KEY,
accessKeyId: process.env.ACCESS_KEY,
region: "us-east-1"
});
const s3 = new aws.S3();
const awsStorage = multerS3({
s3: s3,
bucket: process.env.BUCKET_NAME,
key: function(req, file, cb) {
console.log(file);
cb(null, file.originalname);
}
});
const upload = multer({
storage: awsStorage(),
/** in above line if you are using local storage in ./public/uploads folder than use
******* storage: fileStorage,
* if you are using aws s3 bucket storage than use
******* storage: awsStorage(),
*/
limits: { fileSize: 5000000 },
fileFilter: function(req, file, cb) {
checkFileType(file, cb);
}
});
app.post("/user-profile-image", upload.single("profile"), (req, res, err) => {
try {
res.send(req.file);
} catch (err) {
res.send(400);
}
});
const checkFileType = (file, cb) => {
const filetypes = /jpeg|jpg|png|gif/;
const extname = filetypes.test(path.extname(file.originalname).toLowerCase());
const mimetype = filetypes.test(file.mimetype);
if (mimetype && extname) {
return cb(null, true);
} else {
cb("Error: Images Only!");
}
};

//here is the function for upload the images on aws bucket using multer
const path = require('path');
const fs = require('fs');
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
aws.config.update({
secretAccessKey: '**************************',
accessKeyId: '********************',
region: '**********************'
});
s3 = new aws.S3();
const storage = multerS3({
s3: s3,
bucket: 'bucket-name',
key: function(req, file, cb) {
console.log(file);
cb(null, file.originalname);
}
})
//export the created function
exports.uploadVideo = multer({ storage: storage }).single('file_name');
//================================================================================
//import uploadVideo function whenever you need to upload the file on aws s3 bucket
const { uploadVideo } = require('../../services/upload');
exports.videoUpload = (req, res) => {
uploadVideo(req, res, function(err) {
if (err) {
console.log(err);
res.json({ status: 401, msg: err.message });
} else {
const image = getImagePath(req.file.filename);
res.json({ status: 200, msg: 'Image uploaded sucess.', data: image });
}
});
}
//================================================================================
//here is route file
router.post('/video-upload',uploadController.videoUpload);

I was passing S3 to mutler in caps, like
S3: {object}
Changing it to small s3 works for me:-
s3: {object}

WE can Upload IMAGE/ CSV/ EXCEL files to AWS s3 using multer-s3.
Im using .single(fieldname) method for uploading single file.
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const s3 = new aws.S3({
accessKeyId: process.env.AWS_ACCESS_KEY,
secretAccessKey: process.env.AWS_SECRET_KEY,
region: process.env.REGION,
});
const upload = multer({
storage: multerS3({
s3: s3,
bucket: process.env.AWS_S3_BUCKET,
metadata: function (req, file, cb) {
cb(null, {fieldName: 'Meta_Data'});
},
key: function (req, file, cb) {
cb(null, file.originalname);
},
limits: {
fileSize: 1024 * 1024 * 5 // allowed only 5 MB files
}
})
}).single('file');
exports.uploadfile = async(req,res,next)=>{
try{
upload(req,res, function(err){
if(err){
console.log(err);
}
console.log(req.file.location);
})
})
}catch (err){
res.status(400).json({
status : 'fail',
message : err.message
});
}
}
In Routes file
router.route('/')
.post(imageController.uploadfile);

Related

MulterS3 is giving this.client.send is not a function error

Note:
Getting below error in multer s3. This error is pointing NPM module and I really don't understand the issue with this module. I have used upload.single as well as upload.array method to check the working of this module. But not working.
Code:
const bodyParser = require('body-parser');
const cors = require('cors');
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const http = require('http');
const app = express();
app.set('port', 3000);
app.use(
bodyParser.json({
limit: '50mb'
})
);
app.use(
bodyParser.urlencoded({
limit: '50mb',
extended: true
})
);
app.use(cors());
aws.config.update({
accessKeyId: '',
secretAccessKey: ''
});
s3 = new aws.S3();
const upload = multer({
storage: multerS3({
s3: s3,
bucket: '',
acl: 'public-read',
contentType: multerS3.AUTO_CONTENT_TYPE,
metadata: function (req, file, cb) {
cb(null, {
fieldName: file.fieldname
});
},
key: function (req, file, cb) {
let extArray = file.mimetype.split('/');
let ext = extArray[extArray.length - 1];
console.log(`ext ->> `, ext, ` file.fieldname ->> `, file.fieldname);
cb(null, "test/" + Date.now().toString() + '.' + ext);
},
/* limits: {
fileSize: 1024 * 1024 * 10
} */
})
});
(() => {
server = http.createServer(app).listen(app.get('port'), () => {
console.debug(`Server started ->> `);
app.get('/test', (req, res) => {Mul
res.send('Hello');
});
app.post('/media', upload.single('media'), (req, res) => {
console.log(`req.files ->> `, req.file);
res.send('Thanks for you waiting time');
});
});
})();
Error:
ext ->> jpeg file.fieldname ->> media
TypeError: this.client.send is not a function
at Upload.__uploadUsingPut (/home/tristate/Jay/Force/node_modules/#aws-sdk/lib-storage/dist-cjs/Upload.js:52:25)
at Upload.__doConcurrentUpload (/home/tristate/Jay/Force/node_modules/#aws-sdk/lib-storage/dist-cjs/Upload.js:99:39)
at process._tickCallback (internal/process/next_tick.js:68:7)
Looking solution with multer-s3. Thank you in advance.
Have you checked this issues ?
https://github.com/anacronw/multer-s3/issues/169
please check your aws-sdk, multer-s3 version is compatible
Just hit the enter:
npm i multer-s3#2.10.0
3.x.x releases of multer-s3 use AWS JavaScript SDK v3. Specifically, it uses the Upload class from #aws-sdk/lib-storage which in turn calls the modular S3Client.
check their github readme
require('dotenv').config();
const { S3Client } = require('#aws-sdk/client-s3');
const multer = require('multer');
const multerS3 = require('multer-s3');
const shortId = require('shortid');
let s3 = new S3Client({
region: 'ca-central-1',
credentials: {
accessKeyId: process.env.ACCESS_KEY_AWS,
secretAccessKey: process.env.ACCESS_SECRET_AWS,
},
sslEnabled: false,
s3ForcePathStyle: true,
signatureVersion: 'v4',
});
exports.upload = multer({
storage: multerS3({
s3: s3,
bucket: process.env.BUCKET_AWS,
contentType: multerS3.AUTO_CONTENT_TYPE,
acl: 'public-read',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, shortId.generate() + '-' + file.originalname);
},
}),
});
I had this same issue. Downgrading Multer.s3 to 2x version solve the problem!
Multer S3 2.x is compatible with AWS SDK 2.x and
Multer S3 3.x is compatible with AWS SDK 3.x.
For me, removing this line worked
acl: 'public-read',

Set Multer object key in node.js on the fly

I have node express routes, where I need to set a aws bucket name in a Multer Object.
When the users get logged in my app, I have an API to get the correct bucket name to user.
I call that API in a middleware to all routes and I set req.folder_do_bucket='XXXX' from API result.
After that, when the user access others routes such as below, I need to set the bucket in the Multer Object. But, my problem is that "req" is not available to set it in the Multer.
Maybe I can create the multer object into the route, but I am not sure if this is a better solution, because I will have to do it in all routes(get, post....)
How could I solve it?
const express = require("express");
const router = express.Router();
const multer = require("multer");
const aws = require("aws-sdk");
const s3 = new aws.S3();
const multerS3 = require("multer-s3");
const storageImagemClienteS3 = multerS3({
s3: s3,
bucket: req.folder_do_bucket, // AWS_BUCKET_NAME -->Here is my problem, req is not available here
contentType: multerS3.AUTO_CONTENT_TYPE,
acl: "public-read",
key: (req, file, cb) => {
const fileName =
`${req.conta}/clientes/cliente-${req.params.id}/imagens/` +
file.originalname;
cb(null, fileName);
},
});
console.log(storageImagemClienteS3);
const uploadImagemClienteS3 = multer({
storage: storageImagemClienteS3,
limits: { fileSize: MAX_SIZE },
fileFilter: function (req, file, cb) {
checkFileTypeImage(file, cb);
},
}).single("file");
router.post("/upload/:id", function (req, res) {
if (!req.params.id || req.params.id == "undefined") {
return res
.status(400)
.send({ message: "Um erro ocorreu selecionando cliente" });
}
uploadImagemClienteS3(req, res, async (err) => {
if (err) {
....
} else {
......
}
});
});

"MulterError: Unexpected field\n at wrappedFileFilter using AWS, Multer S3, NodeJS, Mongo

I have spent days trying to figure out how to post photos to my AWS bucket and have read a lot of similar questions on SO but nothing seems to be working and I almost always get an "unexpected field" error.
I have confirmed the libraries are the correct versions; added bodyParser; added region / signatureVersion; and have tried changing 'upload.single('image') a variety of ways.
In my Model I am trying to save the image url to a field name 'image-field' but using it doesn't change the error I am getting. I am only trying to have this work on my backend before attempting the FE and am using with PostMan.
Please help, I am losing my mind over here.
import express from 'express';
import aws from 'aws-sdk';
import multer from 'multer';
import multerS3 from 'multer-s3';
import bodyParser from 'body-parser';
const router = express.Router();
router.use(bodyParser.json())
aws.config.update({
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
region: 'us-west-2',
signatureVersion: 'v4'
})
const s3 = new aws.S3()
const upload = multer({
storage: multerS3({
s3: s3,
bucket: 'bucket-name',
acl: "public-read",
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, Date.now().toString() + '-' + file.originalname);
},
}),
})
router.post('/', upload.single('image'), (req, res, next) => {
console.log('Uploaded!')
res.send(req.file)
})
export default router;
PostMan
enter image description here
Note: If it helps, I am including old code I was using successfully for my FE and BE to save images locally and then the urls on Mongo.
import path from 'path';
import express from 'express';
import multer from 'multer';
const router = express.Router();
const storage = multer.diskStorage({
destination(req, file, cb) {
cb(null, 'uploads/')
},
filename(req, file, cb) {
cb(null, `${file.fieldname}-${Date.now()}${path.extname(file.originalname)}`)
}
})
function checkFileType(file, cb) {
const filetypes = /jpg|jpeg|png/
const extname = filetypes.test(path.extname(file.originalname).toLowerCase())
const mimetype = filetypes.test(file.mimetype)
if(extname && mimetype) {
return cb(null, true)
} else {
cb('Images only!')
}
}
const upload = multer({
storage,
fileFilter: function(req, file, cb) {
checkFileType(file, cb)
}
})
router.post('/', upload.single('image'), (req, res) => {
res.send(`/${req.file.path}`)
})
export default router;
We did it! See below code below, but the update really pertains to changing the POST API from .single() to .any() and then setting fileFilter to remove any files you do not want to include. I will note that PostMan gives me an error on my route when I do post the file, but the image does successfully upload to the s3 bucket.
Said another way, .any() "Accepts all files that comes over the wire. An array of files will be stored in req.files". Since we know we are taking any file, we must use fileFilter to define which files we want to be able to post. Essentially, if you don't use fileFilter, you can post any file you want.
import express from 'express';
import aws from 'aws-sdk';
import multer from 'multer';
import multerS3 from 'multer-s3';
import path from 'path';
const router = express.Router();
aws.config.update({
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
region: 'us-west-2',
signatureVersion: 'v4'
})
const s3 = new aws.S3()
function checkFileType(file, cb) {
const filetypes = /jpg|jpeg|png/
const extname = filetypes.test(path.extname(file.originalname).toLowerCase())
const mimetype = filetypes.test(file.mimetype)
if(extname && mimetype) {
return cb(null, true)
} else {
cb('Images only!')
}
}
const upload = multer({
limits: { fileSize: 2000000 },
fileFilter: function(req, file, cb) {
checkFileType(file, cb)
},
storage: multerS3({
s3: s3,
bucket: 'recipebook-recipe-cover-images',
acl: "public-read",
contentType: multerS3.AUTO_CONTENT_TYPE,
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, Date.now().toString() + '-' + file.originalname);
},
}),
})
router.post('/', upload.any())
export default router;

How to Upload image wih destination Path as file.filename to Amazon s3, with Multer and expressjs

I am uploading images to Amazon S3 with Multers3 and express.
The Image is uploading but I am facing challenges uploading images to the Right Path.
For example, before integrating Multer with s3, with this code, the image will be uploaded to public/uploads/postImages/the new filename.jpg
const uploadPath = path.join('public', Post.postImageBasePath)
const imageMineTypes = ['image/jpeg', 'image/png', 'image/gif']
const upload = multer({
dest: uploadPath,
fileFilter: (req, file, callback) => {
callback(null, imageMineTypes.includes(file.mimetype) )
}
})
But after integrating Multer with Amazon S3, with this code, the image is now being uploaded to public/uploads/postImages/original file name.jpg
const uploadPath = path.join('public', Post.postImageBasePath)
const upload = multer({
storage: multerS3({
s3: s3,
bucket: bucketname,
s3BucketEndpoint:true,
endpoint:"http://" + bucketname + ".s3.amazonaws.com",
key: function (req, file, cb) {
cb(null, uploadPath + '/' + file.originalname);
}
})
})
If I want to get the new file name then I can do something like this
req.file.filename but this is only possible inside my upload and that is after the image has been successfully uploaded. if try to do it inside the above code it will show undefined, because the new file name has not yet been generated. below is my code for upload.
router.post('/', upload.single('cover'), async (req, res, next) => {
const fileName = req.file != null ? req.file.filename : null
const post = new Post({
title: req.body.title,
description: req.body.description,
from: req.body.from,
postImage: fileName
})
try {
const newPost = await post.save()
res.redirect('/posts')
} catch {
if (post.postImage != null) {
removePostImage(post.postImage)
}
renderNewPage(res, post, true)
}
});
How do I add dest: uploadPath, path with S3? so that the path will be
public/uploads/postImages/new file name.jpg
This is what i want to achieve
const uploadPath = path.join('public', Post.postImageBasePath)
const imageMineTypes = ['image/jpeg', 'image/png', 'image/gif']
const upload = multer({
storage: multerS3({
dest: uploadPath,
s3: s3,
bucket: bucketname,
s3BucketEndpoint:true,
endpoint:"http://" + bucketname + ".s3.amazonaws.com",
key: function (req, file, cb) {
cb(null, uploadPath + '/' + req.file.filename);
},
fileFilter: (req, file, callback) => {
callback(null, imageMineTypes.includes(file.mimetype) )
}
})
})
In Summary, I want the image path to be the new image name that has been generated

I want to upload files with extension to aws s3 but it will not upload file with extension

This is my File Upload.js but when i upload file it will get uploaded succesfully but its extension is missing i want to upload file with extension .Please help me
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
aws.config.update({
secretAccessKey: '',
accessKeyId: '',
region: 'us-east-2'
});
const s3 = new aws.S3({ })
const fileFilter = (req, file, cb) => {
if (file.mimetype === 'music/AAC' || file.mimetype === 'music/AVI' ||file.mimetype === 'music/mp3' ||file.mimetype === 'music/AVI') {
cb(null, true)
} else {
cb(new Error('Invalid Mime Type, only JPEG and PNG'), false);
}
}
const upload = multer({
storage: multerS3({
fileFilter,
s3,
bucket: 'admin-song',
acl: 'public-read',
metadata: function (req, file, cb) {
cb(null, {fieldName: 'TESTING_META_DATA!'});
},
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
})
module.exports = upload;
And This is my Api End Point Guide Me Please
app.post('/v1/admin/upload/song',function(req, res) {
singleUpload(req, res, function(err) {
if (err) {
return res.status(422).send({errors: [{title: 'File Upload Error', detail: err.message}] });
}
console.log(res)
return res.json({'imageUrl': req['file'].location});
});
});
You have to add whatever string you want as filename in your multer function.You can add file extension to the file with the following method
const upload = multer({
storage: multerS3({
fileFilter,
s3,
bucket: 'admin-song',
acl: 'public-read',
metadata: function (req, file, cb) {
cb(null, {fieldName: 'TESTING_META_DATA!'});
},
key: function (req, file, cb) {
cb(null, Date.now().toString() + '.' + fileExtension)
}
})
})

Resources