Im uploading image files to my S3 bucket with multer S3. Everthing goes smooth, file does get uploaded but bucket name gets duplicated in returned FILE LOCATION:
Here's the file location is s3: CORRECT in S3
https://MYBUCKET.s3.eu-west-3.amazonaws.com/1669200254736-img-intro-bg.jpg
Here's what I get as file.location in my node app: Bucket name is doubled:
https://MYBUCKET.MYBUCKET.s3.eu-west-3.amazonaws.com/1669200254736-img-intro-bg.jpg'
HERE's MY APP CODE;
const { S3Client } = require('#aws-sdk/client-s3');
const multer = require('multer');
const multerS3 = require('multer-s3');
require('dotenv').config();
const credentials = {
region: process.env.region,
credentials: {
accessKeyId: process.env.accessKeyId,
secretAccessKey: process.env.secretAccessKey,
},
};
const s3 = new S3Client(credentials);
const imageFilter = (req, file, cb) => {
if (file.mimetype.startsWith('image')) {
cb(null, true);
} else {
cb('Please upload only images.', false);
}
};
const uploadFile = multer({
storage: multerS3({
s3: s3,
bucket: 'MYBUCKET',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, `${Date.now()}-img-${file.originalname}`);
},
}),
fileFilter: imageFilter,
limits: {
fieldSize: '50mb',
},
});
module.exports = uploadFile;
Thank you in advance for your support.
I've been searching online for a few days but of no avail.
Apparently the problem stems form underlying Amazon S3 sdk. The most recent s3 client sdk still has the issue.
Downgrade Amazon sdk to < 3.180.0, untill issue is fixed by AWS.
npm i #aws-sdk/client-s3#3.180.0
Link to Original Issue at Github
Note:
Getting below error in multer s3. This error is pointing NPM module and I really don't understand the issue with this module. I have used upload.single as well as upload.array method to check the working of this module. But not working.
Code:
const bodyParser = require('body-parser');
const cors = require('cors');
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const http = require('http');
const app = express();
app.set('port', 3000);
app.use(
bodyParser.json({
limit: '50mb'
})
);
app.use(
bodyParser.urlencoded({
limit: '50mb',
extended: true
})
);
app.use(cors());
aws.config.update({
accessKeyId: '',
secretAccessKey: ''
});
s3 = new aws.S3();
const upload = multer({
storage: multerS3({
s3: s3,
bucket: '',
acl: 'public-read',
contentType: multerS3.AUTO_CONTENT_TYPE,
metadata: function (req, file, cb) {
cb(null, {
fieldName: file.fieldname
});
},
key: function (req, file, cb) {
let extArray = file.mimetype.split('/');
let ext = extArray[extArray.length - 1];
console.log(`ext ->> `, ext, ` file.fieldname ->> `, file.fieldname);
cb(null, "test/" + Date.now().toString() + '.' + ext);
},
/* limits: {
fileSize: 1024 * 1024 * 10
} */
})
});
(() => {
server = http.createServer(app).listen(app.get('port'), () => {
console.debug(`Server started ->> `);
app.get('/test', (req, res) => {Mul
res.send('Hello');
});
app.post('/media', upload.single('media'), (req, res) => {
console.log(`req.files ->> `, req.file);
res.send('Thanks for you waiting time');
});
});
})();
Error:
ext ->> jpeg file.fieldname ->> media
TypeError: this.client.send is not a function
at Upload.__uploadUsingPut (/home/tristate/Jay/Force/node_modules/#aws-sdk/lib-storage/dist-cjs/Upload.js:52:25)
at Upload.__doConcurrentUpload (/home/tristate/Jay/Force/node_modules/#aws-sdk/lib-storage/dist-cjs/Upload.js:99:39)
at process._tickCallback (internal/process/next_tick.js:68:7)
Looking solution with multer-s3. Thank you in advance.
Have you checked this issues ?
https://github.com/anacronw/multer-s3/issues/169
please check your aws-sdk, multer-s3 version is compatible
Just hit the enter:
npm i multer-s3#2.10.0
3.x.x releases of multer-s3 use AWS JavaScript SDK v3. Specifically, it uses the Upload class from #aws-sdk/lib-storage which in turn calls the modular S3Client.
check their github readme
require('dotenv').config();
const { S3Client } = require('#aws-sdk/client-s3');
const multer = require('multer');
const multerS3 = require('multer-s3');
const shortId = require('shortid');
let s3 = new S3Client({
region: 'ca-central-1',
credentials: {
accessKeyId: process.env.ACCESS_KEY_AWS,
secretAccessKey: process.env.ACCESS_SECRET_AWS,
},
sslEnabled: false,
s3ForcePathStyle: true,
signatureVersion: 'v4',
});
exports.upload = multer({
storage: multerS3({
s3: s3,
bucket: process.env.BUCKET_AWS,
contentType: multerS3.AUTO_CONTENT_TYPE,
acl: 'public-read',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, shortId.generate() + '-' + file.originalname);
},
}),
});
I had this same issue. Downgrading Multer.s3 to 2x version solve the problem!
Multer S3 2.x is compatible with AWS SDK 2.x and
Multer S3 3.x is compatible with AWS SDK 3.x.
For me, removing this line worked
acl: 'public-read',
require("dotenv").config();
const AWS = require("aws-sdk");
const multer = require("multer");
const multerS3 = require("multer-s3");
const uuid = require("uuid").v4;
const path = require("path");
const s3 = new AWS.S3({
accessKeyId: process.env.AWS_S3_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_S3_SECRET_ACCESS_KEY
});
const upload = multer({
storage: multerS3({
s3: s3,
Bucket: process.env.AWS_S3_BUCKET_NAME,
ACL: "public-read",
metadata: (req, file, cd) => {
cd(null, { fieldName: file.fieldname });
},
key: (req, file, cb) => {
const ext = path.extname(file.originalname);
const uniqueName = `${uuid()}${ext}`;
cb(null, uniqueName);
},
}),
});
module.exports = {
upload
}
router.post("/photo-upload", upload.array('photos'), (req, res) => {
return res.status(200).send({
success: true,
result: 'Images Uploaded',
});
});
After adding this code my code is crashing and getting below errors
**node_modules/multer-s3/index.js:94
case 'undefined': throw new Error('bucket is required')
Error: bucket is required
at new S3Storage**
is there any way to upload multiple file at a time not using loop.
Body: buffer, can I send it as a [buffer, buffer]?
I would suggest you maybe consider using multer and multerS3 libraries, this would look as follows.
fileUpload.js
const aws = require("aws-sdk")
const multer = require("multer")
const multerS3 = require("multer-s3")
const uuid = require("uuid").v4
const path = require("path")
const s3 = new aws.S3({
accessKeyId: <secret-id>,
secretAccessKey: <secret-key>,
region: <server-region>,
apiVersion: "2012-10-17"
})
const upload = multer({
storage: multerS3({
s3:s3,
bucket: <bucket-name>,
acl: "public-read",
metadata: (req, file, cd) => {
cd(null, {fieldName: file.fieldname})
},
key: async (req, file, cb) => {
const ext = path.extname(file.originalname)
const uniqueName = `${uuid()}${ext}`
cb(null, uniqueName)
},
})
})
You then import the file into your routes and add upload.array to the route you want to upload images on
imageRoutes.js
const express = require("express");
const router = express.Router();
const upload = require("./fileUpload")
router.post("/", upload.array("image"), (req, res) => {
res.send("uploaded")
}
module.exports = router;
Got to do a crc32 checksum for a uploaded file before saving to s3 disk.
I am using multer and multer-s3 to upload a file to s3 which was easy to achive.
Here In middle trying to do a crc32 check during file upload. Which was partially completed The below code does the job of crc32 check perfectly but the uploaded file is not having all the chunks.
var AWS = require("aws-sdk");
const multer = require("multer");
const multerS3 = require("multer-s3");
const { crc32 } = require('crc');
AWS.config.update({
secretAccessKey: AWS_XXXXXX_SecretAccessKey,
accessKeyId: AWS_XXXXXX_AccessKeyID,
//region: 'us-east-1'
});
const s3 = new AWS.S3();
var upload = multer({
limits: { fileSize: 100000 },
storage: multerS3({
s3: s3,
bucket: constants.AWS_Cred.bucketName,
/* metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
//tried moving stream logic here still no success
}, */
key: function (req, file, cb) {
console.log("KEY");
var buffer = new Buffer.alloc(0);
// const b = file.stream.pipe( new PassThrough()); -- tried to clone a stream here still failed
file.stream.on("data", function(data) {
buffer = Buffer.concat([buffer, data]);
});
file.stream.on('end', function() {
let fileCRCCheckSum = crc32(buffer).toString(16);
console.log(fileCRCCheckSum);
});
cb(null, constants.Firmware_Saved_Location + file.originalname);
},
}),
});
Router
router.post(
"/uploadSong",
upload.single("files"),
uploadController.DoesSomeDBEntriesHere
);
Will Take care of Controller which just saves other Entries to Db.
Happy to Hear Any Suggestions.
I am trying to upload an image to amazon s3 using multer-s3, but I am getting this error:
TypeError: Expected opts.s3 to be object
node_modules/multer-s3/index.js:69:20
This is my server code:
var upload = multer({
storage: s3({
dirname: '/',
bucket: 'bucket',
secretAccessKey: 'key',
accessKeyId: 'key',
region: 'us-west-2',
filename: function (req, file, cb) {
cb(null, file.originalname);
}
})
});
app.post('/upload', upload.array('file'), function (req, res, next) {
res.send("Uploaded!");
});
Why I am getting this error?
[Update Mar 2022] It works perfectly fine till-date and now also shows the uploaded file public URL as well.
Complete and working Node Cheat | Upload to s3 using multer-s3 available.
Code:
var express = require('express'),
aws = require('aws-sdk'),
bodyParser = require('body-parser'),
multer = require('multer'),
multerS3 = require('multer-s3');
aws.config.update({
secretAccessKey: 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX',
accessKeyId: 'XXXXXXXXXXXXXXX',
region: 'us-east-1'
});
var app = express(),
s3 = new aws.S3();
app.use(bodyParser.json());
var upload = multer({
storage: multerS3({
s3: s3,
acl: 'public-read',
bucket: 'bucket-name',
key: function (req, file, cb) {
console.log(file);
cb(null, file.originalname); //use Date.now() for unique file keys
}
})
});
//open in browser to see upload form
app.get('/', function (req, res) {
res.sendFile(__dirname + '/index.html');//index.html is inside node-cheat
});
//use by upload form
app.post('/upload', upload.array('upl', 25), function (req, res, next) {
res.send({
message: "Uploaded!",
urls: req.files.map(function(file) {
return {url: file.location, name: file.key, type: file.mimetype, size: file.size};
})
});
});
app.listen(3000, function () {
console.log('Example app listening on port 3000!');
});
For complete repo:
Clone node-cheat express_multer_s3, run node app followed by npm install express body-parser aws-sdk multer multer-s3.
Happy Helping!
#V31 has answered very well still I want to add my 2 cents.
I believe in keeping one responsibility into one file, for better code organization and debugging purpose.
I have created a file for uploading upload.js.
require('dotenv').config();
const AWS = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const s3Config = new AWS.S3({
accessKeyId: process.env.AWS_IAM_USER_KEY,
secretAccessKey: process.env.AWS_IAM_USER_SECRET,
Bucket: process.env.AWS_BUCKET_NAME
});
const fileFilter = (req, file, cb) => {
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png') {
cb(null, true)
} else {
cb(null, false)
}
}
// this is just to test locally if multer is working fine.
const storage = multer.diskStorage({
destination: (req, res, cb) => {
cb(null, 'src/api/media/profiles')
},
filename: (req, file, cb) => {
cb(null, new Date().toISOString() + '-' + file.originalname)
}
})
const multerS3Config = multerS3({
s3: s3Config,
bucket: process.env.AWS_BUCKET_NAME,
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
console.log(file)
cb(null, new Date().toISOString() + '-' + file.originalname)
}
});
const upload = multer({
storage: multerS3Config,
fileFilter: fileFilter,
limits: {
fileSize: 1024 * 1024 * 5 // we are allowing only 5 MB files
}
})
exports.profileImage = upload;
Which is imported inside my routes routes.js
const express = require('express');
const ProfileController = require('../profile/controller');
const { profileImage } = require('../utils/upload.js');
const routes = (app) => {
const apiRoutes = express.Router();
apiRoutes.use('/profile', profileRoutes);
profileRoutes.post('/',profileImage.single('profileImage'), ProfileController.saveProfile);
app.use('/api', apiRoutes);
}
module.exports = routes
Postman screen shot for post body
I just want to add my cents,
There are many comments in all answers like how to get public URL after uploading and S3 response object and lets see implementation and cases,
// INITIALIZE NPMS
var AWS = require('aws-sdk'),
multer = require('multer'),
multerS3 = require('multer-s3'),
path = require('path');
// CONFIGURATION OF S3
AWS.config.update({
secretAccessKey: '***********************************',
accessKeyId: '****************',
region: 'us-east-1'
});
// CREATE OBJECT FOR S3
const S3 = new AWS.S3();
// CREATE MULTER FUNCTION FOR UPLOAD
var upload = multer({
// CREATE MULTER-S3 FUNCTION FOR STORAGE
storage: multerS3({
s3: S3,
acl: 'public-read',
// bucket - WE CAN PASS SUB FOLDER NAME ALSO LIKE 'bucket-name/sub-folder1'
bucket: 'bucket-name',
// META DATA FOR PUTTING FIELD NAME
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
// SET / MODIFY ORIGINAL FILE NAME
key: function (req, file, cb) {
cb(null, file.originalname); //set unique file name if you wise using Date.toISOString()
// EXAMPLE 1
// cb(null, Date.now() + '-' + file.originalname);
// EXAMPLE 2
// cb(null, new Date().toISOString() + '-' + file.originalname);
}
}),
// SET DEFAULT FILE SIZE UPLOAD LIMIT
limits: { fileSize: 1024 * 1024 * 50 }, // 50MB
// FILTER OPTIONS LIKE VALIDATING FILE EXTENSION
fileFilter: function(req, file, cb) {
const filetypes = /jpeg|jpg|png/;
const extname = filetypes.test(path.extname(file.originalname).toLowerCase());
const mimetype = filetypes.test(file.mimetype);
if (mimetype && extname) {
return cb(null, true);
} else {
cb("Error: Allow images only of extensions jpeg|jpg|png !");
}
}
});
There are three cases, if we want to retrieve files res object from S3 after upload:
Case 1: When we are using .single(fieldname) method it will return file object in req.file
app.post('/upload', upload.single('file'), function (req, res, next) {
console.log('Uploaded!');
res.send(req.file);
});
Case 2: When we are using .array(fieldname[, maxCount]) method it will return file object in req.files
app.post('/upload', upload.array('file', 1), function (req, res, next) {
console.log('Uploaded!');
res.send(req.files);
});
Case 3: When we are using .fields(fields) method it will return file object in req.files
app.post('/upload', upload.fields([
{ name: 'avatar', maxCount: 1 },
{ name: 'gallery', maxCount: 8 }
]), function (req, res, next) {
console.log('Uploaded!');
res.send(req.files);
});
s3 needs to be an object to be passed. According to the docs, the object needs to be like this:
var upload = multer({
storage: multerS3({
s3: s3,
bucket: 'some-bucket',
metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
},
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
})
MulterS3 Docs
/*** Using Multer To Upload Image image is uploading */
const fileStorage = multer.diskStorage({
destination: function(req, file, cb) {
cb(null, "./public/uploads");
},
filename: function(req, file, cb) {
cb(null, file.originalname);
}
});
/** AWS catalog */
aws.config.update({
secretAccessKey: process.env.SECRET_KEY,
accessKeyId: process.env.ACCESS_KEY,
region: "us-east-1"
});
const s3 = new aws.S3();
const awsStorage = multerS3({
s3: s3,
bucket: process.env.BUCKET_NAME,
key: function(req, file, cb) {
console.log(file);
cb(null, file.originalname);
}
});
const upload = multer({
storage: awsStorage(),
/** in above line if you are using local storage in ./public/uploads folder than use
******* storage: fileStorage,
* if you are using aws s3 bucket storage than use
******* storage: awsStorage(),
*/
limits: { fileSize: 5000000 },
fileFilter: function(req, file, cb) {
checkFileType(file, cb);
}
});
app.post("/user-profile-image", upload.single("profile"), (req, res, err) => {
try {
res.send(req.file);
} catch (err) {
res.send(400);
}
});
const checkFileType = (file, cb) => {
const filetypes = /jpeg|jpg|png|gif/;
const extname = filetypes.test(path.extname(file.originalname).toLowerCase());
const mimetype = filetypes.test(file.mimetype);
if (mimetype && extname) {
return cb(null, true);
} else {
cb("Error: Images Only!");
}
};
//here is the function for upload the images on aws bucket using multer
const path = require('path');
const fs = require('fs');
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
aws.config.update({
secretAccessKey: '**************************',
accessKeyId: '********************',
region: '**********************'
});
s3 = new aws.S3();
const storage = multerS3({
s3: s3,
bucket: 'bucket-name',
key: function(req, file, cb) {
console.log(file);
cb(null, file.originalname);
}
})
//export the created function
exports.uploadVideo = multer({ storage: storage }).single('file_name');
//================================================================================
//import uploadVideo function whenever you need to upload the file on aws s3 bucket
const { uploadVideo } = require('../../services/upload');
exports.videoUpload = (req, res) => {
uploadVideo(req, res, function(err) {
if (err) {
console.log(err);
res.json({ status: 401, msg: err.message });
} else {
const image = getImagePath(req.file.filename);
res.json({ status: 200, msg: 'Image uploaded sucess.', data: image });
}
});
}
//================================================================================
//here is route file
router.post('/video-upload',uploadController.videoUpload);
I was passing S3 to mutler in caps, like
S3: {object}
Changing it to small s3 works for me:-
s3: {object}
WE can Upload IMAGE/ CSV/ EXCEL files to AWS s3 using multer-s3.
Im using .single(fieldname) method for uploading single file.
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const s3 = new aws.S3({
accessKeyId: process.env.AWS_ACCESS_KEY,
secretAccessKey: process.env.AWS_SECRET_KEY,
region: process.env.REGION,
});
const upload = multer({
storage: multerS3({
s3: s3,
bucket: process.env.AWS_S3_BUCKET,
metadata: function (req, file, cb) {
cb(null, {fieldName: 'Meta_Data'});
},
key: function (req, file, cb) {
cb(null, file.originalname);
},
limits: {
fileSize: 1024 * 1024 * 5 // allowed only 5 MB files
}
})
}).single('file');
exports.uploadfile = async(req,res,next)=>{
try{
upload(req,res, function(err){
if(err){
console.log(err);
}
console.log(req.file.location);
})
})
}catch (err){
res.status(400).json({
status : 'fail',
message : err.message
});
}
}
In Routes file
router.route('/')
.post(imageController.uploadfile);