How to upload multiple files with multer-s3 in Nodejs - node.js

I am trying to upload multiple images with Nodejs, Expressjs and Multer-s3, but it's not working.
I have a model called Program and the Program model has an array image attribute but when I try to upload multiple images my req.file returns undefined.
Here is my model
const programSchema = new mongoose.Schema({
programtype: {
type: String,
required: true,
},
title: {
type: String,
required: true,
},
description: {
type: String,
required: true,
},
createdAt: {
type: Date,
required: true,
default: Date.now,
},
programImage: {
type: Array,
require: true,
},
});
and my routes
const Program = require("../models/program");
const fs = require("fs");
const multer = require("multer");
const path = require("path");
var AWS = require("aws-sdk");
var multerS3 = require("multer-s3");
AWS.config.update({
secretAccessKey: process.env.S3_SECRECT,
accessKeyId: process.env.AWS_ACCESS_KEY,
region: process.env.S3_REGION,
});
const uploadPath = path.join("public", Program.programImageBasePath);
const imageMineTypes = ["image/jpeg", "image/png", "image/gif"];
const bucketname = "mybucketname";
s3 = new AWS.S3();
const upload = multer({
storage: multerS3({
s3: s3,
acl: "public-read",
bucket: bucketname,
s3BucketEndpoint: true,
endpoint: "http://" + bucketname + ".s3.amazonaws.com",
key: function (req, file, cb) {
const uploadPathWithOriginalName = uploadPath + "/" + file.originalname;
cb(null, uploadPathWithOriginalName);
},
}),
});
router.post("/create", upload.array("cover", 10), async (req, res, next) => {
console.log(req.file);
const program = new Program({
programtype: req.body.programtype,
title: req.body.title,
description: req.body.description,
programImage: req.file.location,
});
try {
const programs = await program.save();
res.redirect("/programs");
} catch {
if (program.programImage != null) {
removeprogramImage(program.programImage);
}
res.render("programs/new");
}
});
and my views
<h2 style="padding-top: 90px;" > New Programs</h2>
<form action="/programs/create" method="POST" enctype="multipart/form-data">
<div>
<label>Image</label>
<input type="file" name="cover" multiple />
</div>
Cacel
<button type="submit">Create</button>
</form>

You can refer to this example.
const s3 = new AWS.S3({
accessKeyId: 'xxxxxxxxx',
secretAccessKey: 'xxxxxxxxx'
});
const uploadS3 = multer({
storage: multerS3({
s3: s3,
acl: 'public-read',
bucket: 'xxxxxxxx',
metadata: (req, file, callBack) => {
callBack(null, { fieldName: file.fieldname })
},
key: (req, file, callBack) => {
var fullPath = 'products/' + file.originalname;//If you want to save into a folder concat de name of the folder to the path
callBack(null, fullPath)
}
}),
limits: { fileSize: 2000000 }, // In bytes: 2000000 bytes = 2 MB
fileFilter: function (req, file, cb) {
checkFileType(file, cb);
}
}).array('photos', 10);
exports.uploadProductsImages = async (req, res) => {
uploadS3(req, res, (error) => {
console.log('files', req.files);
if (error) {
console.log('errors', error);
res.status(500).json({
status: 'fail',
error: error
});
} else {
// If File not found
if (req.files === undefined) {
console.log('uploadProductsImages Error: No File Selected!');
res.status(500).json({
status: 'fail',
message: 'Error: No File Selected'
});
} else {
// If Success
let fileArray = req.files,
fileLocation;
const images = [];
for (let i = 0; i < fileArray.length; i++) {
fileLocation = fileArray[i].location;
console.log('filenm', fileLocation);
images.push(fileLocation)
}
// Save the file name into database
return res.status(200).json({
status: 'ok',
filesArray: fileArray,
locationArray: images
});
}
}
})
};

Related

After uploading an image to S3 bucket, it looks strange most of the time, but the same images looks OK sometimes

After uploading an image to S3 bucket, it looks strange most of the time, but the same images looks OK sometimes, so I'm not sure what's going on. It started happening from last week and before that it was working fine.
Here is a sample image that I uploaded, and after uploading, it appears as follows.
https://hub-uploads-stage.s3.us-east-2.amazonaws.com/gallery-documents/file-1675056885128-alphabet-gcf3617eaf_1920.jpg
Original Image
https://i.imgur.com/ChbHNn9.png
I'm using packages: multer, multer-s3 with node js 16.19.0
I'm using following dependencies
multer: ^1.4.2
multer-s3: ^2.10.0
aws-sdk: ^2.1002.0
file-type: ^16.5.3
Here is the code I used to upload files.
import AWS from "aws-sdk";
import multer from "multer";
import multerS3 from "multer-s3";
import { fromBuffer } from 'file-type';
import stream from 'stream';
AWS.config.update({
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
region: process.env.AWS_REGION,
signatureVersion: "s3v4",
});
const s3 = new AWS.S3();
export const basePath = {
profile: "profile-images",
post: "post-documents",
slider: "slider-images",
gallery: "gallery-documents",
icons: "icons",
posters: "product-posters",
chat: "chat"
}
const buckets = {
uploads: process.env.S3_BUCKET_UPLOADS || 'hub-uploads-stage'
}
const mimeTypes = {
image: [
"image/gif",
"image/jpg",
"image/jpeg",
"image/png",
"image/tiff",
"image/pipeg",
"image/svg+xml",
"image/bmp",
"image/x-xbitmap",
"image/x-icon"
]
}
const storage = (Bucket: any, uploadBasePath: any) => multerS3({
s3,
bucket: Bucket,
acl: 'public-read',
contentType: function (req: any, file: any, callback: any) {
file.stream.once('data', async function (firstChunk: any) {
var type = await fromBuffer(firstChunk)
var mime = (type === null ? 'application/octet-stream' : type?.mime)
var outStream = new stream.PassThrough()
outStream.write(firstChunk)
file.stream.pipe(outStream)
callback(null, mime, outStream)
})
},
metadata: function (req: any, file: any, callback: any) {
callback(null, { fieldName: file.originalname })
},
key: function (req: any, file: any, callback: any) {
let fileName = `${file.fieldname}-${Date.now()}-${file.originalname}`;
let filePath = uploadBasePath + '/' + fileName;
callback(null, filePath)
}
})
export const uploader = {
profile: multer({
storage: storage(buckets.uploads, basePath.profile),
fileFilter: function (req, file, callback) {
if (mimeTypes.image.includes(file.mimetype)) {
callback(null, true)
} else {
callback(null, false)
}
}
}),
post: multer({
storage: storage(buckets.uploads, basePath.post)
}),
slide: multer({
storage: storage(buckets.uploads, basePath.slider),
fileFilter: function (req, file, callback) {
if (mimeTypes.image.includes(file.mimetype)) {
callback(null, true)
} else {
callback(null, false)
}
}
}),
gallery: multer({
storage: storage(buckets.uploads, basePath.gallery),
fileFilter: function (req, file, callback) {
console.log("File in filter =>", file)
callback(null, true)
}
}),
icon: multer({
storage: storage(buckets.uploads, basePath.icons)
}),
productPoster: multer({
storage: storage(buckets.uploads, basePath.posters),
fileFilter: function (req, file, callback) {
if (mimeTypes.image.includes(file.mimetype)) {
callback(null, true)
} else {
callback(null, false)
}
}
}),
chat: multer({
storage: storage(buckets.uploads, basePath.chat)
})
};
Is there something i need to upgrade or change?
Thanks

Upload a profile pic with Reactjs and Nodejs to MongoDb

I am trying to let the user update their profile with their own avatar but for some reason, it doesn't work. I am sending a FormData from the front-end and catch it with nodeJs before storing it to MongoDB. user has 3 options to update: name, about, avatar. Whenever I try to update the name and about it works just fine but when I am uploading an avatar I get a 500 error. Can you guys take a look? FYI I am a begginer.
Here is my code:
FRONT-END
const handleFileInputChange = (e) => {
const file = e.target.files[0];
previewFile(file);
setUser({
...user,
avatar: file,
});
};
const onSubmit = (e) => {
e.preventDefault();
const formData = new FormData();
formData.append('name', user.name);
formData.append('about', user.about);
formData.append('avatar', user.avatar);
editUserProfile(formData); // external axios.patch f.
};
const editUserProfile = async (formData) => {
try {
await axios.patch('api/v1/users/me', formData, {
headers: {
Authorization: `Bearer ${localStorage.getItem('token')}`,
},
});
dispatch({
type: EDIT_USER,
});
} catch (err) {
console.log(err);
}
};
<form onSubmit={onSubmit} encType="multipart/form-data">
<input
// accept="image/*"
accept=".png, .jpg, .jpeg"
type="file"
name="avatar"
// value={fileInputState}
onChange={handleFileInputChange}
style={{ display: 'none' }}
id="icon-button-file"
/>
...
BACK-END
router.patch('api/v1/users/me', protect, upload.single('avatar'), getMe, editUser = async (req, res) => {
try {
const { name, about } = req.body;
const profileFileds = {};
if (name) profileFileds.name = name;
if (about) profileFileds.about = about;
if (req.file) profileFileds.avatar = req.file.filename;
const user = await User.findByIdAndUpdate(req.params.id, profileFileds, {
new: true,
runValidators: true,
});
if (!user) {
return next(
new custom error...
);
}
res.status(200).json({
status: 'success',
data: {
user,
},
});
} catch (err) {
console.log(err) error 400 ...
}
}););
Sorry guys if it's a long code and I really appreciate it, I am struggling 2 days now and can't figure it out
//MULTER CONFIG just in case but it shouldn't be a problem with it
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, '/images');
},
filename: function (req, file, cb) {
cb(null, uuidv4() + '-' + Date.now() + path.extname(file.originalname));
},
});
const fileFilter = (req, file, cb) => {
const allowedFileTypes = ['image/jpeg', 'image/jpg', 'image/png'];
if (allowedFileTypes.includes(file.mimetype)) {
cb(null, true);
} else {
cb(null, false);
}
};
const upload = multer({
storage: storage,
limits: {
fileSize: 1024 * 1024 * 2,
},
fileFilter: fileFilter,
});

How to rename my originalname when using multer memoryStorage?

i am trying to rename my file originalname when using multer memoryStorage. I am using multer to upload an array of files and when i console.log(req.files) i get:
{
fieldname: 'images',
originalname: 'snake.jpg',
encoding: '7bit',
mimetype: 'image/jpeg',
buffer: <Buffer ff d8 38134 more bytes>,
size: 38184
}
The reason i want to rename my originalname is i am storing the images in AWS S3 and if the images have the same name it gets updated not added as a new image.
I have tried doing so by adding the date next to the originalname when storing in database but then originalname does not change when the images get added in bucket.
Here is my code:
posts.js
const storage=multer.memoryStorage()
const upload = multer({
storage: storage,
limits: { fieldSize: 25 * 1024 * 1024 },
});
router.post(
"/",
[ upload.array("images", config.get("maxImageCount")),
imageResize,
],
async (req, res) => {
const paths = await req.files.map((file) => ({ originalName: file.originalname + "-" + new
Date().toISOString() + "-" + uuidv4()}));
await Post.create({
title: req.body.title,
userId: req.body.userId,
Post_Images: paths.map((x) => ({ images: x.originalName })),
},
{
include: [Post_Image] }).then(
res.status(201).send())
imageResize.js
const sharp = require("sharp");
require("dotenv").config();
const AWS = require('aws-sdk')
const s3 = new AWS.S3({
accessKeyId: process.env.AWS_ID,
secretAccessKey: process.env.AWS_SECRET,
region:process.env.AWS_REGION
})
module.exports = async (req, res, next) => {
const images = [];
const resizePromises = req.files.map(async (file) => {
console.log(file)
await sharp(file.buffer)
.resize(2000)
.jpeg({ quality: 50 })
.toBuffer()
.then(resized=>s3.upload({
Bucket:process.env.AWS_BUCKET,
Key:file.originalname + "_full.jpg",
Body:file.buffer,
ACL: 'public-read'
}).promise()),
await sharp(file.buffer)
.resize(100)
.jpeg({ quality: 30 })
.toBuffer()
.then(resized=>s3.upload({
Bucket:process.env.AWS_BUCKET,
Key:file.originalname + "_thumb.jpg",
Body:file.buffer,
ACL: 'public-read'
}).promise())
images.push(file.originalname);
});
await Promise.all([...resizePromises]);
req.images = images;
next();
};
in other words, i am trying to change my originalname in my req.files to
originalname + "-" + new Date().toISOString() + "-" + uuidv4()
OR
How do i keep the uuid and the date the same in my posts.js and imageResize.js?

How to send multiple files to AWS S3 using Multer-S3 from different fields

I'm currently using Postman in which I require to upload two files from two different fields to AWS-S3; this is how it looks like:
This is the API route that I'm calling:
router.route('/').post(uploadThumbnail, uploadVideo, createVideo);
That route calls three functions(which is supposed to return the data requested from Postman):
exports.createVideo = asyncHandler(async (req, res, next) => {
// Add user to req,body
req.body.user = req.user.id;
// Bring files
if (req.file) {
console.log(req.file);
}
});
Here are the two other functions(with the aws upload function); one for thumbnail and a second one for video_url:
const upload = multer({
storage: multerS3({
s3: s3,
bucket: process.env.AWS_BUCKET_NAME,
acl: 'public-read',
key: function(req, file, cb) {
const strOne = process.env.WEBSITE_NAME + '-';
const userId = req.user.id + '-';
const userEmail = req.user.email + '-';
const todaysDate = Date.now().toString() + '.';
const extension = file.mimetype.split('/')[1];
const finalStr = strOne.concat(userId, userEmail, todaysDate, extension);
cb(null, finalStr);
}
})
});
exports.uploadThumbnail = upload.single('thumbnail');
exports.uploadVideo = upload.single('video_url');
Everytime I run the post, Postman throws me this error:
{
"status": "error",
"error": {
"name": "MulterError",
"message": "Unexpected field",
"code": "LIMIT_UNEXPECTED_FILE",
"field": "video_url",
"storageErrors": [],
"statusCode": 500,
"status": "error"
},
"message": "Unexpected field",
"stack": "MulterError: Unexpected field\n at wrappedFileFilter (C:\\xampp\\htdocs\\myporn\\node_modules\\multer\\index.js:40:19)\n at Busboy.<anonymous> (C:\\xampp\\htdocs\\myporn\\node_modules\\multer\\lib\\make-middleware.js:114:7)\n at Busboy.emit (events.js:198:13)\n at Busboy.EventEmitter.emit (domain.js:448:20)\n at Busboy.emit (C:\\xampp\\htdocs\\myporn\\node_modules\\busboy\\lib\\main.js:38:33)\n at PartStream.<anonymous> (C:\\xampp\\htdocs\\myporn\\node_modules\\busboy\\lib\\types\\multipart.js:213:13)\n at PartStream.emit (events.js:198:13)\n at PartStream.EventEmitter.emit (domain.js:448:20)\n at HeaderParser.<anonymous> (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\Dicer.js:51:16)\n at HeaderParser.emit (events.js:198:13)\n at HeaderParser.EventEmitter.emit (domain.js:448:20)\n at HeaderParser._finish (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\HeaderParser.js:68:8)\n at SBMH.<anonymous> (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\HeaderParser.js:40:12)\n at SBMH.emit (events.js:198:13)\n at SBMH.EventEmitter.emit (domain.js:448:20)\n at SBMH._sbmh_feed (C:\\xampp\\htdocs\\myporn\\node_modules\\streamsearch\\lib\\sbmh.js:159:14)\n at SBMH.push (C:\\xampp\\htdocs\\myporn\\node_modules\\streamsearch\\lib\\sbmh.js:56:14)\n at HeaderParser.push (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\HeaderParser.js:46:19)\n at Dicer._oninfo (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\Dicer.js:197:25)\n at SBMH.<anonymous> (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\Dicer.js:127:10)\n at SBMH.emit (events.js:198:13)\n at SBMH.EventEmitter.emit (domain.js:448:20)\n at SBMH._sbmh_feed (C:\\xampp\\htdocs\\myporn\\node_modules\\streamsearch\\lib\\sbmh.js:188:10)\n at SBMH.push (C:\\xampp\\htdocs\\myporn\\node_modules\\streamsearch\\lib\\sbmh.js:56:14)\n at Dicer._write (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\Dicer.js:109:17)\n at doWrite (_stream_writable.js:415:12)\n at writeOrBuffer (_stream_writable.js:399:5)\n at Dicer.Writable.write (_stream_writable.js:299:11)"
}
The function works great but only when sending one single file, it can be either thumbnail or video_url but not both...I need both fields to work.
Any idea on how to fix this?
const s3 = new AWS.S3({
accessKeyId: 'xxxxxxxxx',
secretAccessKey: 'xxxxxxxxx'
});
const uploadS3 = multer({
storage: multerS3({
s3: s3,
acl: 'public-read',
bucket: 'xxxxxxxx',
metadata: (req, file, callBack) => {
callBack(null, { fieldName: file.fieldname })
},
key: (req, file, callBack) => {
var fullPath = 'products/' + file.originalname;//If you want to save into a folder concat de name of the folder to the path
callBack(null, fullPath)
}
}),
limits: { fileSize: 2000000 }, // In bytes: 2000000 bytes = 2 MB
fileFilter: function (req, file, cb) {
checkFileType(file, cb);
}
}).array('photos', 10);
exports.uploadProductsImages = async (req, res) => {
uploadS3(req, res, (error) => {
console.log('files', req.files);
if (error) {
console.log('errors', error);
res.status(500).json({
status: 'fail',
error: error
});
} else {
// If File not found
if (req.files === undefined) {
console.log('uploadProductsImages Error: No File Selected!');
res.status(500).json({
status: 'fail',
message: 'Error: No File Selected'
});
} else {
// If Success
let fileArray = req.files,
fileLocation;
const images = [];
for (let i = 0; i < fileArray.length; i++) {
fileLocation = fileArray[i].location;
console.log('filenm', fileLocation);
images.push(fileLocation)
}
// Save the file name into database
return res.status(200).json({
status: 'ok',
filesArray: fileArray,
locationArray: images
});
}
}
})
};
Make use of file.fieldname and .fields to achieve what you need
function uploadImageToS3(){
const multer = require("multer");
const multerS3 = require("multer-s3");
const uuid = require("uuid").v4;
const path = require("path");
const cloudStorage = multerS3({
s3: s3Client,
bucket: process.env.AWS_PRODUCTS_BUCKET,
acl: "public-read",
metadata: (req, file, cb) => {
if (file.fieldname == "thumbnail") {
const extension = ".jpeg";
cb(null, `${uuid()}${extension}`);
return;
}
if (file.fieldname == "video_url") {
const extension = ".jpeg";
cb(null, `${uuid()}${extension}`);
return;
}
cb(
"fieldName other than avatar and productImages are not supported",
null
);
},
// key: name of the file
key: (req, file, cb) => {
const extension = path.extname(file.originalname);
cb(null, `${uuid()}${extension}`);
},
});
const limitFileSize = { fileSize: 1024 * 1024 * 5 }, // 1Byte -->1024Bytes or 1MB --> 5MB
filterFileType = (req, file, cb) => {
const isAllowedFileType =
file.mimetype == "image/jpeg" ||
file.mimetype == "image/jpg" ||
file.mimetype == "image/png";
if (isAllowedFileType) {
cb(null, true);
return;
}
// To reject this file pass `false`
cb(null, false);
};
const upload = multer({
storage: cloudStorage,
limits: limitFileSize,
fileFilter: filterFileType,
});
return upload;
}
In route, use it like so
router.post(
"/upload",
uploadImageToS3().fields([
{ name: "thumbnail", maxCount: 1 },
{ name: "video_url", maxCount: 1 }
]),
handleImagesUploadController
);

multers3 skipping shouldTransform

I am trying to crop images before it get send to s3 bucket. My issue is in my multerS3 options shouldTransform is being skipped hence not taking the transforms.
Here is my entire file of upload
require('multer-s3-transform');
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const sharp = require('sharp');
aws.config.update({
secretAccessKey: secretAccessKey,
accessKeyId: accessKeyId,
region: region,
});
const s3 = new aws.S3();
const fileFilter = (req, file, cb) => {
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png' || file.mimetype === 'image/jpg') {
cb(null, true);
} else {
cb(new Error(message.FAIL.invalidImage), false);
}
};
const upload = multer({
fileFilter,
storage: multerS3({
s3,
bucket: bucket,
acl: 'public-read',
shouldTransform: function (req, file, cb) {
console.log('in should transform ');
cb(null, true);
},
transforms: [
{
id: 'original',
key: function (req, file, cb) {
cb(null, Date.now().toString());
},
transform: function (req, file, cb) {
console.log('og');
cb(null, sharp().jpg())
},
},
{
id: 'resized',
key: function (req, file, cb) {
cb(null, Date.now().toString());
},
transform: function (req, file, cb) {
console.log('thumbnail');
cb(null, sharp().resize(300, 300).jpg())
},
}
],
metadata: function (req, file, cb) {
cb(null, {fieldName: 'some meta'});
},
key: function (req, file, cb) {
cb(null, Date.now().toString());
},
})
});
module.exports = upload;
Here is my route
const photoUpload = upload.fields([{name: 'photo', maxCount: 1}]);
// Route for uploading photo image
app.post(routeRoot + '/upload/photo', function (req, res) {
console.log('in route ');
photoUpload(req, res, function (err) {
if (err) {
return res.status(200).send({error: {message: err.message}});
} else {
account.uploadPhoto(req, res);
}
})
});
The result I get
files [Object: null prototype] {
photo: [
{
fieldname: 'photo',
originalname: '4.jpg',
encoding: '7bit',
mimetype: 'image/jpeg',
size: 84154,
bucket: '...',
key: '...',
acl: 'public-read',
contentType: 'image/jpeg',
contentDisposition: null,
storageClass: 'STANDARD',
serverSideEncryption: null,
metadata: [Object],
location: '...',
etag: '...',
versionId: undefined
}
]
}
File upload to S3 works but it is not being transformed. I have been trying to figure this one out.
Thank you!
I've figured it out.
const multerS3 = require('multer-s3-transform');
instead of
const multerS3 = require('multer-s3');
for some reason the docs has 'multer-s3' require but not 'multer-s3-transform' which is what is needed for shouldTransform to work

Resources