multers3 skipping shouldTransform - node.js

I am trying to crop images before it get send to s3 bucket. My issue is in my multerS3 options shouldTransform is being skipped hence not taking the transforms.
Here is my entire file of upload
require('multer-s3-transform');
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const sharp = require('sharp');
aws.config.update({
secretAccessKey: secretAccessKey,
accessKeyId: accessKeyId,
region: region,
});
const s3 = new aws.S3();
const fileFilter = (req, file, cb) => {
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png' || file.mimetype === 'image/jpg') {
cb(null, true);
} else {
cb(new Error(message.FAIL.invalidImage), false);
}
};
const upload = multer({
fileFilter,
storage: multerS3({
s3,
bucket: bucket,
acl: 'public-read',
shouldTransform: function (req, file, cb) {
console.log('in should transform ');
cb(null, true);
},
transforms: [
{
id: 'original',
key: function (req, file, cb) {
cb(null, Date.now().toString());
},
transform: function (req, file, cb) {
console.log('og');
cb(null, sharp().jpg())
},
},
{
id: 'resized',
key: function (req, file, cb) {
cb(null, Date.now().toString());
},
transform: function (req, file, cb) {
console.log('thumbnail');
cb(null, sharp().resize(300, 300).jpg())
},
}
],
metadata: function (req, file, cb) {
cb(null, {fieldName: 'some meta'});
},
key: function (req, file, cb) {
cb(null, Date.now().toString());
},
})
});
module.exports = upload;
Here is my route
const photoUpload = upload.fields([{name: 'photo', maxCount: 1}]);
// Route for uploading photo image
app.post(routeRoot + '/upload/photo', function (req, res) {
console.log('in route ');
photoUpload(req, res, function (err) {
if (err) {
return res.status(200).send({error: {message: err.message}});
} else {
account.uploadPhoto(req, res);
}
})
});
The result I get
files [Object: null prototype] {
photo: [
{
fieldname: 'photo',
originalname: '4.jpg',
encoding: '7bit',
mimetype: 'image/jpeg',
size: 84154,
bucket: '...',
key: '...',
acl: 'public-read',
contentType: 'image/jpeg',
contentDisposition: null,
storageClass: 'STANDARD',
serverSideEncryption: null,
metadata: [Object],
location: '...',
etag: '...',
versionId: undefined
}
]
}
File upload to S3 works but it is not being transformed. I have been trying to figure this one out.
Thank you!

I've figured it out.
const multerS3 = require('multer-s3-transform');
instead of
const multerS3 = require('multer-s3');
for some reason the docs has 'multer-s3' require but not 'multer-s3-transform' which is what is needed for shouldTransform to work

Related

After uploading an image to S3 bucket, it looks strange most of the time, but the same images looks OK sometimes

After uploading an image to S3 bucket, it looks strange most of the time, but the same images looks OK sometimes, so I'm not sure what's going on. It started happening from last week and before that it was working fine.
Here is a sample image that I uploaded, and after uploading, it appears as follows.
https://hub-uploads-stage.s3.us-east-2.amazonaws.com/gallery-documents/file-1675056885128-alphabet-gcf3617eaf_1920.jpg
Original Image
https://i.imgur.com/ChbHNn9.png
I'm using packages: multer, multer-s3 with node js 16.19.0
I'm using following dependencies
multer: ^1.4.2
multer-s3: ^2.10.0
aws-sdk: ^2.1002.0
file-type: ^16.5.3
Here is the code I used to upload files.
import AWS from "aws-sdk";
import multer from "multer";
import multerS3 from "multer-s3";
import { fromBuffer } from 'file-type';
import stream from 'stream';
AWS.config.update({
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
region: process.env.AWS_REGION,
signatureVersion: "s3v4",
});
const s3 = new AWS.S3();
export const basePath = {
profile: "profile-images",
post: "post-documents",
slider: "slider-images",
gallery: "gallery-documents",
icons: "icons",
posters: "product-posters",
chat: "chat"
}
const buckets = {
uploads: process.env.S3_BUCKET_UPLOADS || 'hub-uploads-stage'
}
const mimeTypes = {
image: [
"image/gif",
"image/jpg",
"image/jpeg",
"image/png",
"image/tiff",
"image/pipeg",
"image/svg+xml",
"image/bmp",
"image/x-xbitmap",
"image/x-icon"
]
}
const storage = (Bucket: any, uploadBasePath: any) => multerS3({
s3,
bucket: Bucket,
acl: 'public-read',
contentType: function (req: any, file: any, callback: any) {
file.stream.once('data', async function (firstChunk: any) {
var type = await fromBuffer(firstChunk)
var mime = (type === null ? 'application/octet-stream' : type?.mime)
var outStream = new stream.PassThrough()
outStream.write(firstChunk)
file.stream.pipe(outStream)
callback(null, mime, outStream)
})
},
metadata: function (req: any, file: any, callback: any) {
callback(null, { fieldName: file.originalname })
},
key: function (req: any, file: any, callback: any) {
let fileName = `${file.fieldname}-${Date.now()}-${file.originalname}`;
let filePath = uploadBasePath + '/' + fileName;
callback(null, filePath)
}
})
export const uploader = {
profile: multer({
storage: storage(buckets.uploads, basePath.profile),
fileFilter: function (req, file, callback) {
if (mimeTypes.image.includes(file.mimetype)) {
callback(null, true)
} else {
callback(null, false)
}
}
}),
post: multer({
storage: storage(buckets.uploads, basePath.post)
}),
slide: multer({
storage: storage(buckets.uploads, basePath.slider),
fileFilter: function (req, file, callback) {
if (mimeTypes.image.includes(file.mimetype)) {
callback(null, true)
} else {
callback(null, false)
}
}
}),
gallery: multer({
storage: storage(buckets.uploads, basePath.gallery),
fileFilter: function (req, file, callback) {
console.log("File in filter =>", file)
callback(null, true)
}
}),
icon: multer({
storage: storage(buckets.uploads, basePath.icons)
}),
productPoster: multer({
storage: storage(buckets.uploads, basePath.posters),
fileFilter: function (req, file, callback) {
if (mimeTypes.image.includes(file.mimetype)) {
callback(null, true)
} else {
callback(null, false)
}
}
}),
chat: multer({
storage: storage(buckets.uploads, basePath.chat)
})
};
Is there something i need to upgrade or change?
Thanks

How to Resize Image before upload to S3 using multer

This is my code, it work fine, I just wish resize the image before up to s3 amazon cloud.
import multer from 'multer';
import path from 'path';
import multerS3 from 'multer-s3';
import aws from 'aws-sdk';
const storageType = {
s3: multerS3({
s3: new aws.S3(),
bucket: 'restsystemplatesimage',
contentType: multerS3.AUTO_CONTENT_TYPE,
acl: 'public-read',
key: (req, file, cb) => {
console.log(file);
//const restCod = req.params.data.split('|')
cb(null, file.originalname);
}
})
}
export default {
storage: storageType['s3'],
limits: {
filesize: 1 * 1024 * 1024
},
fileFilter: (req, file, cb) => {
const allowedMimes = [
"image/jpeg",
"image/jpg",
"image/png"
];
if (allowedMimes.includes(file.mimetype)) {
cb(null, true);
}else {
cb( new Error("Invalid file type"));
}
}
};
When I print the file on console, it show ...
{
fieldname: 'image',
originalname: 'espetinho-de-carne.jpg',
encoding: '7bit',
mimetype: 'image/jpeg'
}
Shouldn´t the buffer object appear? I can't modify the image if I don't know where it is
please somebody help
I found the solution, instead of using the package multer-s3, I used the package multer-sharp-s3, and make these changes to my code
import multer from 'multer';
import path from 'path';
import multerS3 from 'multer-sharp-s3';
import aws from 'aws-sdk';
const storageType = {
s3: multerS3({
s3: new aws.S3(),
Bucket: 'restsystemplatesimage',
resize: {
width: 320,
height: 192
},
ACL: 'public-read',
key: (req, file, cb) => {
console.log(file);
//const restCod = req.params.data.split('|')
cb(null, file.originalname);
}
})
}
export default {
storage: storageType['s3'],
limits: {
filesize: 1 * 1024 * 1024
},
fileFilter: (req, file, cb) => {
const allowedMimes = [
"image/jpeg",
"image/jpg",
"image/png"
];
if (allowedMimes.includes(file.mimetype)) {
cb(null, true);
}else {
cb( new Error("Invalid file type"));
}
}
};
´´´

How to loop into multiple fieldname to get the files.path

I have Node.js app, served by express, with my frontend being made with React.js. My issue is i got to different fieldname for my images. Cover Photo and Avatar Photo. I'm having hard time to figure out how to loop into to different fieldname and get the path of the image. The result that i want is the backend will res.send(path of the image either avatar or cover or both of them).
// imageuploadroutes
import express from 'express';
const router = express.Router();
import multer from 'multer';
import path from 'path';
const storage = multer.diskStorage({
destination: function (req, file, cb) {
if (file.fieldname === 'coverPhoto') {
cb(null, 'public/images/cover');
} else {
cb(null, 'public/images/avatar');
}
},
filename: function (req, file, cb) {
cb(
null,
`${file.fieldname}-${Date.now()}${path.extname(file.originalname)}`
);
},
});
const upload = multer({
storage,
limits: {
fileSize: '2mb',
},
fileFilter: (req, file, cb) => {
if (
file.mimetype == 'image/png' ||
file.mimetype == 'image/jpg' ||
file.mimetype == 'image/jpeg'
) {
cb(null, true);
} else {
cb(null, false);
return cb(new Error('Only .png, .jpg and .jpeg format allowed!'));
}
},
});
router.post(
'/user/profile',
upload.fields([
{
name: 'coverPhoto',
maxCount: 1,
},
{
name: 'avatarPhoto',
maxCount: 1,
},
]),
function (req, res) {
var file = req.files[Object.keys(req.files)[0]];
console.log(file)
}
);
export default router;
Result
[
{
fieldname: 'avatarPhoto',
originalname: 'Screen Shot 2021-03-02 at 11.49.56 AM.png',
encoding: '7bit',
mimetype: 'image/png',
destination: 'public/images/avatar',
filename: 'avatarPhoto-1614704247624.png',
path: 'public/images/avatar/avatarPhoto-1614704247624.png',
size: 597941
}
]
but the problem is I can't get the .path
The file is an array, you should iterate over it:
var file = req.files[Object.keys(req.files)[0]];
console.log(file);
file.forEach(fileData => {
console.log(fileData.path);
})

How to upload multiple files with multer-s3 in Nodejs

I am trying to upload multiple images with Nodejs, Expressjs and Multer-s3, but it's not working.
I have a model called Program and the Program model has an array image attribute but when I try to upload multiple images my req.file returns undefined.
Here is my model
const programSchema = new mongoose.Schema({
programtype: {
type: String,
required: true,
},
title: {
type: String,
required: true,
},
description: {
type: String,
required: true,
},
createdAt: {
type: Date,
required: true,
default: Date.now,
},
programImage: {
type: Array,
require: true,
},
});
and my routes
const Program = require("../models/program");
const fs = require("fs");
const multer = require("multer");
const path = require("path");
var AWS = require("aws-sdk");
var multerS3 = require("multer-s3");
AWS.config.update({
secretAccessKey: process.env.S3_SECRECT,
accessKeyId: process.env.AWS_ACCESS_KEY,
region: process.env.S3_REGION,
});
const uploadPath = path.join("public", Program.programImageBasePath);
const imageMineTypes = ["image/jpeg", "image/png", "image/gif"];
const bucketname = "mybucketname";
s3 = new AWS.S3();
const upload = multer({
storage: multerS3({
s3: s3,
acl: "public-read",
bucket: bucketname,
s3BucketEndpoint: true,
endpoint: "http://" + bucketname + ".s3.amazonaws.com",
key: function (req, file, cb) {
const uploadPathWithOriginalName = uploadPath + "/" + file.originalname;
cb(null, uploadPathWithOriginalName);
},
}),
});
router.post("/create", upload.array("cover", 10), async (req, res, next) => {
console.log(req.file);
const program = new Program({
programtype: req.body.programtype,
title: req.body.title,
description: req.body.description,
programImage: req.file.location,
});
try {
const programs = await program.save();
res.redirect("/programs");
} catch {
if (program.programImage != null) {
removeprogramImage(program.programImage);
}
res.render("programs/new");
}
});
and my views
<h2 style="padding-top: 90px;" > New Programs</h2>
<form action="/programs/create" method="POST" enctype="multipart/form-data">
<div>
<label>Image</label>
<input type="file" name="cover" multiple />
</div>
Cacel
<button type="submit">Create</button>
</form>
You can refer to this example.
const s3 = new AWS.S3({
accessKeyId: 'xxxxxxxxx',
secretAccessKey: 'xxxxxxxxx'
});
const uploadS3 = multer({
storage: multerS3({
s3: s3,
acl: 'public-read',
bucket: 'xxxxxxxx',
metadata: (req, file, callBack) => {
callBack(null, { fieldName: file.fieldname })
},
key: (req, file, callBack) => {
var fullPath = 'products/' + file.originalname;//If you want to save into a folder concat de name of the folder to the path
callBack(null, fullPath)
}
}),
limits: { fileSize: 2000000 }, // In bytes: 2000000 bytes = 2 MB
fileFilter: function (req, file, cb) {
checkFileType(file, cb);
}
}).array('photos', 10);
exports.uploadProductsImages = async (req, res) => {
uploadS3(req, res, (error) => {
console.log('files', req.files);
if (error) {
console.log('errors', error);
res.status(500).json({
status: 'fail',
error: error
});
} else {
// If File not found
if (req.files === undefined) {
console.log('uploadProductsImages Error: No File Selected!');
res.status(500).json({
status: 'fail',
message: 'Error: No File Selected'
});
} else {
// If Success
let fileArray = req.files,
fileLocation;
const images = [];
for (let i = 0; i < fileArray.length; i++) {
fileLocation = fileArray[i].location;
console.log('filenm', fileLocation);
images.push(fileLocation)
}
// Save the file name into database
return res.status(200).json({
status: 'ok',
filesArray: fileArray,
locationArray: images
});
}
}
})
};

How to send multiple files to AWS S3 using Multer-S3 from different fields

I'm currently using Postman in which I require to upload two files from two different fields to AWS-S3; this is how it looks like:
This is the API route that I'm calling:
router.route('/').post(uploadThumbnail, uploadVideo, createVideo);
That route calls three functions(which is supposed to return the data requested from Postman):
exports.createVideo = asyncHandler(async (req, res, next) => {
// Add user to req,body
req.body.user = req.user.id;
// Bring files
if (req.file) {
console.log(req.file);
}
});
Here are the two other functions(with the aws upload function); one for thumbnail and a second one for video_url:
const upload = multer({
storage: multerS3({
s3: s3,
bucket: process.env.AWS_BUCKET_NAME,
acl: 'public-read',
key: function(req, file, cb) {
const strOne = process.env.WEBSITE_NAME + '-';
const userId = req.user.id + '-';
const userEmail = req.user.email + '-';
const todaysDate = Date.now().toString() + '.';
const extension = file.mimetype.split('/')[1];
const finalStr = strOne.concat(userId, userEmail, todaysDate, extension);
cb(null, finalStr);
}
})
});
exports.uploadThumbnail = upload.single('thumbnail');
exports.uploadVideo = upload.single('video_url');
Everytime I run the post, Postman throws me this error:
{
"status": "error",
"error": {
"name": "MulterError",
"message": "Unexpected field",
"code": "LIMIT_UNEXPECTED_FILE",
"field": "video_url",
"storageErrors": [],
"statusCode": 500,
"status": "error"
},
"message": "Unexpected field",
"stack": "MulterError: Unexpected field\n at wrappedFileFilter (C:\\xampp\\htdocs\\myporn\\node_modules\\multer\\index.js:40:19)\n at Busboy.<anonymous> (C:\\xampp\\htdocs\\myporn\\node_modules\\multer\\lib\\make-middleware.js:114:7)\n at Busboy.emit (events.js:198:13)\n at Busboy.EventEmitter.emit (domain.js:448:20)\n at Busboy.emit (C:\\xampp\\htdocs\\myporn\\node_modules\\busboy\\lib\\main.js:38:33)\n at PartStream.<anonymous> (C:\\xampp\\htdocs\\myporn\\node_modules\\busboy\\lib\\types\\multipart.js:213:13)\n at PartStream.emit (events.js:198:13)\n at PartStream.EventEmitter.emit (domain.js:448:20)\n at HeaderParser.<anonymous> (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\Dicer.js:51:16)\n at HeaderParser.emit (events.js:198:13)\n at HeaderParser.EventEmitter.emit (domain.js:448:20)\n at HeaderParser._finish (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\HeaderParser.js:68:8)\n at SBMH.<anonymous> (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\HeaderParser.js:40:12)\n at SBMH.emit (events.js:198:13)\n at SBMH.EventEmitter.emit (domain.js:448:20)\n at SBMH._sbmh_feed (C:\\xampp\\htdocs\\myporn\\node_modules\\streamsearch\\lib\\sbmh.js:159:14)\n at SBMH.push (C:\\xampp\\htdocs\\myporn\\node_modules\\streamsearch\\lib\\sbmh.js:56:14)\n at HeaderParser.push (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\HeaderParser.js:46:19)\n at Dicer._oninfo (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\Dicer.js:197:25)\n at SBMH.<anonymous> (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\Dicer.js:127:10)\n at SBMH.emit (events.js:198:13)\n at SBMH.EventEmitter.emit (domain.js:448:20)\n at SBMH._sbmh_feed (C:\\xampp\\htdocs\\myporn\\node_modules\\streamsearch\\lib\\sbmh.js:188:10)\n at SBMH.push (C:\\xampp\\htdocs\\myporn\\node_modules\\streamsearch\\lib\\sbmh.js:56:14)\n at Dicer._write (C:\\xampp\\htdocs\\myporn\\node_modules\\dicer\\lib\\Dicer.js:109:17)\n at doWrite (_stream_writable.js:415:12)\n at writeOrBuffer (_stream_writable.js:399:5)\n at Dicer.Writable.write (_stream_writable.js:299:11)"
}
The function works great but only when sending one single file, it can be either thumbnail or video_url but not both...I need both fields to work.
Any idea on how to fix this?
const s3 = new AWS.S3({
accessKeyId: 'xxxxxxxxx',
secretAccessKey: 'xxxxxxxxx'
});
const uploadS3 = multer({
storage: multerS3({
s3: s3,
acl: 'public-read',
bucket: 'xxxxxxxx',
metadata: (req, file, callBack) => {
callBack(null, { fieldName: file.fieldname })
},
key: (req, file, callBack) => {
var fullPath = 'products/' + file.originalname;//If you want to save into a folder concat de name of the folder to the path
callBack(null, fullPath)
}
}),
limits: { fileSize: 2000000 }, // In bytes: 2000000 bytes = 2 MB
fileFilter: function (req, file, cb) {
checkFileType(file, cb);
}
}).array('photos', 10);
exports.uploadProductsImages = async (req, res) => {
uploadS3(req, res, (error) => {
console.log('files', req.files);
if (error) {
console.log('errors', error);
res.status(500).json({
status: 'fail',
error: error
});
} else {
// If File not found
if (req.files === undefined) {
console.log('uploadProductsImages Error: No File Selected!');
res.status(500).json({
status: 'fail',
message: 'Error: No File Selected'
});
} else {
// If Success
let fileArray = req.files,
fileLocation;
const images = [];
for (let i = 0; i < fileArray.length; i++) {
fileLocation = fileArray[i].location;
console.log('filenm', fileLocation);
images.push(fileLocation)
}
// Save the file name into database
return res.status(200).json({
status: 'ok',
filesArray: fileArray,
locationArray: images
});
}
}
})
};
Make use of file.fieldname and .fields to achieve what you need
function uploadImageToS3(){
const multer = require("multer");
const multerS3 = require("multer-s3");
const uuid = require("uuid").v4;
const path = require("path");
const cloudStorage = multerS3({
s3: s3Client,
bucket: process.env.AWS_PRODUCTS_BUCKET,
acl: "public-read",
metadata: (req, file, cb) => {
if (file.fieldname == "thumbnail") {
const extension = ".jpeg";
cb(null, `${uuid()}${extension}`);
return;
}
if (file.fieldname == "video_url") {
const extension = ".jpeg";
cb(null, `${uuid()}${extension}`);
return;
}
cb(
"fieldName other than avatar and productImages are not supported",
null
);
},
// key: name of the file
key: (req, file, cb) => {
const extension = path.extname(file.originalname);
cb(null, `${uuid()}${extension}`);
},
});
const limitFileSize = { fileSize: 1024 * 1024 * 5 }, // 1Byte -->1024Bytes or 1MB --> 5MB
filterFileType = (req, file, cb) => {
const isAllowedFileType =
file.mimetype == "image/jpeg" ||
file.mimetype == "image/jpg" ||
file.mimetype == "image/png";
if (isAllowedFileType) {
cb(null, true);
return;
}
// To reject this file pass `false`
cb(null, false);
};
const upload = multer({
storage: cloudStorage,
limits: limitFileSize,
fileFilter: filterFileType,
});
return upload;
}
In route, use it like so
router.post(
"/upload",
uploadImageToS3().fields([
{ name: "thumbnail", maxCount: 1 },
{ name: "video_url", maxCount: 1 }
]),
handleImagesUploadController
);

Resources