upload multiple files using multerS3 and NodeJs - node.js

I've been trying for days but can't seem to figure out where I'm going wrong. My code works for a single file upload in a local folder but can't seem to upload multiple files even if I use upload.array() and req.files in multer and controller function respectively. Whereas when I try to upload a single/multiple files in my s3 bucket nothing gets uploaded at all. Below I'll also provide my model structure in case you need to see it.
::UPDATE::
The files are getting uploaded in s3 bucket but the database can't catch the file path. imagePath is coming blank inside database
route.js:
const ProductController = require('./../controllers/productController');
const appConfig = require("./../config/appConfig");
const multer = require('multer');
const aws = require('aws-sdk');
const multerS3 = require('multer-s3');
const path = require('path');
const s3 = new aws.S3({ accessKeyId: "***", secretAccessKey: "***" });
var upload = multer({
storage: multerS3({
s3: s3,
bucket: 'mean-ecom',
metadata: function (req, file, cb) {
cb(null, {fieldName: file.originalname + path.extname(file.fieldname)});
},
key: function (req, file, cb) {
cb(null, Date.now().toString())}})});
let setRouter = (app) => {
let baseUrl = appConfig.apiVersion;
app.post(baseUrl+'/post-product', upload.single('imagePath'), ProductController.postProduct)}
module.exports = {
setRouter: setRouter}
ProductController.postProduct:
const ProductModel = mongoose.model('Product')
let postProduct = (req, res) => {
let newProduct = new ProductModel({
imagePath: req.file && req.file.path})
newProduct.save((err, result) => {
if (err) { console.log('Error at saving new Product :: ProductController', err);
res.send(err);
} else {
console.log('Successfully saved new Product'); res.send(result) }
})}
productModel.js
const mongoose = require('mongoose');
const Schema = mongoose.Schema;
let productSchema = new Schema(
{
imagePath: {
type: String,
default: '' }})
mongoose.model('Product', productSchema);

The first thing you need to do is change the model's imagePath property from a string to an array of strings, so you can store multiple images
const Schema = mongoose.Schema;
let productSchema = new Schema(
{
imagePath: {type: Array}
})
mongoose.model('Product', productSchema)
Then, in the router, you should use upload.any() (upload.array() should work the same way).
Finaly, if you want to store the files locations in the model array, you loop throw the files updating the model.
let postProduct = (req, res) => {
// We create a temporal array with the files locations
let files = [];
// We loop throw the req.files array and store their locations in the temporal files array
for (let i = 0; i < req.files.length; i++) {
let file = req.files[i].location
files.push(file)
}
let newProduct = new ProductModel({
imagePath: files}) //We create the model with the temp array
newProduct.save((err, result) => {
if (err) {
console.log('Error at saving new Product :: ProductController', err);
res.send(err);
}
else{
console.log('Successfully saved new Product');
res.send(result)
}
})

Related

Error: "Undefined". Getting Undefined while using multer multiple file uploads. Tried most options including restart and npm reinstalling

I am trying to upload multiple images using a get request to update the initial post request that accepts a single upload. Not able to upload multiple images despite using the "files" as suggested in the docs and on here. The (file.filename) comes back undefined for some reason. Also tried changing the base path manually instead of getting it dynamically. Did not work. The images are uploaded to the __dir but shows up as undefined in the database and the postman put console.
const express = require("express");
const { Category } = require("../models/category");
const router = express.Router(); //comes with express
const {Product} = require("../models/product")
const mongoose = require("mongoose");
const multer = require("multer");
//MIME TYPE LOOKUP
const FILE_TYPE_MAP = {
"image/png" : "png",
"image/jpeg" : "jpeg",
"image/jpg" : "jpg"
}
const storage = multer.diskStorage({
destination: function (req, file, cb) {
const isValid = FILE_TYPE_MAP[file.mimetype]
let uploadError = new Error("invalid Image Format");
if(isValid){
uploadError = null;
}
cb(uploadError, __dirname + '/public/uploads')
},
filename: function (req, file, cb) {//const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9)
const extension = FILE_TYPE_MAP[file.mimetype]
const fileName = file.originalname.split(" ").join("_");
cb(null, Date.now() + "-" + fileName + "." + extension )
}
})
const uploadOptions = multer({ storage: storage })
//SINGLE UPLOADS
router.post(`/`, uploadOptions.single("image"), async (req, res)=>{
let category = await Category.findById(req.body.category);
if(!category) return res.status(400).send("What the fuck? Invalid category");
const file = req.file;
if(!file) return res.status(400).send("What the fuck? I need a file"); //compulsary image file upload similar for the category
const fileName = req.file.filename;
const basePath = `${req.protocol}://${req.get("host")}/public/uploads/`;
const product = new Product({
name: req.body.name,
description: req.body.description,
richDescription: req.body.richDescription,
image: `${basePath}${fileName}`,//"http://localhost:3000/public/upload/image-213213"
//images: req.body.images,
brand: req.body.brand,
price: req.body.price,
category: req.body.category,
countInStock: req.body.countInStock,
rating: req.body.rating,
numReviews: req.body.numReviews,
isFeatured: req.body.isFeatured
})
product1 = await product.save();
if(!product1)
return res.status(500).send("The product cannot be created");
res.send(product1);
})
//MULTIPLE UPLOADS
router.put(
"/gallery-images/:id",
uploadOptions.any("images"),
async (req, res) => {
if(!mongoose.isValidObjectId(req.params.id)){
res.status(400).send("Invalid product id")
}
const files = req.files
let imagePaths = [];
const basePath = `${req.protocol}://${req.get("host")}/public/uploads/`;
if(files){
files.map(file => {
imagePaths.push(`${basePath}${files.fileName}`); })
}
const product = await Product.findByIdAndUpdate(
req.params.id,
{
images: imagePaths
},
{ new : true}
)
if(!product)
return res.status(404).send("The product cannot be updated")
res.send(product);
})
when you were doing the mapping you used file as an object and you put files to get information from filename which is not good. there also it is not fileName but rather filename.
copy and paste this code you should be fine. next time do a console log of the object for more details it can help you in the future
if(files){
files.map(file=>{
//console.log(file)
imagesPaths.push(`${basePath}${file.filename}`);
})
}
and I also give you a bonus for the extension of the original name which does not fade after this :
const fileName = file.originalname.split(" ").join("_");
try that and admire the results
const fileName = file.originalname.replace(/(.png|.jpeg|.jpg)/,'')

How to use Multer middleware to upload array of images

Im trying to use Multer to upload an array of images. At the client side i have a FormData called pictures.
pictures array, from react-native-image-picker:
const [pictures, setPictures] = useState([]);
const imagePickerCallBack = data => {
const picturesData = [...pictures];
const index = picturesData.length;
const image = {
image: data.uri,
fileName: data.fileName,
type: data.type,
index: index,
};
picturesData.push(image);
setPictures(picturesData);
setLoad(false);
};
Step 1 - Create formData with all images:
const data = new FormData();
pictures.forEach(pic => {
data.append('pictures', {
fileName: pic.fileName,
uri: pic.image,
type: pic.type,
});
});
const headers = {
'Content-Type': 'multipart/form-data',
'x-access-token': token,
};
const diaryUpdatePost = await post(`diary/uploadPictures/${diary}`, body, {
headers,
});
Step 2 - Get the request at server side. Im setting up multer and routers:
const router = express.Router();
const multer = require('multer');
const storage = multer.diskStorage({
destination(req, file, cb) {
cb(null, 'uploads/');
},
filename(req, file, cb) {
cb(null, `${file.fieldname}-${Date.now()}`);
},
});
const upload = multer({ storage, limits: { fieldSize: 25 * 1024 * 1024 } });
// Multer with the same FormData (client)
router.post('/uploadPictures/:name', upload.array('pictures'), diaryController.uploadDiaryPictures);
And finally my diaryController, where i need to get all files:
exports.uploadDiaryPictures = async (req, res) => {
// Logging []. I cant access files from here
console.log(`files ${req.files}...`);
};
I already tried to use express-fileupload, but req.files return undefined. Some ideia to help? Thx.
You need to give a count of files you expect to upload:
upload.array('pictures', <number_of_pictures>)
Or if it is allowed to be any number:
upload.any('pictures')
You should also add the file itself to your form data
data.append('pictures', {
name: pic.fileName,
file: pic.image,
type: pic.type,
});
None of the answers here helped. The solution for me was to iteratively append EACH file object from the files array to the same field name given in Multer, instead of appending the files array itself to the field name given in Multer.
So from this:
export const addFiles= createAsyncThunk(
"addFiles",
async (payload: any, thunkApi) => {
const formData = new FormData();
// Here was the problem -- I was appending the array itself
// to the "files" field
formData.append("files", payload.files);
formData.append("data", JSON.stringify(payload?.data || {}));
const response = await axios.post('/user/products/files', formData);
if(response){
return response;
}
return thunkApi.rejectWithValue("");
}
);
I did this:
export const addFiles= createAsyncThunk(
"addFiles",
async (payload: any, thunkApi) => {
const formData = new FormData();
// The following loop was the solution
for (const file of payload.files) {
formData.append("files", file);
}
formData.append("data", JSON.stringify(payload?.data || {}));
const response = await axios.post('/user/products/files', formData);
if(response){
return response;
}
return thunkApi.rejectWithValue("");
}
);
This was my Multer configuration:
multer({ dest: "/uploads" }).array("files")
The files posted to my endpoint were then available to me at:
req.files
PS: Although the accepted answer kind of did that, he did not mention that you cannot append the entire array at once, which was the main problem for me.

How to upload multiple files in nodejs to AWS S3 and save file url into database?

Hi i need to upload multiple images at a time on s3.
Currently i am using express-fileupload to upload single image on AWS, and i want to use same approach to make it upload multiple files to s3 and update images array with urls on mongodb.
My schema property:
const ServiceSchema = new mongoose.Schema(
{
photo: [
{
type: String,
default: 'no-photo.jpg',
},
],
});
module.exports = mongoose.model('Service', ServiceSchema);
My Controller:
// #desc Upload photo for service
// #route PUT /api/v1/services/:id/photo
// #access Private
exports.servicePhotoUpload = asyncHandler(async (req, res, next) => {
const service = await Service.findById(req.params.id);
if (!service) {
return next(new ErrorResponse(`Service not found with id of ${req.params.id}`, 404));
}
// Make sure user adding service is business owner
if (service.user.toString() !== req.user.id && req.user.role !== 'admin') {
return next(
new ErrorResponse(
`User ${req.user.id} is not authorized to update this service to business ${service._id}`,
401
)
);
}
// File Upload validation
if (!req.files) {
return next(new ErrorResponse(`Please upload a file.`, 400));
}
const file = req.files.file;
// Make sure it is a valid image file
if (!file.mimetype.startsWith('image')) {
return next(new ErrorResponse(`Please upload a valid image file.`, 400));
}
//Check File Size
if (file.size > process.env.MAX_FILE_UPLOAD) {
return next(
new ErrorResponse(
`Please upload an image less then ${process.env.MAX_FILE_UPLOAD / 1024}KB in size.`,
400
)
);
}
// Create custom filename
file.name = `service-uploads/servicePhoto_${service._id}${path.parse(file.name).ext}`;
uploadToS3({
fileData: req.files.file.data,
fileName: file.name,
})
.then(async (result) => {
console.log('Success Result: ', result);
await Service.findByIdAndUpdate(service._id, { photo: result.Location });
return res
.status(200)
.json({ success: true, message: 'Service photo added successfully', url: result.Location });
})
.catch((err) => {
console.log(err);
return next(new ErrorResponse('Failed to upload file to S3', 500));
});
});
My Utility File to upload File to S3:
const AWS = require('aws-sdk');
const uploadToS3 = (options) => {
// Set the AWS Configuration
AWS.config.update({
accessKeyId: process.env.AWS_S3_ACCESS_KEY,
secretAccessKey: process.env.AWS_S3_SECRET_KEY,
region: 'us-east-2',
});
// Create S3 service object
const s3 = new AWS.S3({ apiVersion: '2006-03-01' });
// Setting up S3 upload parameters
const params = {
Bucket: 'toolbox-uploads',
Key: options.fileName, // File name you want to save as in S3
Body: options.fileData, //
};
// Return S3 uploading function as a promise so return url can be handled properly
return s3.upload(params).promise();
};
module.exports = uploadToS3;
My Router:
const express = require('express');
const {
servicePhotoUpload
} = require('../controllers/service');
const Service = require('../models/Service');
router.route('/:id/photo').put(protect, authorize('publisher', 'business', 'admin'), servicePhotoUpload);
module.exports = router;
This above code is workng 100%.
I am bit confused as there were different approach and none worked for me from google and stack overflow and none of them is getting return url and saving into database.
I want to make separate utility file to upload multiple files to 3 same as i did for single files to use them anywhere. That file should return uploaded urls so i can update my database.
I have tried multer-s3 but no solution works for me.
This approach might be different for you but that is how I was able to resolve the same issue.
First you'll need
Multer
multer-s3
aws-sdk
I made a FileUpload class that handles both single and multi-upload (I also needed to be able to upload pdf and video files) and this is the code in my constructor, note that I also specified the s3-bucket in question from aws.
this.s3 = new AWS.S3({
accessKeyId: process.env.S3_ACCESS_KEY_ID,
secretAccessKey: process.env.S3_SECRET_KEY,
Bucket: 'name_of_s3_bucket',
});
I created a method called upload in the class. Code below
upload(path, type) {
let ext = 'jpeg';
const multerFilter = (req, file, cb) => {
if (type === 'image') {
if (file.mimetype.startsWith(this.type)) {
cb(null, true);
} else {
cb(
new AppError(
'Not an Image! Please upload only images',
400
),
false
);
}
} else if (type === 'pdf') {
ext = 'pdf';
const isPdf = file.mimetype.split('/')[1];
if (isPdf.startsWith(this.type)) {
cb(null, true);
} else {
cb(
new AppError('Not a pdf! Please upload only pdf', 400),
false
);
}
}
};
const upload = multer({
storage: multers3({
acl: 'public-read',
s3: this.s3,
bucket: 'name_of_s3_bucket',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
let filename = `user-${
req.user.id
}/${path}/${uuid.v4()}-${Date.now()}.${ext}`;
// eslint-disable-next-line camelcase
const paths_with_sub_folders = [
'auditions',
'biography',
'movies',
];
if (paths_with_sub_folders.includes(path)) {
filename = `user-${req.user.id}/${path}/${
req.params.id
}/${uuid.v4()}-${Date.now()}.${ext}`;
}
cb(null, filename);
},
}),
fileFilter: multerFilter,
limits: {
fileSize: 5000000,
},
});
return upload;
}
To consume the above, I import the class into any controller that I needed an upload feature and called the following.
Side Note : Ignore the paths code (It was just a way to generate unique file name for the files)
const upload = new FileUpload('image').upload('profile-images', 'image');
exports.uploadUserPhoto = upload.array('photos', 10);
I then used the uploadUserPhoto as a middleware before calling the following
exports.addToDB = catchAsync(async (req, res, next) => {
if (!req.files) return next();
req.body.photos = [];
Promise.all(
req.files.map(async (file, i) => {
req.body.photos.push(file.key);
})
);
next();
});
On a high-level overview, this is the flow, First, upload your photos to s3 and get the req.files, then look through that req.files object passing them into an array field on your req object then finally save them on your DB.
NOTE: You must promisify the req.file loop since the task is asynchrnous
My final router looked like this
router
.route('/:id')
.put(uploadUserPhoto, addToDB, updateProfile)
Item.js
Your model can have a field called images thats type array.
const mongoose = require("mongoose");
const ItemSchema = mongoose.Schema({
images: {
type: [],
},
});
module.exports = mongoose.model("Items", ItemSchema);
You map through the array of object and only extract the data you want to store, in this example it is the key which is the unique name given to every image thats uploaded.
route.js
router.post("/", verify, upload.array("image"), async (req, res) => {
const { files } = req;
const images = [];
files.map((file) => {
images.push(file.key);
});
try {
new Item({
images,
}).save();
res.status(200).send({message: "saved images to db"})
}catch(err){
res.status(400).send({message: err})
}
});
Let me know if this does what you wanted

How to get files uploaded via multer using mongoose?

I'm using the below function to get the files uploaded by multer in mongodb.The request is returning empty array.
exports.getPhotos = async (req, res) => {
const photos = await Photo.find()
.then(photos => {
res.status(200).json(photos);
})
.catch(err => res.status(500).json({message: "Something went wrong"}));
};
and this is the schema of the image. Is there any way to get the files without specifying the schema?
const mongoose = require("mongoose");
const {ObjectId} = mongoose.Schema;
const photoSchema = new mongoose.Schema({
lenght: {
type: String,
},
chunkSize: {
type: String,
required: true
},
uploadDate: {
type: Date,
},
filename: {
type: String,
},
md5: {
type: String,
},
contentType: {
type: String,
},
});
module.exports = mongoose.model("Photo", photoSchema);
i use gridfs so it could upload larger files too . a piece of sample code below
//Connecting to mongo
const conn = mongoose.createConnection(mongoURI);
//Init gfs
let gfs;
conn.once('open', ()=>{
gfs = GridFsStream(conn.db, mongoose.mongo);
gfs.collection('uploads');
})
//Creating Storage engine
const storage = new GridFsStorage({
url:mongoURI,
file: (req, file) => {
return new Promise((resolve, reject)=>{
crypto.randomBytes(16,(err, buf)=>{
if(err){
return reject(err)
}
const fileName = buf.toString('hex') + path.extname(file.originalname)
//bucket name should match the collection name
const fileInfo = {
filename:fileName,
bucketName:'uploads'
}
resolve(fileInfo);
})
})
}
})
const upload = multer({storage})
now use this upload const in your paths like the one below there are a few methods for upload like array , single and ... depends on number of files you are uploading . the 'uploadedFile' argument is the name of the file input and you should be consider setting it in your frontend.
app.post('/',upload.single('uploadedFile'),(req, res)=>{
res.json('file uploaded')
})
this upload middleware adds a files to your request which you can use to put file names in your database and later fetch it by that uniqe names with a route like the one below .
app.get('/:filename', (req, res)=>{
gfs.files.findOne({filename:req.params.filename},(err,file)=>{
if(!file || file.length === 0){
return res.status(404).json({
err:'No file Exists'
})
}
const readStream = gfs.createReadStream(file.filename);
readStream.pipe(res)
})
})

uploading images in mlab

currently i am storing images which are uploaded by node rest server in local directory "/uploads" . this is continuously increasing my repo size .
to avoid this , i want to store image files in mongoDB atlas or mlab just like service.
const express = require("express");
const router = express.Router();
const mongoose = require("mongoose");
const multer = require('multer');
const storage = multer.diskStorage({
destination: function(req, file, cb) {
cb(null, './uploads/');
},
filename: function(req, file, cb) {
cb(null, new Date().toISOString() + file.originalname);
}
});
const fileFilter = (req, file, cb) => {
// reject a file
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png')
{
cb(null, true);
} else {
cb(null, false);
}
};
const upload = multer({
storage: storage,
limits: {
fileSize: 1024 * 1024 * 5
},
fileFilter: fileFilter
});
Please do help me in this. Thanks in advance.
You can achieve this by using a mongoose Schema and the fs core module to encode the image and unlink the file from /uploads.
I would start by creating a Mongoose Schema to set the model of all information you want to store pertaining to your uploaded file.
I'm going to use base64 encoding for this example.
uploadModel.js
const mongoose = require('mongoose');
const fs = require('fs');
const Schema = mongoose.Schema;
mongoose.set('useCreateIndex', true);
let uploadSchema = new Schema({
name: {
type: String,
},
mimetype: {
type: String,
},
size: {
type: Number,
},
base64: {
type: String,
}
})
module.exports = mongoose.model('upload',uploadSchema);
After setting up a model create a function to base64 encode and module.exports that as well.
To encode your file, use fs.readFileSync(path_to_file, encode_type). After the file has been encoded and saved in a variable you can use fs.unlink(path_to_file) to delete the file out of your /uploads folder.
uploadModel.js
module.exports.base64_encode = function(file) {
return new Promise((resolve, reject) => {
if(file == undefined){
reject('no file found');
} else {
let encodedData = fs.readFileSync(file, 'base64');
fs.unlink(file);
resolve(encodedData.toString('base64'));
}
})
}
Now inside your route file require your model.
route.js
const Upload = require('path_to_uploadModel');
router.post('/path_to_upload', upload.single('form_name_of_file'), (req, res) => {
let img = req.file;
let model = new Upload({
name: img.originalname,
size: img.size,
mimetype: img.mimetype,
})
Upload.base64_encode(img.path)
.then((base64) => {
model['base64'] = base64;
model.save((err)=> {
if(err) throw err;
});
}
})
Hope this helps

Resources