How to use Multer middleware to upload array of images - node.js

Im trying to use Multer to upload an array of images. At the client side i have a FormData called pictures.
pictures array, from react-native-image-picker:
const [pictures, setPictures] = useState([]);
const imagePickerCallBack = data => {
const picturesData = [...pictures];
const index = picturesData.length;
const image = {
image: data.uri,
fileName: data.fileName,
type: data.type,
index: index,
};
picturesData.push(image);
setPictures(picturesData);
setLoad(false);
};
Step 1 - Create formData with all images:
const data = new FormData();
pictures.forEach(pic => {
data.append('pictures', {
fileName: pic.fileName,
uri: pic.image,
type: pic.type,
});
});
const headers = {
'Content-Type': 'multipart/form-data',
'x-access-token': token,
};
const diaryUpdatePost = await post(`diary/uploadPictures/${diary}`, body, {
headers,
});
Step 2 - Get the request at server side. Im setting up multer and routers:
const router = express.Router();
const multer = require('multer');
const storage = multer.diskStorage({
destination(req, file, cb) {
cb(null, 'uploads/');
},
filename(req, file, cb) {
cb(null, `${file.fieldname}-${Date.now()}`);
},
});
const upload = multer({ storage, limits: { fieldSize: 25 * 1024 * 1024 } });
// Multer with the same FormData (client)
router.post('/uploadPictures/:name', upload.array('pictures'), diaryController.uploadDiaryPictures);
And finally my diaryController, where i need to get all files:
exports.uploadDiaryPictures = async (req, res) => {
// Logging []. I cant access files from here
console.log(`files ${req.files}...`);
};
I already tried to use express-fileupload, but req.files return undefined. Some ideia to help? Thx.

You need to give a count of files you expect to upload:
upload.array('pictures', <number_of_pictures>)
Or if it is allowed to be any number:
upload.any('pictures')
You should also add the file itself to your form data
data.append('pictures', {
name: pic.fileName,
file: pic.image,
type: pic.type,
});

None of the answers here helped. The solution for me was to iteratively append EACH file object from the files array to the same field name given in Multer, instead of appending the files array itself to the field name given in Multer.
So from this:
export const addFiles= createAsyncThunk(
"addFiles",
async (payload: any, thunkApi) => {
const formData = new FormData();
// Here was the problem -- I was appending the array itself
// to the "files" field
formData.append("files", payload.files);
formData.append("data", JSON.stringify(payload?.data || {}));
const response = await axios.post('/user/products/files', formData);
if(response){
return response;
}
return thunkApi.rejectWithValue("");
}
);
I did this:
export const addFiles= createAsyncThunk(
"addFiles",
async (payload: any, thunkApi) => {
const formData = new FormData();
// The following loop was the solution
for (const file of payload.files) {
formData.append("files", file);
}
formData.append("data", JSON.stringify(payload?.data || {}));
const response = await axios.post('/user/products/files', formData);
if(response){
return response;
}
return thunkApi.rejectWithValue("");
}
);
This was my Multer configuration:
multer({ dest: "/uploads" }).array("files")
The files posted to my endpoint were then available to me at:
req.files
PS: Although the accepted answer kind of did that, he did not mention that you cannot append the entire array at once, which was the main problem for me.

Related

Saving uploaded file to Pinata IPFS in NodeJS

I've been trying to save uploaded image files to IPFS in NodeJs , while it seems Pinata saves them, the files are pretty much gibberish (after downloading the images are broken).
My code :
// Nodejs route.
exports.postImage = async (req, res, next) => {
// Using multer to get the file.
fileUploadMiddleware(req, res, async (err) => {
// getting bunch of data from query string.
let meta = {
origin,
originid,
context,
ownerid,
format
} = req.query;
if(!meta.format || !req.files) {
return next(new ErrorResponse("File format not specified", 404));
}
if(!meta.originid) {
meta.originid = uuidv4();
}
// NOTE: is this the right way to get the data of the file ?
const buffer = req.files[0].buffer;
const filename = `${metadata.origin}_${metadata.originid}.${ metadata.format }`;
let stream;
try {
stream = Readable.from(buffer);
// HACK to make PINATA WORK.
stream.path = filename;
}
catch(e) {
logger.logError(e);
return false;
}
const options = {
pinataMetadata: {
name: filename,
keyvalues: {
context: metadata.context,
ownerid: metadata.ownerid
}
},
pinataOptions: {
cidVersion: 0
}
};
try {
var result = await pinata.pinFileToIPFS(stream, options);
console.log("SUCCESS ", result);
return result;
}
catch(e) {
logger.logError(e);
return null;
}
res.status(200).json({
success: true,
data: 'You got access'
})
});
}
So basically creating the stream based on the uploaded file buffer and sending it away to Pinata. Where do I go wrong?
const buffer = req.files[0].buffer;
If you used MemoryStorage. buffer property would be available. It is not available for diskStorage because it will save the file locally.:
const storage = multer.memoryStorage()
const upload = multer({ storage: storage })
Also I think it not req.files[0]
const buffer = req.file.buffer;
after I get the buffer, I convert it to FormData using form-data npm package:
import FormData from "form-data";
const formData = new FormData();
formData.append("file", buffer, {
contentType,
filename: fileName + "-" + uuidv4(),
});
then you send a post request to pinata
const url = `https://api.pinata.cloud/pinning/pinFileToIPFS`;
const fileRes = await axios.post(url, formData, {
maxBodyLength: Infinity,
headers: {
// formData.getBoundary() is specific to npm package. native javascript FormData does not have this method
"Content-Type": `multipart/form-data: boundary=${formData.getBoundary()}`,
pinata_api_key: pinataApiKey,
pinata_secret_api_key: pinataSecretApiKey,
},
});

How to upload multiple files in nodejs to AWS S3 and save file url into database?

Hi i need to upload multiple images at a time on s3.
Currently i am using express-fileupload to upload single image on AWS, and i want to use same approach to make it upload multiple files to s3 and update images array with urls on mongodb.
My schema property:
const ServiceSchema = new mongoose.Schema(
{
photo: [
{
type: String,
default: 'no-photo.jpg',
},
],
});
module.exports = mongoose.model('Service', ServiceSchema);
My Controller:
// #desc Upload photo for service
// #route PUT /api/v1/services/:id/photo
// #access Private
exports.servicePhotoUpload = asyncHandler(async (req, res, next) => {
const service = await Service.findById(req.params.id);
if (!service) {
return next(new ErrorResponse(`Service not found with id of ${req.params.id}`, 404));
}
// Make sure user adding service is business owner
if (service.user.toString() !== req.user.id && req.user.role !== 'admin') {
return next(
new ErrorResponse(
`User ${req.user.id} is not authorized to update this service to business ${service._id}`,
401
)
);
}
// File Upload validation
if (!req.files) {
return next(new ErrorResponse(`Please upload a file.`, 400));
}
const file = req.files.file;
// Make sure it is a valid image file
if (!file.mimetype.startsWith('image')) {
return next(new ErrorResponse(`Please upload a valid image file.`, 400));
}
//Check File Size
if (file.size > process.env.MAX_FILE_UPLOAD) {
return next(
new ErrorResponse(
`Please upload an image less then ${process.env.MAX_FILE_UPLOAD / 1024}KB in size.`,
400
)
);
}
// Create custom filename
file.name = `service-uploads/servicePhoto_${service._id}${path.parse(file.name).ext}`;
uploadToS3({
fileData: req.files.file.data,
fileName: file.name,
})
.then(async (result) => {
console.log('Success Result: ', result);
await Service.findByIdAndUpdate(service._id, { photo: result.Location });
return res
.status(200)
.json({ success: true, message: 'Service photo added successfully', url: result.Location });
})
.catch((err) => {
console.log(err);
return next(new ErrorResponse('Failed to upload file to S3', 500));
});
});
My Utility File to upload File to S3:
const AWS = require('aws-sdk');
const uploadToS3 = (options) => {
// Set the AWS Configuration
AWS.config.update({
accessKeyId: process.env.AWS_S3_ACCESS_KEY,
secretAccessKey: process.env.AWS_S3_SECRET_KEY,
region: 'us-east-2',
});
// Create S3 service object
const s3 = new AWS.S3({ apiVersion: '2006-03-01' });
// Setting up S3 upload parameters
const params = {
Bucket: 'toolbox-uploads',
Key: options.fileName, // File name you want to save as in S3
Body: options.fileData, //
};
// Return S3 uploading function as a promise so return url can be handled properly
return s3.upload(params).promise();
};
module.exports = uploadToS3;
My Router:
const express = require('express');
const {
servicePhotoUpload
} = require('../controllers/service');
const Service = require('../models/Service');
router.route('/:id/photo').put(protect, authorize('publisher', 'business', 'admin'), servicePhotoUpload);
module.exports = router;
This above code is workng 100%.
I am bit confused as there were different approach and none worked for me from google and stack overflow and none of them is getting return url and saving into database.
I want to make separate utility file to upload multiple files to 3 same as i did for single files to use them anywhere. That file should return uploaded urls so i can update my database.
I have tried multer-s3 but no solution works for me.
This approach might be different for you but that is how I was able to resolve the same issue.
First you'll need
Multer
multer-s3
aws-sdk
I made a FileUpload class that handles both single and multi-upload (I also needed to be able to upload pdf and video files) and this is the code in my constructor, note that I also specified the s3-bucket in question from aws.
this.s3 = new AWS.S3({
accessKeyId: process.env.S3_ACCESS_KEY_ID,
secretAccessKey: process.env.S3_SECRET_KEY,
Bucket: 'name_of_s3_bucket',
});
I created a method called upload in the class. Code below
upload(path, type) {
let ext = 'jpeg';
const multerFilter = (req, file, cb) => {
if (type === 'image') {
if (file.mimetype.startsWith(this.type)) {
cb(null, true);
} else {
cb(
new AppError(
'Not an Image! Please upload only images',
400
),
false
);
}
} else if (type === 'pdf') {
ext = 'pdf';
const isPdf = file.mimetype.split('/')[1];
if (isPdf.startsWith(this.type)) {
cb(null, true);
} else {
cb(
new AppError('Not a pdf! Please upload only pdf', 400),
false
);
}
}
};
const upload = multer({
storage: multers3({
acl: 'public-read',
s3: this.s3,
bucket: 'name_of_s3_bucket',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
let filename = `user-${
req.user.id
}/${path}/${uuid.v4()}-${Date.now()}.${ext}`;
// eslint-disable-next-line camelcase
const paths_with_sub_folders = [
'auditions',
'biography',
'movies',
];
if (paths_with_sub_folders.includes(path)) {
filename = `user-${req.user.id}/${path}/${
req.params.id
}/${uuid.v4()}-${Date.now()}.${ext}`;
}
cb(null, filename);
},
}),
fileFilter: multerFilter,
limits: {
fileSize: 5000000,
},
});
return upload;
}
To consume the above, I import the class into any controller that I needed an upload feature and called the following.
Side Note : Ignore the paths code (It was just a way to generate unique file name for the files)
const upload = new FileUpload('image').upload('profile-images', 'image');
exports.uploadUserPhoto = upload.array('photos', 10);
I then used the uploadUserPhoto as a middleware before calling the following
exports.addToDB = catchAsync(async (req, res, next) => {
if (!req.files) return next();
req.body.photos = [];
Promise.all(
req.files.map(async (file, i) => {
req.body.photos.push(file.key);
})
);
next();
});
On a high-level overview, this is the flow, First, upload your photos to s3 and get the req.files, then look through that req.files object passing them into an array field on your req object then finally save them on your DB.
NOTE: You must promisify the req.file loop since the task is asynchrnous
My final router looked like this
router
.route('/:id')
.put(uploadUserPhoto, addToDB, updateProfile)
Item.js
Your model can have a field called images thats type array.
const mongoose = require("mongoose");
const ItemSchema = mongoose.Schema({
images: {
type: [],
},
});
module.exports = mongoose.model("Items", ItemSchema);
You map through the array of object and only extract the data you want to store, in this example it is the key which is the unique name given to every image thats uploaded.
route.js
router.post("/", verify, upload.array("image"), async (req, res) => {
const { files } = req;
const images = [];
files.map((file) => {
images.push(file.key);
});
try {
new Item({
images,
}).save();
res.status(200).send({message: "saved images to db"})
}catch(err){
res.status(400).send({message: err})
}
});
Let me know if this does what you wanted

how to upload a image file in node.js?

var title="this is title";
var content="this is content";
const config = { headers: {'Accept': 'application/json', 'Content-Type': 'multipart/form-data' } };
const form = new FormData()
let file =event.target.files[0]
form.append('file', file)
form.append('title', title)
form.append('content', content)`enter code here`
Axios.post("http://localhost:3001/article/get/123", form,config ).then((res)=>{
console.log(res.data)
})
in node I have used multer for upload image or anything.
Below is the code for upload which I have used as a middleware.
const util = require("util");
const path = require("path");
const multer = require("multer");
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, "./Uploads") // folder path where to upload
},
filename: function (req, file, cb) {
cb(null, file.originalname + "-" + Date.now() + path.extname(file.originalname))
}
});
const maxSize = 1 * 20000 * 20000; // file size validation
const uploadFiles = multer({ storage: storage, limits: { fileSize: maxSize } }).array("myfiles", 10); // key name should be myfiles in postman while upload
const uploadFilesMiddleware = util.promisify(uploadFiles);
module.exports = uploadFilesMiddleware;
Below is the function which I have created in controller for upload and file check.
fileUpload = async (req, res) => {
try {
let userCode = req.headers.user_code;
await upload(req, res);
if (req.files.length <= 0) {
return res.status(httpStatusCode.OK).send(responseGenerators({}, httpStatusCode.OK, 'Kindly select a file to upload..!!', true));
}
let response = [];
for (const element of req.files) {
let data = await service.addFileData(element, userCode);
response.push(data); // for file path to be stored in database
}
if (response && response.length > 0) {
return res.status(httpStatusCode.OK).send(responseGenerators(response, httpStatusCode.OK, 'File uploaded sucessfully..!!', false));
} else {
return res.status(httpStatusCode.OK).send(responseGenerators({}, httpStatusCode.OK, 'Failed to upload file kindly try later..!!', true));
}
} catch (error) {
logger.warn(`Error while fetch post data. Error: %j %s`, error, error)
return res.status(httpStatusCode.INTERNAL_SERVER_ERROR).send(responseGenerators({}, httpStatusCode.INTERNAL_SERVER_ERROR, 'Error while uploading file data', true))
}
}
and the route will go like this.
router.post('/upload/file', fileUploadController.fileUpload);
And be sure to keep same name in postman while file upload as in middleware.
The above code is in react.js. I want to do same work in node.js and the file will be upload from the public folder. main issue is how to upload image file in format like we have in frontend event.target.files[0]

How to resize and image before saving it with gridfs?

i'm a beginner with mongoDB and gridfs. I have a code which save in the database an incoming image. But I'd like to resize the image before saving it. It tried to use sharp, but I don't really understand how it works.
I tried the following code, but it's not working
const storage = new GridFsStorage({
url: mongoURI,
file: (req, file) => {
return new Promise((resolve, reject) => {
const filename = req.params.userID;
const fileInfo = {
filename: filename,
bucketName: "ProfilePic",
};
resolve(fileInfo);
});
}
});
const upload = multer({ storage });
router.post("/setProfilePic/:userID", upload.single("picture"), async(req, res) => {
//code not working
await sharp(req.file.path)
.resize(500)
.jpeg({quality: 50})
.toFile(
path.resolve(req.file.destination,'resized',image)
)
// end of non working code
return res.status(200).json({state: "ok"})
});
My solution is to use the fromStream method from 'multer-gridfs-storage'
The idea goes like this:
Use multer to parse the file, get the file buffer (raw data)
Use sharp to catch the input buffer and create another buffer that contains the resized image.
Use Readable to create a readable stream, then apply fromStream method to pipe the stream to the database.
Implementation
const multer = require('multer')
const { GridFsStorage } = require("multer-gridfs-storage");
const { Readable } = require("stream"); // from nodejs
const sharp = require("sharp");
const storage = new GridFsStorage({
url: mongoURI,
options: { useUnifiedTopology: true }, // silence the warning
file: () => ({ bucketName: "ProfilePic",filename: req.params.userID }),
});
router.post("/setProfilePic/:userID",multer().single('picture'),(req,res) => {
const file = req.file
const buffer = file.buffer
// chain sharp methods
sharp(buffer)
.resize(500)
.jpeg({ quality: 50 })
.toBuffer((err, data, info) => {
// data here directly contains the buffer object.
const fileStream = Readable.from(data);
// write the resized stream to the database.
storage.fromStream(fileStream, req, file).then(() => {
res.json({ state: 'ok' });
});
});
})
Instead of making full use of the multer's storage engine, I create my own stream.
I am also a MongoDB beginner, so I am happy enough to make it work. If it doesn't work, please leave a comment and I will try my best to solve it.
I used the previous answer. I made a few changes to make it work. My req.file.buffer was always undefined. So I made few changes.
There are two things I want to highlight below
Make sure you use multer version "^2.0.0-rc.1".
You can create buffer by
const buffer = await getStream.buffer(req.file.stream);
rest of solution is exactly same. Here is the full code sample.
const multer = require('multer')
const { GridFsStorage } = require("multer-gridfs-storage");
const { Readable } = require("stream"); // from nodejs
const sharp = require("sharp");
const getStream = require("get-stream");
const storage = new GridFsStorage({
url: mongoURI,
options: { useUnifiedTopology: true }, // silence the warning
file: () => ({ bucketName: "ProfilePic",filename: req.params.userID }),
});
app.post("/setProfilePic/:userID", multer.single("image"), async (req, res) => {
const file = req.file;
const buffer = await getStream.buffer(req.file.stream);
// chain sharp methods
sharp(buffer)
.resize(200, 100)
.jpeg({ quality: 50 })
.toBuffer((err, data, info) => {
// data here directly contains the buffer object.
const fileStream = Readable.from(data);
// write the resized stream to the database.
storage.fromStream(fileStream, req, file).then(() => {
res.json({ state: "ok" });
});
});
});

uploading images in mlab

currently i am storing images which are uploaded by node rest server in local directory "/uploads" . this is continuously increasing my repo size .
to avoid this , i want to store image files in mongoDB atlas or mlab just like service.
const express = require("express");
const router = express.Router();
const mongoose = require("mongoose");
const multer = require('multer');
const storage = multer.diskStorage({
destination: function(req, file, cb) {
cb(null, './uploads/');
},
filename: function(req, file, cb) {
cb(null, new Date().toISOString() + file.originalname);
}
});
const fileFilter = (req, file, cb) => {
// reject a file
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png')
{
cb(null, true);
} else {
cb(null, false);
}
};
const upload = multer({
storage: storage,
limits: {
fileSize: 1024 * 1024 * 5
},
fileFilter: fileFilter
});
Please do help me in this. Thanks in advance.
You can achieve this by using a mongoose Schema and the fs core module to encode the image and unlink the file from /uploads.
I would start by creating a Mongoose Schema to set the model of all information you want to store pertaining to your uploaded file.
I'm going to use base64 encoding for this example.
uploadModel.js
const mongoose = require('mongoose');
const fs = require('fs');
const Schema = mongoose.Schema;
mongoose.set('useCreateIndex', true);
let uploadSchema = new Schema({
name: {
type: String,
},
mimetype: {
type: String,
},
size: {
type: Number,
},
base64: {
type: String,
}
})
module.exports = mongoose.model('upload',uploadSchema);
After setting up a model create a function to base64 encode and module.exports that as well.
To encode your file, use fs.readFileSync(path_to_file, encode_type). After the file has been encoded and saved in a variable you can use fs.unlink(path_to_file) to delete the file out of your /uploads folder.
uploadModel.js
module.exports.base64_encode = function(file) {
return new Promise((resolve, reject) => {
if(file == undefined){
reject('no file found');
} else {
let encodedData = fs.readFileSync(file, 'base64');
fs.unlink(file);
resolve(encodedData.toString('base64'));
}
})
}
Now inside your route file require your model.
route.js
const Upload = require('path_to_uploadModel');
router.post('/path_to_upload', upload.single('form_name_of_file'), (req, res) => {
let img = req.file;
let model = new Upload({
name: img.originalname,
size: img.size,
mimetype: img.mimetype,
})
Upload.base64_encode(img.path)
.then((base64) => {
model['base64'] = base64;
model.save((err)=> {
if(err) throw err;
});
}
})
Hope this helps

Resources