I am trying to upload excel file with 250000 rows with multer in nodejs. I am able to upload file with 50000 rows successfully but when upload file with 250000 rows which is 7.3 MB in size. My code is below--
var storage = multer.diskStorage({ //multers disk storage settings
destination: function (req, file, cb) {
cb(null, env.uploadPath)
},
filename: function (req, file, cb) {
var servFileName = Date.now()+'_'+file.originalname;//Work same as +new Date();
//cb(null, file.fieldname + '-' + datetimestamp + '.' + file.originalname.split('.')[file.originalname.split('.').length -1])
cb(null, servFileName)
}
});
var upload = multer({ //multer settings
storage: storage,
limits: { fileSize: 10000000 },
fileFilter : function(req, file, callback) { //file filter
// if (['xls','xlsx'].indexOf(file.originalname.split('.')[file.originalname.split('.').length-1]) === -1) {
// return callback(new Error('Wrong extension type'));
// }
//matching extension after filename
console.log("multer console 1");
var regex = new RegExp(/.(?=xlsx|xls|xlsx.zip|xls.zip)/);
if(regex.exec(file.originalname)){
callback(null, true);
}else{
return callback(new Error('Wrong extension type'));
}
}
}).single('file');
module.exports = upload;
I am not able to find out where is the problem. Please help.
Thanks in advance!!!
Can you try to set some value to maximum size, using:
var upload = multer({
limits: { fileSize: YOUR_MAX_SIZE},
....
})
Related
I have created a multer function that uploads a file of following type "jpeg, png, jpg and pdf" in the documents field for uploading documents like PAN, aadhar card etc.(both routes are separate)
Now using that same multer function I want to give permission to user to upload their profile picture but that should be only in image format. It should not accept pdf or txt or any other file extension.
How can I do that?
The multer function:
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, "./src/uploads/");
},
filename: function (req, file, cb) {
cb(
null,
file.fieldname + "-" + Date.now() + path.extname(file.originalname)
);
},
});
var upload = multer({
storage: storage,
fileFilter: function (req, file, cb) {
var filetypes = /jpeg|jpg|png|pdf/;
var mimetype = filetypes.test(file.mimetype);
var extname = filetypes.test(path.extname(file.originalname).toLowerCase());
if (mimetype && extname) {
return cb(null, true);
}
cb("Please upload valid file");
},
});
both uploading documents fields are different. For uploading documents the field is documents and for display picture it is profile_picture.
The uploadDocuments service:
exports.uploadDocuments = async (_id, file) => {
let document = await User.findByIdAndUpdate(
{ _id },
{
$set: { registration_process_value: 99 },
$push: { documents: { $each: file } },
},
{ new: true }
);
return document;
};
I wonder how to stop uploading file or creating a folder when error happens with multer.
Is there any way to solve this issue?
Here is my code of multer.diskStorage configuration.
const storage = multer.diskStorage({
destination: (req, file, cb) => {
const isValid = FILE_TYPE_MAP[file.mimetype];
let uploadError = new Error('Invalid image type');
if (!fs.existsSync(userPath)) {
fs.mkdirSync(userPath, { recursive: true });
}
if (isValid) {
uploadError = null;
}
cb(uploadError, userPath);
},
filename: (req, file, cb) => {
const extension = FILE_TYPE_MAP[file.mimetype];
cb(null, `${uniqueId}-${Date.now()}-${imageName}.${extension}`);
},
});
const maxSize = 1 * 1024 * 1024;
const uploadOptions = multer({
storage: storage,
limits: { fileSize: maxSize },
}); code here
const express = require("express");
const multer = require("multer");
const makeDir = require('make-dir');
const fs = require('fs');
const path = require('path');
const upload = multer({
storage: Storage,
fileFilter: (req, file, cb) => {
console.log("upload fileFilter req ::", req.file);
checkFileType(file, cb);
}
}).fields(
[
{
name:'comppic',
maxCount:1
},
{
name:'userpic',
maxCount:1
},
{
name:'usercv',
maxCount:1
}
]
);
function checkFileType(file, cb) {
console.log("file:::::::::::::::", file);
}
I am using above code to upload file. In that how do I check for size of file which is uploading and returns error if file size exceeds.
console.log("upload fileFilter req ::", req.file); // output undefined
console.log("file:::::::::::::::", file);
output
file::::::::::::::: {
fieldname: 'userpic',
originalname: 'DSC01416.JPG',
encoding: '7bit',
mimetype: 'image/jpeg'
}
Please help me to resolve it.
Different file size limitation based on file type, there is size of file in req.rawHeaders so I write a logic in fileFilter based on type file, It's work to me,multer set .any for multer. req.rawHeaders is array, one of the item is size of file, in my request,size of file is in last item in Array so I used req.rawHeaders.slice(-1)[0]
in this logic, size of .pdf < 1 Mb and .png,.jpeg,.jpg < 4 Mb
const multer = require('multer');
const { v4: uuid } = require("uuid");
const TYPE_IMAGE = {
'image/png': 'png',
'image/jpeg': 'jpeg',
'image/jpg': 'jpg'
};
const TYPE_File = {
'application/pdf': 'pdf',
};
const fileUpload =
multer({
limits: 500000,
storage: multer.diskStorage({
destination: (req, file, cb) => {
cb(null, 'uploads/images');
},
filename: (req, file, cb) => {
const ext = (TYPE_IMAGE[file.mimetype]) ? TYPE_IMAGE[file.mimetype] : TYPE_File[file.mimetype];
cb(null, uuid() + '.' + ext);
}
}),
fileFilter: (req, file, cb) => {
let size = +req.rawHeaders.slice(-1)[0]
let isValid =false;
if(!!TYPE_IMAGE[file.mimetype] && size < 4 * 1024 * 1024 ){
isValid = true
}
if(!!TYPE_File[file.mimetype] && size < 1 * 1024 * 1024 ){
isValid = true
}
let error = isValid ? null : new Error('Invalid mime type!');
cb(error, isValid);
}
}).any();
module.exports = fileUpload;
I have an issue, I succeed to upload a picture to a folder, but how to upload to Google Cloud?
Here is my code:
const multer = require('multer');
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, './uploads/');
},
filename: function (req, file, cb) {
const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9);
const original = file.originalname; //original name yg di upload
const ext = original.substr(original.length - 5);//ambil 5 string terakhir pasti itu sudah termasuk extension
const regex = /[^\w\s]/g;//temukan selain word/kata atau whitepsace
const dot = ext.search(regex);
cb(null, uniqueSuffix + ext.substr(dot));//ambil setelah . sebagai extension
}
});
const fileFilter = (req, file, cb) => {
// jika jpg dan png maka di terima, selain itu riject
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png') {
cb(null, true);
} else {
cb(null, false);
}
}
upload_image = multer({
storage: storage,
limits: {fileSize: 1024 * 1024 * 5},
fileFilter: fileFilter
});
module.exports = upload_image;
What I need is only change/upload to Google Cloud.
Thank you in advance!
finally i found the solution
the basic idea is, first i upload picture to /upload folder, after that i upload from /upload to google cloud, last i delete file in /upload folder, but for delete i put in the end of middle ware cause if delete after upload to google cloud, file is deleted before upload to google cloud finish
this my complete code
upload to /upload folder
const multer = require('multer');
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, './uploads/');
},
filename: function (req, file, cb) {
const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9);
const original = file.originalname; //original name yg di upload
const ext = original.substr(original.length - 5);//ambil 5 string terakhir pasti itu sudah termasuk extension
const regex = /[^\w\s]/g;//temukan selain word/kata atau whitepsace
const dot = ext.search(regex);
cb(null, uniqueSuffix + ext.substr(dot));//ambil setelah . sebagai extension
}
});
const fileFilter = (req, file, cb) => {
// jika jpg dan png maka di terima, selain itu riject
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png') {
cb(null, true);
} else {
cb(null, false);
}
}
upload_image = multer({
storage: storage,
limits: {fileSize: 1024 * 1024 * 5},
fileFilter: fileFilter
});
module.exports = upload_image;
this is code to upload to google cloud
const util = require('util')
const gc = require('../../config')
const bucket = gc.bucket('mybucket')//bucket name di google drive
const path = require('path')
const { format } = util
//promise di panggil di midleware
const uploadImage = (files) => new Promise((resolve, reject) => {
//perlu di looping di sini karena yang di kirim berupa array
let arrayPicture = [];
files.forEach(element => {
const {filename} = element
// console.log(filename);
const picture = path.join(__dirname,'../../uploads',filename);
//ini peruntah untuk uplaod
bucket.upload(picture);
// ini yang di kirim ke return
// const publicUrl = format(
// `https://storage.googleapis.com/${bucket.name}/${filename}`
// )
arrayPicture.push(picture);
});
resolve(arrayPicture);
reject(err=>(err))
})
module.exports = uploadImage
this is my code to delete in /upload folder
const fs = require('fs');
module.exports = (req,res,next)=>{
// const arrayPicture = req.picture;
const removeImage = (arrayPicture)=> new Promise( (resolve,rejects)=>{
arrayPicture.forEach(element => {
fs.unlink(element,(err)=>{
if(err){
console.log(err);
res.status(500).json({
message:"remove data from folder fail",
error: err, status:500,
});
}
});
});
const message = "unlink succed"
resolve(message);
rejects(err=>(err))
});
removeImage(req.picture)
.then(result=>{
// res.send(result);
next();
})
.catch(err => {
console.log(err);
res.status(500).json({
error: err, status:500, message:"unlink unsucced"
});
})
}
please note that i send array of picture name in req.picture with location directory
and this is my route midleware
router.post("/", checkAuth, checkProfile, UploadService.array('productImage',4), PostImageToCloud
,ProductPostController,ProductQuantityPostController ,ProductImagePostController ,RemoveImageWasUpload
,ProductResource);
I'm making a simple file upload system with multer:
var maxSize = 1 * 1000 * 1000;
var storage = multer.diskStorage({
destination: function (req, file, callback) {
callback(null, 'public/upload');
},
filename: function (req, file, callback) {
callback(null, file.originalname);
},
onFileUploadStart: function(file, req, res){
if(req.files.file.length > maxSize) {
return false;
}
}
});
var upload = multer({ storage : storage}).single('bestand');
router.post('/upload',function(req,res){
upload(req,res,function(err) {
if(err) {
return res.end("Error uploading file.");
}
console.log(req.file);
res.redirect(req.baseUrl);
});
});
This all works fine and the file gets uploaded. The only thing that is not working is the limit on the max size. I made it so that onfileupload start the size of the file gets checked and if its to big it will return false. But the file still just gets uploaded.
It seems that onFileUploadStart isn't doing anything at all. I tried to console.log something in it, but nothing.
What am I doing wrong? How can I limit the file size when uploading with multer?
There is no onFileUploadStart with the new multer API. If you want to limit the file size, you should instead add limits: { fileSize: maxSize } to the object passed to multer():
var upload = multer({
storage: storage,
limits: { fileSize: maxSize }
}).single('bestand');
I think it is you are looking.
Have a great day.
const fileFilterMiddleware = (req, file, cb) => {
const fileSize = parseInt(req.headers["content-length"])
if ((file.mimetype === "image/png" || file.mimetype === "image/jpg" || file.mimetype === "image/jpeg" || file.mimetype === "application/octet-stream") && fileSize <= 1282810) {
cb(null, true)
} else if (file.mimetype === "video/mp4" && fileSize <= 22282810) {
cb(null, true)
} else {
cb(null, false)
}
}
For anyone who uses Multer with Ts.ED. When I tried to upload a too large file (no matter what extension it had), I ended up with the following error showing up:
Cannot read properties of undefined (reading 'replace')
I did use the following code to fix this:
#Configuration({
...
multer: {
limits: {
fieldNameSize: 300,
fileSize: 1048576, // 10 Mb
},
fileFilter: (req, file, callback) => {
const acceptableExtensions = ['.png', '.jpg'];
if (!(acceptableExtensions.includes(Path.extname(file.originalname)))) {
return callback(new Error('...'));
}
// added this
const fileSize = parseInt(req.headers['content-length']);
if (fileSize > 1048576) {
return callback(new Error('...'));
}
// --
callback(null, true);
}
}
})
Now it works.