I am a bit overstrained with handling file uploads using multer. What I want to do is:
Uploading one or multiple files via ajax
Storing these files at a dynamic destination folder
Processing the contents files with a specific format (.json/.xml/.txt)
1) is already done and I managed storing these files inside of a tmp folder.
2) As far as I understood I need to store these files first and then move them to my desired location (which is a dynamic location)
3) I want to parse the files if possible, otherwise the files should just be stored as asset.
My questions:
How can I store (or move) the files at a dynamic target?
How can I process the file contents (for example to store the parsed contents in a database)?
My code:
var storage = multer.diskStorage({
destination: function (req, file, cb) {
fs.mkdirsSync(config.fileHandler.uploadTempDir)
cb(null, config.fileHandler.uploadTempDir)
},
filename: function (req, file, cb) {
cb(null, Date.now() + '-' + file.originalname)
}
})
const upload = multer({ storage: storage })
var stringsUpload = upload.fields([{ name: 'sourceStrings[]', maxCount: 8 }])
router.post('/:projectId/update/sourcestrings', isLoggedIn, stringsUpload, processSourceStrings)
function processSourceStrings(req, res, next) {
var projectId = req.params.projectId
Project.findOne({ project_id: projectId }).populate('owner').then(function (project) {
if (!project)
return res.send(404, { error: "Couldn't find a project with this id" })
if (!isAllowedToEditProject(req, project))
return res.send(403, { error: "You are not allowed to edit this project" })
// Move files to the right destination
var targetpath = 'user/1/project/4/'
// Process file
return res.status(200).send()
})
}
Related
Folder Structure image# Multer.js File
const multer = require("multer");
const path = require("path");
const fs = require("fs");
const httpStatus = require("http-status");
const ApiError = require("../utils/ApiError")
const logger = require("../utils/logger");
const multerUpload = async (req, res, next) => {
let fileName = "";
let storage = multer.diskStorage({
destination: function (req, file, callback) {
fs.mkdir(path.join(path.resolve(), "/tmp"), (err) => {
if (err) {
logger.error("mkdir tmp %o", err);
}
callback(null, path.join(path.resolve(), "/tmp"));
});
},
filename: function (req, file, callback) {
fileName = file.fieldname + "-" + req.query.eventId + Date.now() + path.extname(file.originalname);
logger.info("filename of uploadSheet===> %s", fileName);
callback(null, fileName);
},
});
// below code is to read the added data to DB from file
var upload = multer({
storage: storage,
fileFilter: function (req, file, callback) {
var ext = path.extname(file.originalname);
if (ext !== '.xlsx') {
return callback(new Error('Only Excel sheets are allowed'))
}
callback(null, true)
},
}).single("sheet");
upload(req, res, async function (err) {
if (err) {
next(new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message));
} else {
req.fileName = fileName;
next();
}
})
}
module.exports = multerUpload;
It gives error of EORFS read only file in vercel production but the code works fine in local.
I'm trying to upload the excel sheet file from the Api and then read the data from it and add it into the Mongodb.
I once encountered this same problem working with Heroku a long time ago, I haven't worked with vercel but with quick research, I will say this is the cause, vercel does not provide storage for you to upload files to in production, you need a separate service for that like Amazon S3, but there also exists Azure File Storage and Google Cloud Storage.
alternatively, if you don't want to add more services to your project, you can just convert the image to base64 string and save it as text(but need to make the field/column read-only so it does not get corrupted) NOT the best alternative but it was something I once did
To use /tmp in server functions, you should just use /tmp/your-file. Remove path.resolve().
Only if you need to store something temporarily, you may try to use /tmp directory.
Limit 512 MB + no guaranty - https://github.com/vercel/vercel/discussions/5320
I'm new with uploading files, I want to upload images of different products into Firebase storage and another file that required in the app, one product can have many images, so I want to create a folder for every product, the name of the folder will be the id of the product.
In code: I use #google-cloud/storage library to upload the file into firebase storage, but I search in documentation, no way that I can be able to create a folder then upload it to folder.
here my code :
I create middleware of multer to pass it in an endpoint, with check type of file.
const express = require("express");
const Multer = require("multer");
const { Storage } = require("#google-cloud/storage")
const storage = new Storage({
projectId: process.env.PROJECT_FIREBASE_ID,
keyFilename: "hawat-service.json",
});
const bucket = storage.bucket(process.env.BUCKET_NAME);
const multer = Multer({
storage: Multer.memoryStorage(),
fileFilter: (req, file, cb) => {
checkFileType(req, file, cb);
}
})
const checkFileType = (req ,file, cb) => {
if (file.fieldname == 'cover' || file.fieldname == 'images') {
if (!file.originalname.match(/\.(jpg|JPG|jpeg|JPEG|png|PNG|gif|GIF)$/)) {
req.error = new Error( "Only images are allowed")
return cb(null, false);
}
} else if (file.fieldname == 'card' || file.fieldname == 'licence') {
if (!file.originalname.match(/\.(pdf|jpg|JPG|jpeg|JPEG|png|PNG|gif|GIF)$/)) {
req.error = new Error("Only images and pdf are allowed")
return cb(null, false);
}
}
return cb(null, true)
}
module.exports = (req, res, next) => {
return multer.fields([{ name: 'cover', maxCount: 1 },
{ name: 'images', maxCount: 5 }, { name: 'card', maxCount: 1 },
{ name: 'licence', maxCount: 1 }
])
(req, res, () => {
if (req.error) return res.status(400).send( {message : req.error.message })
next()
})
}
the function to upload file is
const express = require("express");
const Multer = require("multer");
const { Storage } = require("#google-cloud/storage");
const storage = new Storage({
projectId: process.env.PROJECT_FIREBASE_ID,
keyFilename: "hawat-service.json",
});
const bucket = storage.bucket(process.env.BUCKET_NAME);
module.exports = {
upload: async ( file) => {
return new Promise((resolve, reject) => {
let newFileName = `${file.originalname}_${Date.now()}`;
let fileUpload = bucket.file(newFileName);
const createStream = fileUpload.createWriteStream({
metadata: {
contentType: file.mimetype
}
});
createStream.on('error', (error) => {
console.log("error in uploading is" , error)
reject('Something is wrong! Unable to upload at the moment.');
});
createStream.on('finish', () => {
// The public URL can be used to directly access the file via HTTP.
const url = `https://storage.googleapis.com/${bucket.name}/${fileUpload.name}`;
// storage.bucket(process.env.BUCKET_NAME).file(fileUpload.name).makePublic();
resolve(url);
});
createStream.end(file.buffer);
});
the endpoint is
router.post('/add-product' , auth, multer , seller.onAddProduct)
the function onAddProduct is a function that can receive multiple files from the user.
So How can I create a folder for every product, then upload files in the folder?
also, How can I delete the folder after created it?
I am not using the same method you are using but you could use my solution as a case study
await storage.bucket(bucketName).upload(filename, {
destination:"{Foldername}/{Filename}",
})
Folders in Google Cloud Storage are not really a thing. As you can see in this documentation:
gsutil provides the illusion of a hierarchical file tree atop the "flat" name space supported by the Cloud Storage service. To the service, the object gs://your-bucket/abc/def.txt is just an object that happens to have "/" characters in its name. There is no "abc"directory, just a single object with the given name
So what you see as a folder in Cloud Storage is simply another object that is emulating a folder structure, what really matters are the object paths.
In your case there are 2 ways you can go about what you want to do, you can either:
Create an emulated empty directory by creating an object that ends in a trailing slash. For example, to create a subdirectory called foo at the root of a bucket, you would create an empty object (size 0) called foo/ and then upload the file with it's full path.
Simply upload the file with it's full path including the desired "subdirectory" and when you fetch it from GCS it will look like it is located at the emulated directory.
Personally I would use the latter, as you will achieve the same results with only 1 step instead of 2.
If you want to create an empty folder in Cloud Storage, you can do this:
const userId = "your_user_id"
// Folder name. Notice the slash at the end of the path
const folderName = `users/${userId}/`;
// Create a folder
await bucket.file(folderName).save("");
After creating the new folder, you can upload your file there by setting its destination:
const destination = `${folderName}${fileName}`;
await bucket.upload(file, {
destination,
})
But actually you don't need to create a folder as a separate step. You can just set full destination for your file in bucket.upload(...) as described above.
I have a function that uses multer that uploads files:
exports.create = (req, res) => {
console.log("req.body1 : ")
console.log(req.body)
var fileFilter = function (req, file, cb) {
console.log("req.body2: ")
console.log(req.body)
// supported image file mimetypes
var allowedMimes = ['image/jpeg', 'image/pjpeg', 'image/png', 'image/gif'];
if (_.includes(allowedMimes, file.mimetype)) {
// allow supported image files
cb(null, true);
} else {
// throw error for invalid files
cb(new Error('Invalid file type. Only jpg, png and gif image files are allowed.'));
}
};
let upload = multer({
storage: multer.diskStorage({
destination: (req, file, callback) => {
console.log("req.body3 : ")
console.log(req.body)
let userId = req.params.userId;
let pathToSave = `./public/uploads/${userId}/store`;
fs.mkdirsSync(pathToSave);
callback(null, pathToSave);
},
filename: (req, file, callback) => {
callback(null, uuidv1() + path.extname(file.originalname));
}
}),
limits: {
files: 5, // allow only 1 file per request
fileSize: 5 * 1024 * 1024, // 5 MB (max file size)
},
fileFilter: fileFilter
}).array('photo');
upload(req, res, function (err) {
console.log("req.body4 : ")
console.log(req.body)
...
...
As you see there are lots of console.log that prints out info of incoming data by POST method. Strange thing is that fields other than files does not appear until it enters to last upload function.
So the problem is I can't validate things using those fields until it reaches to last upload function. So I cannot cancel and delete the uploded files if there exists any errors in other fields than files.
Following is outputs of above code:
req.body :
{}
req.body2:
{}
req.body3 :
{}
req.body4 :
{ name: '1111111111',
price: '1212',
cid: '1',
...
File uploads are done around console.log("req.body3 : "). Then it outputs other fields in console.log("req.body4 : "). I need those other fields that appears in req.body4 to validate stuffs before actually uploading the files. But I can't because those fields are retrieved after the file is uploaded.
How can I get fields of others before multer actually uploads the file?
===============================================
APPENDED:
I found that if I use .any() instead of .array('photo') then I can access both fields and files. But still, the problem is that it uploads those files first then gives me the access to the those field in uploads function at the bottom where console.log("req.body4 : ") is. so still the problem is that files get uploaded first before I need to validate using those fields.
you should be receiving an object in your body only one object so you should be able to access it like any other object
{
object1: 'something here',
object2: {
nestedObj1: {
something: 123,
arrInObj: ['hello', 123, 'cool'],
}
}
then you would be able to access these things like so:
console.log(req.body.object1) // 'something here'
console.log(req.body.object2.nestedObj1[0]) // 'hello'
console.log(req.body.object2.nestedObj1.forEach(item => console.log(item)) // 'hello' 123 'cool'
I have an application and in that application, I want to use some file upload mechanism.
My requirement is:
Once the file is uploaded, its name will be changed to something unique, like uuid4(). I will store this name in the database later.
I have written something like that, however I have a couple of questions:
const multer = require('multer');
const upload = multer();
router.post('/', middleware.checkToken, upload.single('file'), (req,res,next)=>{
// key:
// file : "Insert File Here"
console.log("req:");
console.log(req.file);
const str = req.file.originalname
var filename = str.substring(0,str.lastIndexOf('.'));
// I will use filename and uuid for storing it in the database
// I will generate unique uuid for the document and store the document
// with that name
var extension = str.substring(str.lastIndexOf('.') + 1, str.length);
// HERE!
res.status(200).json();
})
I have seen examples of storing it in the diskStorage:
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, '/tmp/my-uploads')
},
filename: function (req, file, cb) {
cb(null, file.fieldname + '-' + Date.now())
}
})
var upload = multer({ storage: storage })
However, as far as I understood, this is a configuration outside of the API call. Meaning that I cannot modify it every time I call this API. I want to assign different names to the file, and I need that name(uuid) to save that name in the database.
How can I preserve such functionality?
Thanks to #Rashomon and #Eimran Hossain Eimon, I have solved the issue. In case of anyone wonders the solution, here it is:
const multer = require('multer');
var storage = multer.diskStorage({
destination: function (req, file, cb) {
// the file is saved to here
cb(null, '/PATH/TO/FILE')
},
filename: function (req, file, cb) {
// the filename field is added or altered here once the file is uploaded
cb(null, uuidv4() + '.xlsx')
}
})
var upload = multer({ storage: storage })
router.post('/', middleware.checkToken, upload.single('file'), (req,res,next)=>{
// the file is taken from multi-form and the key of the form must be "file"
// visible name of the file, which is the original, uploaded name of the file
const name = req.file.originalname;
// name of the file to be stored, which contains unique uuidv4
const fileName = req.file.filename;
// get rid of the extension of the file ".xlsx"
const file_id = fileName.substring(0, fileName.lastIndexOf('.'));
// TODO
// Right now, only xlsx is supported
const type = "xlsx";
const myObject = new DatabaseObject({
_id : new mongoose.Types.ObjectId(),
file_id: file_id,
name : name,
type: "xlsx"
})
myObject .save()
.then(savedObject=>{
// return some meaningful response
}).catch(err=>{
// return error response
})
})
This solves my current issue. Thanks for helping. For future improvements, I'll add error case for:
In case uuidv4 returns an id which already exists(I believe it is highly unlikely since the object contains some timestamp data), rerun the renaming function.
In case there is an error in saving to the database, I should delete the uploaded file to avoid future conflicts.
If you have solutions for those problems too, I'm much appreciated.
I think you got it wrong... you said that
I cannot modify it every time I call this API.
But actually, the filename is called every time for every file. Let me explain this portion of code...
filename: function (req, file, cb) {
cb(null, file.fieldname + '-' + Date.now())
}
Here look at the callback function (denoted by cb) :
1st argument null in callback function is like convention. You always pass null as the first argument in a callback function. See this Reference
2nd argument determines what the file should be named inside the destination folder. So, here you can specify any function, which can return you a unique filename every time.
Since you are using mongoose...
I think it would be better if you implement your function uniqueFileName() using the mongoose method within your Schema (in which you want to save the file path) and call it in your route handler. Learn More
No need. Because you're using timestamps.
In case of an error in saving to the database, you can delete the uploaded file using this code to avoid future conflicts. Try this:
const multer = require('multer');
const fs = require('fs'); // add this line
var storage = multer.diskStorage({
destination: function (req, file, cb) {
// the file is saved to here
cb(null, '/PATH/TO/FILE')
},
filename: function (req, file, cb) {
// the filename field is added or altered here once the file is uploaded
cb(null, uuidv4() + '.xlsx')
}
})
var upload = multer({ storage: storage })
router.post('/', middleware.checkToken, upload.single('file'), (req,res,next)=>{
// the file is taken from multi-form and the key of the form must be "file"
// visible name of the file, which is the original, uploaded name of the file
const name = req.file.originalname;
// name of the file to be stored, which contains unique uuidv4
const fileName = req.file.filename;
// get rid of the extension of the file ".xlsx"
const file_id = fileName.substring(0, fileName.lastIndexOf('.'));
// TODO
// Right now, only xlsx is supported
const type = "xlsx";
const myObject = new DatabaseObject({
_id : new mongoose.Types.ObjectId(),
file_id: file_id,
name : name,
type: "xlsx"
})
myObject .save()
.then(savedObject=>{
// return some meaningful response
}).catch(err=>{
// add this
// Assuming that 'path/file.txt' is a regular file.
fs.unlink('path/file.txt', (err) => {
if (err) throw err;
console.log('path/file.txt was deleted');
});
})
})
also see NodeJS File System Doc
I'm uploading file using multer in my nodejs (express js) application which is working fine. I have put a mime type check there also to allow only png files but if I change the ext of the uploaded file from abc.exe to abc.png it also gets uploaded which is wrong.
here is my code.
var multer = require('multer');
var imagefolder = __base + 'public/complaintimages/';
var diskstorage = multer.diskStorage({
destination: function (req, file, cb) {
if (common.ImageMimeTypes.indexOf(file.mimetype) < 0) {
common.ActionOutput.Status = common.ActionStatus.WrongFileUploaded;
common.ActionOutput.Message = 'Invalid image file: ' + file.originalname;
cb(new Error('FileUpload:' + common.ActionStatus.WrongFileUploaded), null);
} else
cb(null, imagefolder);
},
filename: function (req, file, cb) {
var filenm = randomstring.generate(10);
//console.log(filenm + file.originalname);
cb(null, filenm + file.originalname);
}
});
var upload = multer({
storage: diskstorage
});
It should check the file content for mime type. Renaming other into png should not be uploaded. It seems to be bug in the library.
Please advice.
In your route handler when you have the saved file name, you can use the mmmagic module:
var mmm = require('mmmagic'),
var magic = new mmm.Magic(mmm.MAGIC_MIME_TYPE);
magic.detectFile(fileName, function (err, mime) {
if (err) {
// handle error
} else {
// check the mime
// and remove the file if you don't like it
// plus send a correct response to the client
}
});
Update
If mmmagic doesn't work for you then you can use the file-type module but it works on buffers so you first will have to read the file (or some part of it) into a buffer and check the mime type with file-type. The read-chunk module can be handy to read part of the file.
See:
https://www.npmjs.com/package/file-type
https://www.npmjs.com/package/read-chunk