How to upload and read excel file in nodejs? - node.js

I want to build an API "/upload/excel" that will allow users to import an excel file and inside it after receiving an excel file, it will read its field and save it into database.
How to achieve this?

I am using multer for uploading the file and xlsx to process it and mongodb as a database (mongoose for the model):
Lead is the data model, you have to add the Excel sheet columns name. See mongoose documentation for more information
const express = require("express");
const multer = require("multer");
const connectDB = require("./config/db");
const Lead = require("./models/Lead");
connectDB();
const uploadXLSX = async (req, res, next) => {
try {
let path = req.file.path;
var workbook = XLSX.readFile(path);
var sheet_name_list = workbook.SheetNames;
let jsonData = XLSX.utils.sheet_to_json(
workbook.Sheets[sheet_name_list[0]]
);
if (jsonData.length === 0) {
return res.status(400).json({
success: false,
message: "xml sheet has no data",
});
}
let savedData = await Lead.create(jsonData);
return res.status(201).json({
success: true,
message: savedData.length + " rows added to the database",
});
} catch (err) {
return res.status(500).json({ success: false, message: err.message });
}
};
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, "uploads");
},
filename: function (req, file, cb) {
cb(null, Date.now() + "-" + file.originalname);
},
});
const upload = multer({ storage: storage });
app.post("/upload", upload.single("xlsx"), uploadXLSX);
const port = process.env.PORT || 5000;
const server = app.listen(port, () => {
console.log("app running on port", port);
});
So from here when you make a call to localhost:5000/upload with postman see picture below

You can upload a file through multer or formidable
https://www.npmjs.com/package/multer
https://www.npmjs.com/package/formidable
And you can read xl files though any one of these below npm packges
https://www.npmjs.com/package/xlsx
https://www.npmjs.com/package/read-excel-file

Related

Express Multer File Upload

I am trying to use Multer file upload in my React/Express application. But, I am getting an error that says that the file object is undefined. I have moved all of my Express server api functions into their own file, which has been working fine. I would like to keep the file upload API function in the same file as the rest of the API functions. This is what I have:
server.js
const express = require("express");
var cors = require("cors");
const config = require('config');
const { errorHandler } = require("./middleware/errorMiddleware");
const PORT = config.get('app_port') || 8000;
const app = express();
app.use(express.json());
app.use(cors());
app.use(express.urlencoded({ extended: false }));
app.use("/api/meshnodes", require("./routes/myCrudRoutes"));
app.listen(PORT, () => console.log(`Server started on port ${PORT}`));
app.use(errorHandler);
myCrudRoutes.js
const express = require('express')
const router = express.Router()
const {uploadFile} = require('../controllers/myCrudContoller')
router.post('/catalog/files/upload/', uploadFile)
module.exports = router
myCrudControllers.js
const ansyncHandler = require("express-async-handler");
const multer = require("multer");
let storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, "./uploads");
},
filename: function (req, file, cb) {
let extension = getFileExtention(file.mimetype);
cb(null, file.fieldname + "-" + Date.now() + "." + extension);
},
});
const upload = multer({ storage: storage });
const uploadFile = (upload.single("File"), (req,res, next)=>{
console.log("got file2 ")
const file = req.body;
console.log(req)
if (!file) {
const error = new Error("No File");
error.httpStatusCode = 400;
return next(error);
}
console.log("server upload ")
});
I believe the issue is with my myCrudControllers.js uploadFile function. If I have this same functionality placed directly in my server.js file, like this:
app.post(
"/catalog/files/upload",
upload.single("File"),
(req, res, next) => {
const file = req.file;
//...
It works fine, but I want to be consistent in where I have my API functions.
thanks
I refactored my code and this works:
First, I created a helper file:
uploader.js
const multer = require('multer');
let storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, process.env.REACT_APP_UPLOAD_LOCATION);
},
filename: function (req, file, cb) {
let extension = getFileExtention(file.mimetype);
cb(null, file.fieldname + "-" + Date.now() + "." + extension);
},
});
const upload = multer({ storage: storage });
myCrudRoutes.js
const uploadHelper = require('../helpers/uploader');
router.post('/catalog/files/upload', uploadHelper.upload.single('File'), uploadFile);
myCrudControllers.js
const uploadFile = ansyncHandler(async (req,res)=>{
const file = req.file;
if (!file) {
const error = new Error("No File");
error.httpStatusCode = 400;
return error;
}
//other stuff
}, (error, req, res, next) => {
res.status(400).send({ error: error.message });
}
);

How to upload compressed image to mongodb using sharp & nodeJS

I want to upload compressed image to database. I'm using sharp ( npm package) for that purpose & using nodeJS as backend stack. I'm using multer to upload image.
My Code :
const express = require('express')
const mongoose = require('mongoose')
const path = require('path')
const cors = require('cors')
const sharp = require('sharp')
const bodyParser = require('body-parser')
const fs = require('fs')
const multer = require('multer')
const app = express()
const { HelpModel, VolunteerModel } = require('./models/model')
require('dotenv').config()
app.use(cors())
app.use(express.static('./build'))
app.use(bodyParser.urlencoded({ extended: true }));
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, 'upload-data')
},
filename: (req, file, cb) => {
cb(null, Date.now() + '-' + file.originalname);
}
})
const fileFilter = (req, file, cb) => {
if (file.mimetype.startsWith('image')) cb(null, true)
else cb('Invalid File', false)
}
const Upload = multer({ storage, fileFilter })
const dataBase = "mongodb url"
mongoose.connect(dataBase,
{ useNewUrlParser: true, useUnifiedTopology: true })
.then(() => console.log("Mongoose is connected"))
.catch(e => console.log("could not connect"))
app.post('/help', Upload.single('image'), async (req, res) => {
const unique = Date.now() + '-' + '.jpeg'
const imgPath = path.resolve(__dirname + '/upload-data/' + unique);
const resize = async () => {
await sharp(req.file.path).resize(188, 280).jpeg({
quality: 100,
chromaSubsampling: ('4:4:4')
}).toFile(imgPath, (err, info) => {
if (err) console.log(err)
else console.log(info)
})
}
resize()
const newData = new HelpModel({
location: req.body.location,
contact: req.body.contact,
about: req.body.about,
img: {
data: fs.readFileSync('upload-data/' + unique),
contentType: 'image/png'
}
})
newData.save().then( () =>console.log('UPLOADED')).catch(err=>console.log(err,'ERROR OCCURED'))
const rec = await HelpModel.find({})
// res.json(rec)
res.redirect('/help')
})
const port = process.env.PORT || 8000
app.listen(port, () => console.log(`SERVER RUNNING AT PORT ${port}`))
Code is almost working as I desire, it resizes & compress the image uploaded by user and saves it in required destination in local storage.
But while uploading file to database, it is unable to access the file from local storage.
And when I'm hardcoding the file it is able to upload to database.
I'm not able to figure out why it is happening. My guess is the code might be asyncronous & data: fs.readFileSync('upload-data/' + unique) is trying to access file before it is actually upoaded & stored to local storage. After making it syncronous it is still performing in same way.

Uploading image and video into a form to create an object

Hope you're doing well.
I have a question regarding my Node.js backend app. I am trying to upload an image and a video but I get this error :
"error": "Unexpected field" on postman
Here is my code :
create:(req,res)=>{
console.log("Test1");
console.log("Test1bis");
//const exercise=JSON.parse(req.body.exercise);
//delete exercise._id;
console.log("Test2")
var exerciseB = new Exercise({
...req.body,
image:`${req.protocol}://${req.get('host')}/images/exercise/${req.file.filename1}`,
video:`${req.protocol}://${req.get('host')}/videos/exercise/${req.file.filename2}`
})
console.log("Test3")
exerciseB.save((err,exercise)=>{
if(err){
console.log("C'est le 2eme err");
return res.status(500).json({
status:500,
message:err.message
})
}
console.log("Ca arrive jusqu'au 200");
return res.status(201).json({
status:200,
message:"Exercise Created Successfully !"
})
})
}
And this is the multer file I am using to generate the image in a separate file ( I am using a similar multer for the video) :
const multer=require('multer');
const MIME_TYPES={
'image/jpg':'jpg',
'image/jpeg':'jpg',
'image/png':'png'
}
const storage=multer.diskStorage({
destination:(req,file,callback)=>{
callback(null,'public/images/exercise');
},
filename1:(req,file,callback)=>{
var name=Math.floor(Math.random()*Math.floor(123456789)).toString();
name+=Math.floor(Math.random()*Math.floor(123456789)).toString();
name+=Math.floor(Math.random()*Math.floor(123456789)).toString();
name+=Math.floor(Math.random()*Math.floor(123456789)).toString();
name+=Date.now()+".";
const extension=MIME_TYPES[file.mimetype];
name+=extension;
callback(null,name)
}
})
module.exports=multer({storage}).single('image');
And here is my postman screenshot :
This is my other multer file for the video :
const multer=require('multer');
const MIME_TYPES={
'video/mp4':'mp4',
'video/mpeg':'mpeg',
'video/ogg':'ogv',
'video/mp2t':'ts',
'video//webm':'webm',
}
const storage=multer.diskStorage({
destination:(req,file,callback)=>{
callback(null,'public/videos/exercise');
},
filename2:(req,file,callback)=>{
var name=Math.floor(Math.random()*Math.floor(123456789)).toString();
name+=Math.floor(Math.random()*Math.floor(123456789)).toString();
name+=Math.floor(Math.random()*Math.floor(123456789)).toString();
name+=Math.floor(Math.random()*Math.floor(123456789)).toString();
name+=Date.now()+".";
const extension=MIME_TYPES[file.mimetype];
name+=extension;
callback(null,name)
}
})
module.exports=multer({storage}).single('video');
And this is my routes file :
var express = require('express');
var router = express.Router();
var exerciseController = require('../controllers/exerciseController');
const multerImage=require('../middlewares/multer-image');
const multerVideo=require('../middlewares/multer-video');
/* GET home page. */
router.get('/show', exerciseController.show);
router.post('/create',multerImage, multerVideo, exerciseController.create);
router.put('/update/:id',multerImage, exerciseController.update);
module.exports = router;
I think I can't proceed the way I do in the routes file in the second router call.
Hope you can help me ! Thank you for trying too :)
remove this line
module.exports=multer({storage}).single('image');
and set this line because image is not your name filename1/filename2 your image name
module.exports=multer({storage}).single('filename1');
This is running code but make sure you already created dir public/uploads, you edit your upload location and upload multiple file
const express = require('express');
const bodyParser = require('body-parser');
const multer = require('multer');
const app = express();
app.use(bodyParser.urlencoded({ limit: '10mb', extended: true }))
app.use(bodyParser.json({ limit: '10mb', extended: true }))
app.use(express.static('public/uploads'))
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, "public/uploads")
},
filename: function (req, file, cb) {
cb(null, file.originalname);
}
})
app.post('/', (req, res) => {
const upload = multer({ storage: storage }).single('image_url');
upload(req, res, function (err) {
if (err) {
console.log(err)
return
}
var exerciseB = new Exercise({
...req.body,
image: `${req.protocol}://${req.get('host')}/images/exercise/${req.file.originalname}`,
video: `${req.protocol}://${req.get('host')}/videos/exercise/${req.file.originalname}`
})
exerciseB.save((err, exercise) => {
if (err) {
console.log("C'est le 2eme err");
return res.status(500).json({
status: 500,
message: err.message
})
}
console.log("Ca arrive jusqu'au 200");
return res.status(201).json({
status: 200,
message: "Exercise Created Successfully !"
})
})
})
})
app.listen(9001, () => {
console.log(`listening on port ${9001}`)
});

How to upload images directly on cloudinary without storing it into local directory?

I am new in ExpressJs and working on creating api for one of a dashboard created in reactjs. There is a form in a dashboard which is collecting some of information from the users like "title", "description" and "image". I have created an express server to collect that information and to save it into mongodb. For images What I have done is that, I am uploading image to Cloudinary and storing uploaded url and public_id into database.
So after following some of tutorials I have done something like this.
index.js
const express = require("express");
const bodyParser = require("body-parser");
const cors = require("cors");
const db = require("./db");
// Api router import goes here
const sectionTypesRouter = require("./routes/section-types-router");
const app = express();
const apiPort = 3000;
app.use(bodyParser.urlencoded({ extended: true }));
app.use(cors());
app.use(bodyParser.json());
db.on("error", console.error.bind(console, "MongoDB connection error:"));
app.get("/", (req, res) => {
res.send("Hello World!");
});
app.use("/api", sectionTypesRouter);
app.listen(apiPort, () => console.log(`Server running on port ${apiPort}`));
Than, First I have create a file multer.js :
const multer = require("multer");
const storage = multer.diskStorage({
destination: "public/uploads",
filename: (req, file, cb) => {
cb(null, file.fieldname + "-" + Date.now());
},
});
const fileFilter = (req, file, cb) => {
if (file.mimetype === "image/jpeg" || file.mimetype === "image/png") {
cb(null, true);
} else {
//reject file
cb({ message: "Unsupported file format" }, false);
}
};
const upload = multer({
storage: storage,
fileFilter: fileFilter,
});
module.exports = upload;
Below is my api router section-type-router.js :
const express = require("express");
const upload = require("../utils/multer");
const SectionTypesCtrl = require("../controllers/section-types-ctrl");
const router = express.Router();
router.post(
"/section-type",
upload.single("image"),
SectionTypesCtrl.createSectionType
);
router.get("/section-types", SectionTypesCtrl.getSectionTypes);
module.exports = router;
This is the section-type-ctrl.js :
const SectionType = require("../models/section-type-model");
const fs = require("fs");
const path = require("path");
const cloudinaryUploader = require("../utils/cloudinaryUploader");
const createSectionType = async (req, res) => {
const body = req.body;
if (!body) {
return res.status(400).json({
success: false,
error: "Required parameter are missing",
});
}
cloudinaryUploader
.uploads(req.file.path, "Section-Types")
.then((result) => {
const sectionType = new SectionType({
title: body.title,
description: body.description,
image: {
url: result.url,
publicId: result.public_id,
},
});
sectionType
.save()
.then(() => {
return res.status(201).json({
success: true,
id: sectionType._id,
message: "Section type created!",
});
})
.catch((error) => {
return res.status(400).json({
error,
message: "Section type not created!",
});
});
})
.catch((error) => {
res.status(500).send({
message: "failure",
error,
});
});
};
module.exports = {
createSectionType,
};
And lastly this is cloudinaryUpload.js :
const cloudinary = require("../config/cloudinary");
exports.uploads = (file, folder) => {
return new Promise((resolve) => {
cloudinary.uploader.upload(
file,
{
resource_type: "auto",
folder: folder,
},
(err, result) => {
if (!err) {
resolve({
url: result.url,
public_id: result.public_id,
});
} else {
throw err;
}
}
);
}).catch((error) => {
throw error;
});
};
Now, everything is working properly. Images is uploading to the cloudinary and returned url and public_id is storing in database. But the problem is that image that I have uploaded is also upload on local directory public/uploads/. This will may create a storage issue while host a site. So Is there any best way to upload image directly to the cloudinary without creating a copy in local directory which also should work on production mode ?
In your example, the file is being stored to public/uploads on your server because you're telling multer to do so via multer.diskStorage
As #Molda's comment above says, you can avoid this by using the multer-storage-cloudinary package to have Multer store the file in Cloudinary automatically.
Another possibility is to change how you're using Multer so it doesn't store the file anywhere, then take the uploaded file while it's in memory and pass it to Cloudinary's SDK as a stream.
There's an example of this in this blog post on the Cloudinary site: https://cloudinary.com/blog/node_js_file_upload_to_a_local_server_or_to_the_cloud
In your case, you can stop using multer.diskStorage, in favour of just using multer() then use streamifier or another library to turn the uploaded file into a stream, and pass that to cloudinary.uploader.upload_stream()

How can i upload an image after verifying user data in express.js

I am trying to create a user profile in express.js and MongoDB. I am using multer for image uploading. Multer middleware always uploads the image before verifying my user data. If user validation is failed, nevertheless image is uploaded. But, I want to upload an image after validating user data. That means, I will check user data in the controller, and if it is valid then I will upload image and store user data to MongoDB. How can I do that? Thanks in advance!
multerConfig.js
exports.multerConfig = (multer) => {
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, './uploads/');
},
filename: (req, file, cb) => {
cb(null, 'img-' + new Date().toISOString() + '-' + file.originalname);
}
});
const fileFilter = (req, file, cb) => {
(file.mimetype === 'image/png' || file.mimetype === 'image/jpeg' || file.mimetype === 'image/jpg')
? cb(null, true)
: cb(null, false)
};
return multer({
storage: storage,
limits: { fileSize: 1048576 },
fileFilter: fileFilter
});
};
user.js(Routes)
const multer = require('multer');
const express = require('express');
const userController = require('../controllers/user');
const { multerConfig } = require('../utility/multerConfig');
const validateObjectId = require('../middleware/validateObjectId');
const asyncErrorHandler = require('../middleware/asyncErrorHandler');
const router = express.Router();
const upload = multerConfig(multer);
router.post('/', upload.single('image'), asyncErrorHandler(userController.createUser));
module.exports = router;
user.js(Controller)
const { User, validate } = require('../models/user');
const { deleteFile } = require('../utility/fileUtility');
const { failed, success } = require('../utility/utility');
exports.createUser = async (req, res) => {
const { error } = validate(req.body);
if (error) return res.status(400).send({ ...failed, message: error.details[0].message });
const { name, address, mobile, email, password } = req.body;
if (!req.file) return res.status(400).send({ ...failed, message: `you have to upload an image!` });
const isUserExist = await User.find().or([{ mobile }, { email }]);
if (isUserExist.length > 0) return res.status(409).send({ ...failed, message: `${name} is already exists!` });
const image = req.file.path;
const newUser = new User({ name, address, mobile, email, password, image });
const savedUser = await newUser.save();
if (!savedUser) return res.status(500).send({ ...failed, message: `user ${name} is failed to save!` });
res.send({
...success,
data: savedUser,
message: `user ${name} is saved successfully`
});
};
You can use two multer middleware (one for parsing text, one for uploading your file).
Let's say you have a form with a name (text field) and avatar (file field), you can do this:
var express = require('express');
var multer = require('multer');
var app = express();
var upload = multer({ dest: 'uploads/' });
app.post('/profile',
upload.none(), function (req, res, next) {
// validate `req.body.name` here
// and call next(err) if it fails
next();
},
upload.single('avatar'), function (req, res, next) {
// file is now uploaded, save the location to the database
res.end('done!');
});
app.listen(9000);

Resources