How to perform bulk upload in mongoose (node.js) - node.js

Question

I am trying to perform bulk upload in node js and mongodb is my db,can anyone suggest me some best articles regarding this.Thanks in advance.

Answer 1

You can use Model.collection.insert or Model.insertMany as below where collections is array of items in bulk.
Model.collection.insert(collections, function (err, models) {
next(err, models);
});
OR,
Model.insertMany(collections, function (err, models) {
next(err, models);
});
Mongoose reference: http://mongoosejs.com/docs/api.html#model_Model.insertMany
Mongo reference: https://docs.mongodb.com/v3.2/reference/method/db.collection.insert/

Answer 2

You can insert multiple records with batch/bulk insert in mongoose.
var arr = [{ name: 'Star Wars' }, { name: 'The Empire Strikes Back' }];
Movies.insertMany(arr, function(error, docs) {});
Let's say i have an excel file employees.xlsx with following data and i want perform bulk write.
There are several libraries out there for converting excel data to json in node, i use xlsx but it's personal taste you can use whatever is convenient for you.
Here is a helper i use for reading "/public/employees.xlsx" file.I found the content from here here.
//** helper/excel-reader.js **//
var excelReader = {};
excelReader.readExcel = function(filePath){
var XLSX = require('xlsx');
var workbook = XLSX.readFile(filePath);
var sheet_name_list = workbook.SheetNames;
var data = [];
sheet_name_list.forEach(function(y) {
var worksheet = workbook.Sheets[y];
var headers = {};
for(z in worksheet) {
if(z[0] === '!') continue;
//parse out the column, row, and value
var tt = 0;
for (var i = 0; i < z.length; i++) {
if (!isNaN(z[i])) {
tt = i;
break;
}
};
var col = z.substring(0,tt);
var row = parseInt(z.substring(tt));
var value = worksheet[z].v;
//store header names
if(row == 1 && value) {
headers[col] = value;
continue;
}
if(!data[row]) data[row]={};
data[row][headers[col]] = value;
}
//drop those first two rows which are empty
data.shift();
data.shift();
});
return data;
}
module.exports = excelReader;
Now the employee model somehow looks like this.
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var employee = new Schema({
name: String,
adderess: String,
phonenumber: String
});
module.exports = mongoose.model('Employee', employee);
Now let's use the above code so here is my users.js route whenever i type "localhost:3000/users" it write the csv content to database.
var express = require('express');
var router = express.Router();
var excelReader = require('../helpers/excel-reader');
var mongoose = require('mongoose');
var Employee = require('../models/employee');
/* GET users listing. */
router.get('/', function(req, res, next) {
var employeesJsonArray = excelReader.readExcel('./public/employees.xlsx')
Employee.insertMany(employeesJsonArray,function(error, docs) {
if(error){
next(error)
}
else{
res.json(docs);//just rendering the document i got
}
});
});
module.exports = router;
Hope this helps!!