I am trying to perform bulk upload in node js and mongodb is my db,can anyone suggest me some best articles regarding this.Thanks in advance.
You can use Model.collection.insert or Model.insertMany as below where collections is array of items in bulk.
Model.collection.insert(collections, function (err, models) {
next(err, models);
});
OR,
Model.insertMany(collections, function (err, models) {
next(err, models);
});
Mongoose reference: http://mongoosejs.com/docs/api.html#model_Model.insertMany
Mongo reference: https://docs.mongodb.com/v3.2/reference/method/db.collection.insert/
You can insert multiple records with batch/bulk insert in mongoose.
var arr = [{ name: 'Star Wars' }, { name: 'The Empire Strikes Back' }];
Movies.insertMany(arr, function(error, docs) {});
Let's say i have an excel file employees.xlsx with following data and i want perform bulk write.
There are several libraries out there for converting excel data to json in node, i use xlsx but it's personal taste you can use whatever is convenient for you.
Here is a helper i use for reading "/public/employees.xlsx" file.I found the content from here here.
//** helper/excel-reader.js **//
var excelReader = {};
excelReader.readExcel = function(filePath){
var XLSX = require('xlsx');
var workbook = XLSX.readFile(filePath);
var sheet_name_list = workbook.SheetNames;
var data = [];
sheet_name_list.forEach(function(y) {
var worksheet = workbook.Sheets[y];
var headers = {};
for(z in worksheet) {
if(z[0] === '!') continue;
//parse out the column, row, and value
var tt = 0;
for (var i = 0; i < z.length; i++) {
if (!isNaN(z[i])) {
tt = i;
break;
}
};
var col = z.substring(0,tt);
var row = parseInt(z.substring(tt));
var value = worksheet[z].v;
//store header names
if(row == 1 && value) {
headers[col] = value;
continue;
}
if(!data[row]) data[row]={};
data[row][headers[col]] = value;
}
//drop those first two rows which are empty
data.shift();
data.shift();
});
return data;
}
module.exports = excelReader;
Now the employee model somehow looks like this.
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var employee = new Schema({
name: String,
adderess: String,
phonenumber: String
});
module.exports = mongoose.model('Employee', employee);
Now let's use the above code so here is my users.js route whenever i type "localhost:3000/users" it write the csv content to database.
var express = require('express');
var router = express.Router();
var excelReader = require('../helpers/excel-reader');
var mongoose = require('mongoose');
var Employee = require('../models/employee');
/* GET users listing. */
router.get('/', function(req, res, next) {
var employeesJsonArray = excelReader.readExcel('./public/employees.xlsx')
Employee.insertMany(employeesJsonArray,function(error, docs) {
if(error){
next(error)
}
else{
res.json(docs);//just rendering the document i got
}
});
});
module.exports = router;
Hope this helps!!
I'm learning to use the mean stack and trying to build a url shortener. I've got a module that takes the req.params.UserUrl checks and makes sure it's a valid url then creates a random number that I want to use as the short route. I can't seem to find a way to save the random number so that I can check their next url request against it. After a google search it seemed maybe the most effecient way would be to save an object in the database with the long_url and the short_url:randomNumber. My code doesn't throw any errors but when I check my heroku database it has a new entry but only has the _id and __v that mLabs generates itself. Can someone tell me where I'm going wrong.
Route File
var express = require('express');
var router = express.Router();
var mongoose = require('mongoose');
var URLShortener = require(process.cwd()+'/public/Modules/urlShortener.module.js');
var ShortURL = require('../models/shortUrl.js');
router.get('/', function(req, res) {
res.render('index', { title: 'FreeCodeCamp Projects' });
});
router.get('/urlShortener', function(req, res){
res.render('freecodecamp/urlShortener', { title: 'Url Shortener Site'});
});
router.get('/urlShortener/:userUrl', function(req, res){
if(URLShortener.checkValidUrl(req.params.userUrl))
{
var UserUrl = req.params.userUrl;
var randNbr = URLShortener.assignRanNbr();
ShortURL.create(URLShortener.createUrlObj(UserUrl, randNbr), function (err, smallUrl) {
if (err) return console.log(err);
else res.json(smallUrl);
});
}
else
{
res.send('Invalid url');
}
});
router.get('/:short', function(req, res){
if(randNbr == req.params.short)
{
res.redirect(userUrl);
}
else
{
res.send('Not the correct shortcut');
}
});
module.exports = router;
Url Schema
var mongoose = require('mongoose')
var Schema = mongoose.Schema
var shortUrlSchema = new Schema({
long_id:String,
short_id:Number
}, {collection: 'shortUrl'});
module.exports = mongoose.model('shortUrl', shortUrlSchema);
urlShortener Module
'use strict'
module.exports.checkValidUrl = function(url){
var pattern = new RegExp(/((([A-Za-z]{3,9}:(?:\/\/)?)(?:[-;:&=\+\$,\w]+#)?[A-Za-z0-9.-]+|(?:www.|[-;:&=\+\$,\w]+#)[A-Za-z0-9.-]+)((?:\/[\+~%\/.\w-_]*)?\??(?:[-\+=&;%#.\w_]*)#?(?:[\w]*))?)/);
return pattern.test(url);
}
module.exports.assignRanNbr = function(){
var randNbr = Math.floor(Math.random() * (9999 - 1 + 1)) + 1;
return randNbr;
}
module.exports.createUrlObj = function(url, num){
var urlObj = {};
urlObj.original_url = url;
urlObj.short_url = 'https://rawlejuglal-me-rawlejuglal-1.c9users.io/freecodecamp/'+num;
return urlObj;
}
Your createUrlObj method is returning an object with the properties original_url and short_url, but your shortUrlSchema properties are long_id and short_id. The property names in your create method need to match your schema. The property value types must also match your schema types (currently short_url is a string and short_id is a number). I think what you really want is for your createUrlObj method to be
module.exports.createUrlObj = function(url, num){
var urlObj = {};
urlObj.long_url = url;
urlObj.short_id = num;
return urlObj;
}
and your schema to be
var shortUrlSchema = new mongoose.Schema({
long_url: String,
short_id: Number
}, {collection: 'shortUrl'});
Additionally, your '/:short' route should have a call to the database since the randNbr and userUrl variables are not defined in that route.
router.get('/:short', function(req, res){
ShortUrl.findOne({short_id: req.params.short}, function(err, shortUrl){
if(err) res.send('Invalid Url');
res.redirect(shortUrl.long_url)
})
});
I have a connector.js file which using which I want to export dbResult object.
(function(){
var Massive = require("massive");
var connectionString = "postgres://postgres:postgres#localhost/postgres";
var db = Massive.connectSync({connectionString : connectionString});
var dbResult ;
db.query("Select * from company", function (err, data) {
dbResult = data;
console.log(data);
});
})(module.exports);
Now in Another file I am trying to get the dbResult and display the data:
var express = require("express");
var app = express();
var connectorObject = require("./Connector.js");
var Massive = require("massive");
app.get("/api/Steves",function(req,res){
res.set("Content-Type","application/json");
res.send(connectorObject.dbResult);
});
app.listen(3000);
console.log("Server Started on port 3000...");
But when I start the URL , not able to see any response .
Am I missing anything here.
What you want to do, is return a function that can be evaluated later for the result:
var Massive = require("massive");
var connectionString = "postgres://postgres:postgres#localhost/postgres";
var db = Massive.connectSync({connectionString : connectionString});
module.exports.getCompanies = function(callback) {
db.query("Select * from company", callback);
}
Then you can access it from your other files as:
var connector = require('./Connector');
connector.getCompanies(function( err, data ) {
if ( err ) return console.error( err );
console.log( data );
});
I'm trying to use mongoose to control my db logic and transactions. I already got Schema definitions and I'm exporting the models.
Howver when i try to use a model, it will fail witl a message like:
return mongoose.model('Report', reportSchema);
} has no method 'find'...
This is my Model export:
module.exports = (function() {
var mongoose = require('mongoose'),
Schema = mongoose.Schema,
ObjectId = Schema.ObjectId;
var reportSchema = mongoose.Schema({
category: ObjectId,
authorName: String,
authorEmail: String,
text: String,
address: String,
coordinates: {
type: "Point",
coordinates: [Number,Number]
},
date: {
type: Date,
default: new Date()
},
comments: Array
});
return mongoose.model('Report', reportSchema);
});
And this is how my controller functions are coded using mongoose inside:
module.exports = (function() {
var mongoose = require('mongoose');
var Report = require('../models/Report');
var Category = require('../models/Category');
function _getReports (request,response,next) {
var take = request.query.take;
var skip = request.query.skip;
Report.find({}).limit(take).skip(skip).exec(function (err,reports) {
callback(err,reports,response);
});
}
function _getReport (request,response,next) {
var id = request.params.id;
Report.findById({_id: id}, function (err,report) {
callback(err,report);
});
}
function _createReport (request,response) {
var newReport = new Report();
newReport.text = request.body.text;
newReport.category = request.body.category;
newReport.author = request.session.userId;
newReport.save(function (err,savedReport) {
callback(err,savedReport._id,response);
});
}
function _updateReport (request,response) {
var id = request.params.id;
var update = request.body.report;
Report.findByIdAndUpdate(id, update, function (err,foundReport) {
callback(err,foundReport,response);
});
}
function _deleteReport (request,response) {
var id = request.params.id;
Report.findByIdAndRemove(id, function (err,foundReport) {
callback(err,foundReport,response);
});
}
function _getReports (request,response,next) {
var take = request.query.take;
var skip = request.query.skip;
Report.find({}).limit(take).skip(skip).exec(function (err,reports){
callback(err,reports,response);
});
}
function _getCategories (request,response) {
var take = request.query.take;
var skip = request.query.skip;
Report.find({}).limit(take).skip(skip).exec(function (err,reports) {
callback(err,reports,response);
});
}
function _getCategoryReports (argument) {
var _id = mongoose.Types.ObjectId(request.params.id);
Report.find({category:id},{category:false},function (err, foundReports) {
callback(err,foundReports,response);
});
}
function _createCategory (request,response) {
var newCategory = new Category();
newCategory.name = request.body.name;
newCategory.save(function (err,savedCategory) {
callback(err,savedCategory._id,response);
});
}
function _updateCategory (request,response) {
var id = request.params.id;
var update = request.body.category;
Category.findByIdAndUpdate(id, update, function (err,foundCategory) {
callback(err,foundCategory,response);
});
}
function _deleteCategory (request,response) {
var id = request.params.id;
Category.findByIdAndRemove(id, function (err,foundCategory) {
callback(err,foundCategory,response);
});
}
function callback (err,object,response) {
if (err)
response.status(500).send(JSON.stringify(err));
response.send(JSON.stringify(object));
}
var apiController = {
getReports: _getReports,
getReport: _getReport,
createReport: _createReport,
updateReport: _updateReport,
deleteReport: _deleteReport,
getCategories: _getCategories,
getCategoryReports: _getCategoryReports,
createCategory: _createCategory,
updateCategory: _updateCategory
}
return apiController;
})();
Before this, a mongoose connection is ensured:
var connectToMongoose = function (mongoose,app) {
var connect = function () {
var options = { server: { socketOptions: { keepAlive: 1 } } };
mongoose.connect( 'mongodb://localhost/data4', options);
}
mongoose.connection.on('connected', function () {
console.log('Connected to db');
app.listen(32884, function() {
console.log("Listening at \"data4 port\" #:32884");
});
})
mongoose.connection.on('error', function (err) {
console.log(err);
});
mongoose.connection.on('disconnected', function () {
console.log('Disconnected from db, attempting to connect again...');
app.close();
connect();
});
connect();
};
module.exports = connectToMongoose;
Which is invoked by require('./db/mongoose-connect.js')(mongoose,app);
What am I doing wrong?
There are a couple issues here that I caught off the bat.
First off, I don't see a mongoose.connect() line that explicitly connects your mongoose ODM to a mongo server+database. An example would be:
var mongoose = require( 'mongoose' ),
Schema = mongo.Schema,
ObjectId = mongo.Schema.ObjectId;
mongoose.connect( 'mongodb://localhost/db_name' );
Your schema export looks fine. But you're using an anonymous function as your export. Since you're doing that, your require statement needs to change a little:
var Report = require('../models/Report')();
var Category = require('../models/Category')();
Notice the () at the end of the require statements. You need to execute the function that you're defining as your model file's module.export.
EDIT: I see that you added your mongoose connect code. At this point, executing the module.exports function that you assign in the model file should allow your mongoose models to function as intended.
When you export a function;
// file: A.js
module.exports = function () {
//some logic
};
And you want to use it on another file, when you require the A file, you are importing a function and in order to use that function, you need to to invoke it.
// file: B.js
var A = require('./A.js');
A();
so your model is exporting a function
module.exports = (function() {
var mongoose = require('mongoose'),
Schema = mongoose.Schema,
ObjectId = Schema.ObjectId;
// ..
// some code
// ..
return mongoose.model('Report', reportSchema);
});
and when you are importing your model from your controller, you need to execute your imported function so that your Report variable contains the model created:
module.exports = (function() {
var mongoose = require('mongoose');
var Report = require('../models/Report') ();
I have created a gist of how you could write your code using modules without using IIFE.
https://gist.github.com/wilsonbalderrama/d5484f3f530899f101dc
actually if you download all those files on a folder and run:
$ sudo npm install
$ mocha
You could see that all the tests created for the controller are passing.
In addition you don't need to use IIFE in Node.JS since when you are creating a module because you already have an isolated scope in Node.JS using modules.
// IIFE
module.exports = (function() {
var apiController = {
getReport: function () {}
}
return apiController;
})();
In Node.JS you can export a object,
// GOOD WAY
module.exports = {
getReport: function () {}
};
Hey so I am trying to use gridfs to store profile pictures... I am not sure what I am doing wrong, I am kind of new to this as well.
This is what I got so far, but it seems to be erroring this:
TypeError: Cannot read property 'primary' of undefined
at Stream.GridStore.open (..\node_modules\mongodb\lib\mongodb\gridfs\gridstore.js:146:69)
This is my code:
var mongoose = require('mongoose'),
Schema = mongoose.Schema,
ObjectId = mongoose.Schema.Types.ObjectId;
var GridStore = require('mongodb').GridStore;
var Step = require('step');
exports.newProfilePicturePost = function(req, res) {
var db = mongoose.createConnection('mongodb://localhost/test5');
var fileId = new ObjectId();
var gridStore = new GridStore(db, fileId, "w", {
"metadata": {
category: 'image'
}, content_type: 'image'});
gridStore.chunkSize = 1024 * 256;
gridStore.open(function(err, gridStore) {
Step(
function writeData() {
var group = this.group();
for(var i = 0; i < 1000000; i += 5000) {
gridStore.write(new Buffer(5000), group());
}
},
function doneWithWrite() {
gridStore.close(function(err, result) {
console.log("File has been written to GridFS");
});
}
)
});
};
Any help or fixes I should make to my code is welcomed, also I am not sure how to specify the collection I want to store the picture in and I want to also add the userId to the picture being saved in the collection for fetching it later.