Adding value to MongoDB using NodeJS - node.js

currently i have data that needs to be inserted to mongodb. The data have been successfully inserted into mongodb, however there is some value that i would like to add to the data and append it into mongodb.
How can i do so? This is my code for inserting data into mongodb
var MongoClient = require('mongodb').MongoClient
, format = require('util').format;
MongoClient.connect('mongodb://127.0.0.1:27017/test', function(err, db) {
if(err) throw err;
var collection = db.collection('test_insert');
collection.insert({ values:parsestring() }, function(err, docs) {
collection.count(function(err, count) {
console.log(format("count = %s", count));
});
});
// Locate all the entries using find
collection.find().toArray(function(err, results) {
console.dir(results);
// Let's close the db
db.close();
});
});
// "1,61,54,87,20,12/3/2016,8:39AM" this default value
function parsestring(str="1,30,40,50,20,10/10/2016,10:39AM")
{
return str.split(",");
}
I would like to add value to the text string.
For example:
Machine Unit: 1,
Air Temperature °C: 30,
Water Temperature °C: 40,
Heat Temperature °C: 50,
Room Temperature °C: 20,
Date: 10/10/2016,
Time: 10:39AM

Like already mentioned in the comments, you should store your data as objects (or arrays). The line str.split(",") already returns you an array, which you store. In your code you also fetch your data as an Array. When you console.dir() your results, you could map your results.entries to a specific output string if you prefer.
collection.find().toArray(function(err, results) {
console.dir(results); // <= results.entries
// Let's close the db
db.close();
});
See also:
https://developer.mozilla.org/de/docs/Web/JavaScript/Reference/Global_Objects/Array/map &
https://developer.mozilla.org/de/docs/Web/JavaScript/Reference/Global_Objects/Array/entries
However, if you want to store objects (e.g. for accessing data via identifier) you could simply do it like:
function parsestring(str="1,30,40,50,20,10/10/2016,10:39AM"){
var dataArr = str.split(",");
var dbEntry = {};
dbEntry.machine = dataArr[0];
dbEntry.airTemp = dataArr[1];
dbEntry.waterTemp = dataArr[2];
dbEntry.heatTemp = dataArr[3];
dbEntry.roomTemp = dataArr[4];
dbEntry.date = dataArr[5];
dbEntry.time = dataArr[6];
return dbEntry;
}
When returning a simple object from mongoDB, you won't need .toArray()

Related

node js how to sort a collection

I have a mongodb collection which has a large quantity of entries. + 10000.
All of these have the same paramters:
Val_string
T1_string
T2_string
I am receiving this via a get request from my mobile app however the order is not correct.
How can I sort my collection by the Val string which is a number going from 1 to 10000 in this case.
I am using mongoose mongodb and node js.
Presently I query the collection and get 5 results send them and then loop another 5 until I have read all the results.
So I need to sort the collection out prior to sending this data.
app.get("/testmethod", function(req, res)
{
mongoose.connect("mongodb://localhost:27017/CTI", {useNewUrlParser: true},function(err, client){
if(err) {
console.log(err)
}
else
{
client.db.collection("datas").find().limit(items_to_send).skip(Sent_data).toArray(function(err, result) {
if (err) {throw err;}
//console.log(result);
Sent_data = Sent_data + items_to_send;
db.close();
var D1_T1 = result[0].T1_String;
var D1_T2 = result[0].T2_String;
var D2_T1 = result[1].T1_String;
var D2_T2 = result[1].T2_String;
var D3_T1 = result[2].T1_String;
var D3_T2 = result[2].T2_String;
var D4_T1 = result[3].T1_String;
var D4_T2 = result[3].T2_String;
var D5_T1 = result[4].T1_String;
var D5_T2 = result[4].T2_String;
res.status(200).send({
D1_T1: D1_T1,
D1_T2: D1_T2,
D2_T1: D2_T1,
D2_T2: D2_T2,
D3_T1: D3_T1,
D3_T2: D3_T2,
D4_T1: D4_T1,
D4_T2: D4_T2,
D5_T1: D5_T1,
D5_T2: D5_T2
});
});
}//End of else
});//End of connect
});
What I want is a sort to order the entire collection by the val string field which goes from 1-10000
You have as below
client.db.collection("datas").find().limit(items_to_send).skip(Sent_data)
You can add .sort({fieldName:sortOrder})
sort order: 1 / -1(desc)
client.db.collection("datas").find().limit(items_to_send)
.skip(Sent_data).sort({"val_string":1})
Sort follows lexicographical order so no problem of being string.

Retrieve data from MongoDB and save it to global object in Node.js and Express.js

I'm trying to get data from MongoDB collection and then save it to a global object.Later I need to parse it to HTML template.
Here is my code:
When user log onto his profile: then we need to get his projects and here we call findeprojects() function
usrRouter.route('/profile')
.all(function (req,res,next) {
if(!req.user){
res.redirect('/');
}
next();
})
.get(function (req,res,userObj) {
// var proj = findprojects();
userObj = req.user;
var pro = {};
pro = findprojects(userObj);
res.render('index',{name:userObj.username, email:userObj.email});
//res.sendFile('profile.html',{root:path.join(__dirname,'../public'),},{name:userObj.username});
});
Here is findeprojects function code:
var findprojects = function(obj) {
var usern = obj.username;
mongodb.connect(url,function(err, db){
if(err) throw err;
var collection = db.collection('projects');
//console.log(usern);
collection.find({'pusername':usern});
cursor =db.collection('projects').find({ 'pusername': usern }).toArray(function(err,items){
//console.log(items);
var i;
for(i=0; i<items.length;){
userProjects.createdBy = items[i].pusername;
userProjects.proName = items[i].projectName;
userProjects.proType = items[i].projectType;
userProjects.proDesc = items[i].projectDesc;
//return userProjects;
i = i+1;
}
});
console.log(userProjects);
});
};
I have declared global object at the top like:
userProjects = {
createdBy:'',
proName:'',
proType:'',
proDesc:''
};
But when I console userprojects object after calling the findeprojects() function it displays empty values.
why dont you use mongoose to model your stuff.
its more intuitive and you no need to declare the global object and do the mapping in the for loop that you are doing.
also your approach is a bit wrong in terms of when you iterate through for aren't you overwriting ?
say you have two documents where pusername is abdul.
so in your case you loose first object which will get overwritten by the second one.
i see that you commented out a return statement but even that wont work properly.
from a design point of view your approach is not efficient.
in mongoose you can do:
{
var userProjectSchema = new mongoose.Schema({
createdBy: { type: String }
, proName: String
, proType: String
, proDesc: String
});
// Find a single document by username.
userProjectSchema.findOne({ pusername : 'abdul' }, function(err, resDoc) {
if (err) return console.error(err);
// do your html stuff here
});
// Find all documents.
userProjectSchema.find(function(err, results) {
if (err) return console.error(err);
// do your html stuff here
});
}

Is my date in the correct format to Query a DateRange using MongoDB and Node JS

I've tried to find a solution looking at similar problems. One main issue that I've come across is that the Date is in a String format or not in the some altered form. I have a large file of data in JSON format. First I read through that file and convert each String to a date OBJ , the push all my data to my MongoDB.
convert to date.
var fs = require('fs');
fs.readFile('data.txt',function(err,data){
var x = [];
if(err) throw err;
var buf = new Buffer(data);
x = buf.toString();
x = JSON.parse(x);
//console.log(x);
for(var a in x){
x[a].TrxDate=new Date(x[a].TrxDate);
}
x = JSON.stringify(x);
fs.writeFile('data.txt',x,callback());
});
function callback(){
console.log('finished');
}
my date field in the DB
"TxDate": "2015-01-01T05:00:00.000Z"
Here is my query. It returns nothing found
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var url = 'blank';
MongoClient.connect(url,function(err,db){
if(err){
console.log('unable to connect to the mongodb server Error:',err);
}else {
console.log('connection good:',url);
var collection = db.collection('records');
collection.find({TxDate: { $gte: new Date('2015-07-01')}}).toArray(function(err,result){
if(err){
console.log(err);
}else if (result.length){
console.log('Found',result);
}else{
console.log('Nothing found');
}
});
}
});
What am I doing wrong here? Something I just thought of, Is it possible for me to check the type of the object int the DB?

Store values from spookyjs environment into mongoDB

I am trying to scrape data from site by spookyjs and store in mongoDB.I am able to get data from the website.But not able to save scraped data from spookyjs environment to mongoDB.To save scraped data,I passed my database model instance to spookyjs .I refered below link for it.
https://github.com/SpookyJS/SpookyJS/wiki/Introduction
Below is my code where I extracted data in prod_link_info variable and pass its values into mongoDB
var product_model = require('./product').product_model;
//get results
spooky.then([{product_model:product_model},function(){
this.waitForSelector('li[id^="product_"]', function() {
// Get info on all elements matching this CSS selector
var prod_link_info = this.evaluate(function() {
var nodes = document.querySelectorAll('li[id^="product_"]');
return [].map.call(nodes, function(node) { // Alternatively: return Array.prototype.map.call(...
return node.querySelector('a').getAttribute('href')+"\n";
});
});
//insert values in mongodb
for (var i = 0; i < prod_link_info.length; i++) {
product_model.create(
{
prod_link_info:prod_link_info[i],
}, function(err, product){
if(err) console.log(err);
else console.log(product);
});
} });
}]);
Below is the code of database schema and model used in above code.
var mongoose=require('mongoose');
var Schema = mongoose.Schema;
// create a schema
var productSchema = new Schema({
prod_link_info: String,
});
var product_model= mongoose.model('product_model', productSchema);
module.exports = {
product_model: product_model
}
But when I run above code it gives me following error ReferenceError: Can't find variable: product_model.
I want to store the data extracted from spookyjs to mongoDB.Please suggest where am I doing wrong.
When you pass hash of variables to spooky, it is converted to a string using JSON.stringify and then gets converted back to an object using JSON.parse in casper environment (please refer docs); so it is impossible to pass mongoose model to casper environment (moreover there is no actual reason for that).
To solve the problem, you should pass the data from Spooky (casper) environment. As far as I know, the only way to do is to emit data and then handle it using spooky.on. Your example should look like:
var product_model = require('./product').product_model;
//get results
spooky.then([{},function(){
this.waitForSelector('li[id^="product_"]', function() {
// Get info on all elements matching this CSS selector
var prod_link_info = this.evaluate(function() {
var nodes = document.querySelectorAll('li[id^="product_"]');
return [].map.call(nodes, function(node) { // Alternatively: return Array.prototype.map.call(...
return node.querySelector('a').getAttribute('href')+"\n";
});
});
this.emit('data.ready', prod_link_info);
});
}]);
spooky.on('data.ready', function (prod_link_info) {
//insert values in mongodb
for (var i = 0; i < prod_link_info.length; i++) {
product_model.create(
{
prod_link_info:prod_link_info[i],
}, function(err, product){
if(err) console.log(err);
else console.log(product);
});
}
});

Mongoose (mongodb) batch insert?

Does Mongoose v3.6+ support batch inserts now? I've searched for a few minutes but anything matching this query is a couple of years old and the answer was an unequivocal no.
Edit:
For future reference, the answer is to use Model.create(). create() accepts an array as its first argument, so you can pass your documents to be inserted as an array.
See Model.create() documentation
Model.create() vs Model.collection.insert(): a faster approach
Model.create() is a bad way to do inserts if you are dealing with a very large bulk. It will be very slow. In that case you should use Model.collection.insert, which performs much better. Depending on the size of the bulk, Model.create() will even crash! Tried with a million documents, no luck. Using Model.collection.insert it took just a few seconds.
Model.collection.insert(docs, options, callback)
docs is the array of documents to be inserted;
options is an optional configuration object - see the docs
callback(err, docs) will be called after all documents get saved or an error occurs. On success, docs is the array of persisted documents.
As Mongoose's author points out here, this method will bypass any validation procedures and access the Mongo driver directly. It's a trade-off you have to make since you're handling a large amount of data, otherwise you wouldn't be able to insert it to your database at all (remember we're talking hundreds of thousands of documents here).
A simple example
var Potato = mongoose.model('Potato', PotatoSchema);
var potatoBag = [/* a humongous amount of potato objects */];
Potato.collection.insert(potatoBag, onInsert);
function onInsert(err, docs) {
if (err) {
// TODO: handle error
} else {
console.info('%d potatoes were successfully stored.', docs.length);
}
}
Update 2019-06-22: although insert() can still be used just fine, it's been deprecated in favor of insertMany(). The parameters are exactly the same, so you can just use it as a drop-in replacement and everything should work just fine (well, the return value is a bit different, but you're probably not using it anyway).
Reference
Mongo documentation
Aaron Heckman on Google Groups discussing bulk inserts
Mongoose 4.4.0 now supports bulk insert
Mongoose 4.4.0 introduces --true-- bulk insert with the model method .insertMany(). It is way faster than looping on .create() or providing it with an array.
Usage:
var rawDocuments = [/* ... */];
Book.insertMany(rawDocuments)
.then(function(mongooseDocuments) {
/* ... */
})
.catch(function(err) {
/* Error handling */
});
Or
Book.insertMany(rawDocuments, function (err, mongooseDocuments) { /* Your callback function... */ });
You can track it on:
https://github.com/Automattic/mongoose/issues/723
https://github.com/Automattic/mongoose/blob/1887e72694829b62f4e3547283783cebbe66b46b/lib/model.js#L1774
Indeed, you can use the "create" method of Mongoose, it can contain an array of documents, see this example:
Candy.create({ candy: 'jelly bean' }, { candy: 'snickers' }, function (err, jellybean, snickers) {
});
The callback function contains the inserted documents.
You do not always know how many items has to be inserted (fixed argument length like above) so you can loop through them:
var insertedDocs = [];
for (var i=1; i<arguments.length; ++i) {
insertedDocs.push(arguments[i]);
}
Update: A better solution
A better solution would to use Candy.collection.insert() instead of Candy.create() - used in the example above - because it's faster (create() is calling Model.save() on each item so it's slower).
See the Mongo documentation for more information:
http://docs.mongodb.org/manual/reference/method/db.collection.insert/
(thanks to arcseldon for pointing this out)
Here are both way of saving data with insertMany and save
1) Mongoose save array of documents with insertMany in bulk
/* write mongoose schema model and export this */
var Potato = mongoose.model('Potato', PotatoSchema);
/* write this api in routes directory */
router.post('/addDocuments', function (req, res) {
const data = [/* array of object which data need to save in db */];
Potato.insertMany(data)
.then((result) => {
console.log("result ", result);
res.status(200).json({'success': 'new documents added!', 'data': result});
})
.catch(err => {
console.error("error ", err);
res.status(400).json({err});
});
})
2) Mongoose save array of documents with .save()
These documents will save parallel.
/* write mongoose schema model and export this */
var Potato = mongoose.model('Potato', PotatoSchema);
/* write this api in routes directory */
router.post('/addDocuments', function (req, res) {
const saveData = []
const data = [/* array of object which data need to save in db */];
data.map((i) => {
console.log(i)
var potato = new Potato(data[i])
potato.save()
.then((result) => {
console.log(result)
saveData.push(result)
if (saveData.length === data.length) {
res.status(200).json({'success': 'new documents added!', 'data': saveData});
}
})
.catch((err) => {
console.error(err)
res.status(500).json({err});
})
})
})
You can perform bulk insert using mongoose, as the highest score answer.
But the example cannot work, it should be:
/* a humongous amount of potatos */
var potatoBag = [{name:'potato1'}, {name:'potato2'}];
var Potato = mongoose.model('Potato', PotatoSchema);
Potato.collection.insert(potatoBag, onInsert);
function onInsert(err, docs) {
if (err) {
// TODO: handle error
} else {
console.info('%d potatoes were successfully stored.', docs.length);
}
}
Don't use a schema instance for the bulk insert, you should use a plain map object.
It seems that using mongoose there is a limit of more than 1000 documents, when using
Potato.collection.insert(potatoBag, onInsert);
You can use:
var bulk = Model.collection.initializeOrderedBulkOp();
async.each(users, function (user, callback) {
bulk.insert(hash);
}, function (err) {
var bulkStart = Date.now();
bulk.execute(function(err, res){
if (err) console.log (" gameResult.js > err " , err);
console.log (" gameResult.js > BULK TIME " , Date.now() - bulkStart );
console.log (" gameResult.js > BULK INSERT " , res.nInserted)
});
});
But this is almost twice as fast when testing with 10000 documents:
function fastInsert(arrOfResults) {
var startTime = Date.now();
var count = 0;
var c = Math.round( arrOfResults.length / 990);
var fakeArr = [];
fakeArr.length = c;
var docsSaved = 0
async.each(fakeArr, function (item, callback) {
var sliced = arrOfResults.slice(count, count+999);
sliced.length)
count = count +999;
if(sliced.length != 0 ){
GameResultModel.collection.insert(sliced, function (err, docs) {
docsSaved += docs.ops.length
callback();
});
}else {
callback()
}
}, function (err) {
console.log (" gameResult.js > BULK INSERT AMOUNT: ", arrOfResults.length, "docsSaved " , docsSaved, " DIFF TIME:",Date.now() - startTime);
});
}
You can perform bulk insert using mongoDB shell using inserting the values in an array.
db.collection.insert([{values},{values},{values},{values}]);
Sharing working and relevant code from our project:
//documentsArray is the list of sampleCollection objects
sampleCollection.insertMany(documentsArray)
.then((res) => {
console.log("insert sampleCollection result ", res);
})
.catch(err => {
console.log("bulk insert sampleCollection error ", err);
});

Resources