Querying a MongoDB documents in real-time - node.js

I'm building a web application that will work with Big Data.
I will mine Twitter data using the Apache Storm, subsequently saving them in a MongoDB database.
At same time, this data has to be fetched via Node.js in real time and be sent via socket.io to my front-end.
Exist a way to querying MongoDB via Node.js in real time?
Thanks.

I am working on a project with mongoDB, I used the mongodb npm module to query the database in real time.
First I get a list of collections which are in my database:
//My server controller page
var MongoClient = require('mongodb').MongoClient
, assert = require('assert');
exports.getCollections = function(req,res){
mongoose.connection.db.collectionNames(function(err, names) {
if (err){
console.log(err)
} else {
res.status(200).send({collections: names});
}
});
};
On the front end, I do an Angular ng-repeat to list my collections, then when I click on the collection name, I run the following code:
MongoClient.connect(url, function (err, db) {
assert.equal(null, err);
var collection = db.collection(req.body.collName);
collection.find({}).limit(req.body.limit).toArray(function (err, docs) {
if (err) {
console.log(err)
} else {
res.status(200).send({r: docs, count: docs.length});
db.close();
}
})
});
Here is my client side angular code:
//get the collection list upon page load
$http.get('collections')
.success(function(c){
$scope.collList = c.collections;
})
.error(function(err){
$scope.error = err;
});
//function run when collection is selected
$scope.doFind = function(coll){
$scope.collViewing = coll;
$http.post('/basicfind',{collName: coll,limit:$scope.limiter})
.success(function(data){
$scope.results = data.r;
$scope.countOfResults = data.count;
})
.error(function(err){
$scope.error = err.message;
});
};
Hope that helps, let me know if you need me to share any more code

Related

How to continuously query a mongodb database efficiently

I'm new to Node and MongoDB and I have a seemingly simple request. I've managed to connect to my database, and use a query to get my desired results. Now, I want to have this query continue indefinitely, since the end goal for my project is to plot data real time.
I would have thought a simple 'while (true)' loop would suffice, but that doesn't seem to be the case.
const MongoClient = require('mongodb').MongoClient;
// Connection URL
const url = 'mongodb://<username>:<password>#ds157614.mlab.com:57614/flight_data';
// Use connect method to connect to the Server
MongoClient.connect(url, { useNewUrlParser: true }, function(err, db) {
if (err) throw err;
var dbo = db.db("flight_data").collection("data");
while(true)
{
dbo.find().sort({_id: 1}).limit(1).toArray(function(err, result) {
if (err) throw err;
console.log("Temperature: " + result[0].data.temperature);
});
}
db.close();
});
I have found that the while loop is indeed running, but for some reason, the query just doesn't happen when inside the while loop. If you remove the while loop, the code functions fine. I just want it to continually print the results of the query being repeated.
Querying a DB continuously is inefficient and resource wasting, instead use change streams. It watches collection for any changes and will make the db call then only. Works only for Mongo 3.6+.
const MongoClient = require("mongodb").MongoClient;
// Connection URL
const url =
"mongodb://<username>:<password>#ds157614.mlab.com:57614/flight_data";
// Use connect method to connect to the Server
MongoClient.connect(url, { useNewUrlParser: true }, function(err, db) {
if (err) throw err;
const collection = db.collection("data");
const changeStream = collection.watch();
changeStream.on("change", next => {
// process next document
collection
.find()
.sort({ _id: 1 })
.limit(1)
.toArray(function(err, result) {
if (err) throw err;
console.log("Temperature: " + result[0].data.temperature);
});
});
db.close();
});

db.getSiblingDB is not a function [duplicate]

I have been trying W3schools tutorial on nodeJS with MongoDB.
When I try to implement this example in a nodeJS environment and invoke the function with an AJAX call, I got the error below:
TypeError: db.collection is not a function
at c:\Users\user\Desktop\Web Project\WebService.JS:79:14
at args.push (c:\Users\user\node_modules\mongodb\lib\utils.js:431:72)
at c:\Users\user\node_modules\mongodb\lib\mongo_client.js:254:5
at connectCallback (c:\Users\user\node_modules\mongodb\lib\mongo_client.js:933:5)
at c:\Users\user\node_modules\mongodb\lib\mongo_client.js:794:11
at _combinedTickCallback (internal/process/next_tick.js:73:7)
at process._tickCallback (internal/process/next_tick.js:104:9)
Please find below my implemented code:
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/mytestingdb";
MongoClient.connect(url, function(err, db) {
if (err) throw err;
db.collection("customers").findOne({}, function(err, result) {
if (err) throw err;
console.log(result.name);
db.close();
});
});
Note that the error occurs whenever the execution hits:
db.collection("customers").findOne({}, function(err, result) {}
Also, note (in case it matters) that I have installed the latest MongoDB package for node JS (npm install mongodb), and the MongoDB version is MongoDB Enterprise 3.4.4, with MongoDB Node.js driver v3.0.0-rc0.
For people on version 3.0 of the MongoDB native NodeJS driver:
(This is applicable to people with "mongodb": "^3.0.0-rc0", or a later version in package.json, that want to keep using the latest version.)
In version 2.x of the MongoDB native NodeJS driver you would get the database object as an argument to the connect callback:
MongoClient.connect('mongodb://localhost:27017/mytestingdb', (err, db) => {
// Database returned
});
According to the changelog for 3.0 you now get a client object containing the database object instead:
MongoClient.connect('mongodb://localhost:27017', (err, client) => {
// Client returned
var db = client.db('mytestingdb');
});
The close() method has also been moved to the client. The code in the question can therefore be translated to:
MongoClient.connect('mongodb://localhost', function (err, client) {
if (err) throw err;
var db = client.db('mytestingdb');
db.collection('customers').findOne({}, function (findErr, result) {
if (findErr) throw findErr;
console.log(result.name);
client.close();
});
});
I encountered the same thing. In package.json, change mongodb line to "mongodb": "^2.2.33". You will need to uninstall mongodb npm by removing MongoDB Driver/ node_modules or etc , then install npm to install this version.
This resolved the issue for me. Seems to be a bug or docs need to be updated.
For those that want to continue using version ^3.0.1 be aware of the changes to how you use the MongoClient.connect() method. The callback doesn't return db instead it returns client, against which there is a function called db(dbname) that you must invoke to get the db instance you are looking for.
const MongoClient = require('mongodb').MongoClient;
const assert = require('assert');
// Connection URL
const url = 'mongodb://localhost:27017';
// Database Name
const dbName = 'myproject';
// Use connect method to connect to the server
MongoClient.connect(url, function(err, client) {
assert.equal(null, err);
console.log("Connected successfully to server");
const db = client.db(dbName);
client.close();
});
MongoClient.connect(url (err, client) => {
if(err) throw err;
let database = client.db('databaseName');
database.collection('name').find()
.toArray((err, results) => {
if(err) throw err;
results.forEach((value)=>{
console.log(value.name);
});
})
})
The only problem with your code is that you are accessing the object that's holding the database handler. You must access the database directly (see database variable above). This code will return your database in an array and then it loops through it and logs the name for everyone in the database.
Piggy backing on #MikkaS answer for Mongo Client v3.x, I just needed the async / await format, which looks slightly modified as this:
const myFunc = async () => {
// Prepping here...
// Connect
let client = await MongoClient.connect('mongodb://localhost');
let db = await client.db();
// Run the query
let cursor = await db.collection('customers').find({});
// Do whatever you want on the result.
}
I did a little experimenting to see if I could keep the database name as part of the url. I prefer the promise syntax but it should still work for the callback syntax. Notice below that client.db() is called without passing any parameters.
MongoClient.connect(
'mongodb://localhost:27017/mytestingdb',
{ useNewUrlParser: true}
)
.then(client => {
// The database name is part of the url. client.db() seems
// to know that and works even without a parameter that
// relays the db name.
let db = client.db();
console.log('the current database is: ' + db.s.databaseName);
// client.close() if you want to
})
.catch(err => console.log(err));
My package.json lists monbodb ^3.2.5.
The 'useNewUrlParser' option is not required if you're willing to deal with a deprecation warning. But it is wise to use at this point until version 4 comes out where presumably the new driver will be the default and you won't need the option anymore.
It used to work with the older versions of MongoDb client ~ 2.2.33
Option 1: So you can either use the older version
npm uninstall mongodb --save
npm install mongodb#2.2.33 --save
Option 2: Keep using the newer version (3.0 and above) and modify the code a little bit.
let MongoClient = require('mongodb').MongoClient;
MongoClient.connect('mongodb://localhost:27017', function(err, client){
if(err) throw err;
let db = client.db('myTestingDb');
db.collection('customers').find().toArray(function(err, result){
if(err) throw err;
console.log(result);
client.close();
});
});
I solved it easily via running these codes:
npm uninstall mongodb --save
npm install mongodb#2.2.33 --save
Happy Coding!
If someone is still trying how to resolve this error, I have done this like below.
const MongoClient = require('mongodb').MongoClient;
// Connection URL
const url = 'mongodb://localhost:27017';
// Database Name
const dbName = 'mytestingdb';
const retrieveCustomers = (db, callback)=>{
// Get the customers collection
const collection = db.collection('customers');
// Find some customers
collection.find({}).toArray((err, customers) =>{
if(err) throw err;
console.log("Found the following records");
console.log(customers)
callback(customers);
});
}
const retrieveCustomer = (db, callback)=>{
// Get the customers collection
const collection = db.collection('customers');
// Find some customers
collection.find({'name': 'mahendra'}).toArray((err, customers) =>{
if(err) throw err;
console.log("Found the following records");
console.log(customers)
callback(customers);
});
}
const insertCustomers = (db, callback)=> {
// Get the customers collection
const collection = db.collection('customers');
const dataArray = [{name : 'mahendra'}, {name :'divit'}, {name : 'aryan'} ];
// Insert some customers
collection.insertMany(dataArray, (err, result)=> {
if(err) throw err;
console.log("Inserted 3 customers into the collection");
callback(result);
});
}
// Use connect method to connect to the server
MongoClient.connect(url,{ useUnifiedTopology: true }, (err, client) => {
console.log("Connected successfully to server");
const db = client.db(dbName);
insertCustomers(db, ()=> {
retrieveCustomers(db, ()=> {
retrieveCustomer(db, ()=> {
client.close();
});
});
});
});
I have MongoDB shell version v3.6.4, below code use mongoclient, It's good for me:
var MongoClient = require('mongodb').MongoClient,
assert = require('assert');
var url = 'mongodb://localhost:27017/video';
MongoClient.connect(url,{ useNewUrlParser: true }, function(err, client)
{
assert.equal(null, err);
console.log("Successfully connected to server");
var db = client.db('video');
// Find some documents in our collection
db.collection('movies').find({}).toArray(function(err, docs) {
// Print the documents returned
docs.forEach(function(doc) {
console.log(doc.title);
});
// Close the DB
client.close();
});
// Declare success
console.log("Called find()");
});
MongoDB queries return a cursor to an array stored in memory. To access that array's result you must call .toArray() at the end of the query.
db.collection("customers").find({}).toArray()
Late answer but maybe someone will need it in future
we can create async function which one will return our collection and db instances
const dBInstances = async () => {
const collection = await db
.then((client) => {
const db = client.db();
const collection = db.collection("AGGREGATION");
return { collection: collection, db: db };
})
.catch((err) => {
console.log(`Data base instances error ${err}`);
});
return collection;
};
and after we can use result of execution dBInstances() by this way i used JS destructurisation in example below
const test = async (req, res) => {
const { collection, db } = await dBInstances();
console.log(collection);
console.log(db);
};
now we have separated access to our db and collection.
Recently I had the same issue, I finally resolved it using MongoDB official website documentation and sample codes.
My MongoDB client version is "mongodb": "^4.4.1" and I managed to insert a document finally without needing to downgrade my MongoDB package according to the approved answer which seems to be obsolete.
import { MongoClient } from "mongodb";
// Replace the uri string with your MongoDB deployment's connection string.
const uri = "<connection string uri>";
const client = new MongoClient(uri);
async function run() {
try {
await client.connect();
const database = client.db("insertDB");
const haiku = database.collection("haiku");
// create a document to insert
const doc = {
title: "Record of a Shriveled Datum",
content: "No bytes, no problem. Just insert a document, in MongoDB",
}
const result = await haiku.insertOne(doc);
console.log(`A document was inserted with the _id: ${result.insertedId}`);
} finally {
await client.close();
}
}
run().catch(console.dir);

MongoDB - How can I get the size of a collection using Node.js?

I'm writing a server side code using node.js, I'm trying to get the MongoDB collection size using count method which is not working.
This is my code
var mongo = require('mongodb');
var host = "127.0.0.1";
var port = mongo.Connection.DEFAULT_PORT;
function getStock(name, callback) {
var db = new mongo.Db("testDB", new mongo.Server(host, port, {}));
db.open (function(error){
console.log("We are connected! " + host + ":" +port);
db.collection("stocks", function(error, collection){
console.log("We have a collection");
**var numOfDocs = db.collection('stocks').count()**
**console.log("The num of Docs in our collection is: ",numOfDocs)**
collection.find({"name":name.toString()}, function(error, cursor) {
cursor.toArray(function(error, stocks) {
if (stocks.length==0) {
//console.log("No Stocks found!");
callback(false);
}
else {
callback(stocks[0]);
//console.log("Found a stock -> ",stocks[0]);
}
});
});
});
});
}
.count() is an asynchronous function, just like .find() is.
Try the following code:
collection.count({}, function(error, numOfDocs) {
console.log('I have '+numOfDocs+' documents in my collection');
// ..
});
The first argument to .count() should be a search query. Since you want to count all documents, I'm passing an empty object {} as a search query.
Since mongo 4, count() is deprecated. You should use countDocuments() instead:
MongoDB drivers compatible with the 4.0 features deprecate their
respective cursor and collection count() APIs in favor of new APIs for
countDocuments() and estimatedDocumentCount(). For the specific API
names for a given driver, see the driver documentation.
db.orders.countDocuments({})

NodeJS MongoDB : Multiple save requests not working

I am using node-mongodb-native in my application. I send multiple POST requests to nodejs server to save/update each documents, but only one document is getting updated and all other document are not changing. The data received in the server is correct.
save : function(req,res) {
data = req.body;
if(!data._id){
data._id = new ObjectID();
}else{
data._id = ObjectID(data._id);
}
mColl(req.params.collname, function (collection,db) {
collection.save(data, {safe:true}, function(err, result) {
if(err){
res.send(err);
}
else {
res.send(result);
}
});
});
}
I am not getting the response for the request also.
For starters, don't do this:
data = req.body;
When a new request comes in, you're overwriting the (global!) data variable, and all kinds of undefined stuff can happen. So always declare a new variable:
var data = req.body;

Find data from mongodb databse and display it in new html page

Through ajax request I have get the data form client and save it in mongodb database (mongoose) through save query .Now I want to know how to find data and display it new page.
I have get the data from client and save it in database. Now I want that when callback function calls it find data from the database and display it in new page using response.redirect.Please guide me.
$("#saveChanges5").live('click',function(){
var sendingObj = {
regOperation: $("#area_operation").val()
,regFieldAct: $("#field_activity").val()
,regOther: $("#other_details").val()
};
$.ajax({
url:'/WorkDetails'
,type:'post'
,data: sendingObj
,success:function(){
alert('Successfully saved')
},
error: function(){
alert("Saving Failed")
}
})
});
app.post("/WorkDetails",function(req ,res){
console.log(req.body);
saveWorkDetails(req.body.regOperation ,req.body.regFieldAct ,req.body.regOther ,function(){
res.send("");//here i want to add new code
});
});
function saveWorkDetails(area_operation ,field_activity ,other_details , callback){
console.log("save CALLED");
var receivedObj = new WorkDetailInfo({
area_operation:area_operation ,
field_activity:field_activity ,
other_details:other_details
});
console.log(receivedObj);
receivedObj.save(function(err){
console.log("inside Save ");
if(err){
//res.send(err);
console.log(err);
}
else{
callback();
}
});
}
You can make use of narwhal-mongodb APIs
Following is the example usage:
var MongoDB = require("mongodb");
var db = new MongoDB.Mongo().getDB("mydb");
var colls = db.getCollectionNames();
colls.forEach(function(el) { print(el); });
var coll = db.getCollection("testCollection");

Resources