How to continuously query a mongodb database efficiently - node.js

I'm new to Node and MongoDB and I have a seemingly simple request. I've managed to connect to my database, and use a query to get my desired results. Now, I want to have this query continue indefinitely, since the end goal for my project is to plot data real time.
I would have thought a simple 'while (true)' loop would suffice, but that doesn't seem to be the case.
const MongoClient = require('mongodb').MongoClient;
// Connection URL
const url = 'mongodb://<username>:<password>#ds157614.mlab.com:57614/flight_data';
// Use connect method to connect to the Server
MongoClient.connect(url, { useNewUrlParser: true }, function(err, db) {
if (err) throw err;
var dbo = db.db("flight_data").collection("data");
while(true)
{
dbo.find().sort({_id: 1}).limit(1).toArray(function(err, result) {
if (err) throw err;
console.log("Temperature: " + result[0].data.temperature);
});
}
db.close();
});
I have found that the while loop is indeed running, but for some reason, the query just doesn't happen when inside the while loop. If you remove the while loop, the code functions fine. I just want it to continually print the results of the query being repeated.

Querying a DB continuously is inefficient and resource wasting, instead use change streams. It watches collection for any changes and will make the db call then only. Works only for Mongo 3.6+.
const MongoClient = require("mongodb").MongoClient;
// Connection URL
const url =
"mongodb://<username>:<password>#ds157614.mlab.com:57614/flight_data";
// Use connect method to connect to the Server
MongoClient.connect(url, { useNewUrlParser: true }, function(err, db) {
if (err) throw err;
const collection = db.collection("data");
const changeStream = collection.watch();
changeStream.on("change", next => {
// process next document
collection
.find()
.sort({ _id: 1 })
.limit(1)
.toArray(function(err, result) {
if (err) throw err;
console.log("Temperature: " + result[0].data.temperature);
});
});
db.close();
});

Related

db.getSiblingDB is not a function [duplicate]

I have been trying W3schools tutorial on nodeJS with MongoDB.
When I try to implement this example in a nodeJS environment and invoke the function with an AJAX call, I got the error below:
TypeError: db.collection is not a function
at c:\Users\user\Desktop\Web Project\WebService.JS:79:14
at args.push (c:\Users\user\node_modules\mongodb\lib\utils.js:431:72)
at c:\Users\user\node_modules\mongodb\lib\mongo_client.js:254:5
at connectCallback (c:\Users\user\node_modules\mongodb\lib\mongo_client.js:933:5)
at c:\Users\user\node_modules\mongodb\lib\mongo_client.js:794:11
at _combinedTickCallback (internal/process/next_tick.js:73:7)
at process._tickCallback (internal/process/next_tick.js:104:9)
Please find below my implemented code:
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/mytestingdb";
MongoClient.connect(url, function(err, db) {
if (err) throw err;
db.collection("customers").findOne({}, function(err, result) {
if (err) throw err;
console.log(result.name);
db.close();
});
});
Note that the error occurs whenever the execution hits:
db.collection("customers").findOne({}, function(err, result) {}
Also, note (in case it matters) that I have installed the latest MongoDB package for node JS (npm install mongodb), and the MongoDB version is MongoDB Enterprise 3.4.4, with MongoDB Node.js driver v3.0.0-rc0.
For people on version 3.0 of the MongoDB native NodeJS driver:
(This is applicable to people with "mongodb": "^3.0.0-rc0", or a later version in package.json, that want to keep using the latest version.)
In version 2.x of the MongoDB native NodeJS driver you would get the database object as an argument to the connect callback:
MongoClient.connect('mongodb://localhost:27017/mytestingdb', (err, db) => {
// Database returned
});
According to the changelog for 3.0 you now get a client object containing the database object instead:
MongoClient.connect('mongodb://localhost:27017', (err, client) => {
// Client returned
var db = client.db('mytestingdb');
});
The close() method has also been moved to the client. The code in the question can therefore be translated to:
MongoClient.connect('mongodb://localhost', function (err, client) {
if (err) throw err;
var db = client.db('mytestingdb');
db.collection('customers').findOne({}, function (findErr, result) {
if (findErr) throw findErr;
console.log(result.name);
client.close();
});
});
I encountered the same thing. In package.json, change mongodb line to "mongodb": "^2.2.33". You will need to uninstall mongodb npm by removing MongoDB Driver/ node_modules or etc , then install npm to install this version.
This resolved the issue for me. Seems to be a bug or docs need to be updated.
For those that want to continue using version ^3.0.1 be aware of the changes to how you use the MongoClient.connect() method. The callback doesn't return db instead it returns client, against which there is a function called db(dbname) that you must invoke to get the db instance you are looking for.
const MongoClient = require('mongodb').MongoClient;
const assert = require('assert');
// Connection URL
const url = 'mongodb://localhost:27017';
// Database Name
const dbName = 'myproject';
// Use connect method to connect to the server
MongoClient.connect(url, function(err, client) {
assert.equal(null, err);
console.log("Connected successfully to server");
const db = client.db(dbName);
client.close();
});
MongoClient.connect(url (err, client) => {
if(err) throw err;
let database = client.db('databaseName');
database.collection('name').find()
.toArray((err, results) => {
if(err) throw err;
results.forEach((value)=>{
console.log(value.name);
});
})
})
The only problem with your code is that you are accessing the object that's holding the database handler. You must access the database directly (see database variable above). This code will return your database in an array and then it loops through it and logs the name for everyone in the database.
Piggy backing on #MikkaS answer for Mongo Client v3.x, I just needed the async / await format, which looks slightly modified as this:
const myFunc = async () => {
// Prepping here...
// Connect
let client = await MongoClient.connect('mongodb://localhost');
let db = await client.db();
// Run the query
let cursor = await db.collection('customers').find({});
// Do whatever you want on the result.
}
I did a little experimenting to see if I could keep the database name as part of the url. I prefer the promise syntax but it should still work for the callback syntax. Notice below that client.db() is called without passing any parameters.
MongoClient.connect(
'mongodb://localhost:27017/mytestingdb',
{ useNewUrlParser: true}
)
.then(client => {
// The database name is part of the url. client.db() seems
// to know that and works even without a parameter that
// relays the db name.
let db = client.db();
console.log('the current database is: ' + db.s.databaseName);
// client.close() if you want to
})
.catch(err => console.log(err));
My package.json lists monbodb ^3.2.5.
The 'useNewUrlParser' option is not required if you're willing to deal with a deprecation warning. But it is wise to use at this point until version 4 comes out where presumably the new driver will be the default and you won't need the option anymore.
It used to work with the older versions of MongoDb client ~ 2.2.33
Option 1: So you can either use the older version
npm uninstall mongodb --save
npm install mongodb#2.2.33 --save
Option 2: Keep using the newer version (3.0 and above) and modify the code a little bit.
let MongoClient = require('mongodb').MongoClient;
MongoClient.connect('mongodb://localhost:27017', function(err, client){
if(err) throw err;
let db = client.db('myTestingDb');
db.collection('customers').find().toArray(function(err, result){
if(err) throw err;
console.log(result);
client.close();
});
});
I solved it easily via running these codes:
npm uninstall mongodb --save
npm install mongodb#2.2.33 --save
Happy Coding!
If someone is still trying how to resolve this error, I have done this like below.
const MongoClient = require('mongodb').MongoClient;
// Connection URL
const url = 'mongodb://localhost:27017';
// Database Name
const dbName = 'mytestingdb';
const retrieveCustomers = (db, callback)=>{
// Get the customers collection
const collection = db.collection('customers');
// Find some customers
collection.find({}).toArray((err, customers) =>{
if(err) throw err;
console.log("Found the following records");
console.log(customers)
callback(customers);
});
}
const retrieveCustomer = (db, callback)=>{
// Get the customers collection
const collection = db.collection('customers');
// Find some customers
collection.find({'name': 'mahendra'}).toArray((err, customers) =>{
if(err) throw err;
console.log("Found the following records");
console.log(customers)
callback(customers);
});
}
const insertCustomers = (db, callback)=> {
// Get the customers collection
const collection = db.collection('customers');
const dataArray = [{name : 'mahendra'}, {name :'divit'}, {name : 'aryan'} ];
// Insert some customers
collection.insertMany(dataArray, (err, result)=> {
if(err) throw err;
console.log("Inserted 3 customers into the collection");
callback(result);
});
}
// Use connect method to connect to the server
MongoClient.connect(url,{ useUnifiedTopology: true }, (err, client) => {
console.log("Connected successfully to server");
const db = client.db(dbName);
insertCustomers(db, ()=> {
retrieveCustomers(db, ()=> {
retrieveCustomer(db, ()=> {
client.close();
});
});
});
});
I have MongoDB shell version v3.6.4, below code use mongoclient, It's good for me:
var MongoClient = require('mongodb').MongoClient,
assert = require('assert');
var url = 'mongodb://localhost:27017/video';
MongoClient.connect(url,{ useNewUrlParser: true }, function(err, client)
{
assert.equal(null, err);
console.log("Successfully connected to server");
var db = client.db('video');
// Find some documents in our collection
db.collection('movies').find({}).toArray(function(err, docs) {
// Print the documents returned
docs.forEach(function(doc) {
console.log(doc.title);
});
// Close the DB
client.close();
});
// Declare success
console.log("Called find()");
});
MongoDB queries return a cursor to an array stored in memory. To access that array's result you must call .toArray() at the end of the query.
db.collection("customers").find({}).toArray()
Late answer but maybe someone will need it in future
we can create async function which one will return our collection and db instances
const dBInstances = async () => {
const collection = await db
.then((client) => {
const db = client.db();
const collection = db.collection("AGGREGATION");
return { collection: collection, db: db };
})
.catch((err) => {
console.log(`Data base instances error ${err}`);
});
return collection;
};
and after we can use result of execution dBInstances() by this way i used JS destructurisation in example below
const test = async (req, res) => {
const { collection, db } = await dBInstances();
console.log(collection);
console.log(db);
};
now we have separated access to our db and collection.
Recently I had the same issue, I finally resolved it using MongoDB official website documentation and sample codes.
My MongoDB client version is "mongodb": "^4.4.1" and I managed to insert a document finally without needing to downgrade my MongoDB package according to the approved answer which seems to be obsolete.
import { MongoClient } from "mongodb";
// Replace the uri string with your MongoDB deployment's connection string.
const uri = "<connection string uri>";
const client = new MongoClient(uri);
async function run() {
try {
await client.connect();
const database = client.db("insertDB");
const haiku = database.collection("haiku");
// create a document to insert
const doc = {
title: "Record of a Shriveled Datum",
content: "No bytes, no problem. Just insert a document, in MongoDB",
}
const result = await haiku.insertOne(doc);
console.log(`A document was inserted with the _id: ${result.insertedId}`);
} finally {
await client.close();
}
}
run().catch(console.dir);

Using NodeJS promise to query MongoDB

I am building a chatbot using WATSON API which sends artist data give users' input. I am trying to use nodejs promise in order to query my DB and print out the data, since DB accessing is asynchronous.
So the artpromise function is a function which takes in the artist's name and query the db to save the result in the 'result' variable. Then I am trying to print out the result (in chatbot i actually print out the result to the user).
However I am not getting the result I want and keep getting a syntax error. Any help would be appreciated.
let arttistinfo;
function artpromise (artist) {
return new Promise(function(resolve, reject) {
const MongoClient = require("mongodb").MongoClient;
const url = 'mongodb://majac.co.kr:27017/artbot';
MongoClient.connect(url, function(err, db) {
if (err) throw err;
var dbo = db.db("artbot");
var query = {name: artist};
artistinfo = dbo.collection("artistdb").find(query)
.toArray(function(err, result) {
if (err) throw reject(err);
resolve(result);
});
db.close();
}
});
)};
let artist = "Jan Tarasin";
artpormise.then(function(artist) {
console.log(result);
});
I'd rewrite like so, I can see there were a small number of issues with your code, but this works for me now:
function artpromise (artist) {
return new Promise(function(resolve, reject) {
const MongoClient = require("mongodb").MongoClient;
const url = 'mongodb://majac.co.kr:27017/artbot';
MongoClient.connect(url, function(err, db) {
if (err) throw err;
var dbo = db.db("artbot");
var query = {name: artist};
artistinfo = dbo.collection("artistdb").find(query)
.toArray(function(err, result) {
if (err) throw reject(err);
resolve(result);
});
db.close();
});
});
};
let artist = "Jan Tarasin";
artpromise(artist).then(function(result) {
console.log(result);
});
I get the result below:
[{
_id: 5abdbc18423795deaaff0d8e,
nationality: 'Polish',
art_link: 'https: //media.mutualart.com/Images/2016_06/29/20/203606422/0532d043-71f6-47bc-945e-aeededd2d483_570.Jpeg',
years: '1926',
name: 'JanTarasin',
art_title: '"Falujace watki I",
2003r.'
}]
MongoDB Node driver is natively supporting promises from v3 on. So you may greatly simplify your code by using them.
Here is how i would approach to your problem;
function artpromise (artist) {
const MongoClient = require("mongodb").MongoClient;
return MongoClient.connect('mongodb://majac.co.kr:27017') // connect to mongo server
.then(mc => mc.db('artbot') // get mongoClient object and connect to artbot db
.collection('artistdb') // connect to the artistdb collection
.find({name: artist}) // perform your query
.toArray() // convert the results into an array
.then(as => (mc.close(), as))) // close db and return array from query result
.catch(e => console.log(e)); // catch errors
}
let artist = "Jan Tarasin";
artpromise(artist).then(as => as.forEach(a => console.log(a)));
[nodemon] starting `node maeror.js`
{ _id: 5abdbc18423795deaaff0d8e,
nationality: 'Polish',
art_link: 'https://media.mutualart.com/Images/2016_06/29/20/203606422/0532d043-71f6-47bc-945e-aeededd2d483_570.Jpeg',
years: '1926',
name: 'Jan Tarasin',
art_title: ' "Falujące wątki I", 2003 r. ' }
[nodemon] clean exit - waiting for changes before restart
It might be useful to remind that cursor.toArray() returns a promise as it has to iterate all the query results at once before consturcting the results array. Sometimes this operation might be time consuming yielding delayed server response. So you may instead use the cursor.forEach() method to process the documents returned from the query one by one like a stream. Which means processing the first document and then iterating to the next one. Here is another example to show how it might be implemented.
function artpromise (artist) {
const MongoClient = require("mongodb").MongoClient;
return MongoClient.connect('mongodb://majac.co.kr:27017') // connect to mongo server
.then(function(mc){
var cursor = mc.db('artbot') // get mongoClient object and connect to artbot db
.collection('artistdb') // connect to the artistdb collection
.find({name: artist}); // get the cursor
return [mc, cursor]; // return mongoClient and cursor objects
});
}
let artist = "Italian";
artpromise(artist).then(function([mc,docs]){
docs.forEach(doc => console.log(doc), // process a document and then iterate to the next
() => mc.close()); // close db session when all documents are processed
})
.catch(e => console.log(e)); // catch errors
[nodemon] starting `node maeror_v2.js`
{ _id: 5abdbc18423795deaafeff13,
nationality: 'Dutch',
art_link: 'https://media.mutualart.com/Images/2012_04/15/13/132154856/ddf14e9d-85b1-4b5a-b621-00583e013879_570.Jpeg',
years: '1839 - 1902',
name: 'Frederick Hendrik Kaemmerer',
art_title: ' A Beach Stroll ' }
[nodemon] clean exit - waiting for changes before restart

MongoDB not closing connections from NodeJS app

I have a node application that makes a call to mongoDB every 10 seconds, but looking at the output in my terminal, the connections just keep counting up and never seem to close:
My code to hit the ddb every 10 seconds:
const MongoClient = require("mongodb").MongoClient
setInterval(function(){
MongoClient.connect(uri, (err, client) => {
if (err){
console.log(err);
}
database = client.db(databaseName)
getData(function(data){
if(data.length > 0){
db_response = data;
params["fieldA"] = db_response[0]['fieldA'];
}
})
})
}, 10000)
function getData(callback){
var query = { fieldA: "foo" };
database.collection(CollectionName).find(query).toArray(function(err, result){
if (err){
throw err;
}
callback(result);
})
}
(The vars uri, CollectionName and databaseName are declared earlier) I guess what i need to do (and havent yet figured out) is to connect to the DB once when the server starts, and then run the getData() function on successful connection, does that mean the database variable needs to be a global var??
As you correctly identified you only need to create your db connection once. So rather than wrapping the the db connection creation with setInterval, wrap setInterval around the only function you want to repeat, in this case getData.
On your other question, the database variable doesn't need to global but you are right getData does need to use it. Therefore pass it as an argument along with your callback function.
If you want to close your connection use client.close(); inside MongoClient.connect
const MongoClient = require("mongodb").MongoClient
MongoClient.connect(uri, (err, client) => {
if (err){
console.log(err);
}
const database = client.db(databaseName);
setInterval(function(){
getData(database, function(data){
if(data.length > 0){
db_response = data;
params["fieldA"] = db_response[0]['fieldA'];
}
})
}, 10000)
})
function getData(db, callback){
var query = { fieldA: "foo" };
db.collection(CollectionName).find(query).toArray(function(err, result){
if (err){
throw err;
}
callback(result);
})
}

Serverless - AWS Lamda with MongoDB Atlas Queries Not Running

I've created an AWS account and want to use MongoDB Atlas with AWS Lambda.
The only dependency I've downloaded is mongodb locally.
npm install mongodb
Driver based connection string given from mongoDB Atlas for Nodejs is
var uri = "mongodb+srv://kay:myRealPassword#cluster0.mongodb.net/test";
MongoClient.connect(uri, function(err, client) {
const collection = client.db("test").collection("devices");
// perform actions on the collection object
client.close();
});
I think the connection is successful, because err parameter is NULL.
But I cannot figure out how to create collection, how to find results, how to insert documents.
I've tried this code
module.exports.hello = (event, context, callback) => {
var MongoClient = require('mongodb').MongoClient;
var uri = "mongodb+srv://kay:myRealPassword#cluster0.mongodb.net/test";
MongoClient.connect(uri, function(err, client) {
const collection = client.db("test").collection("devices");
collection.insert( { "msg" : "My First Document" } );
var results = client.db("test").collection("devices").find();
console.log(results);
client.close();
callback(null, { message: 'Go Serverless v1.0! Your function executed successfully!', event });
});
};
but it returns (in Windows console) a huge Object in JSON format, its like a configuration data (not a query result)
enter image description here
I'm executing this code locally by
sls invoke local --function hello
The general idea is to check if there is an error in the connection, the insert, and so on. Take at look at this error checking:
if (error) return 1;
There are more sophisticated methods, but for your case this should do the work.
This is a example of how it show look your script:
MongoClient.connect(uri, (error, client) => {
if (error) return 1; // Checking the connection
console.log('Connection Successful');
var db = client.db('mydb'); // Your DB
let newDocument = { "msg" : "My First Document" }; // Your document
db.collection('mycollection').insert(newDocument, (error, results) => { // Your collection
if (error) return 1; // Checking the insert
console.log('Insert Successful');
})
db.collection('mycollection')
.find({})
.toArray((error, accounts) => {
if (error) return 1; // Checking the find
console.log('Find Successful');
console.log(accounts);
return 0;
})
})
And you should have an output like this:
Connection Successful
Insert Successful
Find Successful
[ { _id: 5a857dd2c940040d85cbe5f2, msg: 'My First Document' } ]
If your output is not like this, well the missing log would point the place where you have your error.

Querying a MongoDB documents in real-time

I'm building a web application that will work with Big Data.
I will mine Twitter data using the Apache Storm, subsequently saving them in a MongoDB database.
At same time, this data has to be fetched via Node.js in real time and be sent via socket.io to my front-end.
Exist a way to querying MongoDB via Node.js in real time?
Thanks.
I am working on a project with mongoDB, I used the mongodb npm module to query the database in real time.
First I get a list of collections which are in my database:
//My server controller page
var MongoClient = require('mongodb').MongoClient
, assert = require('assert');
exports.getCollections = function(req,res){
mongoose.connection.db.collectionNames(function(err, names) {
if (err){
console.log(err)
} else {
res.status(200).send({collections: names});
}
});
};
On the front end, I do an Angular ng-repeat to list my collections, then when I click on the collection name, I run the following code:
MongoClient.connect(url, function (err, db) {
assert.equal(null, err);
var collection = db.collection(req.body.collName);
collection.find({}).limit(req.body.limit).toArray(function (err, docs) {
if (err) {
console.log(err)
} else {
res.status(200).send({r: docs, count: docs.length});
db.close();
}
})
});
Here is my client side angular code:
//get the collection list upon page load
$http.get('collections')
.success(function(c){
$scope.collList = c.collections;
})
.error(function(err){
$scope.error = err;
});
//function run when collection is selected
$scope.doFind = function(coll){
$scope.collViewing = coll;
$http.post('/basicfind',{collName: coll,limit:$scope.limiter})
.success(function(data){
$scope.results = data.r;
$scope.countOfResults = data.count;
})
.error(function(err){
$scope.error = err.message;
});
};
Hope that helps, let me know if you need me to share any more code

Resources