I am trying to find the best way to write this code. I fetch locations from a remote resource and need to check if there are any new locations present in the data, if there are I should add them to my database, if they are not new I just want to update them.
const http = require('http');
const timeout = 5000; //5 seconds
const MongoClient = require('mongodb').MongoClient;
// Database Name
const dbName = 'weatherApp';
const url = 'mongodb://localhost:27017';
// Connect using MongoClient
MongoClient.connect(url, function(err, client) {
if(err){
console.log(err);
return;
}
const locationsCollection = client.db(dbName).collection('locations');
(function fetchTemperatureLoop(){
console.log('Started http request..');
http.get('remote url..', function(resp){
var data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
if(data.isJson()){
var locations = JSON.parse(data).toArray();
(function locationsLoop(){
var location = locations.pop();
locationsCollection.findOne({location: location.location}, function(err, result){
if(err){
console.log(err);
return;
}
if(result){
//Exists
var measurements = result.measurements;
measurements.push({timestamp: +new Date, temperature: location.temperature})
locationsCollection.update({location: location.location}, {$set: {measurements: measurements}}, function(err){
if(err){
console.log(err);
return;
}
console.log('Added new temperature for location: ' + location.location);
continueLocationsLoop();
});
}else{
//Doesnt exist
location.measurements = [];
location.measurements.push({timestamp: +new Date, temperature: location.temperature});
locationsCollection.insert(location, function(err){
if(err){
console.log(err);
return;
}
console.log('Created new location: ' + location.location);
continueLocationsLoop();
});
}
});
function continueLocationsLoop(){
if(locations.length){
locationsLoop()
}else{
setTimeout(fetchTemperatureLoop, timeout);
}
}
})();
}
});
}).on("error", (err) => {
console.log("Error: " + err.message);
console.log("Continue anyways..");
setTimeout(fetchTemperatureLoop, timeout);
});
})();
});
String.prototype.isJson = function(){
try{
JSON.parse(this);
}catch(e){
return false;
}
return true;
}
Object.prototype.toArray = function(){
var arr = [];
for(var key in this){
if(this.hasOwnProperty(key)){
arr.push(this[key]);
}
}
return arr;
}
I really want to avoid using so many closures but I dont want to repeat myself either. Any help rewriting this code in an optimal way is much appriciated.
My main problem was illiterating through the locations and doing the calls to the database.
I am trying to connect to mongodb from node and I am getting below error
node_modules\mongodb\lib\mongo_client.js:458
throw err
^
ReferenceError: connect is not defined
I am using the mongodb module version
2.0.48
I am trying to run a simple test code
(function (dbase) {
var mdb = require('mongodb');
var mongoUrl = "mongodb://localhost:27017/theBoard";
var connection;
dbase.dbConnection = function (next) {
if (connection) {
next(null, connection);
} else {
mdb.MongoClient.connect(mongoUrl, function(err, db) {
if (err) {
next(err, null);
} else {
console.log("connected");
connection = { db: db , notes: db.collection("notes")};
next(null, connection);
}
});
}
}
Can someone please help me understand this issue.
---Additional information
data module -
(function (data) {
var mdb = require('./db.js');
data.GetCategory = function() {
mdb.dbConnection(function(err, db) {
if (err)
console.log("Error connecting to mango");
if (connect) {
db.notes.count(function(err, count) {
if (err)
console.log("Failed to retreive collection");
else
console.log("Count - "+count);
});
console.log("Connected");
}
});
}})(module.exports);
db.js
(function (dbase) {
var mdb = require('mongodb');
var mongoUrl = "mongodb://localhost:27017/theBoard";
var connection;
dbase.dbConnection = function (next) {
if (connection) {
next(null, connection);
} else {
mdb.MongoClient.connect(mongoUrl, function(err, db) {
if (err) {
next(err, null);
} else {
console.log("connected");
connection = { db: db , notes: db.collection("notes") };
next(null, connection);
}
});
}
} })(module.exports);
Controller -
(function (controller) {
var data = require('.././data');
controller.init = function (app) {
app.get("/", handleRequest);
}
var handleRequest = function (req, res) {
data.GetCategory();
var a = {};
a.send = "Mamma is coming home";
res.send(a);
}
})(module.exports);
Just in case some one runs into an issue like this due to bad coding practice even if it for test purpose is to never have function names which are the same as the function names in the API. In db.js I had an undefined variable named connect which trowing and error when it was accessed and since it was called through the API function called "connect" the error was thrown by the API leading me to believe that the API function had an issue
below is my couchbase nodejs code
kdatabase.js
var couchbase = require('couchbase');
var db = new couchbase.Connection({
host: "http://127.0.0.1:8091",
bucket: "default",
},
function(err) {
if (err) throw err;
db.get('id1', function(err, result) {
if (err) throw err;
console.log(result.value);
process.exit(0);
});
});
it works
but I hope to wrap it to object that can be easily to operate
module.exports = function(app) {
return new KDatabase(app);
};
var KDatabase = function(app) {
this.app = app;
};
//couchbase
KDatabase.prototype.query = function(userName) {
var couchbase = require('couchbase');
var db = new couchbase.Connection({
host: "http://127.0.0.1:8091",
bucket: "default",
},
function(err) {
if (err) throw err;
console.log(userName + '!!!!--');
db.get(userName, function(err, result) {
if (err) throw err;
var o = result.value;
console.log(o['password'] + '***--');
return o['password'];
});
});
};
then I call
var db = require('kdatabase.js')();
var s = db.query(msg.username, function(err) {
if (err) {
console.log('aaa');
}
console.log('bbb');
return;
});
the lines
console.log(userName + '!!!!--');
console.log(o['password'] + '***--');
display correctly
but
console.log('aaa');
console.log('bbb');
are never executed
Your query method does not take a callback argument, so you never call it.
KDatabase.prototype.query = function(userName, cb) {
/* snip */
console.log(o['password'] + '***--');
cb(err, result);
I have my express code::
var express=require('express');
var fs=require('fs');
var http=require('http');
var crypto=require('crypto');
var mysql=require('mysql');
var async=require('async');
var app=express();
var connection=mysql.createConnection({
host: 'localhost',
user: '************',
password: '************',
database: 'ImagePostingDB'
});
connection.connect(function(err) {
if ( !err ) {
console.log("Connected to MySQL");
} else if ( err )
{
console.log(err);
} });
app.set('port',process.env.PORT||7004);
app.use('/Details',express.static(__dirname+'/public/images'));
app.use(express.bodyParser());
app.get('/DescriptionSortedPrice/',function(request,response){
connection.query('SELECT * FROM ImagePostingtable ORDER BY Sl_no', function(err, rows, fields) {
if (err) {
return response.send(500, err.message);
}
console.log('Found results:', rows);
response.json({
'restaurants' : rows
});
});
});
app.post('/Details/',function(req,res,next) {
var file_name=req.files.key.originalFilename;
var file_name1=req.body.key1;
var name;
console.log(file_name);
console.log(file_name1);
async.series( [
// Get the first table contents
function ( callback ) {
crypto.randomBytes(8, function(ex, buf) {
var array = req.files.key.originalFilename.split('.');
var type = array[array.length - 1];
name = buf.toString('hex') + '.' + type;
fs.rename(req.files.key.path, './public/images/' + name, function(e) {
if (e) {
res.send(500, e.message);
} else {
res.send("I got the message - This i confirm");
}
return callback(null);
});
});
},
// Updating the database
function ( callback ) {
connection.query('INSERT INTO ImagePostingtable (Image_Name,Person_Name) VALUES (?,?)', [name,file_name1], function (err, rows, fields) {
console.log('Connection result error ' + err);
return callback(null);
});
}
]);
});
app.get('/Details/',function(req,res){
res.send("Image displayed");
});
http.createServer(app).listen(app.get('port'),function(){
console.log('Express server listening on port'+app.get('port'));
});
Snapshot shows i am connected to mysql and to server
I already have few images as shown in the snapshot below in the location /public/images
Now if i test in my browser for one of the images
http://54.218.73.244:7004/c92beeaf5ba50e65.jpg
i get error as below and image is not displayed in browser
Cannot GET /c92beeaf5ba50e65.jpg
HOW TO RESOLVE THIS ! ! !
Hoping this might help someone
With the help of Andrew in one of the answers i resolved this
I just changed the line of code to below::
app.use(express.static(__dirname+'/public/images'));
I am having trouble understanding node.js.
Example, MongoDB access, here's what I've got (mydb.js):
var mongodb = require('mongodb'),
server = new mongodb.Server('staff.mongohq.com', 10030, {
auto_reconnect: true
}),
db = new mongodb.Db('mydb', server);
function authenticateAndGo(db, handle) {
db.authenticate('username', 'password', function(err) {
if (err) {
console.log(err);
return;
}
console.log('Database user authenticated');
var collection = new mongodb.Collection(db, 'test');
handle(collection);
});
}
function query(handle) {
db.open(function(err, db) {
if( err ) {
console.log(err);
return;
}
console.log('Database connected');
authenticateAndGo(db, handle);
});
};
exports.query = query;
So, if I want to use it later, I would
var mydb = require('./mydb');
mydb.query(function(collection) {
collection.find({}, {
limit: 10
}).toArray(function(err, docs) {
console.log(docs);
});
});
But, If I do multiple calls, like so:
var mydb = require('./mydb');
mydb.query(function(collection) {
collection.find({}, {
limit: 10
}).toArray(function(err, docs) {
console.log(docs);
});
});
mydb.query(function(collection) {
collection.find({}, {
limit: 10
}).toArray(function(err, docs) {
console.log(docs);
});
});
I get an exception:
Error: db object already connecting, open cannot be called multiple times
I think that there is really something fundamental that I do not understand about all this and it is probable that this question is stupid ...
Anyway, all help is welcome.
Thanks in advance.
mydb.js:
var mongodb= require('mongodb'),
server = new mongodb.Server('staff.mongohq.com', 10030, {
auto_reconnect: true
}),
db1 = new mongodb.Db('mydb', server);
// callback: (err, db)
function openDatabase(callback) {
db1.open(function(err, db) {
if (err)
return callback(err);
console.log('Database connected');
return callback(null, db);
});
}
// callback: (err, collection)
function authenticate(db, username, password, callback) {
db.authenticate(username, password, function(err, result) {
if (err) {
return callback (err);
}
if (result) {
var collection = new mongodb.Collection(db, 'test');
// always, ALWAYS return the error object as the first argument of a callback
return callback(null, collection);
} else {
return callback (new Error('authentication failed'));
}
});
}
exports.openDatabase = openDatabase;
exports.authenticate = authenticate;
use.js:
var mydb = require('./mydb');
// open the database once
mydb.openDatabase(function(err, db) {
if (err) {
console.log('ERROR CONNECTING TO DATABASE');
console.log(err);
process.exit(1);
}
// authenticate once after you opened the database. What's the point of
// authenticating on-demand (for each query)?
mydb.authenticate(db, 'usernsame', 'password', function(err, collection) {
if (err) {
console.log('ERROR AUTHENTICATING');
console.log(err);
process.exit(1);
}
// use the returned collection as many times as you like INSIDE THE CALLBACK
collection.find({}, {limit: 10})
.toArray(function(err, docs) {
console.log('\n------ 1 ------');
console.log(docs);
});
collection.find({}, {limit: 10})
.toArray(function(err, docs) {
console.log('\n------ 2 ------');
console.log(docs);
});
});
});
Result:
on success:
Database connected
Database user authenticated
------ 1 ------
[ { _id: 4f86889079a120bf04e48550, asd: 'asd' } ]
------ 2 ------
[ { _id: 4f86889079a120bf04e48550, asd: 'asd' } ]
on failure:
Database connected
{ [MongoError: auth fails] name: 'MongoError', errmsg: 'auth fails', ok: 0 }
[Original Answer]:
You're opening the db multiple times (once in each query). You should open the database just once, and use the db object in the callback for later use.
You're using the same variable name multiple times, and that might've caused some confusion.
var mongodb = require('mongodb'),
server = new mongodb.Server('staff.mongohq.com', 10030, {
auto_reconnect: true
}),
db1 = new mongodb.Db('mydb', server);
function authenticateAndGo(db, handle) {
db.authenticate('username', 'password', function(err) {
if (err) {
console.log(err);
return;
}
console.log('Database user authenticated');
var collection = new mongodb.Collection(db, 'test');
handle(collection);
});
}
function query(handle) {
db1.open(function(err, db2) {
if( err ) {
console.log(err);
return;
}
console.log('Database connected');
authenticateAndGo(db2, handle);
});
};
exports.query = query;
I've changed the above code a little (db1 for the original db, db2 for the opened db). As you can see, you're opening db1 multiple times, which is not good. extract the code for opening into another method and use it ONCE and use the db2 instance for all your queries/updates/removes/...
You can only call "open" once. When the open callback fires, you can then do your queries on the DB object it returns. So one way to handle this is to queue up the requests until the open completes.
e.g MyMongo.js
var mongodb = require('mongodb');
function MyMongo(host, port, dbname) {
this.host = host;
this.port = port;
this.dbname = dbname;
this.server = new mongodb.Server(
'localhost',
9000,
{auto_reconnect: true});
this.db_connector = new mongodb.Db(this.dbname, this.server);
var self = this;
this.db = undefined;
this.queue = [];
this.db_connector.open(function(err, db) {
if( err ) {
console.log(err);
return;
}
self.db = db;
for (var i = 0; i < self.queue.length; i++) {
var collection = new mongodb.Collection(
self.db, self.queue[i].cn);
self.queue[i].cb(collection);
}
self.queue = [];
});
}
exports.MyMongo = MyMongo;
MyMongo.prototype.query = function(collectionName, callback) {
if (this.db != undefined) {
var collection = new mongodb.Collection(this.db, collectionName);
callback(collection);
return;
}
this.queue.push({ "cn" : collectionName, "cb" : callback});
}
and then a sample use:
var MyMongo = require('./MyMongo.js').MyMongo;
var db = new MyMongo('localhost', 9000, 'db1');
var COL = 'col';
db.query(COL, function(collection) {
collection.find({}, {
limit: 10
}).toArray(function(err, docs) {
console.log("First:\n", docs);
});
});
db.query(COL, function(collection) {
collection.find({}, {
limit: 10
}).toArray(function(err, docs) {
console.log("\nSecond:\n", docs);
});
});
I simply call the open function once directly after the db init:
var mongodb = require('mongodb');
var server = new mongodb.Server('foo', 3000, {auto_reconnect: true});
var db = new mongodb.Db('mydb', server);
db.open(function(){});
After that I do not have to care about that anymore because of auto_reconnect is true.
db.collection('bar', function(err, collection) { [...] };