Automatically insert data to MongoDB with NodeJS - node.js

I'm trying to create a script which is going to automatically send data from API every day at specific time and store it in MongoDB collection. For that purpose I use NodeJS-Schedule.
I just started to exploring MongoDB and now I'm looking for a way to automatically receive data from API and store it in MongoDB.
I saw that there is MongoDB-Cron
This is part of the code that I use to receive a data from API.
var j = schedule.scheduleJob("*/55 20 * * *", function() {
request(
"GET",
"http://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest?CMC_PRO_API_KEY=API-KEY-HERE"
)
.then((r1) => {
var x1 = JSON.parse(r1.target.responseText);
var BTCdata = x1.data.find((d) => d.symbol === "BTC").quote.USD.volume_24h; // creating a variable to store a BTC request from API
console.log(BTCdata);
})
.catch((err) => {
console.log(err);
});
});
function request(method, url) {
return new Promise(function(resolve, reject) {
var xhr = new XMLHttpRequest();
xhr.open(method, url);
xhr.onload = resolve;
xhr.onerror = reject;
xhr.send();
});
}
Here is the part to insert a data to MongoDB collection.
I a little bit don't understand how I can automate this insert function and can I make this automatisation to send data to mongodb every day at specific time?
var url = "mongodb+srv://name:password#cluster0-1kunr.mongodb.net/<dbname>?retryWrites=true&w=majority";
MongoClient.connect(url, { useNewUrlParser: true, useUnifiedTopology: true }, function(err, db) {
if (err) throw err;
var dbo = db.db("Crypto");
var myobj = { Name: "BTC", Volume: "BTCdata" };
dbo.collection("Crypto-Values").insertOne(myobj, function(err, res) {
if (err) throw err;
console.log("1 document inserted");
db.close();
});
});
EDIT1:
Is the code supposed to be like this?
var MongoClient = require('mongodb').MongoClient;
var MongoCron = require('mongodb-cron');
const saveToDatabase = (BTCdata) => {
var url = "mongodb+srv://name:password#cluster0-1kunr.mongodb.net/<dbname>?retryWrites=true&w=majority";
MongoClient.connect(url, { useNewUrlParser: true, useUnifiedTopology: true }, function(err, db) {
var j = schedule.scheduleJob("*/55 20 * * *", function() {
request(
"GET",
"http://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest?CMC_PRO_API_KEY=API-KEY-HERE"
)
.then((r1) => {
var x1 = JSON.parse(r1.target.responseText);
var BTCdata = x1.data.find((d) => d.symbol === "BTC").quote.USD.volume_24h; // creating a variable to store a BTC request from API
console.log(BTCdata);
// Saving to database
saveToDatabase(BTCdata);
})
.catch((err) => {
console.log(err);
});
});
});
};

I don't think you need to disturb yourself with automating the db insertion, you can include the insertion in the corn job itself. Perhaps, encapsulate the data persisting logic in a function:
const saveToDatabase = (BTCdata) => {
var url = "mongodb+srv://name:password#cluster0-1kunr.mongodb.net/<dbname>?retryWrites=true&w=majority";
MongoClient.connect(url, { useNewUrlParser: true, useUnifiedTopology: true }, function(err, db) {
if (err) throw err;
var dbo = db.db("Crypto");
var myobj = { Name: "BTC", Volume: BTCdata };
dbo.collection("Crypto-Values").insertOne(myobj, function(err, res) {
if (err) throw err;
console.log("1 document inserted");
db.close();
});
});
}
You should call the function with the BTCData after the API call in the corn job. The program should be something like this:
const { MongoClient } = require('mongodb');
const schedule = require('node-schedule');
const saveToDatabase = function (BTCdata) {
const url = 'mongodb+srv://name:password#cluster0-1kunr.mongodb.net/<dbname>?retryWrites=true&w=majority';
MongoClient.connect(url, { useNewUrlParser: true, useUnifiedTopology: true }, (err, db) => {
if (err) throw err;
const dbo = db.db('Crypto');
const myobj = { Name: 'BTC', Volume: BTCdata };
dbo.collection('Crypto-Values').insertOne(myobj, (error, res) => {
if (error) throw error;
console.log('1 document inserted');
db.close();
});
});
};
function request(method, url) {
return new Promise(((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open(method, url);
xhr.onload = resolve;
xhr.onerror = reject;
xhr.send();
}));
}
const j = schedule.scheduleJob('*/55 20 * * *', () => {
request(
'GET',
'http://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest?CMC_PRO_API_KEY=API-KEY-HERE',
)
.then((r1) => {
const x1 = JSON.parse(r1.target.responseText);
const BTCdata = x1.data.find((d) => d.symbol === 'BTC').quote.USD.volume_24h; // creating a variable to store a BTC request from API
console.log(BTCdata);
// Saving to database
saveToDatabase(BTCdata);
})
.catch((err) => {
console.log(err);
});
});
Remember to catch errors where necessary

Related

Nodejs formidable async mongoose await issue

I'm trying to make a OCR parse of an image.
All the things works well but I have a problem this mongoose and syncronysm.
But cannot use "await" on the mongoose find call as the function is not async. How do I solve that.
Here is my code:
// post processImage
router.post('/', async (req, res) => {
try {
var baseUrl;
const form = formidable({ multiples: true });
form.parse(req, function (error, fields, files) {
var imatgeAProcessar = files.image.path;
var extname = path.extname(files.image.name);
getTextFromImage(imatgeAProcessar) // OCR process of the image
.then(res => {
const boss_name_req = res.boss_name;
const boss = Boses.findOne({"name" : boss_name_req}).exec();
// ERROR HERE // return nothing althought it exist on database (no await?)
console.log(JSON.stringify(boss)); // writes "{}"
const processedImage = {
"success": true,
"boss_name": boss.name,
"boss_image": baseUrl + 'images/' + boss.num + ".png"
}
res.json(processedImage);
})
});
} catch (err) {
res.json({message: err});
}
});
*edited
// post processImage
router.post('/', async(req, res) => {
try {
var baseUrl;
const form = formidable({ multiples: true });
var formfields = await new Promise(function(resolve, reject) {
form.parse(req, function(err, fields, files) {
if (err) {
reject(err);
return;
}
resolve(files);
}); // form.parse
});
var imatgeAProcessar = formfields.image.path;
var extname = path.extname(formfields.image.name);
const res = await getTextFromImage(imatgeAProcessar)
const boss_name_req = res.boss_name;
const boss = await Boses.findOne({ "name": boss_name_req }).limit(4).skip(0).exec();
const processedImage = {
"success": true,
"boss_name": boss.name,
"boss_image": baseUrl + 'images/' + boss.num + ".png"
}
res.json(processedImage)
} catch (err) {
res.json({ message: err });
}
});
Finally I found the way... I wrote a callback on the findOne call as :
const boss = Boss.findOne({"name" : boss_name_req})
.then( resMongoose => {
try {
const processedImage = {
"success": true,
"gym": resOCR.gym,
"boss_name": resMongoose.name,
}
res.json(processedImage);
} catch (err) {
res.json({message: err});
}
});

Access Function in a Exported Module

I would like to create a module, where i have a function where i can insert a sql statement and get the result as recordset. Im new to nodeJs and have some trouble in accessing the functions right.
my Modul (sql.js)
var exports = module.exports = {};
sql = require('mssql');
let config = {
user: '###',
password '###'
server: '###',
database: '###',
driver: "###",
options: {
trustedConnection: true
}
};
var db = function (query) {
var rc;
console.log('verbinde');
sql.connect(config, function (err) {
console.log('verbinde');
if (err) console.log(err);
let request = new sql.request();
request.query(query, function (err, recordset) {
if (err) console.log(err);
console.log(recordset);
rc = recordset;
});
})
sql.close();
}
exports.db = db;
call
const ipc = require('electron').ipcMain;
const sql = require('../../customModules/sql.js');
console.log(sql.db('SELECT * FROM devices'));
i get an undefined.
it seems like, the sql.connect isnt rly called.
You might get better results using a Promise. Perhaps something like this (untested):
// sql.js
var db = (query) => new Promise((resolve, reject) => {
console.log('verbinde');
sql.connect(config, function (err) {
console.log('verbinde');
if (err) reject(err);
let request = new sql.request();
request.query(query, function (err, recordset) {
if (err) reject(err);
console.log('got those records!');
resolve(recordset);
});
})
sql.close();
})
// call
const ipc = require('electron').ipcMain;
const sql = require('../../customModules/sql.js');
sql.db('SELECT * FROM devices')
.then(data => console.log(data))
.catch(e => console.log(e));

async.eachSeries runs only once with async.waterfall inside for each iteration

I am new to async library. I have used async.eachSeries and async.waterfall for each iteration. I see, the async.waterfall runs only once.
Here is my code :
var fs = require('fs'),
async = require('async'),
Client = require('node-rest-client').Client;
// REST API Call and output in jsonOutput.results
console.log(jsonOutput.results.length); // jsonOutput.results has 124 records.
async.eachSeries(jsonOutput.results, function(account, callback) {
var dataObject = {};
dataObject.updatetime = new Date();
var setAccountInfoURL = ""; // Data Update REST API Request
async.waterfall([
function setAccountInfo(updateCallback) {
// client.get(setAccountInfoURL, function (data, response) {
// var jsonOutput = JSON.parse(data.toString('utf8'));
updateCallback(null, "output", account)
// });
},
function saveAccountInfo(jsonOutput, account, updateCallback) {
var debuglog = JSON.stringify(account) + "\n" + jsonOutput;
fs.appendFile("debuginfo.json", debuglog + "\n", function (err) {
if(err) {
console.log(err);
}
console.log("JSON saved to " + "debuginfo.json");
updateCallback(null);
});
}
],function asyncComplete(err) {
if (err) {
console.warn('Error setting account info.', err);
}
console.log('async completed');
});
}, function(err){
if (err) {
console.log('error in loop');
}
console.log('loop completed');
});
Output:
124
JSON saved to debuginfo.json
async completed
Any help is really appreciated.
I found my mistake. I missed calling the callback after each iteration just after async is completed.
var fs = require('fs'),
async = require('async'),
Client = require('node-rest-client').Client;
// REST API Call and output in jsonOutput.results
console.log(jsonOutput.results.length); // jsonOutput.results has 124 records.
async.eachSeries(jsonOutput.results, function(account, callback) {
var dataObject = {};
dataObject.updatetime = new Date();
var setAccountInfoURL = ""; // Data Update REST API Request
async.waterfall([
function setAccountInfo(updateCallback) {
// client.get(setAccountInfoURL, function (data, response) {
// var jsonOutput = JSON.parse(data.toString('utf8'));
updateCallback(null, "output", account)
// });
},
function saveAccountInfo(jsonOutput, account, updateCallback) {
var debuglog = JSON.stringify(account) + "\n" + jsonOutput;
fs.appendFile("debuginfo.json", debuglog + "\n", function (err) {
if(err) {
console.log(err);
}
console.log("JSON saved to " + "debuginfo.json");
updateCallback(null);
});
}
],function asyncComplete(err) {
if (err) {
console.warn('Error setting account info.', err);
}
console.log('async completed');
callback(null); // this is the change.
});
}, function(err){
if (err) {
console.log('error in loop');
}
console.log('loop completed');
});

wrap couchbase access function

below is my couchbase nodejs code
kdatabase.js
var couchbase = require('couchbase');
var db = new couchbase.Connection({
host: "http://127.0.0.1:8091",
bucket: "default",
},
function(err) {
if (err) throw err;
db.get('id1', function(err, result) {
if (err) throw err;
console.log(result.value);
process.exit(0);
});
});
it works
but I hope to wrap it to object that can be easily to operate
module.exports = function(app) {
return new KDatabase(app);
};
var KDatabase = function(app) {
this.app = app;
};
//couchbase
KDatabase.prototype.query = function(userName) {
var couchbase = require('couchbase');
var db = new couchbase.Connection({
host: "http://127.0.0.1:8091",
bucket: "default",
},
function(err) {
if (err) throw err;
console.log(userName + '!!!!--');
db.get(userName, function(err, result) {
if (err) throw err;
var o = result.value;
console.log(o['password'] + '***--');
return o['password'];
});
});
};
then I call
var db = require('kdatabase.js')();
var s = db.query(msg.username, function(err) {
if (err) {
console.log('aaa');
}
console.log('bbb');
return;
});
the lines
console.log(userName + '!!!!--');
console.log(o['password'] + '***--');
display correctly
but
console.log('aaa');
console.log('bbb');
are never executed
Your query method does not take a callback argument, so you never call it.
KDatabase.prototype.query = function(userName, cb) {
/* snip */
console.log(o['password'] + '***--');
cb(err, result);

Node.js reuse MongoDB reference

I am having trouble understanding node.js.
Example, MongoDB access, here's what I've got (mydb.js):
var mongodb = require('mongodb'),
server = new mongodb.Server('staff.mongohq.com', 10030, {
auto_reconnect: true
}),
db = new mongodb.Db('mydb', server);
function authenticateAndGo(db, handle) {
db.authenticate('username', 'password', function(err) {
if (err) {
console.log(err);
return;
}
console.log('Database user authenticated');
var collection = new mongodb.Collection(db, 'test');
handle(collection);
});
}
function query(handle) {
db.open(function(err, db) {
if( err ) {
console.log(err);
return;
}
console.log('Database connected');
authenticateAndGo(db, handle);
});
};
exports.query = query;
So, if I want to use it later, I would
var mydb = require('./mydb');
mydb.query(function(collection) {
collection.find({}, {
limit: 10
}).toArray(function(err, docs) {
console.log(docs);
});
});
But, If I do multiple calls, like so:
var mydb = require('./mydb');
mydb.query(function(collection) {
collection.find({}, {
limit: 10
}).toArray(function(err, docs) {
console.log(docs);
});
});
mydb.query(function(collection) {
collection.find({}, {
limit: 10
}).toArray(function(err, docs) {
console.log(docs);
});
});
I get an exception:
Error: db object already connecting, open cannot be called multiple times
I think that there is really something fundamental that I do not understand about all this and it is probable that this question is stupid ...
Anyway, all help is welcome.
Thanks in advance.
mydb.js:
var mongodb= require('mongodb'),
server = new mongodb.Server('staff.mongohq.com', 10030, {
auto_reconnect: true
}),
db1 = new mongodb.Db('mydb', server);
// callback: (err, db)
function openDatabase(callback) {
db1.open(function(err, db) {
if (err)
return callback(err);
console.log('Database connected');
return callback(null, db);
});
}
// callback: (err, collection)
function authenticate(db, username, password, callback) {
db.authenticate(username, password, function(err, result) {
if (err) {
return callback (err);
}
if (result) {
var collection = new mongodb.Collection(db, 'test');
// always, ALWAYS return the error object as the first argument of a callback
return callback(null, collection);
} else {
return callback (new Error('authentication failed'));
}
});
}
exports.openDatabase = openDatabase;
exports.authenticate = authenticate;
use.js:
var mydb = require('./mydb');
// open the database once
mydb.openDatabase(function(err, db) {
if (err) {
console.log('ERROR CONNECTING TO DATABASE');
console.log(err);
process.exit(1);
}
// authenticate once after you opened the database. What's the point of
// authenticating on-demand (for each query)?
mydb.authenticate(db, 'usernsame', 'password', function(err, collection) {
if (err) {
console.log('ERROR AUTHENTICATING');
console.log(err);
process.exit(1);
}
// use the returned collection as many times as you like INSIDE THE CALLBACK
collection.find({}, {limit: 10})
.toArray(function(err, docs) {
console.log('\n------ 1 ------');
console.log(docs);
});
collection.find({}, {limit: 10})
.toArray(function(err, docs) {
console.log('\n------ 2 ------');
console.log(docs);
});
});
});
Result:
on success:
Database connected
Database user authenticated
------ 1 ------
[ { _id: 4f86889079a120bf04e48550, asd: 'asd' } ]
------ 2 ------
[ { _id: 4f86889079a120bf04e48550, asd: 'asd' } ]
on failure:
Database connected
{ [MongoError: auth fails] name: 'MongoError', errmsg: 'auth fails', ok: 0 }
[Original Answer]:
You're opening the db multiple times (once in each query). You should open the database just once, and use the db object in the callback for later use.
You're using the same variable name multiple times, and that might've caused some confusion.
var mongodb = require('mongodb'),
server = new mongodb.Server('staff.mongohq.com', 10030, {
auto_reconnect: true
}),
db1 = new mongodb.Db('mydb', server);
function authenticateAndGo(db, handle) {
db.authenticate('username', 'password', function(err) {
if (err) {
console.log(err);
return;
}
console.log('Database user authenticated');
var collection = new mongodb.Collection(db, 'test');
handle(collection);
});
}
function query(handle) {
db1.open(function(err, db2) {
if( err ) {
console.log(err);
return;
}
console.log('Database connected');
authenticateAndGo(db2, handle);
});
};
exports.query = query;
I've changed the above code a little (db1 for the original db, db2 for the opened db). As you can see, you're opening db1 multiple times, which is not good. extract the code for opening into another method and use it ONCE and use the db2 instance for all your queries/updates/removes/...
You can only call "open" once. When the open callback fires, you can then do your queries on the DB object it returns. So one way to handle this is to queue up the requests until the open completes.
e.g MyMongo.js
var mongodb = require('mongodb');
function MyMongo(host, port, dbname) {
this.host = host;
this.port = port;
this.dbname = dbname;
this.server = new mongodb.Server(
'localhost',
9000,
{auto_reconnect: true});
this.db_connector = new mongodb.Db(this.dbname, this.server);
var self = this;
this.db = undefined;
this.queue = [];
this.db_connector.open(function(err, db) {
if( err ) {
console.log(err);
return;
}
self.db = db;
for (var i = 0; i < self.queue.length; i++) {
var collection = new mongodb.Collection(
self.db, self.queue[i].cn);
self.queue[i].cb(collection);
}
self.queue = [];
});
}
exports.MyMongo = MyMongo;
MyMongo.prototype.query = function(collectionName, callback) {
if (this.db != undefined) {
var collection = new mongodb.Collection(this.db, collectionName);
callback(collection);
return;
}
this.queue.push({ "cn" : collectionName, "cb" : callback});
}
and then a sample use:
var MyMongo = require('./MyMongo.js').MyMongo;
var db = new MyMongo('localhost', 9000, 'db1');
var COL = 'col';
db.query(COL, function(collection) {
collection.find({}, {
limit: 10
}).toArray(function(err, docs) {
console.log("First:\n", docs);
});
});
db.query(COL, function(collection) {
collection.find({}, {
limit: 10
}).toArray(function(err, docs) {
console.log("\nSecond:\n", docs);
});
});
I simply call the open function once directly after the db init:
var mongodb = require('mongodb');
var server = new mongodb.Server('foo', 3000, {auto_reconnect: true});
var db = new mongodb.Db('mydb', server);
db.open(function(){});
After that I do not have to care about that anymore because of auto_reconnect is true.
db.collection('bar', function(err, collection) { [...] };

Resources