How to iterate mongodb database in node.js to send to Algolia? - node.js

In the documentation of Algolia, for the node.js part they specified to use MySQL for indexing but not MongoDB, I have another question regarding this issue but it is more a general question , check here
Some folks ask me to use mongo-connector but tried it and I got some unknown error, which got me to square one
My real question is, how do i iterate a list of collections in mongodb to algolia?
This is the Algolia's version of MySQL in Node.js
var _ = require('lodash');
var async = require('async');
var mysql = require('mysql');
var algoliasearch = require('algoliasearch');
var client = algoliasearch("RQGLD4LOQI", "••••••••••••••••••••••••••••••••");
var index = client.initIndex('YourIndexName');
var connection = mysql.createConnection({
host: 'localhost',
user: 'mysql_user',
password: 'mysql_password',
database: 'YourDatabaseName'
});
connection.query('SELECT * FROM TABLE_TO_IMPORT', function(err, results, fields) {
if (err) {
throw err;
}
// let's use table IDS as Algolia objectIDs
results = results.map(function(result) {
result.objectID = result.id;
return result;
});
// split our results into chunks of 5,000 objects, to get a good indexing/insert performance
var chunkedResults = _.chunk(results, 5000);
// for each chunk of 5,000 objects, save to algolia, in parallel. Call end() when finished
// or if any save produces an error
// https://github.com/caolan/async#eacharr-iterator-callback
async.each(chunkedResults, index.saveObjects.bind(index), end);
});
function end(err) {
if (err) {
throw err;
}
console.log('MySQL<>Algolia import done')
};
To be specific I'm using mongoose as my ORM, so I have no experience in other libraries. Please help me on this, so that I could some searching interface already :(.

You can use the following code to iterate over the whole MongoDB mydb.myCollection collection + create batches that will be sent to the Algolia index:
var Db = require('mongodb').Db,
Server = require('mongodb').Server,
algoliasearch = require('algoliasearch');
// init Algolia index
var client = algoliasearch("*********", "••••••••••••••••••••••••••••••••");
var index = client.initIndex('YourIndexName');
// init connection to MongoDB
var db = new Db('mydb', new Server('localhost', 27017));
db.open(function(err, db) {
// get the collection
db.collection('myCollection', function(err, collection) {
// iterate over the whole collection using a cursor
var batch = [];
collection.find().forEach(function(doc) {
batch.push(doc);
if (batch.length > 10000) {
// send documents by batch of 10000 to Algolia
index.addObjects(batch);
batch = [];
}
});
// last batch
if (batch.length > 0) {
index.addObjects(batch);
}
});
});

Related

Node.Js Retrieve specific data from SQL Server and encode it into JSON Array

I know this question have many duplicates, but I have already wasted too much time searching for the right solution.
First take a look at my Node.JS:
var express = require('express');
var app = express();
app.get('/', function (req, res) {
var sql = require("mssql");
// config for your database
var config = {
user: 'myuser',
password: 'mypass',
server: 'myip',
database: 'mydatabase'
};
sql.close();
// connect to your database
sql.connect(config, function (err) {
if (err) console.log(err);
var dataqu = '';
// create Request object
var request = new sql.Request();
// query to the database and get the records
request.query("select * from AR_Invoices", function (err, recordset) {
if (err) console.log(err)
res.json(recordset);
sql.close();
});
});
});
var server = app.listen(5000,'0.0.0.0', function () {
console.log('Server is running..');
});
This code runs fine, but the json result structure is like this :
{"recordsets":[[{"Tipe":"Invoices","InvoiceID":411891,"InvoiceNumber":"SR.1701.0001"}]],"recordset":[{"Tipe":"Invoices","InvoiceID":411891,"InvoiceNumber":"SR.1701.0001"}],"output":{},"rowsAffected":[1]}
I don't know why but for some reason the result is always resulting in duplicate.
And how to just select InvoiceID and InvoiceNumber ?
I already tested using recordset.InvoiceID or recordset[0].InvoiceID but all is always in vain, and the result always in duplicate.
Can anyone explain how to do this properly?
I want the final result became like this :
[
{ "InvoiceID":"1", "InvoiceNumber":"mynumber" }
]
For the future reference, i finally got how to do this here is my full code
var express = require('express');
var app = express();
var dateFormat = require('dateformat');
app.get('/', function (req, res) {
var sql = require("mssql");
// config for your database
var config = {
user: 'myuser',
password: 'mypassword',
server: 'myip',
database: 'mydb'
};
sql.close();
// connect to your database
sql.connect(config, function (err) {
if (err) console.log(err);
// create Request object
var request = new sql.Request();
// query to the database and get the records
request.query("select top 2 'Invoices' as Tipe,InvoiceID,InvoiceNumber,InvoiceDate,(select top 1DriverPicture from dbDigitalApp.dbo.tbdriver) as Blob from AR_Invoices", function (err, result) {
if (err) console.log(err)
var myarr = new Array();
for (var i = 0; i < result.recordset.length; ++i) {
var InvoiceNumber = result.recordset[i].InvoiceNumber;
var InvoiceDate = dateFormat(result.recordset[i].InvoiceDate, "dd mmmm yyyy");
var Blob = result.recordset[i].Blob;
myarr.push({'InvoiceNumber':InvoiceNumber,'InvoiceDate':InvoiceDate,'Blob':Buffer.from(Blob).toString('base64')});
}
res.json(myarr);
sql.close();
});
});
});
var server = app.listen(5000,'0.0.0.0', function () {
console.log('Server is running..');
});
and the result of above code is like this :
answers
with above code you can get specific field only and do whatever you want with those specific data, such as change date format or encode base64.
i don't know if this the cleanest way to do this since the node.js has its own function using res.json that can set all field of retrieved data without need to loop through it.
But at least here is my kind of solution, hope it will be helpful to there future people who wondering the same thing like me.

How to get mongoDB collection size in node js?

I'm trying to connect to a mongoDB DB, and make some processes on a collection, and close the collection when all the collection items were processed. when I'm trying to receive array.length, I get undefined.
Db = require('mongodb').Db;
Server = require('mongodb').Server;
const db = new Db(DB_NAME, new Server(HOST, PORT));
// connect to mongoDB
db.open(function (err, db) {
const Collection = db.collection(COLLECTION_NAME);
var items = Collection.find({});
var itemsLength = items.lebgth;
var itemsProcessed = 0;
items.forEach((item, index, array) => {
// some process like:
Collection.update({query}, {set}, callback)
itemsProcessed++;
if(itemsProcessed == array.length){
db.close();
// close connection if all items were processed
}
});
});
Is there any other way to do it?
Just use db.collection.count()
EDIT:
You can use the optional callback of forEach, as documented here:
items.forEach((item) => {
// some process...
}, (err) => db.close()); // close connection if all items were processed
check this!
Collection.stat()

MongoDB | Node.js Connection Pooling w/ module.exports

Hey guys so I'm pretty new to creating modules, I'm having a bit of trouble accessing my mongodb connection pool from my main application.
Here's the module:
// mongo-pool.js
// -------------
var assert = require('assert');
var mongodb = require('mongodb');
var MongoClient = mongodb.MongoClient;
var url = 'connection_url';
var mongoPool = {
start: function() {
MongoClient.connect(url, function(err, db) {
assert.equal(null, err);
console.log("Successfully connected to mongo");
// Make the db object accessible here?
});
}
}
module.exports = mongoPool;
When I require mongo-pool.js and call mongoPool.start() It says it successfully connected to mongo, although the db object is not accessible to make queries. Here is the main js file:
var mongoPool = require('./mongo-pool.js');
var pool = mongoPool.start();
var collection = pool.db.collection('accounts');
collection.update(
{ _id: 'DiyNaiis' },
{ $push: { children: 'JULIAN' } }
)
The variable pool is undefined. I can't seem to figure out why, I've tried return db in the module, didn't seem to work.
Any help is appreciated, thank you!
A buddy of mine helped me figure out what the problem was, here's the solution incase anyone runs into it.
I updated my mongo-pool.js module and assigned the db property to itself:
var assert = require('assert');
var mongodb = require('mongodb');
var MongoClient = mongodb.MongoClient;
var url = 'my_database_url';
var mongoPool = {
start: function() {
MongoClient.connect(url, function(err, db) {
assert.equal(null, err);
var self = this;
self.db = db;
// THESE ASSIGNMENTS
console.log("Successfully connected to mongo");
// Make the db object accessible here?
});
}
}
module.exports = mongoPool;
Then in my main.js file:
var mongoPool = require('./mongo-pool.js');
// Include My mongo global module
new mongoPool.start();
// Initialize the new MongoDB object globally
setTimeout(function() {
console.log(db);
}, 3000);
// Set a 3 second timeout before testing the db object...
// It will return undefined if it's called before the mongo connection is made
Now the db object is globally available from a module.

sharing mongodb connection object globally in a node js app

how do i share the db object returned from when i call db.open or db.connect across the entire app?
i have a dbconnect.js module as follows :
var mongodb = require('mongodb');
var global_db = '';
// Define options. Note poolSize.
var serverOptions = {
'auto_reconnect': true,
'poolSize': 5
};
// Now create the server, passing our options.
var serv = new mongodb.Server('localhost', 27017, serverOptions);
// At this point, there is no connection made to the server.
// Create a handle to the Mongo database called 'myDB'.
var dbManager = new mongodb.Db('myDB', serv);
// NOW we initialize ALL 5 connections:
dbManager.open(function (error, db) {
// Do something with the connection.
global_db = db;
// Make sure to call db.close() when ALL connections need
// to be shut down.
db.close();
});
function getConnection()
{
return global_db;
}
exports.getConnection = getConnection;
and i am using this dbconnect.js in my app.js as:
var http = require('http');
var db = require('./dbconnect').getConnection();
var collection = db.collection('testcollection');
console.log(db);
console.log(collection);
var server = http.createServer();
server.on('request',route);
server.listen(8000,'127.0.0.1');
console.log('Server running at http://127.0.0.1:8000');
function route(request,response)
{
var url = request.url;
var doc = {};
doc[url] = 'ok';
collection.insert(doc,{w:1},function(err,result)
{
if(err) console.log(err);
else console.log(result);
});
}
in the console, the db and collection variable show empty values, i also tried removing the db.close() call in dbconnect.js but to no use, however the insertion works when i place it inside dbconnect.js file in the dbManager.open function, how do i do this?or any similar alternatives?
You can't do that, because dbManager.open( is async method, but you trying to get data from module synchronously.
Try this:
In dbconnect.js
var on_db_ready = null;
module.exports = {
db_ready:function(db_ready_callback){
on_db_ready = db_ready_callback;
//here we call callback if already have db
if (global_db) on_db_ready(global_db);
},
getConnection:getConnection
};
dbManager.open(function (error, db) {
if (on_db_ready) on_db_ready(db);
global_db= db;
})
in app.js:
var db = require('./dbconnect').db_ready(function(db){
//Here i have my database
//or can use getConnection method
});
this is not very beautiful way, but, I hope, explain your mistake

Getting data from MongoDB+Node.js to a client side JavaScript

How to connect to MongoDB with Node.js? And then pass the result to a client side JavaScript and display in HTML.
var http = require('http');
var URL = require('url');
var Db = require('mongodb').Db;
var Server = require('mongodb').Server;
var client = new Db('people', new Server("127.0.0.1", 27017, {}), { safe: false });
client.open(function (err, client) {
client.collection('people', listAllData);
});
var listAllData = function (err, collection) {
collection.find().toArray(function (err, results) {
console.log(results);
});
}
You should use Mongoose - elegant mongodb object modeling for node.js. http://mongoosejs.com
The quickstart guide is really cool, you should read it.
According to the documentation, here is a small example of how to use Mongoose:
var mongoose = require('mongoose');
var db = mongoose.createConnection('localhost', 'test');
var schema = mongoose.Schema({ name: 'string' });
var Cat = db.model('Cat', schema);
var kitty = new Cat({ name: 'Zildjian' });
kitty.save(function (err) {
if (err) // ...
console.log('meow');
});
I prefer MongoJS to Mongoose because it uses the same syntax used by the MongoDB Client syntax
https://github.com/gett/mongojs
// simple usage for a local db
var db = mongojs('mydb', ['mycollection']);
// the db is on a remote server (the port default to mongo)
var db = mongojs('example.com/mydb', ['mycollection']);
// we can also provide some credentials
var db = mongojs('username:password#example.com/mydb', ['mycollection']);
// connect now, and worry about collections later
var db = mongojs('mydb');
var mycollection = db.collection('mycollection');
Then you can use the same syntax as the Mongo Client
db.mycollection.find({}, function(err, docs) { ... });
db.mycollection.find({}).limit(2).skip(1, function(err, docs) { ... });

Resources