Node.js + Mongoose consume all memory on inserts - node.js

I need to fill Mongo collection with about 80 million of records like
//Example
{
"_id" : "4gtvCPATZ",
"isActivated" : false
}
Where _id is randomly generated. I do this using Node.js Express and Mongoose:
app.get('/api/codes', function(req, res) {
for (var i = 0; i < 100000; i++) {
var code = new Code({
_id: randomStr(9),
isActivated: 0
});
code.save();
code = null;
}
res.render('index');
});
function randomStr(m) {
var m = m || 9;
s = '', r = 'ABCDEFGHIJKLMNPQRSTUVWXYZabcdefghijklmnpqrstuvwxyz123456789';
for (var i = 0; i < m; i++) {
s += r.charAt(Math.floor(Math.random() * r.length));
}
return s;
};
On 'index' jade layout has JS code that reloads the page to generate next 100000 records:
script(type='text/javascript').
location.href='http://localhost:3000/api/codes'
node.js process starts to consume memory and after 4-5 page reloads hangs up having 1GB of memory.
What am I doing wrong?
UPDATE:
Considering the robertklep's comment I've updated the code and it works properly:
app.get('/api/codes', function(req, res) {
var count = 0;
async.whilst(
function () { return count < 100000; },
function (callback) {
count++;
var code = new Code({
_id: randomStr(9),
isActivated: 0
});
code.save(function(){
callback();
});
},
function (err) {
res.render('index');
}
);
});
UPDATE 2: I've tested the freakish's advice and made my app work faster:
var MongoDB = require("mongodb");
app.get('/api/codes', function(req, res) {
var MongoClient = require('mongodb').MongoClient
, format = require('util').format;
MongoClient.connect('mongodb://127.0.0.1:27017/code', function(err, db) {
var collection = db.collection('codes');
var count = 0;
async.whilst(
function () { return count < 10; },
function (callback) {
count++;
var docs = [];
for (var i = 0; i < 100000; i++) {
docs.push({
_id: randomStr(9),
isActivated: 0
});
}
collection.insert(docs, function(err, docs) {
callback();
});
},
function (err) {
res.render('index');
}
);
})
});
Now it writes about 1M records in 60-70 seconds.
Thanks!

Related

Array is empty, no data after for loop added

I'm using redis and nodejs for the first time, without success. Trying to loop insert data but got an empty array.
const redis = require("redis");
const client = redis.createClient({
retry_strategy: function(options) {
if (options.error && options.error.code === "ECONNREFUSED") {
// End reconnecting on a specific error and flush all commands with
// a individual error
return new Error("The server refused the connection");
}
if (options.total_retry_time > 1000 * 60 * 60) {
// End reconnecting after a specific timeout and flush all commands
// with a individual error
return new Error("Retry time exhausted");
}
if (options.attempt > 10) {
// End reconnecting with built in error
return undefined;
}
// reconnect after
return Math.min(options.attempt * 100, 3000);
},
});
var data_value = {
id: '235235',
totalrv: 'WAIT',
product: 'productName2',
url: 'url2',
process: 'proc',
task: 'msg'
};
client.set("key0", JSON.stringify(data_value));
client.set("key1", JSON.stringify(data_value));
client.set("key2", JSON.stringify(data_value));
client.set("key3", JSON.stringify(data_value));
client.set("key4", JSON.stringify(data_value));
//client.get("key2", redis.print);
var logger_data = {
logger: []
};
client.keys('*', function (err, keys) {
if (err) return console.log(err);
for(var i = 0, len = keys.length; i < len; i++) {
var values_v = client.get(keys[i].toString(), function(err, val) {
// console.log(logger_data);
// data is exist ...
logger_data.logger.push(JSON.parse(val));
});
}
});
// empty data
console.log(logger_data);
I wrote 2 print data result, in the loop it's working ok, but end of function, there are no data in array.
you can call a function inside the callback if you want to print the logger_data with values outside the asynchronous callback, like this
function printLoggerData(){
console.log(logger_data);
}
client.keys('*', function (err, keys) {
if (err) return console.log(err);
for(var i = 0, len = keys.length; i < len; i++) {
var values_v = client.get(keys[i].toString(), function(err, val) {
// console.log(logger_data);
// data is exist ...
logger_data.logger.push(JSON.parse(val));
// calling the function here so that it contains the latest values
printLoggerData();
});
}
});
Ok, with help of CertainPerformance, it's now working in asynchronous.
Thank you very much ...
async function getMessage () {
var logger_data = {
logger: []
};
return new Promise(function(resolve, reject) {
client.keys('*', function (err, keys) {
if (err) return console.log(err);
for(var i = 0, len = keys.length; i < len; i++) {
var values_v = client.get(keys[i].toString(), function(err, val) {
logger_data.logger.push(JSON.parse(val));
resolve(logger_data);
});
}
});
});
}
async function main() {
let message = await getMessage();
console.log(message);
}
main();

Redis in Nodejs for loop not working properly

I have a for loop like below which isn't getting executed as expected.
var redis = require('redis');
var client = redis.createClient();
var arr = [{title:"title1"},{title:"title2"},{title:"title3"},{title:"title4"}];
for(var i =0; i<arr.length; i++){
//console.log(arr[i]);
var obj1 = arr[i];
client.get(obj1.title, function(err, response){
if(err){
console.log(err);
}
if(response){
if(i%3==0){
client.del(obj1.title);
}else{
client.incr(obj1.title);
}
}else{
client.set(obj1.title, 1);
}
});
}
The output on running the below code afterwards was
for(var i=0; i<arr.length; i++){
client.get(arr[i].title, redis.print);
}
The output:
Reply: null
Reply: null
Reply: null
Reply: null
Reply: null
Reply: null
Reply: 2
which was not what i expected, since all values except the one divisible by 3 should be atleast 1;
Please create a new function. In the new function, you can delete, increment or creating the new key.
The below code works fine for me. Please check.
var redis = require('redis');
var client = redis.createClient();
var arr = [ {
title : "title1"
}, {
title : "title2"
}, {
title : "title3"
}, {
title : "title4"
} ];
function delOrIncr(obj1, i) {
client.get(obj1.title, function(err, response) {
if (err) {
console.log(err);
}
if (response) {
if (i % 3 === 0) {
console.log('Deleting >' + obj1.title);
client.del(obj1.title);
} else {
console.log('Increment >' + obj1.title);
client.incr(obj1.title);
}
} else {
console.log('Creating new >' + obj1.title);
client.set(obj1.title, 1);
}
});
}
for (var i = 0; i < arr.length; i++) {
delOrIncr(arr[i], i);
}
Note:-
Please run the get as a separate program to check the result of the above program.
var redis = require('redis');
var client = redis.createClient();
var arr = [{title:"title1"},{title:"title2"},{title:"title3"},{title:"title4"}];
for(var i =0; i<arr.length; i++){
//console.log(arr[i]); // this is cool
var obj1 = arr[i];
client.get(obj1.title, function(err, response){
if(err){
console.log(err);
}
if(response){
if(i%3==0){
// mistake 1:
// due to async op,loop will already be over and i will be 3 here
// mistake 2:
// obj1 will be arr[3] here, not what you were expecting :D
client.del(obj1.title);
}else{
client.incr(obj1.title);
}`enter code here`
}else{
// so only the last obj gets stored.
client.set(obj1.title, 1);
}
});
}

For loop not working properly in node js

I want to insert multiple data by for loop , but this code add 2 or 3 data only , then its continue loading loading but nothing happen ...
router.post('/addclient' , function (req , res){
var finished = false;
var user = req.body.username;
for(var i = 0 ; i <30 ; i++){
req.body.username = user + i ;
objDB.insert("clientList" ,req.body, function(err, data){
//if(err) console.log(err);
if(i>20){
finished = true;
}
});
}
if(finished){
res.redirect('/client/client-list');
}
});
you are doing it wrong. insert is asynchronous, so when insert for i=1 completes and callback gets called, maybe i is equal to 20 and more and you can't guess it. you are missing synchronous and asynchronous concepts and differences.
here is two solutions for your problem :
first, adding 30 items at once :
var clientsList = [];
for (var i = 0 ; i < 30 ; i++) {
clientsList[i] = Ith client object; // create object you want to insert for client i
}
// for example insertAtOnce is a function to insert your list
objDB.insertAtOnce(clientsList,function(err,callback) {
if (err) console.log("error : ", err);
res.redirect('/client/client-list');
});
second, using bind
function insertCallback(itr, res, err,callback) {
if (itr == 29) {
res.redirect('/client/client-list');
}
}
----
for(var i = 0 ; i <30 ; i++){
req.body.username = user + i ;
objDB.insert("clientList" ,req.body,insertCallback.bind(null, i, res);
}
Would be best if you use the async node.js library which provides control flows like running each function in series. You could use the async.whilst() method and restructure your code as:
router.post('/addclient', function(req, res) {
var user = req.body.username,
count = 0,
result = { finished = false };
async.whilst(
function () { return count < 30; },
function (callback) {
count++;
req.body.username = user + count.toString();
objDB.insert("clientList", req.body, function(err, data){
if (err) { return callback(err); }
result["data"] = data;
if( count > 20 ) result.finished = true;
callback(null, result);
});
},
function (err, result) {
if (err) { /* handle err */};
if (result.finished) res.redirect('/client/client-list');
}
);
});
You have a callback called after running insert function. It runs not exactly after running insert, and not in loop(it asynchronous). So in your code res.redirrect will never be called, because finished will be true after all loop will run. If you don't need to have all this inserts finished before answer, you can go this way:
if(i == 29){
res.redirect('/client/client-list');
}
Otherwise you need to create some queue, for example, i'm using async library https://github.com/caolan/async to create queues and running callbacks after queue ends.
router.post('/addclient', function(req, res) {
var finished = false;
var user = req.body.username;
var i = 0;
while( true ) {
if ( i > 20 ) {
res.redirect('/client/client-list');
}
req.body.username = user + i;
var promise = new Promise();
objDB.insert("clientList", req.body, function(err, data) {
promise.resolve();
});
promise.then(function() {
i++;
continue;
});
}
});
I have written it from the top of my head. But the basic idea here is that you need a loop to run infinitely, then resolve a promise; the resolution triggers continuing the loop, and increment the counter.
Use Promise. I would have done something like below
router.post('/addclient' , function (req , res){
var finished = false;
var promise = [];
var user = req.body.username;
for(var i = 0 ; i <30 ; i++) {
req.body.username = user + i;
promise.push(new Promise(resolve, reject) {
objDB.insert("clientList" ,req.body, function(err, data) {
if(err) {
reject(err)
return
}
resolve(data)
})
})
}
Promise.all(promise).then(function() {
finished = true
})
if(finished){
res.redirect('/client/client-list');
}
});

Random selection of rows in MongoDB async

So I have read the other posts on how to get a selection of random rows but none work for me.
Random record from MongoDB
and How to find random records in mongodb both suggest dated solutions that don't compile anymore.
Adding an extra random field to my database is not an option.
I've come up with the following code, but it also doesn't work.
exports.randomX = function(req, res)
{
var x = req.params.x;
db.collection('table', function(err, collection) {
collection.count(function(err, count) {
var r;
var arr = [];
for(var i=1; i<=x; i++)
{
r = Math.floor(Math.random() * count);
collection.find().limit(1).skip(r).toArray(function(err, items)
{
arr.push(items[0]);
if(arr.length === x)
res.send(arr);
});
}
});
});
}
Not sure if it's your only problem, but x will be a string, so if you want to compare it to arr.length you should either parse it into a number or just use == instead of ===.
exports.randomX = function(req, res)
{
var x = req.params.x;
db.collection('table', function(err, collection) {
collection.count(function(err, count) {
var r;
var arr = [];
for(var i=1; i<=x; i++)
{
r = Math.floor(Math.random() * count);
collection.find().limit(1).skip(r).toArray(function(err, items)
{
arr.push(items[0]);
if(arr.length == x)
res.send(arr);
});
}
});
});
}

Nodejs MongoDB updating documents with asynchronous function

I'm new to NodeJS and MongoDB. Is there a way to update a document using asynchronous functions in nodejs? I have a collection of websites with documents: _id, name and registrant. I want to insert a value in registrar using an asynchronous function. The function is a system call, and I just parse the output that I want. Running the code, the async function logs the information I want but does not store in the database. Any suggestions on how I can solve this? Thanks.
/*
* { _id: 53448014b15c693931000002,
* name: 'google.com',
* registrant: 'defaultval' } */
var MongoClient = require('mongodb').MongoClient;
var ObjectID = require('mongodb').ObjectID;
var id = '53448014b15c693931000002';
var domain = 'google.com';
MongoClient.connect('mongodb://127.0.0.1:27017/mydb', function(err, db) {
if (err) throw err;
var collection = db.collection('websites');
collection.findAndModify({_id:new ObjectID(id)},
{},
{$set: {registrant: test(domain, function (output) {
var t = output.split("\n");
for (var i = 0; i < t.length; ++i) {
if (t[i].indexOf("Registrant Organization:") != -1) {
console.log(t[i].substring(t[i].indexOf(":") + 2, t[i].length));//prints correct value, need this to store in registrant doc
return t[i].substring(t[i].indexOf(":") + 2, t[i].length);
}
}
})}},
{},
function(err, object) {
if (err) console.log(err.message);
else {
console.log(object);
}
db.close();
});
});
var test = function(domain, cb) {
var sys = require('sys');
var exec = require('child_process').exec;
var child = exec('whois ' + domain, function(error, stdout, stderr) {
cb(stdout);
});
}
Start one process on core:
var cluster = require('cluster')
, numCPUs = require('os').cpus().length
, windows = require('os').platform() == 'win32';
if(cluster.isMaster) {
// Fork workers.
for (var i = 0; i < numCPUs; i++) {
cluster.fork();
}
cluster.on('exit', function(worker, code, signal) {
console.log('worker ' + worker.pid + ' died');
});
return;
}
Create function to be async:
function mongodbDriverQuery(callback) {
collection.findOne({ id: getRandomNumber()}, function(err, world) {
callback(err, world);
});
}
http.createServer(function (req, res) {
// JSON response object
var hello = {message: "Hello, World!"};
var helloStr = "Hello, World!";
var path = url.parse(req.url).pathname;
// mysql on windows is not supported
if (windows && (path.substr(0, 3) == '/my' || path == '/update')) {
path = '/doesntexist';
}
switch (path) {
case '/mongodbdriver':
// Database Test
var values = url.parse(req.url, true);
var queries = values.query.queries || 1;
var queryFunctions = new Array(queries);
for (var i = 0; i < queries; i += 1) {
queryFunctions[i] = mongodbDriverQuery;
}
res.writeHead(200, {'Content-Type': 'application/json; charset=UTF-8'});
// and run it
async.parallel(queryFunctions, function(err, results) {
if (queries == 1) {
results = results[0];
}
res.end(JSON.stringify(results));
});
break;

Resources