I want to query sql to get the name and store it in an array.
Next time if it is present in array, then It should not query SQL.
But here it is querying each time to sql.
How can I avoid this ??
var ids = [62519, 1, 2, 3, 2656, 82969, 36201, 82969, 78788, 82969, 97647, 82969, 53745, 54510, 60850, 82969, 29872, 51407, 29347, 105113];
var cnames = {};
for (var i = 0; i < ids.length; i++) {
var nlid = ids[i];
console.log(nlid);
// It should be != not !== according to my knowledge
if (cnames[nlid] != undefined) {
console.log('already present');
}
else {
console.log('not present');
getCname(nlid)
function getCname(nlid) {
sqlcon.query('select name from ss_newsletters where newsletterid=' + nlid + ' limit 1', function (err, rows, fields) {
if (err) {
console.error(err);
}
else {
var x = rows[0].name;
cnames[nlid] = x;
}
});
}
}
}
Nodejs is async, you need to use callbacks, but you run asynchronous code in synchronous loop.
Look at async module: https://github.com/caolan/async . There is async loops, like forEach or eachSerie.
Node JS query function is asynchronous, so first remove the duplicates from the array and then start query.
The following code will remove duplicates from the array ids.
ids.filter(function(elem, index) {
return ids.indexOf(elem) == index;
});
Edited:
Otherwise query synchronously. But it is not a good practice in node js.
var ids = [62519, 1, 2, 3, 2656, 82969, 36201, 82969, 78788, 82969, 97647, 82969, 53745, 54510, 60850, 82969, 29872, 51407, 29347, 105113];
var cnames = {};
getAllNames(ids.length);
function getAllNames(lastIndex) {
var nlid = ids[lastIndex];
console.log(nlid);
if(lastIndex <= 0) // fully iterated
return;
if (cnames[nlid] !== undefined) {
console.log('already present');
} else {
console.log('not present');
sqlcon.query('select name from ss_newsletters where newsletterid=' + nlid + ' limit 1', function(err, rows, fields) {
if (err) {
console.error(err);
} else {
var x = rows[0].name;
cnames[nlid] = x;
}
getAllNames(lastIndex - 1); // Call Synchronously
});
}
}
Related
I was writing a nest code, tried using async.waterfall or async.series but I am getting random results every time I refresh. it seems to be because of the queries of the first 2 functions randomly finishing.
first query was sorting to committed_date DESC. but when I add the 2 sub queries. the sort gets distorted.
Step1: Loop landingpages
Step1.1 - fetch details1 repositories
Step1.2 - fetch details2 versions
Step2: build array
db.collection('landingpages').find({is_deleted:{$ne:1}}).sort({committed_date:-1}).limit(10).toArray(function(err, db_results) {
var data_array = [];
var x=1;
if(db_results.length == 0) {
return_data.lps = data_array;
parallel_done();
}else{
async.each(db_results, function (db_results1, cb) {
async.watefall(
[
function(callback) {
//if this is removed or passed as callback(null, ""); the sort is fixed from committed - 1
var data_repo = {};
db.collection('repositories').find({repository_id: repository_id}).toArray(function(err, db_results1) {
if(db_results1.length == 0){
var data_repo = {};
callback(null, data_repo);
}else{
var data_repo = db_results1[0];
callback(null, data_repo);
}
});
},
function(callback) {
//if this is removed or passed as callback(null, ""); the sort is fixed from committed - 1
var data_version = {};
db.collection('versions').find({landingpage_id: landingpage_id}).sort({_id:-1}).limit(1).toArray(function(err, db_results1) {
if(db_results1.length == 0){
var data_version = {};
callback(null, data_version);
}else{
var data_version = db_results1[0];
callback(null, data_version);
}
});
}
],
function (err, data_repo,data_version) {
var document = {
"x": x++,
"landingpage_id": db_results1.landingpage_id,
"repository_id": db_results1.repository_id,
"version_id": data_version.version_id,
"val": db_results1,
"data_repo": data_repo,
"data_version": data_version,
};
data_array.push(document);
if(data_array.length == db_results.length) {
return_data.lps = data_array;
}
}
);
});
}
});
I Have to Insert about 10,00000 documents in mongodb using nodejs.
I'm generating these documents using a for loop storing them into an array before finally inserting them into mongodb.
var codeArray = new Array();
for (var i = 0; i<1000000; i++){
var token = strNpm.generate();
var now = moment().format('YYYYMMDD hhmmss');
var doc1 = {id:token,
Discount_strId:"pending",
Promotion_strCode:token,
Promotion_strStatus:"I",
Promotion_dtmGeneratedDate:now,
User_strLogin:"test",
Promotion_strMode:"S",
Promotion_dtmValidFrom:"pending",
Promotion_dtmValidTill:"pending",
LastModified_dtmStamp:now
};
codeArray.push(doc1);
db.collection('ClPromoCodeMaster').insert(codeArray, function (err, result) {
if (err){
console.log(err);
}else{
console.log('Inserted Records - ', result.ops.length);
}
});
The problem I'm facing is mongo has an inserting limit of 16mb, so I can't insert the entire array at once.
Please suggest most optimum solutions.
The main problem is in the request size and not the document size, but it amounts to the same limitation. Bulk operations and the async library with async.whilst will handle this:
var bulk = db.collection('ClPromoCodeMaster').initializeOrderedBulkOp(),
i = 0;
async.whilst(
function() { return i < 1000000; },
function(callback) {
var token = strNpm.generate();
var now = moment().format('YYYYMMDD hhmmss');
var doc = {
id:token,
Discount_strId:"pending",
Promotion_strCode:token,
Promotion_strStatus:"I",
Promotion_dtmGeneratedDate:now,
User_strLogin:"test",
Promotion_strMode:"S",
Promotion_dtmValidFrom:"pending",
Promotion_dtmValidTill:"pending",
LastModified_dtmStamp:now
};
bulk.insert(doc);
i++;
// Drain every 1000
if ( i % 1000 == 0 ) {
bulk.execute(function(err,response){
bulk = db.collection('ClPromoCodeMaster').initializeOrderedBulkOp();
callback(err);
});
} else {
callback();
}
},
function(err) {
if (err) throw err;
console.log("done");
}
);
I should note that regardless there is an internal limit on bulk operations to 1000 operations per batch. You can submit in larger sizes, but the driver is just going to break these up and still submit in batches of 1000.
The 1000 is a good number to stay at though, since it is already in line with how the request will be handled, as well as being a reasonable number of things to hold in memory before draining the request queue and sending to the server.
For inserting millions of record at a time, Create node.js child process fork with MongoDb bulk api.
Child Process Creation:(index.js)
const {fork} = require("child_process");
let counter = 1;
function createProcess(data){
const worker = fork("./dbOperation");
worker.send(data);
worker.on("message", (msg) => {
console.log("Worker Message :",counter, msg);
counter++;
})
}
function bulkSaveUser(records) {
const singleBatchCount = 10000; // Save 10,000 records per hit
const noOfProcess = Math.ceil(records/singleBatchCount);
let data = {};
console.log("No of Process :", noOfProcess);
for(let index = 1; index <= noOfProcess; index++) {
data.startCount = (index == 1) ? index : (((index - 1) * singleBatchCount) + 1);
data.endCount = index * singleBatchCount;
createProcess(data);
}
}
bulkSaveUser(1500000);
DB Operation (dbOperation.js)
const MongoClient = require('mongodb').MongoClient;
// Collection Name
const collectionName = "";
// DB Connection String
const connString = "";
process.on("message", (msg) => {
console.log("Initialize Child Process", msg)
const {startCount, endCount} = msg;
inputStudents(startCount, endCount);
});
function initConnection() {
return new Promise(function(r, e) {
MongoClient.connect(connString, function(err, db) {
if (err) e(err)
r(db);
});
});
}
function inputStudents(startCount, endCount) {
let bulkData = [];
for(let index = startCount; index <= endCount; index++ ){
var types = ['exam', 'quiz', 'homework', 'homework'];
let scores = []
// and each class has 4 grades
for (j = 0; j < 4; j++) {
scores.push({'type':types[j],'score':Math.random()*100});
}
// there are 500 different classes that they can take
class_id = Math.floor(Math.random()*501); // get a class id between 0 and 500
record = {'student_id':index, 'scores':scores, 'class_id':class_id};
bulkData.push({ insertOne : { "document" : record } })
}
initConnection()
.then((db) => {
const studentDb = db.db("student");
const collection = studentDb.collection(colName)
console.log("Bulk Data :", bulkData.length);
collection.bulkWrite(bulkData, function(err, res) {
if (err) throw err;
//console.log("Connected Successfully",res);
process.send("Saved Successfully");
db.close();
});
})
.catch((err) => { console.log("Err :", err) });
}
Sample project to insert millions of record in mongodb using child process fork
In NodeJS, I tried to create 2 object of a same class. However, these 2 object are always the same despite having different values. Here is the class.
function reading(){
var readingArr = [];
};
reading.prototype.dbValue = function(counter, limit, type, mIndex) {
db.data.find({ 'type': type }).limit(limit).sort({timestamp:-1}).skip(counter, function(err, docs){
readingArr = [];
if( docs != 'undefined' ){
for(var i=0; i<limit; i++){
readingArr.push(docs[i].measurement[mIndex].value.toFixed(2)); //2 Decimal Placet;
}
}
});
if(typeof readingArr == 'undefined'){
readingArr = [];
}
return readingArr;
};
Here is the object creation.
var spo2 = new reading();
var spo2Arr = spo2.dbValue(0, 5, 'Oximeter', 1);
var temp1 = new reading();
var temp1Arr = temp1.dbValue(0, 5, 'Temperature', 0);
Both spo2Arr and temp1Arr return the same value despite having different value in the database. Example
spo2Arr: 98.00
temp1Arr: 98.00
spo2Arr: 37.91
temp1Arr 37.91
May I know how to create two unique object in NodeJS?
You're performing an asynchronous function call which is not going to complete until some time after dbValue() has finished executing.
Try this:
reading.prototype.dbValue = function(counter, limit, type, mIndex, cb) {
db.data.find({ 'type': type }).limit(limit).sort({timestamp:-1}).skip(counter, function(err, docs){
if (err)
return cb(err);
var readingArr = [];
if (docs !== undefined) {
for (var i = 0; i < limit; i++)
readingArr.push(docs[i].measurement[mIndex].value.toFixed(2));
}
cb(null, readingArr);
});
};
Then you might use it like:
var spo2 = new reading();
spo2.dbValue(0, 5, 'Oximeter', 1, function(err, spo2Arr) {
// check for `err`, if it's falsey, use `spo2Arr`
});
var temp1 = new reading();
temp1.dbValue(0, 5, 'Temperature', 0, function(err, temp1Arr) {
// check for `err`, if it's falsey, use `temp1Arr`
});
If the temperature readings depend on the oximeter readings, you'll have to mode the temperature reading code inside the oximeter reading callback or you can use a module like async to help structure your control flow.
I am pulling information from some collections in mongo that contain node and edge data. First i must get the node so i can grab its edges. Once i have a list of edges i then go back out and grab more nodes (etc.. based on a depth value). The following code is an loose example of how i am attempting to use async.waterfall and the task list.
Initially i have only a single task but once i make my first query i add to the task array. Unfortunately this does not seem to register with async and it does not continue to process the tasks i am adding.
Is there a better way to do this?
var async = require('async')
var mongoose = require('mongoose')
var _ = requrie('underscore')
var Client = this.Mongo.connect(/*path to mongo*/)
var Node = mongoose.Schema({
id : String,
graph_id : String
})
var Edge = mongoose.Schema({
id : String,
source_id : String,
destination_id : String
})
var Nodes = Client.model('myNode', Node)
var Edges = Client.model('myEdge', Edge)
var funcs = []
var round = 1
var depth = 2
var query = {
node : {
id : '12345'
},
edge : {
id : '12345'
}
}
var addTask = function(Nodes, Edges, query, round, depth) {
return function(callback) {
queryData(Nodes, Edges, query, function(err, node_list) {
if(depth > round) {
round++
function_array.push(addTask(Nodes, Edges, query, round, depth))
}
})
}
}
var queryData = function(Nodes, Edges, query, cb) {
async.waterfall([
function(callback) {
Nodes.find(query.node, function(err, nodes) {
var node_keys = _.map(nodes, function(node) {
return node.id
})
callback(null, nodes, node_keys)
})
},
function(nodes, node_keys, callback) {
query.edge.$or = [ {'source_id' : {$in:node_keys}}, {'destination_id' : {$in:node_keys}} ]
Edges.find(query.edge, function(err, edges) {
var edge_keys = _.map(edges, function(edge) {
if(edge['_doc']['source_id'] != query.node.id) {
return edge['_doc']['source_id']
} else {
return edge['_doc']['destination_id']
}
callback(null, nodes, edges, node_keys, edge_keys)
})
})
}
], function(err, nodes, edges, node_keys, edge_keys) {
// update the results object then...
cb(null, _.uniq(edge_keys)
})
}
var function_array = []
function_array.push(addTask(Nodes, Edges, query, round, depth))
async.waterfall(function_array, function(err) {
Client.disconnect()
//this should have run more than just the initial task but does not
})
--------------------- UPDATE ---------------------------
So after playing around with trying to get Async waterfall or series to do this by adding trailing functions I decided to switch to using async.whilst and am now happy with the solution.
function GraphObject() {
this.function_array = []
}
GraphObject.prototype.doStuff = function() {
this.function_array.push(this.buildFunction(100))
this.runTasks(function(err) {
console.log('done with all tasks')
}
}
GraphObject.prototype.buildFunction = function(times) {
return function(cb) {
if(times != 0) {
this.function_array.push(this.buildFunction(times - 1))
}
cb(null)
}
}
GraphObject.prototype.runTasks = function(cb) {
var tasks_run = 0
async.whilst(
function(){
return this.function_array.length > 0
}.bind(this),
function(callback) {
var func = this.function_array.shift()
func.call(this, function(err) {
tasks_run++
callback(err)
})
}.bind(this),
function(err) {
console.log('runTasks ran '+tasks_run+' tasks')
if(err) {
cb(500)
}
cb(null)
}.bind(this)
)
}
A task in your function_array can only add a new task to the array provided it is NOT the last task in the array.
In your case, your function_array contained only 1 task. That task itself cannot add additional tasks since it's the last task.
The solution is to have 2 tasks in the array. A startTask to bootstrap the process, and a finalTask that is more of a dummy task. In that case,
function_array = [startTask, finalTask];
Then startTask would add taskA, taskB will add task C and eventually
function_array = [startTask, taskA, taskB, taskC, finalTask];
The sample code below that illustrates the concepts.
var async = require('async');
var max = 6;
var nodeTask = function(taskId, value, callback){
var r = Math.floor(Math.random() * 20) + 1;
console.log("From Node Task %d: %d", taskId, r);
// add an edge task
if (taskId < max) {
function_array.splice(function_array.length-1, 0, edgeTask);
}
callback(null, taskId + 1, value + r);
};
var edgeTask = function(taskId, value, callback){
var r = Math.floor(Math.random() * 20) + 1;
console.log("From Edge Task %d: %d", taskId, r);
// add a node task
if (taskId < max) {
function_array.splice(function_array.length-1, 0, nodeTask);
}
callback(null, taskId + 1, value + r);
};
var startTask = function(callback) {
function_array.splice(function_array.length-1, 0, nodeTask);
callback(null, 1, 0);
};
var finalTask = function(taskId, value, callback) {
callback(null, value);
};
var function_array = [startTask, finalTask];
async.waterfall(function_array, function (err, result) {
console.log("Sum is ", result);
});
jsfiddle link: http://jsfiddle.net/T8ee7/
When I call Knockout's subscribe method is there a way I can get both the previous and new value? Right now, I can only call get these values separately.
I want to trigger some code if the old and new value are different.
I suppose I could do the following, but it can get messy...
(http://jsfiddle.net/MV3fN/)
var sv = sv || {};
sv.PagedRequest = function (pageNumber, pageSize) {
this.pageNumber = ko.observable(pageNumber || 1);
this.numberOfPages = ko.observable(1);
this.pageSize = ko.observable(pageSize || sv.DefaultPageSize);
};
var _pagedRequest = new sv.PagedRequest();
var oldValue;
_pagedRequest.pageNumber.subscribe(function (previousValue) {
console.log("old: " + previousValue);
oldValue = previousValue;
}, _pagedRequest, "beforeChange");
_pagedRequest.pageNumber.subscribe(function (newValue) {
console.log("new: " + newValue);
if (oldValue != newValue) {
console.log("value changed!");
}
});
_pagedRequest.pageNumber(10);
_pagedRequest.pageNumber(20);
I prefer using an observable extender.
http://jsfiddle.net/neonms92/xybGG/
Extender:
ko.extenders.withPrevious = function (target) {
// Define new properties for previous value and whether it's changed
target.previous = ko.observable();
target.changed = ko.computed(function () { return target() !== target.previous(); });
// Subscribe to observable to update previous, before change.
target.subscribe(function (v) {
target.previous(v);
}, null, 'beforeChange');
// Return modified observable
return target;
}
Example Usage:
// Define observable using 'withPrevious' extension
self.hours = ko.observable().extend({ withPrevious: 1 });
// Subscribe to observable like normal
self.hours.subscribe(function () {
if (!self.hours.changed()) return; // Cancel if value hasn't changed
print('Hours changed from ' + self.hours.previous() + ' to ' + self.hours());
});
This seems to work for me
ko.observable.fn.beforeAndAfterSubscribe = function (callback, target) {
var _oldValue;
this.subscribe(function (oldValue) {
_oldValue = oldValue;
}, null, 'beforeChange');
this.subscribe(function (newValue) {
callback.call(target, _oldValue, newValue);
});
};
See more at: http://ideone.com/NPpNcB#sthash.wJn57567.dpuf
http://jsfiddle.net/MV3fN/3/
var sv = sv || {};
sv.PagedRequest = function (pageNumber, pageSize) {
var self = this;
self.pageNumber = ko.observable(pageNumber || 1);
self.numberOfPages = ko.observable(1);
self.pageSize = ko.observable(pageSize || sv.DefaultPageSize);
self.pageNumber.subscribe(function (previousValue) {
console.log(previousValue);
console.log(self.pageNumber.arguments[0]);
if (previousValue != _pagedRequest.pageNumber.arguments[0]) {
console.log('value changed');
}
else {
//This won't get executed because KO doesn't
//call the function if the value doesn't change
console.log('not changed');
}
}, _pagedRequest, "beforeChange");
};
var _pagedRequest = new sv.PagedRequest();
_pagedRequest.pageNumber(10);
_pagedRequest.pageNumber(20);
_pagedRequest.pageNumber(20);
_pagedRequest.pageNumber(5);
I don't know if you're really supposed to use arguments[0], but it seems to work.
You could also set up your own method to accomplish this in a much cleaner way:
http://jsfiddle.net/PXKgr/2/
...
self.setPageNumber = function(page) {
console.log(self.pageNumber());
console.log(page);
if (self.pageNumber() != page) {
console.log('value changed');
}
else {
console.log('not changed');
}
self.pageNumber(page);
};
...
_pagedRequest.setPageNumber(10);
_pagedRequest.setPageNumber(20);
_pagedRequest.setPageNumber(20);
_pagedRequest.setPageNumber(5);