I'm having problem calling async function inside a while loop.
the problem is 'while' statement will end before its underlying function result appear and thats because it's async function.
the code is like below:
while (end < min) {
db.collection('products').count({
tags: {
$in: ['tech']
}
}, function(err, result) {
if (result) {
a = result;
}
});
max = min;
min = max - step;
myitems.push(a);
}
res.send(myitems);
and at the end i could not send the result because all of while iteration should finish before sending the final result.
how could i modify the code to solve such a problem?
thanks in advance
Without using third party libraries, here's a method of manually sequencing your async operations. Note, because this is async, you have to process the results inside of the next() function when you see that you are done iterating.
// assume that end, max, min and step are all defined and initialized before this
var results = [];
function next() {
if (end < min) {
// something seems missing from the code here because
// this db.collection() call is always the same
db.collection('products').count({tags: {$in: ['tech']}}, function(err, result) {
if (!err && result) {
results.push(result);
max = min;
min - max - step;
next();
} else {
// got an error or a missing result here, provide error response
console.log("db.collection() error or missing result");
}
}
} else {
// all operations are done now
// process the results array
res.send(results);
}
}
// launch the first iteration
next();
You could leverage a 3rd party library to do this as well (non working performQuery example using async):
function performQuery(range, callback) {
// the caller could pre calculate
// the range of products to retrieve
db.collection('products').count({
tags: {
$in: ['tech'],
// could have some sort of range query
$gte: range.min,
$lt: range.max
}
}, function(err, result) {
if (result) {
callback(result)
}
});
}
async.parallel([
performQuery.bind(null, {min: 0, max: 10}),
performQuery.bind(null, {min: 10, max: 20})
], function(err, results) {
res.send(results);
});
Related
Seems like a super basic task, but I just cannot get this to work (not very experienced with mongo or nodeJS).
I have an array of records. I need to check the DB to see if any records with a matching name already exist and if they do grab that record so I can update it.
Right now I am trying this
function hit_the_db(db, record_name, site_id) {
return new Promise((resolve, reject) => {
var record = db.collection('' + site_id + '_campaigns').find({name: record_name}).toArray(function(err, result) {
if (err) {
console.log('...error => ' + err.message);
reject(err);
} else {
console.log('...promise resolved...');
resolve(result);
}
});
console.log('...second layer of select successful, returning data for ' + record.length + ' records...');
return record;
});
}
This query works in another part of the app so I tried to just copy it over, but I am not getting any records returned even though I know there should be with the data I am sending over.
site_id is just a string that would look like ksdlfnsdlfu893hdsvSFJSDgfsdk. The record_name is also just a string that could really be anything but it is previously filtered so no spaces or special characters, most are something along these lines this-is-the-name.
With the names coming through there should be at least one found record for each, but I am getting nothing returned. I just cannot wrap my head around using mongo for these basic tasks, if anyone can help it would be greatly appreciated.
I am just using nodeJS and connecting to mongoDB, there is no express or mongoose or anything like that.
The problem here is that you are mixing callback and promises for async code handling. When you call:
var record = db.collection('' + site_id + '_campaigns').find({name: record_name}).toArray(function(err, result) {
You are passing in a callback function, which will receive the resulting array of mongo records in a parameter called result, but then assigning the immediate returned value to a variable called 'record', which is not going to contain anything.
Here is a cleaned up version of your function.
function hit_the_db(db, site_id, record_name, callback) {
// Find all records matching 'record_name'
db.collection(site_id + 'test_campaigns').find({ name: record_name }).toArray(function(err, results) {
// matching records are now stored in 'results'
if (err) {
console.log('err:', err);
}
return callback(err, results);
});
}
Here is optional code for testing the above function.
// This is called to generate test data
function insert_test_records_callback(db, site_id, record_name, insert_count, callback) {
const testRecords = [];
for (let i = 0; i < insert_count; ++i) {
testRecords.push({name: record_name, val: i});
}
db.collection(site_id + 'test_campaigns').insertMany(testRecords, function(err, result) {
return callback(err);
});
}
// This cleans up by deleting all test records.
function delete_test_records_callback(db, site_id, record_name, callback) {
db.collection(site_id + 'test_campaigns').deleteMany({name: record_name}, function(err, result) {
return callback(err);
});
}
// Test function to insert, query, clean up test records.
function test_callback(db) {
const site_id = 'ksdlfnsdlfu893hdsvSFJSDgfsdk';
const test_record_name = 'test_record_callback';
// First call the insert function
insert_test_records_callback(db, site_id, test_record_name, 3, function(err) {
// Once execution reaches here, insertion has completed.
if (err) {
console.log(err);
return;
}
// Do the query function
hit_the_db(db, site_id, test_record_name, function(err, records) {
// The query function has now completed
console.log('hit_the_db - err:', err);
console.log('hit_the_db - records:', records);
delete_test_records_callback(db, site_id, test_record_name, function(err, records) {
console.log('cleaned up test records.');
});
});
});
}
Output:
hit_the_db - err: null
hit_the_db - records: [ { _id: 5efe09084d078f4b7952dea8,
name: 'test_record_callback',
val: 0 },
{ _id: 5efe09084d078f4b7952dea9,
name: 'test_record_callback',
val: 1 },
{ _id: 5efe09084d078f4b7952deaa,
name: 'test_record_callback',
val: 2 } ]
cleaned up test records.
I am trying to loop through an array and find the amount of tickets assigned to each person.
Unfortunately, I noticed that my taskcount is getting the same values but in different order, because of its asynchronous nature.
Some queries might take long and so the ones that gets finished first gets inserted and hence my array has the same values but in different order. Now, I want to avoid that and make it so, that once a query gets completed, only then the next value from the array is being picked up and pushed to search from the db. How can i modify my existing code.
exports.find_task_count = function(callback) {
var names = ['Evan', 'Surajit', 'Isis', 'Millie', 'Sharon', 'Phoebe', 'Angel', 'Serah']
var taskcount = []
var resultsCount = 0;
for (var i = 0; i < names.length; i++) {
_tasks.find({'assignee': names[i]}, function (err, tickets) {
resultsCount++
if (err) {
console.log(err)
return callback(err)
} else {
taskcount.push(tickets.length)
if (resultsCount === names.length) {
return callback(taskcount);
taskcount=[]
}
}
})
}
}
You can use the async module designed to handle such scenarios.
I have updated the code as follows
var async = require('async');
exports.find_task_count = function (callback) {
var names = ['Evan', 'Surajit', 'Isis', 'Millie', 'Sharon', 'Phoebe', 'Angel', 'Serah'];
async.map(names, function (name, iterateeCallback) {
_tasks.find({ 'assignee': name }, function (err, tickets) {
if (err) {
return iterateeCallback(err);
}
return iterateeCallback(null, tickets.length);
});
}, function (error, results) {
if (error) {
return callback(error);
}
return callback(null, results);
});
}
As per the documentation of async
Note, that since this function applies the iteratee to each item in parallel, there is no guarantee that the iteratee functions will complete in order. However, the results array will be in the same order as the original coll.
if you still want to process the array in series use mapSeries instead of map in the above code
I want to upsert document in the following way
for (var i = 0; i < req.body.app_events.length; i++ ) {
console.log(req.body.app_events[i].event_key);
//delete upsertData._id;
Appusers.update({app_key: req.body.app_key, e_key:req.body.app_events[i].event_key}, {
$set : {
app_key:req.body.app_key,
e_key: req.body.app_events[i].event_key,
e_name: req.body.app_events[i].event_name
}}, { upsert: true}, function(err, data){
if(err) return console.log(err);
console.log(data);
});
}
it is creating a single document with _id only. i want to insert document if it exist otherwise update on the basis of e_key and app_key.
You really should not be calling asynchronous functions inside a synchronous loop. What you need is something that repects the callback on completion of the loop cycle and will alert when the update is complete. This makes incrementing counters externally safe.
Use something like async.whilst for this:
var i = 0;
async.whilst(
function() { return i < req.body.app_events.length; },
function(callback) {
console.log(req.body.app_events[i].event_key);
//delete upsertData._id;
Appusers.findOneAndUpdate(
{ app_key: req.body.app_key, e_key:req.body.app_events[i].event_key},
{
$set : {
app_key:req.body.app_key,
e_key: req.body.app_events[i].event_key,
e_name: req.body.app_events[i].event_name
}
},
{ upsert: true},
function(err,data) {
if (err) callback(err);
console.log(data);
i++;
callback();
}
);
},
function(err) {
if (err)
console.log(err);
else
// done;
}
);
Now the loop is wrapped with a "callback" which is called in itself within the callback to the update method. Also if you expect a "document" back then you should be using .findOneAndUpdate() as .update() just modifies the content and returns the number affected.
When the loop is complete or when an error is passed to the callback, then handling is moved to the last function block, where you complete your call or call other callbacks as required.
Better than above. Dig into the native driver methods for Bulk operations. You need to be careful that you have an open connection to the database already established. If unsure about this, then try to always wrap application logic in:
mongoose.connection('once',function(err) {
// app logic here
});
Which makes sure the connections have been made. The mongoose methods themselves "hide" this away, but the native driver methods have no knowledge.
But this is the fastest possible listing to update the data:
var i = 0;
var bulk = Appusers.collection.initializeOrderedBulkOp();
async.whilst(
function() { return i < req.body.app_events.length; },
function(callback) {
console.log(req.body.app_events[i].event_key);
bulk.find(
{ app_key: req.body.app_key, e_key:req.body.app_events[i].event_key},
).upsert().updateOne({
$set : {
app_key:req.body.app_key,
e_key: req.body.app_events[i].event_key,
e_name: req.body.app_events[i].event_name
}
});
i++;
if ( i % 1000 == 0) {
bulk.execute(function(err,response) {
if (err) callback(err);
console.log(response);
bulk = Appusers.collection.initializeOrderedBulkOp();
callback();
})
} else {
callback();
}
},
function(err) {
if (err)
console.log(err);
else {
if ( i % 1000 != 0 )
bulk.execute(function(err,response) {
if (err) console.log(err)
console.log(response);
// done
});
else
// done
}
}
);
The Bulk methods build up "batches" of results ( in this case 1000 at a time ) and send all to the server in one request with one response ( per batch ). This is a lot more efficient than contacting the database once per every write.
I have a MongoDB databse set up with some objects that have a unique code (not the primary key).
I should also note that I'm using NodeJS and this code is in my server.js to connect to the MongoDB database.
To generate a new code, I generate one randomly and I want to check if it already exists. If not then we use it no problem, but if it already exists I want to generate another code and check it again. This is the code I use to check if the id already exists:
function createPartyId(callback) {
var min = 10000, max = 99999;
var partyId = -1, count = -1;
async.whilst(
function () { return count != 0; },
function (callback) {
partyId = min + Math.floor(Math.random() * (max - min + 1));
partyId = 88888;
getPartyIdCount(partyId, function(num) {
count = num;
});
},
function (err) {
}
);
}
function getPartyIdCount(partyId, callback) {
count = -1;
db.db_name.find({id: partyId}, function(err, records) {
if(err) {
console.log("There was an error executing the database query.");
callback(count);
}
count = records.length;
callback(count);
});
}
First of all, is there any particular reason you're not using a simple number increment sequence? This type of code is prone to inefficiency, the more numbers you generate the more chance you have of collisions which means you're going to be spending more time on generating an ID for your data than you are on the rest of your processing. Not a good idea.
But I can still tell you what's going wrong.
OK, so getPartyIdCount() will only, ever, always, without fail, return undefined (or, basically, nothing).
Your mongo call processes the return value in a callback, and that callback doesn't assign its value to anything, so return records.length just gets lost into nothingness.
You've mixed up createPartyId(), which it appears you want to run synchronously, with your mongo call, which must run asynchronously.
return always goes with the nearest containing function, so in this case it goes with function(err, records), not function getPartyIdCount(partyId).
(Expanding my comment from above)
The issue is that createPartyId is an asynchronous function, but you're trying to return the value synchronously. That won't work. Once you touch an async operation, the rest of the call stack has to be async as well.
You don't include the code that's calling this, but I assume you want it to be something like:
var partyId = createPartyId();
// do stuff...
That's not going to work. Try this:
function createPartyId(callback) {
var min = 10000, max = 99999;
var partyId = -1, count = -1;
async.whilst(
function () { return (count == 0); },
function (callback) {
partyId = min + Math.floor(Math.random() * (max - min + 1));
partyId = 88888;
getPartyIdCount(partyId, function(err, num) {
if (!err) {
count = num;
}
callback(err);
});
},
function (err) {
// this is called when the loop ends, error or not
// Invoke outer callback to return the result
callback(err, count);
}
);
}
function getPartyIdCount(partyId, callback) {
count = -1;
db.db_name.find({id: partyId}, function(err, records) {
if(err) {
console.log("There was an error executing the database query.");
callback(err);
}
count = records.length;
callback(null, count);
});
}
(I've also adopted the default node.js convention of always returning errors as the first argument to callback functions.)
So, to use this you would do:
getPartyId(function (err, num) {
if (err) { return aughItFellOver(err); }
// do stuff
});
I have asynchronous query in Node.js. Variable sq3 is a connection variable.
For example something like this:
for (var i in res) {
if (i == 1) {
sq3.query("SELECT * from students;",
function (err, res) {
if (err) {
throw err;
} else {
if (res.length == 1) {
//do something
} else {
//break for
}
}
});
sq3.end();
}
}
How can I break from callback function?
Thanks
Just do it like this, using recursion instead of loops. Not only does this allow you to achieve the logic you want. It also doesn't spin up a bunch of async requests at once. They execute in turn, but asynchronously, so it's still performant.
function lookatEntireResponse(res) {
function lookAtItemInResponse(item) {
if(item == 1) {
sq3.query("SELECT * from students;",
function(err, res) {
if (err)
{
throw err;
}
else
{
if(res.length==1)
{
doSomething(item);
lookAtItemInResponse(res.shift());
}
else
{
//just don't call the next lookAtItemInResponse function, effectively same thing as "break;"
}
}
});
sq3.end();
} else {
lookAtItemInResponse(res.shift());
}
}
lookAtItemInResponse(res.shift());
}
You can consider throttling simultaneous requests with similar logic (say allowing 10 such requests per lookAtItem call. This way you can achieve a hybrid of the two methods, and then just optimize the number of simultaneous requests for performance. The Async library makes stuff like this easier.
In your code fragment, you can't break from the within the callback function. In node.js, callback functions run at an unspecified later time on the same thread. This means but the time you callback function executes, the for loop has long since finished.
To get the effect you want, you need to restructure you code quite significantly. Here's an example of how you could do it (untested!!). The idea is to keep calling doSomething() with the list of items, shrinking it by one element each time until the desired result is achieved (your break condition).
function doSomething(res)
{
while (res.length > 0)
{
i = res.shift(); // remove the first element from the array and return it
if (i == 1)
{
sq3.query("SELECT * from students;",
function(err, res) {
if (err)
{
throw err;
}
if (res.length==1)
{
//do something
// continue with the remaining elements of the list
// the list will shrink by one each time as we shift off the first element
doSomething(res);
}
else
{
// no need to break, just don't schedule any more queries and nothing else will be run
}
});
sq3.end();
break; // break from the loop BEFORE the query executes. We have scheduled a callback to run when the query completes.
}
}
}
for (var i = 0; i < res.length; i++) {
if (i == 1) {
sq3.query("SELECT * from students;",
function (err, res) {
if (err) {
throw err;
} else {
if (res.length == 1) {
//do something
} else {
i = res.length
}
}
});
sq3.end();
}
}