Nodejs async.eachSeries - node.js

I asked few questions about this few months ago and recently I got back to that script. I figured out some thing and a friend helped with the script but now I have another problem.
This is my script now:
var j = schedule.scheduleJob('*/5 * * * * *', function(){
var steamids = [];
con.query("SELECT * FROM counterStrikeGlobalOffensive", function (err, rows) {
for (var i = 0; i < rows.length; i++) {
steamids.push(rows[i].steam64ID);
}
//const steamIDs = ["2342342341234123", "23423412341234", "2342314123423"]; // Steam IDs to check
eachSeries(steamids, (steamID, callback) => {
CSGOCli.playerProfileRequest(CSGOCli.ToAccountID(steamID));
CSGOCli.on("playerProfile", function(profile) {
console.log(JSON.stringify(profile, null, 2));
callback();
});
}, (err) => {
// error thrown = set, else we're done
});
});
});
When I use constant steamIDs, it works perfectly, but when I use steamids, it gives me an error.(I will explain)...
When I do, console.log(steamids); it returns me this
[ '76561198152643711', '76561198213530057' ]
and steamIDs is
const steamIDs = ["2342342341234123", "23423412341234", "2342314123423"];
so its almost the same as constant SteamIDs but constant has " " around the numbers which shouldn't be why it isn't working but maybe I'm wrong?
Also, I have the callback() but how can I make it stop giving me an error
Error: Callback was already called.
Ask for any other info please :)

You get the Error: Callback was already called. because the CSGOCli.on() is executed multiple times. So it calls the callback once, and later the event fires again. So the callback gets called again but it should only be called once.
For a simple reproduction see this example:
async.eachSeries([1, 2, 3], (data, callback) => {
console.log("Data:", data);
for(let i = 0; i < 2; i++) {
callback();
}
},
(err) => {
console.log("Callback: ", err);
});
But if you add return before the callback like this: return callback();, then the problem disappears because the function will return and no callbacks will be called again.
So change your code to this and see if it works:
CSGOCli.on("playerProfile", function(profile) {
console.log(JSON.stringify(profile, null, 2));
return callback();
});

Related

looping in javascript - callback happening before first loop is fully over

need some help in correcting a loop closure in javascript.
Required flow: MasterGet function is run, data is fetched from a mysql table, for each record fetched a set of 'rules' is run.
Issue faced: if there are two records fetched, while 'rules' are being run for record 1, the record 2 'rules' also get triggered. Need to modify code such at record 2 is checked only after the 'rules' action is completed for record 1.
function MasterGet() {
var countCheckRule = 0;
connection.query(
'SELECT * FROM MASTER',
function selectCb(error, rows, fields) {
if (error) {
console.log('Log 045 - GetData Error: ' + error.message);
return;
}
for (var i = 0; i < rows.length; i++) {
console.log(+ rows[i].INDEX1);
var firstResult = rows[i];
rules.checRules(firstResult, myhttp, function (rules_res) {
firstResult.rules = rules_res;
})
}
});
countCheckRule++;
setTimeout(funcL, 4000);
};
Any guidance will help. Thanks
Added to the issue:
The rules.checrules coding is as below:
exports.checRules = function (A, myhttp, _callback) {
var objrules = { 'rule12': false };
function rule11() {
if (A.NQ > 0 && A.PSQ > 0) {
objrules.rule11 = true;
if (config.execute) {
modifyOrder('S', 'A.BQ', A.TS);
}
} else {
objrules.rule11 = false;
}
}
rule11();
_callback(objrules);
}
So in the loop for 1st record, it checks rule11, and if rule11 is true then it has to execute 'modifyOrder' with the given variables, after 'modifyOrder' is completed, then go back to the loop and check for the 2nd record. If Rule11 is false for the 1st record, then it should automatically go back to the loop and check for the 2nd record.
Currently with the given changes, 2nd record check gets triggered before 'modifyOrder' is complete. Maybe the issue is that the code does not wait for the callback from 'modifyOrder'? Is that the issue? How can I make the code wait till 'modifyOrder' is complete if started.
This code may solve your problem.
This kind of problem accord bez of async nature of js.
function asyncLoop(i, rows, cb) {
if (i < rows.length) {
rules.checRules(rows[i], myhttp, function (rules_res) {
console.log(rules_res);
asyncLoop(i + 1, rows, cb);
//your code
});
} else {
cb();
}
}
function MasterGet() {
connection.query(
'SELECT * FROM MASTER',
function selectCb(error, rows, fields) {
if (error) {
console.log('Log 045 - GetData Error: ' + error.message);
return;
}
asyncLoop(0, rows, () => {
//after async loop complete...
});
});
countCheckRule++;
setTimeout(funcL, 4000);
}

node-schedule job does not working in function

I am trying to work with node-schedule. I want to create a dynamic scheduler. I want to set rule for schedule dynamically. Its working fine when it is called as per the documentation on github. But when I take it in a function through loop then its not working. My code is given below. In code smtp email address has been modified for security purpose.
con.connect(function (err) {
if (err) throw err;
con.query("SELECT * FROM customers", function (err, result, fields) {
if (err) throw err;
for (var i = 0, len = result.length; i < len; i++) {
var dt = dateTime.create(result[i].expire_date);
var formattedDate = dt.format('Y-m-d');
if (result[i].payment_status === 0) {
var dt_str = formattedDate.split('-');
var dt_year = parseInt(dt_str[0]);
var dt_month = parseInt(dt_str[1]);
var dt_date = parseInt(dt_str[2]);
var mail_to = result[i].email, mail_sub='Schedule Of Maintenance', mail_message='Test Mail message Status';
console.log('ScheduleCron Function calling');
scheduleCron(dt_date,dt_month, dt_year, mail_to, mail_sub, mail_message);
} else {
console.log(result[i].payment_status);
}
}
});
});
function scheduleCron(schedule_date,schedule_month,schedule_year,mail_to,mail_sub,mail_message) {
console.log(schedule_date+' '+schedule_month+' '+schedule_year);
console.log('ScheduleCron Function In');
var date = new Date(schedule_year, schedule_month, schedule_date, 13, 33, 0);
schedule.scheduleJob(date, function(){
sendEmail( mail_to, mail_sub, mail_message);
});
}
function sendEmail(mail_to,mail_sub,mail_message) {
var mailOptions = {
from: "email#gmail.com",
to: mail_to,
subject: mail_sub,
text: mail_message
};
transporter.sendMail(mailOptions, function (error, info) {
if (error) {
throw error;
} else {
console.log("Email successfully sent!");
}
});
}
You need to create a job object as you see in documentation https://www.npmjs.com/package/node-schedule
const job = schedule.scheduleJob(date, function(){
sendEmail( mail_to, mail_sub, mail_message);
});
}
You are missing the Job object which is required by the scheduling algorithm.
From the documentation
Jobs and Scheduling
Every scheduled job in Node Schedule is represented by a Job object. You can create jobs manually, then execute the schedule() method to apply a schedule, or use the convenience function scheduleJob() as demonstrated below.
I was confused by this too. I couldn't understand why this wasn't working, I mean "myFunc()" is a function. right? So you throw myFunc() into the second argument of scheduleJob():
function myFunc(){
console.log("stuff");
do.otherStuff();
}
schedule.scheduleJob('*/10 * * * *', myFunc());
But for some strange reason, it has to be called like this:
function myFunc(){
console.log("stuff");
do.otherStuff();
}
schedule.scheduleJob('*/10 * * * *', function(){
myFunc();
});
to avoid the TypeError: this.job.execute is not a function error when it executes.

async.whilst with internal callback

I am trying to loop while the count of some array is less than 50, or if the loop has gone through more than 14 iterations. This seems like a perfect use for async.whilst.
However, my complication is that my work function has an asynchronous query inside of it (a database query).
Here is a simple version of my code:
var items = [];
var key = 20150713;
var iterations = 0;
async.whilst(
function(){
return items.length < 50 || iterations < 14;
},
function(callback){
iterations+=1;
dbQuery("my query", function(err, res){
key -=1;
//add res to items.
callback();
});
},
function(err){
});
Of course this code doesn't work because dbQuery() returns immediately, so async.whilst just blows through 14 iterations and returns an empty array before the first dbQuery even returns.
How do I handle this so that async.whilst waits for the return of the inner function before running again?
Or is async.whilst not suited to my task?
You're using it correctly. Only, from the code you're posting it doesn't look like you're doing anything with the result:
async.whilst(
function(){
return items.length < 50 || iterations < 14;
},
function(callback){
iterations+=1;
dbQuery("my query", function(err, res){
key -=1;
//add res to items.
callback();
});
},
function(err){
// this function will be called when whilst completes
// or when there's an error
if (!err) {
// use items:
console.log(items);
}
else {
console.log('OOps.. something went wrong somewhere');
}
}
);

While loop to check uniqueness of custom id

I have a MongoDB databse set up with some objects that have a unique code (not the primary key).
I should also note that I'm using NodeJS and this code is in my server.js to connect to the MongoDB database.
To generate a new code, I generate one randomly and I want to check if it already exists. If not then we use it no problem, but if it already exists I want to generate another code and check it again. This is the code I use to check if the id already exists:
function createPartyId(callback) {
var min = 10000, max = 99999;
var partyId = -1, count = -1;
async.whilst(
function () { return count != 0; },
function (callback) {
partyId = min + Math.floor(Math.random() * (max - min + 1));
partyId = 88888;
getPartyIdCount(partyId, function(num) {
count = num;
});
},
function (err) {
}
);
}
function getPartyIdCount(partyId, callback) {
count = -1;
db.db_name.find({id: partyId}, function(err, records) {
if(err) {
console.log("There was an error executing the database query.");
callback(count);
}
count = records.length;
callback(count);
});
}
First of all, is there any particular reason you're not using a simple number increment sequence? This type of code is prone to inefficiency, the more numbers you generate the more chance you have of collisions which means you're going to be spending more time on generating an ID for your data than you are on the rest of your processing. Not a good idea.
But I can still tell you what's going wrong.
OK, so getPartyIdCount() will only, ever, always, without fail, return undefined (or, basically, nothing).
Your mongo call processes the return value in a callback, and that callback doesn't assign its value to anything, so return records.length just gets lost into nothingness.
You've mixed up createPartyId(), which it appears you want to run synchronously, with your mongo call, which must run asynchronously.
return always goes with the nearest containing function, so in this case it goes with function(err, records), not function getPartyIdCount(partyId).
(Expanding my comment from above)
The issue is that createPartyId is an asynchronous function, but you're trying to return the value synchronously. That won't work. Once you touch an async operation, the rest of the call stack has to be async as well.
You don't include the code that's calling this, but I assume you want it to be something like:
var partyId = createPartyId();
// do stuff...
That's not going to work. Try this:
function createPartyId(callback) {
var min = 10000, max = 99999;
var partyId = -1, count = -1;
async.whilst(
function () { return (count == 0); },
function (callback) {
partyId = min + Math.floor(Math.random() * (max - min + 1));
partyId = 88888;
getPartyIdCount(partyId, function(err, num) {
if (!err) {
count = num;
}
callback(err);
});
},
function (err) {
// this is called when the loop ends, error or not
// Invoke outer callback to return the result
callback(err, count);
}
);
}
function getPartyIdCount(partyId, callback) {
count = -1;
db.db_name.find({id: partyId}, function(err, records) {
if(err) {
console.log("There was an error executing the database query.");
callback(err);
}
count = records.length;
callback(null, count);
});
}
(I've also adopted the default node.js convention of always returning errors as the first argument to callback functions.)
So, to use this you would do:
getPartyId(function (err, num) {
if (err) { return aughItFellOver(err); }
// do stuff
});

NodeJS async queue too fast (Slowing down async queue method)

I have an HTTP Get request and I want to parse the response and save it to my database.
If i call crawl(i) independentely i get good results. But i have to call crawl() from 1 to 2000.
I get good results but some responses seem to get lost and some responses are duplicates. I don't think I understand how to call thousands of asynchronous functions. I am using the async module queue function but so far I am still missing some data and still have some duplicates. What am I doing wrong here? Thanks for your help.
What i am crawling
My node functions :
function getOptions(i) {
return {
host: 'magicseaweed.com',
path: '/syndicate/rss/index.php?id='+i+'&unit=uk',
method: 'GET'
}
};
function crawl(i){
var req = http.request(getOptions(i), function(res) {
res.on('data', function (body) {
parseLocation(body);
});
});
req.end();
}
function parseLocation(body){
parser.parseString(body, function(err, result) {
if(result && typeof result.rss != 'undefined') {
var locationTitle = result.rss.channel[0].title;
var locationString = result.rss.channel[0].item[0].link[0];
var location = new Location({
id: locationString.split('/')[2],
name: locationTitle
});
location.save();
}
});
}
N = 2 //# of simultaneous tasks
var q = async.queue(function (task, callback) {
crawl(task.url);
callback();
}, N);
q.drain = function() {
console.log('Crawling done.');
}
for(var i = 0; i < 100; i++){
q.push({url: 'http://magicseaweed.com/syndicate/rss/index.php?id='+i+'&unit=uk'});
}
[EDIT] WELL, after a lot of testing it seems that the service I am crawling cannot handle so many request that fast. Because when I do each requests sequentially, I can get all the good responses.
Is there a way to SLOW DOWN ASYNC queue method?
You should have a look at this great module, async which simplifies async tasks like this. You can use queue, simple example:
N = # of simultaneous tasks
var q = async.queue(function (task, callback) {
somehttprequestfunction(task.url, function(){
callback();
}
}, N);
q.drain = function() {
console.log('all items have been processed');
}
for(var i = 0; i < 2000; i++){
q.push({url:"http://somewebsite.com/"+i+"/feed/"});
}
It will have a window of ongoing actions and the tasks room will be available for a future task if you only invoke the callback function. Difference is, your code now opens 2000 connections immidiately and obviously the failure rate is high. Limiting it to a reasonable value, 5,10,20 (depends on site and connection) will result in a better sucess rate. If a request fails, you can always try it again, or push the task to another async queue for another trial. The key point is to invoke callback() in queue function, so that a room will be available when it is done.
var q = async.queue(function (task, callback) {
crawl(task.url);
callback();
}, N);
You'are executing next task immediately after starting the previous one, in this way, the queue is just meaningless. You should modify your code like this:
// first, modify your 'crawl' function to take a callback argument, and call this callback after the job is done.
// then
var q = async.queue(function (task, next/* name this argument as 'next' is more meaningful */) {
crawl(task.url, function () {
// after this one is done, start next one.
next();
});
// or, more simple way, crawl(task.url, next);
}, N);
Another option if you want. Vanilla JS without fancy libraries.
var incrementer = 0;
var resultsArray = [];
var myInterval = setInterval(function() {
incrementer++
if(incrementer == 100){
clearInterval(myInterval)
//when done parse results array
}
//make request here
//push request result to array here
}, 500);
Invokes the function every half second. Easy way to force sync and exit after x requests.
I know I am a little late to the question, however here is a solution I wrote to slow down the number of requests when testing an api endpoint, using node 4 or node 5:
var fs = require('fs');
var supertest = require('supertest');
var request = supertest("http://sometesturl.com/api/test/v1/")
var Helper = require('./check.helper');
var basicAuth = Helper.basicAuth;
var options = Helper.options;
fs.readFile('test.txt', function(err, data){
var parsedItems = JSON.parse(data);
var urlparts = []
// create a queue
for (let year of range(1975, 2016)) {
for (var make in parsedItems[year]){
console.log(year, make, '/models/' + year + '/' + make)
urlparts.push({urlpart:'/models/' + year + '/' + make, year: year, make: make})
}
}
// start dequeue
waitDequeue();
// This function calls itself after the makeRequest promise completes
function waitDequeue(){
var item = urlparts.pop()
if (item){
makeRequest(item)
.then(function(){
// wait this time before next dequeue
setTimeout(function() {
waitDequeue();
}, 3000);
})
} else {
write(parsedItems)
}
}
// make a request, mutate parsedItems then resolve
function makeRequest(item){
return new Promise((resolve, reject)=>{
request
.get(item.urlpart)
.set(options.auth[0], options.auth[1])
.set(options.type[0], options.type[1])
.end(function(err, res) {
if (err) return done1(err);
console.log(res.body)
res.body.forEach(function(model){
parsedItems[item.year][item.make][model] = {}
});
resolve()
})
})
}
// write the results back to the file
function write(parsedItems){
fs.writeFile('test.txt', JSON.stringify(parsedItems, null, 4), function(err){
console.log(err)
})
}
})
A little late but I have found this works!
Using async you can slow down the queue by using whilst inside the task handler eg:
var q = async.priorityQueue(function(task, callback) {
// your code process here for each task
//when ready to complete the task delay it by calling
async.whilst( //wait 6 seconds
function() {
return count < 10;
},
function(callback) {
count++;
setTimeout(function() {
callback(null, count);
}, 1000);
},
function (err, n) {
// n seconds have passed
callback(); //callback to q handler
}
); //whilst
} , 5);

Resources