I am trying to do dns.reverse() on a list of ip using async.parallel().
The code is as follows:
var functions = [];
for (var i = 0; i < data.length; i++) {
var ip = data[i].ip;
var x = function(callback) {
dns.reverse(ip, (err, hostnames) => {
if (err) {
log.error("Error resolving hostname for [" + ip + '] ' + err);
return callback(null, err);
}
callback(null, hostnames);
});
};
functions.push(x);
}
async.parallel(functions, (err, results) => {
for(var i = 0; i < data.length; i++) {
data[i]['hostnames'] = results[i];
}
handler(null, data);
});
What is happening is dns.reverse() is getting called with the same ip (the last one in data array) for all the calls. May be I am doing something wrong. Can somebody explain what is my mistake?
The first callback is executed after the entire for loop finished, because it's async.
The value of ip will be the one in the last iteration of the loop.
You could put some console.log to realize what's really happening.
The correct way of doing it might be:
async.parallel(data.map(({ ip }) => callback => {
dns.reverse(ip, callback)
}), (err, results) => {
for (var i = 0; i < data.length; i++) {
data[i]['hostnames'] = results[i];
}
handler(null, data);
})
Create a new array of functions based on each ip.
Every function will call it's callback as dns.reverse.
Also, it might be better to return a new data array, not changing data inside the loop:
(err, results) => {
const result = data.map((data, index) => ({
...data,
hostnames: results[index]
})
handler(null, result);
})
Thanks to #apokryfos I got a hint. To get the code working I just need to use let instead of var while declaring ip.
var functions = [];
for (var i = 0; i < data.length; i++) {
let ip = data[i].ip;
var x = function(callback) {
dns.reverse(ip, (err, hostnames) => {
if (err) {
log.error("Error resolving hostname for [" + ip + '] ' + err);
return callback(null, err);
}
callback(null, hostnames);
});
};
functions.push(x);
}
async.parallel(functions, (err, results) => {
for(var i = 0; i < data.length; i++) {
data[i]['hostnames'] = results[i];
}
handler(null, data);
});
For anybody interested in understanding following might be helpful: How do JavaScript closures work?
Related
I am trying to finish my script, but for some reason i don't know, it refuses to execute in the order i put it in.
I've tried placing a 'wait' function between the JoinRequest update function and the following code, but when run, it acts as if the function call and wait function were the other way round, countering the point of the wait().
const Roblox = require('noblox.js')
var fs = require('fs');
var joinRequests = []
...
function wait(ms) {
var d = new Date();
var d2 = null;
do { d2 = new Date(); }
while(d2-d < ms*1000);
};
...
function updateJReqs() {
Roblox.getJoinRequests(4745601).then((array) => {
var i;
var final = [];
for(i = 0; i < array.length; i++) {
final.push(array[i].username);
};
if(final === '') {
final = '-None';
};
joinRequests = final
console.log('Updated join requests.')
});
}
function check() {
setTimeout(() => {
fs.readFile('Request.txt',encoding = 'utf-8', function(err, data) {
if (err) {
check();
} else {
updateJReqs(); //for some reason this function is executed alongside the below, not before it.
// Tried putting wait(x) in here.
console.log('Request received: ' + data)
var solution = joinRequests
console.log('Fuffiling request with ' + solution)
fufillRequest(solution)
fs.unlink('Request.txt', function(err) {
if(err) throw err;
});
check();
}
});
}, 400)
}
check();
The script is supposed to wait until a file is created (accomplished), update the list of join requests (accomplished) and then create a new file with the list of join requests in(not accomplished).
if I understand your code you work with async code, you need to return a promise in updateJReqs and add a condition of leaving from the function because you have an infinite recursion
function updateJReqs() {
return new Promise(resolve => {
Roblox.getJoinRequests(4745601).then((array) => {
var i;
var final = [];
for(i = 0; i < array.length; i++) {
final.push(array[i].username);
};
if(final === '') {
final = '-None';
};
joinRequests = final
console.log('Updated join requests.')
resolve();
});
}
}
async function check() {
setTimeout(() => {
fs.readFile('Request.txt',encoding = 'utf-8', function(err, data) {
if (err) {
await check();
} else {
await updateJReqs();
// Tried putting wait(x) in here.
console.log('Request received: ' + data)
var solution = joinRequests
console.log('Fuffiling request with ' + solution)
fufillRequest(solution)
fs.unlink('Request.txt', function(err) {
if(err) throw err;
});
// you dont have an exit from your function check();
return 'Success';
}
});
}, 400)
}
check().then(res => console.log(res));
Hello, I am writing an application where I need to be able to loop through an array of urls. I know there is an example of how to do this but my issue is a little different, I will explain with some code.
nightmare
.goto('some url')
.evaluate(() => {
//evaluate code
})
.then(dataArray => {
var innerRun = function* () {
var returnData = [];
for (var i = 0; i < dataArray.length; i++) {
var item = dataArray[i];
yield nightmare
.goto(item.url)
.evaluate(function () {
return false;
})
.screenshot(item.imgPath)
returnData.push(item);
}
return returnData;
}
vo(innerRun)(function (err, ads) {
if (err) {
console.log("Error running", err)
return;
}
});
});
I would like to be able to loop that code by using an array of urls. I had issues implementing this I believe because I am already doing it inside the then. It would stop running once it hit the yield nightmare inside the then
var mainLoop = function* () {
for (var j = 0; j < urlArray.length; j++) {
var url = urlArray[j];
yield nightmare.goto(url)//same code as in example above
}
}
vo(mainLoop)(function (err, d) {
if (err) {
console.log("Error running", err)
return;
}
});
The above code is what I attempted to do. If anyone has any ideas it would be a huge help thank you!
Maybe try this:
var urls = ['http://example.com', 'http://example2.com', 'http://example3.com'];
var results = [];
urls.forEach(function(url) {
nightmare.goto(url)
.wait('body')
.title()
.then(function(result) {
results.push(result);
});
});
console.dir(results)
Source: https://github.com/rosshinkley/nightmare-examples/blob/master/docs/common-pitfalls/async-operations-loops.md
When I pass an array with 1 element everything works great, when I pass one with two (max for our use case) I get the following error:
There's no write queue for that file descriptor (after write)!
Here is my code:
exports.triggerPhysical = function(state, alerts) {
console.dir("IN PHYSICAL");
console.dir(alerts);
SerialPort.list(function(err, ports) {
var port = {};
for(var i = 0; i < ports.length; i++) {
try {
if(typeof ports[i].manufacturer != 'undefined' && ports[i].manufacturer.includes("Numato")) {
port = ports[i];
}
} catch(err) {
console.dir(err);
}
}
var numato = new SerialPort(port.comName, {baudrate : 19200}, function(err) {
if(err) {
return console.dir(err);
}
console.dir('calling write');
for(var j = 0; j < alerts.length; j++) {
numato.write('relay ' + state + ' ' + alerts[j].index + '\r', function(err) {
if(err) {
console.dir('error writing');
console.dir(err);
}
console.dir('serial message written');
});
}
numato.close();
return true;
});
});
}
First write works great, second one fails. I am guessing there is an obvious solution but I am not finding it. Any insight would be much appreciated.
I ended up doing two things to resolve this issue. First I upgraded to version 5.x of the node-serialport library.
Second, I changed my code to the following:
exports.triggerPhysical = function(state, alerts) {
var port = new SerialPort('/dev/ttyACM0');
port.on("open", function() {
alerts.forEach(function(alert, idx) {
str = 'relay ' + state + ' ' + alert.index + '\r';
port.write(str, function(err, results) {
if(err) {
console.dir("err writing");
console.dir(err);
} else {
console.dir(results);
}
});
});
})
port.drain(writeDone);
function writeDone() {
console.dir("In writeDone");
port.close();
}
}
I am now able to do consecutive writes without causing any errors and the port doesn't end up in a weird locked state.
I'm new into Promises.
I use Bookshelf.js as ORM.
I fetch a number of webpages, get person info (about actors) from those pages and add them into my database if they don't exist.
But there's a problem, even though console.log(name) returns actor names in the right order, my query checks for only one actor, the latest one, which is 9.
What's wrong here?
var entities = require("entities");
var request = require('request');
var cheerio = require('cheerio');
// create promisified version of request()
function requestPromise(options) {
return new Promise(function (resolve, reject) {
request(options, function (err, resp, body) {
if (err) return reject(err);
resolve(body);
});
});
}
var person = require('./models').person;
app.get('/fetch', function (req, res) {
var promises = [];
var headers = {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
};
for (var i = 1; i < 10; i++) {
promises.push(requestPromise({url: "http://www.example.com/person/" + i + "/personname.html", headers: headers}));
}
Promise.all(promises).then(function (data) {
// iterate through all the data here
for (var i = 0; i < data.length; i++) {
if ($ = cheerio.load(data[i])) {
var links = $("#container");
var name = links.find('span[itemprop="name"]').html(); // name
if (name == null) {
console.log("null name returned, do nothing");
} else {
name = entities.decodeHTML(name);
console.log(name); // returns names in the right order
// does this person exist in the database?
person.where('id', i).fetch().then(function (result) {
if (result) {
console.log(i + "exists");
} else {
console.log(i + " doesn't exist");
// returns "9 doesn't exists" 9 times instead of
// checking each ID individually, why?
}
});
}
} else {
console.log("can't open");
}
}
}, function (err) {
// error occurred here
console.log(err);
});
});
EDIT #2
Now the order is broken and my ID's aren't the same with the site's I fetch data from. I see ID's like 11 and 13 even though I iterate from 1 to 5 and it seems to overrule something since it adds duplicate entries.
Here's what I'm trying to do in a nutshell. "Visit these urls in order and add the data you fetch (e.g. names) in the same order (id1 = name1; id2 = name2, etc) to the database".
app.get('/fetch', function (req, res) {
var promises = [];
var headers = {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
};
for (var i = 1; i < 5; i++) {
promises.push(requestPromise({url: "http://example.com/person/ + i + "/personname.html", headers: headers}));
}
Promise.all(promises).then(function (data) {
// iterate through all the data here
data.forEach(function (item, i) {
var $ = cheerio.load(item);
var name = $("#container span[itemprop='name']").text();
if (!name) {
console.log("null name returned, do nothing");
} else {
// name exists
person.where('id', i).fetch({require: true}).then(function (p) {
console.log(i + " exists");
}).catch(function () {
console.log(i + " does not exist");
new person({id: i, name: name}).save(null, {method: 'insert'}).then(function () {
console.log("success" + i);
});
});
}
}, function (err) {
// error occurred here
console.log(err);
});
});
});
When you run your code through jshint, you will see a warning that says
Don't make functions within a loop.
In this piece of code the callback inside then does not run in sync with the enclosing for loop. It runs whenever the database has fetched your result.
person.where('id', i).fetch().then(function (result) {
if (result) {
console.log(i + "exists");
} else {
console.log(i + " doesn't exist");
}
});
Therefore, when that callback runs eventually, the loop has long finished. Your callback function holds a reference to the loop counter i - which, by now, has the value 9.
It's better to use a function that accepts a parameter than to refer to a loop counter.
Luckily node makes this easy, you can use the forEach array function:
data.forEach(function (item, i) {
var $ = cheerio.load(item);
var name = $("#container span[itemprop='name']").text();
if (!name) {
console.log("null name returned, do nothing");
} else {
console.log("successfully scraped name: " + name);
person.where('id', i).fetch({require: true}).then(function (p) {
console.log(i + " exists");
}).catch(function () {
console.log(i + " does not exist");
});
}
});
Note that you can make Bookshelf.js throw instead of silently passing over non-existing records with {require: true}.
More generally speaking, I don't see a real connection between scraping the name from a website and retrieving a model from the database. These two things should probably be done in separate functions that each return an individual promise for the respective thing. That way, requests to the database can run in parallel with requests to the web server.
It looks like you need a closure for person.where('id', i).fetch().
also use node-fetch instead of hand rolling request-promise.
I've got following code
var data=[];
var i = 0,
j = 9;
async.whilst(function () {
return i <= j;
}, function (next) {
connection.query('select * from table', function (err,rows, field) {
data.push(rows.length);
console.log(data);
});
i++;
next();
}, function (err) {
console.log(data);
});
I want to execute the query first and then return result. How should I do it. I've also done it with simple for loop but does not work.
I think you just need to call next() after getting the result of the query:
var data = [];
var i = 0;
var j = 9;
async.whilst(
function () { return i <= j; },
function (next) {
i++;
connection.query('select * from table', function(err, rows, field) {
data.push(rows.length);
console.log(data);
next();
});
},
function (err) {
console.log(data);
}
);
Here is another simple solution with async/await if you use babeljs:
(async function() {
let data = [];
let i = 0;
const j = 9;
while (i <= j) {
let length = await new Promise((resolve, reject) => {
connection.query('select * from table', function(err, rows, field) {
if (err) return reject(err);
resolve(rows.length);
});
});
data.push(length);
}
return data;
})()