Nightmare Looping - node.js

Hello, I am writing an application where I need to be able to loop through an array of urls. I know there is an example of how to do this but my issue is a little different, I will explain with some code.
nightmare
.goto('some url')
.evaluate(() => {
//evaluate code
})
.then(dataArray => {
var innerRun = function* () {
var returnData = [];
for (var i = 0; i < dataArray.length; i++) {
var item = dataArray[i];
yield nightmare
.goto(item.url)
.evaluate(function () {
return false;
})
.screenshot(item.imgPath)
returnData.push(item);
}
return returnData;
}
vo(innerRun)(function (err, ads) {
if (err) {
console.log("Error running", err)
return;
}
});
});
I would like to be able to loop that code by using an array of urls. I had issues implementing this I believe because I am already doing it inside the then. It would stop running once it hit the yield nightmare inside the then
var mainLoop = function* () {
for (var j = 0; j < urlArray.length; j++) {
var url = urlArray[j];
yield nightmare.goto(url)//same code as in example above
}
}
vo(mainLoop)(function (err, d) {
if (err) {
console.log("Error running", err)
return;
}
});
The above code is what I attempted to do. If anyone has any ideas it would be a huge help thank you!

Maybe try this:
var urls = ['http://example.com', 'http://example2.com', 'http://example3.com'];
var results = [];
urls.forEach(function(url) {
nightmare.goto(url)
.wait('body')
.title()
.then(function(result) {
results.push(result);
});
});
console.dir(results)
Source: https://github.com/rosshinkley/nightmare-examples/blob/master/docs/common-pitfalls/async-operations-loops.md

Related

foreach loop in sync function in nodejs

I have code written
function getDetails (req, res) {
const dbQuery = `call spGetSLAReportsDetails('${req.body.domainId}', ${req.body.days},'${req.body.type}','${req.body.app}')`
try {
connectDatabase(dbQuery).then((rows) => {
if (!_.isEmpty(rows.dbData) && !_.isEmpty(rows.dbData[0])) {
const resultList = []
rows.dbData.pop()
var bar = new Promise((resolve, reject) => {
rows.dbData[0].forEach((element, index, array) => {
let query = `select * from YCW.YWFWIC ic where ic.witem=${element.witem} and ic.reqno=${element.reqno};`
connectDatabase(query).then((data) =>{
for (var i = 0; i < data.dbData.length; i++) {
element[data.dbData[i]["cfield"]] = data.dbData[i]["cvalue"]
}
resultList.push(element)
// console.log(resultList)
}).catch((err) => {
console.log(err)
})
if (index === array.length -1) resolve();
});
});
bar.then(() => {
console.log(resultList);
});
res.status(msgCodeJson.ERR004.code).send({
result: resultList })
} else {
console.log("empty array")
res.status(msgCodeJson.ERR004.code).send({
message : "No data found"
})
// httpResponseHandlerError(res, msgCodeJson.ERR001.code, msgCodeJson.ERR001.msg)
}
}).catch(() => {
httpResponseHandlerError(res, msgCodeJson.ERR002.code, msgCodeJson.ERR002.msg)
})
} catch (err) {
httpResponseHandlerError(res, msgCodeJson.ERR009.code, msgCodeJson.ERR009.msg)
}
}
module.exports.getDetails = getDetails
i want data to be fit in resultlist but i get empty list after all operation.
while in foreach loop i am getting proper output.
kindly help in issue.
i tried with async foreach loop but some syntax error is coming.
kindly help
as mentioned in the comment of the code you're using
Best way to wait for .forEach() to complete
This is OK if there is no async processing inside the loop.
yet you have an async function inside your forEach callback, namly this:
connectDatabase(query).then((data) => {
for (var i = 0; i < data.dbData.length; i++) {
element[data.dbData[i]["cfield"]] = data.dbData[i]["cvalue"]
}
resultList.push(element)
}).catch((err) => {
console.log(err)
})
you'll need to resolve the "outer/parent" promise from inside the "inner/child" promise
I suggest using a regular good old for loop and/or checking the count of resolved promises against the rows.dbData[0].length and calling a final code/function once they match

How to get code to execute in order in node.js

I am trying to finish my script, but for some reason i don't know, it refuses to execute in the order i put it in.
I've tried placing a 'wait' function between the JoinRequest update function and the following code, but when run, it acts as if the function call and wait function were the other way round, countering the point of the wait().
const Roblox = require('noblox.js')
var fs = require('fs');
var joinRequests = []
...
function wait(ms) {
var d = new Date();
var d2 = null;
do { d2 = new Date(); }
while(d2-d < ms*1000);
};
...
function updateJReqs() {
Roblox.getJoinRequests(4745601).then((array) => {
var i;
var final = [];
for(i = 0; i < array.length; i++) {
final.push(array[i].username);
};
if(final === '') {
final = '-None';
};
joinRequests = final
console.log('Updated join requests.')
});
}
function check() {
setTimeout(() => {
fs.readFile('Request.txt',encoding = 'utf-8', function(err, data) {
if (err) {
check();
} else {
updateJReqs(); //for some reason this function is executed alongside the below, not before it.
// Tried putting wait(x) in here.
console.log('Request received: ' + data)
var solution = joinRequests
console.log('Fuffiling request with ' + solution)
fufillRequest(solution)
fs.unlink('Request.txt', function(err) {
if(err) throw err;
});
check();
}
});
}, 400)
}
check();
The script is supposed to wait until a file is created (accomplished), update the list of join requests (accomplished) and then create a new file with the list of join requests in(not accomplished).
if I understand your code you work with async code, you need to return a promise in updateJReqs and add a condition of leaving from the function because you have an infinite recursion
function updateJReqs() {
return new Promise(resolve => {
Roblox.getJoinRequests(4745601).then((array) => {
var i;
var final = [];
for(i = 0; i < array.length; i++) {
final.push(array[i].username);
};
if(final === '') {
final = '-None';
};
joinRequests = final
console.log('Updated join requests.')
resolve();
});
}
}
async function check() {
setTimeout(() => {
fs.readFile('Request.txt',encoding = 'utf-8', function(err, data) {
if (err) {
await check();
} else {
await updateJReqs();
// Tried putting wait(x) in here.
console.log('Request received: ' + data)
var solution = joinRequests
console.log('Fuffiling request with ' + solution)
fufillRequest(solution)
fs.unlink('Request.txt', function(err) {
if(err) throw err;
});
// you dont have an exit from your function check();
return 'Success';
}
});
}, 400)
}
check().then(res => console.log(res));

GeneralException on multiple edits in a promise chain

In my Excel add-in I want to perform multiple edits on my document sequentially.
I'm using the promise chain to achive this.
Unfortunately I'm getting a GeneralException: An internal error has occurred. from some of my edits.
The following example performs 250 edits and I get something between 20 and 30 GeneralExceptions in each run. (with Office 2016, in Office online it's much worse)
Example:
var promise;
Office.initialize = function (reason) {
// add awesome addin initialize code here
promise = new OfficeExtension.Promise(function (resolve, reject) { resolve(null); });
for (var i = 0; i < 200; i++) {
insertData("Data" + i);
}
}
function insertData(data) {
if (Office.context.requirements.isSetSupported("ExcelApi", "1.0")) {
//insert the data into the spreadsheet
promise = promise.then(function () {
Excel.run(function (ctx) {
var sheet = ctx.workbook.worksheets.getActiveWorksheet();
var range = ctx.workbook.getSelectedRange();
range.getCell(0, 0).values = data;
range.getCell(1, 0).select();
return ctx.sync()
}).catch(function (error) {
addLogEntry(error.message);
});
});
}
else if (Office.context.requirements.isSetSupported("WordApi", "1.0")) {
promise = promise.then(function () {
Word.run(function (ctx) {
var body = ctx.document.body;
var selectedRange = ctx.document.getSelection();
selectedRange.insertText(data + "\n", 'End');
selectedRange.select('End');
return ctx.sync();
}).catch(function (error) {
addLogEntry(error.message);
});
});
}
}
function addLogEntry(message) {
// log message here
}
What am I doing wrong?
Here the stack trace of the error:
"GeneralException: An internal error has occurred.
at Anonymous function (https://appsforoffice.microsoft.com/lib/1/hosted/word-win32-16.01.debug.js:9329:6)
at lib$es6$promise$$internal$$tryCatch (https://appsforoffice.microsoft.com/lib/1/hosted/word-win32-16.01.debug.js:11207:8)
at lib$es6$promise$$internal$$invokeCallback (https://appsforoffice.microsoft.com/lib/1/hosted/word-win32-16.01.debug.js:11217:8)
at lib$es6$promise$$internal$$publish (https://appsforoffice.microsoft.com/lib/1/hosted/word-win32-16.01.debug.js:11193:9)
at lib$es6$promise$asap$$flush (https://appsforoffice.microsoft.com/lib/1/hosted/word-win32-16.01.debug.js:11027:8)"
I have found a solution, but it is not very elegant and VERY slow (especially in office online)
maybe someone can come up with something better? ;)
Here the fixed insertData function:
var isSending = false;
function insertData(data) {
if (!isSending) {
isSending = true;
if (Office.context.requirements.isSetSupported("ExcelApi", "1.0")) {
//insert the data into the spreadsheet
//promise = promise.then(function () {
Excel.run(function (ctx) {
var sheet = ctx.workbook.worksheets.getActiveWorksheet();
var range = ctx.workbook.getSelectedRange();
range.getCell(0, 0).values = data;
range.getCell(1, 0).select();
return ctx.sync()
}).then(function () {
isSending = false;
if (queue.length > 0) {
insertData(queue.splice(0, 1)[0]);
}
}).catch(function (error) {
addLogEntry(error.message);
});
//});
}
else if (Office.context.requirements.isSetSupported("WordApi", "1.0")) {
//promise = promise.then(function () {
Word.run(function (ctx) {
var body = ctx.document.body;
var selectedRange = ctx.document.getSelection();
selectedRange.insertText(data + "\n", 'End');
selectedRange.select('End');
return ctx.sync();
}).then(function () {
isSending = false;
if (queue.length > 0) {
insertData(queue[0]);
queue = queue.splice(0, 1);
}
}).catch(function (error) {
addLogEntry(error.message);
});
//});
}
}
else {
queue.push(data);
}
}

web scrapy by nightmare about loop ,output is not same every time

var Nightmare = require('nightmare');
var nightmare = Nightmare({ show: true });
var fs = require('fs');
vo = require('vo');
var result;
nightmare
.goto('http://jufa-kyusyu.jp/teams/')
.wait(1000)
.evaluate(function () {
var options = document.querySelectorAll('option'),i;
var values =[]
for (i = 0; i < options.length; ++i) {
values.push(options[i].value)
}
return values;
})
.then(function (values) {
for (var i = 0; i < values.length; i++) {
if(values[i] == "#") values[i] = "/teams/181.html";
nightmare
.goto("http://www.jufa-kyusyu.jp"+values[i])
.evaluate(function () {
var abc = document.querySelector('iframe[class="autoHeight"]').src.toString()
return abc;
})
.then(function (result) {
console.log(result)
})
.catch(function (error) {
console.error('Search failed:', error);
});}
})
.catch(function (error) {
console.error('Search failed:', error);
});
I want to scrapy the web information by nightmarejs looply.I dont know why have two result link is same and the result is changed in running every time.thank you.
You have to be careful when working with async calls inside a loop with Nightmare
Check this answer and this detailed explanation about the concept.
The main idea can be sumarized by this sentence:
Executing the operations in series requires arranging them to execute
in sequential order
The documentation shows how to achieve that using plain, vanilla js and also with vo
Here is a sneak peek on how to solve this loop issue with plain Javascript:
var urls = ['http://example1.com', 'http://example2.com', 'http://example3.com'];
urls.reduce(function(accumulator, url) {
return accumulator.then(function(results) {
return nightmare.goto(url)
.wait('body')
.title()
.then(function(result){
results.push(result);
return results;
});
});
}, Promise.resolve([])).then(function(results){
console.dir(results);
});
Basically what you need to do is queue all your calls in a list and trigger them using Promise.resolve

PhantomJS memory leak and process exit failure

I am currently working on a project with PhantomJS that evaluates a list of web pages specified by a CSV file. I installed NPM and node.js to use in my program.
Here is the program:
var async = require("async");
var webpage = require('webpage'),
fs = require('fs');
var file_h = fs.open('C:\\Users\\morgan\\Documents\\FantasyApp\\URLPlayerListActive.txt', 'r');
var urls = [];
while (!file_h.atEnd()) {
urls.push(file_h.readLine());
}
async.eachSeries(urls, function (url, done) {
console.log(url)
var page = webpage.create();
page.open("http://"+url, function (status) {
if (status !== 'success') {
console.log('Unable to access network');
console.log(status)
var closeresults = page.close();
} else {
var evalresults = page.evaluate(function() {
try {
table2csv('pgl_basic');
try {
ga('send','event','Tool','Action','CSV');
}
catch (e) {}
var list = document.querySelectorAll('#csv_pgl_basic');
var stats = [];
for (var i = 0; i < list.length; i++) {
stats.push(list[i].innerText);
}
return stats;
var closeresults = page.close();
} catch (e) {
console.log(e);
}
});
try {
fs.write("C:\\Users\\morgan\\Documents\\FantasyApp\\Data\\"+url+".txt", evalresults.join('\n'), 'w');
var closeresults = page.close();
} catch(e) {
console.log(e);
var closeresults = page.close();
}
}
done();
});
});
phantom.exit();
My symptoms are either the process memory increases until it reaches my Windows maximum and crashes, OR it finishes my list and the process hangs around forever.
I can implement a work around for either of these problems, but because they both happen, I am unable to put this script to work.
I am looking for assistance preventing the memory leak or simply closing my process when the script is finished. It is possible that these symptoms are from the same root cause.
If the page is not correctly garbage collected, you can try to use the same instance over and over again. The other thing is that you should call phantom.exit when the script actually finished e.g. in the callback of eachSeries.
var page = webpage.create();
async.eachSeries(urls, function (url, done) {
console.log(url)
page.open("http://"+url, function (status) {
if (status !== 'success') {
console.log('Unable to access network');
console.log(status)
} else {
var evalresults = page.evaluate(function() {
try {
table2csv('pgl_basic');
try {
ga('send','event','Tool','Action','CSV');
}
catch (e) {}
var list = document.querySelectorAll('#csv_pgl_basic');
var stats = [];
for (var i = 0; i < list.length; i++) {
stats.push(list[i].innerText);
}
return stats;
} catch (e) {
console.log(e);
}
});
try {
fs.write("C:\\Users\\morgan\\Documents\\FantasyApp\\Data\\"+url+".txt", evalresults.join('\n'), 'w');
} catch(e) {
console.log(e);
}
}
done();
});
}, function(err){
phantom.exit();
});
Some other issues:
page.close doesn't return anything, so closeresults will be undefined.
Any statement that comes after return cannot be executed.
page is not defined in the page context (inside page.evaluate) and therefore page.close(); produces an error which may break your code.
Please register to the onConsoleMessage and onError events to see if there are errors.

Resources