Phantomjs / Node.js - Refactoring block of code - node.js

I am writing a testing application, for the company where I work, using Node.js and Phantomjs. Right now, the relevant part of my code is something like this:
phantom.create(function (ph) {
ph.createPage(function (page) {
page.set('viewportSize', { width: 1920, height: 1080 });
page.set('settings.javascriptEnabled', true);
page.set('settings.loadImages', true);
page.set('settings.localToRemoteUrlAccess', true);
page.set('settings.userAgent', userAgentStrings[randomInt(0, 5)]);
page.set('settings.webSecurityEnabled', false);
page.set('settings.resourceTimeout', 5000);
page.open(URL, function (status) {
if (status == 'success') {
page.evaluate(function (result) {
return document.title;
}, function (result) {
setTimeout(function () {
log.info('Status: ', status);
ph.exit();
}, 60 * 1000);
});
} else if (status == 'fail') {
log.error('Status: ', status);
ph.exit();
}
});
});
});
My question is this: Is there a way to refactor my code in such a way that I can call the "page.open(..." function from outside the "phantom.create(..." block?
I plan to implement node-cron and have one block of code where I set up all the options for the page and another one that I will actually use to open the page. In the end, the opening part will be handled by node-cron, repeating ad infinitum.

Here is a quick example on how to do this. You just need to store your phantom object somewhere and reuse it. Be aware that I made this simple so that you can reuse the concept but that you would need more error handling.
var jobRunner = function() {
// The phantom object will be stored here
this.ph;
// The page object will be stored here
this.page;
};
jobRunner.prototype.start = function(readyCallback) {
var self = this;
phantom.create(function (ph) {
self.ph = ph;
self.ph.createPage(function (page) {
page.set('viewportSize', { width: 1920, height: 1080 });
page.set('settings.javascriptEnabled', true);
page.set('settings.loadImages', true);
page.set('settings.localToRemoteUrlAccess', true);
page.set('settings.userAgent', userAgentStrings[randomInt(0, 5)]);
page.set('settings.webSecurityEnabled', false);
page.set('settings.resourceTimeout', 5000);
self.page = page;
readyCallback();
});
});
};
jobRunner.prototype.doUrl = function(url) {
var self = this;
this.page.open(URL, function (status) {
if (status == 'success') {
page.evaluate(function (result) {
return document.title;
}, function (result) {
setTimeout(function () {
log.info('Status: ', status);
self.ph.exit();
}, 60 * 1000);
});
} else if (status == 'fail') {
log.error('Status: ', status);
self.ph.exit();
}
});
}
var CronJob = require('cron').CronJob;
var phantomJob = new jobRunner();
// Wait for Phantom to be ready then start the Cron Job.
phantomJob.start(function() {
var cron = new CronJob('*/5 * * * * *', function() {
phantomJob.doUrl("http://yoururl.com");
});
});

Related

Memory leak from node/mongoose

The below code dumps all the followers from TWITTER_USERNAME in a mongodb collection. When it's running, the memory utilization of my MongoDB database slowly increases until it crashes. I have to pause the script and restart mongo every few hours to avoid this. Is there a way for the script to get Mongo to release this memory?
var TwitterUserId = require(PATH + "/models/twitterUserId").Model;
var TwitterConnection = require(PATH + "/models/twitterConnection").Model;
util.promiseWhile = Promise.method(function(condition, action, lastValue) {
if (!condition()) return lastValue;
return action().then(promiseWhile.bind(null, condition, action));
});
var go = function(cursor) {
if(!cursor) cursor = -1;
var pollTwitter = function(cnn) {
return new Promise(function(resolve,reject) {
var parms = {
screen_name: TWITTER_USERNAME,
stringify_ids:true
cursor: cursor
};
cnn.get('followers/ids', parms,function(err,res) {
if(err) return reject(err);
cursor = res.next_cursor_str;
resolve(res.ids);
});
});
};
mongoose.connect(config.db.uri,config.db.options)
.then(function() {
return TwitterConnection.findOne({});
})
.then(function(twitterCnnCOnfig) {
return new twitterObject({
consumer_key: twitterCnnCOnfig.consumerKey
,consumer_secret: twitterCnnCOnfig.consumerSecret
,access_token_key: twitterCnnCOnfig.accessTokenKey
,access_token_secret: twitterCnnCOnfig.accessTokenSecret
});
})
.then(function(twitterCnn) {
util.promiseWhile(
function() {
return cursor;
},
function () {
return new Promise(function(outerResolve,outerReject) {
twitter.parms.cursor = cursor;
pollTwitter(twitterCnn)
.each(function(twitterId) {
return new Promise(function(resolve,reject) {
return TwitterUserId.create({twitterId: twitterId})
.then(resolve,reject);
});
})
.then(function() {
setTimeout(function() {
outerResolve();
},50 * 1000);
})
});
}
);
})
};

Nightmare.js conditional browsing

I'm trying to understand how I should make a nightmare.js script using "if-then" logic. For example
var Nightmare = require('nightmare');
var nightmare = Nightmare({
show: true,
paths: {
userData: '/dev/null'
}
});
nightmare
.goto('http://www.example.com/')
.wait('h1')
.evaluate(function() {
return document.querySelector('title').innerText;
})
// here: go to url1 if title == '123' otherwise to url2
.end()
.then(function() {
console.log('then', arguments);
}).catch(function() {
console.log('end', arguments);
});
How do I make this script go to a different url depending on the result of evaluate?
Since Nightmare is thenable, you can return it from a .then() to chain it like you would ordinary Promises.
var Nightmare = require('nightmare');
var nightmare = Nightmare({
show: true,
paths: {
userData: '/dev/null'
}
});
nightmare
.goto('http://www.example.com/')
.wait('h1')
.evaluate(function() {
return document.querySelector('title')
.innerText;
})
.then(function(title) {
if (title == 'someTitle') {
return nightmare.goto('http://www.yahoo.com');
} else {
return nightmare.goto('http://w3c.org');
}
})
.then(function() {
//since nightmare is `then`able, this `.then()` will
//execute the call chain described and returned in
//the previous `.then()`
return nightmare
//... other actions...
.end();
})
.then(function() {
console.log('done');
})
.catch(function() {
console.log('caught', arguments);
});
If you want a more synchronous-looking logic, you may want to consider using generators with vo or co. For example, the above rewritten with vo:
var Nightmare = require('nightmare');
var vo = require('vo');
vo(function * () {
var nightmare = Nightmare({
show: true,
paths: {
userData: '/dev/null'
}
});
var title = yield nightmare
.goto('http://www.example.com/')
.wait('h1')
.evaluate(function() {
return document.querySelector('title')
.innerText;
});
if (title == 'someTitle') {
yield nightmare.goto('http://www.yahoo.com');
} else {
yield nightmare.goto('http://w3c.org');
}
//... other actions...
yield nightmare.end();
})(function(err) {
if (err) {
console.log('caught', err);
} else {
console.log('done');
}
});

node opc-ua - how do i discover a variable in a server?

I am learning node opc-ua and have followed the example provided in the GitHub page for the sample_server.js and simple_client.js.
In the sample_server I add a variable when constructing the address space of the server such as:
//this code is in the server
addressSpace.addVariable({
componentOf: device,
nodeId: "ns=1;s=variable_1",
browseName: "MyVariable1",
dataType: "Double",
value: {
get: function () {
return new opcua.Variant({ dataType: opcua.DataType.Double, value: variable1 });
}
}
});
Here is the whole server code for reference:
var opcua = require("node-opcua");
var server = new opcua.OPCUAServer({
port: 4334, // the port of the listening socket of the server
resourcePath: "UA/MyLittleServer", // this path will be added to the endpoint resource name
buildInfo: {
productName: "MySampleServer1",
buildNumber: "7658",
buildDate: new Date(2014, 5, 2)
}
});
server.initialize(post_initialize);
function post_initialize() {
console.log("initialized");
construct_my_address_space(server, function () {
server.start(function () {
console.log("Server is now listening ... ( press CTRL+C to stop)");
console.log("port ", server.endpoints[0].port);
var endpointUrl = server.endpoints[0].endpointDescriptions()[0].endpointUrl;
console.log(" the primary server endpoint url is ", endpointUrl);
});
});
}
function construct_my_address_space(server, callback) {
var addressSpace = server.engine.addressSpace;
// declare a new object
var device = addressSpace.addObject({
organizedBy: addressSpace.rootFolder.objects,
browseName: "MyDevice"
});
// add a variable named MyVariable1 to the newly created folder "MyDevice"
var variable1 = 1;
// emulate variable1 changing every 500 ms
setInterval(function () { variable1 += 1; }, 500);
addressSpace.addVariable({
componentOf: device,
nodeId: "ns=1;s=variable_1",
browseName: "MyVariable1",
dataType: "Double",
value: {
get: function () {
return new opcua.Variant({ dataType: opcua.DataType.Double, value: variable1 });
}
}
});
callback();
}
Now, in the client i want to discover this variable by name or the full nodeId.
I am using the sample provided to browse the session to the server but in the return i only see FolderType, Objects, Types and Views and can not locate this variable anywhere.
Here is the client code:
var opcua = require("node-opcua");
var async = require("async");
var client = new opcua.OPCUAClient();
var endpointUrl = "opc.tcp://" + require("os").hostname() + ":4334/UA/MyLittleServer";
var the_session, the_subscription;
async.series([
// step 1 : connect to
function (callback) {
client.connect(endpointUrl, function (err) {
if (err) {
console.log(" cannot connect to endpoint :", endpointUrl);
} else {
console.log("connected !");
}
callback(err);
});
},
// step 2 : createSession
function (callback) {
client.createSession(function (err, session) {
if (!err) {
the_session = session;
}
callback(err);
});
},
// step 3 : browse
function (callback) {
the_session.browse("RootFolder", function (err, browse_result) {
if (!err) {
console.log('Result of browsing: ', browse_result);
browse_result[0].references.forEach(function (reference) {
console.log('browsing: ', reference.browseName);
console.log(reference);
console.log("**************************************");
});
}
callback(err);
});
},
/* step 4 : read a variable with readVariableValue
function (callback) {
the_session.readVariableValue("ns=1;s=variable_1", function (err, dataValue) {
if (!err) {
console.log(" var 1 = ", dataValue.toString());
}
callback(err);
});
},
// step 4' : read a variable with read
function (callback) {
var max_age = 0;
var nodes_to_read = [
{ nodeId: "ns=1;s=variable_1", attributeId: opcua.AttributeIds.Value }
];
the_session.read(nodes_to_read, max_age, function (err, nodes_to_read, dataValues) {
if (!err) {
console.log(" variable 1 = ", dataValues[0]);
}
callback(err);
});
},
*/
// step 5: install a subscription and install a monitored item for 10 seconds
function (callback) {
the_subscription = new opcua.ClientSubscription(the_session, {
requestedPublishingInterval: 1000,
requestedLifetimeCount: 10,
requestedMaxKeepAliveCount: 2,
maxNotificationsPerPublish: 10,
publishingEnabled: true,
priority: 10
});
the_subscription.on("started", function () {
console.log("subscription started for 2 seconds - subscriptionId=", the_subscription.subscriptionId);
}).on("keepalive", function () {
console.log("keepalive");
}).on("terminated", function () {
callback();
});
setTimeout(function () {
the_subscription.terminate();
}, 10000);
// install monitored item
var monitoredItem = the_subscription.monitor({
nodeId: opcua.resolveNodeId("ns=1;s=variable_1"),
attributeId: opcua.AttributeIds.Value
},
{
samplingInterval: 100,
discardOldest: true,
queueSize: 10
},
opcua.read_service.TimestampsToReturn.Both
);
console.log("-------------------------------------");
monitoredItem.on("changed", function (dataValue) {
console.log("variable_1 = ", dataValue.value.value);
});
},
// close session
function (callback) {
the_session.close(function (err) {
if (err) {
console.log("session closed failed ?");
} else{
console.log("session closed!");
}
callback();
});
}
],
function (err) {
if (err) {
console.log(" failure ", err);
} else {
console.log("done!");
}
client.disconnect(function () { });
});
Thanks in advance
You need to drill down from from the rootFolder and investigate the various objects until you find the variable you want to monitor, with a series of browseNode.
The best option is to use a free OPCUA client from a commercial vendor such as UAExpert or Prosys OPC UA Client to name a few.
You can also use opcua-commander: This little utility will allow you to explore the address space of the server and find the node you're looking for. ( source code available on github )

PhantomJS memory leak and process exit failure

I am currently working on a project with PhantomJS that evaluates a list of web pages specified by a CSV file. I installed NPM and node.js to use in my program.
Here is the program:
var async = require("async");
var webpage = require('webpage'),
fs = require('fs');
var file_h = fs.open('C:\\Users\\morgan\\Documents\\FantasyApp\\URLPlayerListActive.txt', 'r');
var urls = [];
while (!file_h.atEnd()) {
urls.push(file_h.readLine());
}
async.eachSeries(urls, function (url, done) {
console.log(url)
var page = webpage.create();
page.open("http://"+url, function (status) {
if (status !== 'success') {
console.log('Unable to access network');
console.log(status)
var closeresults = page.close();
} else {
var evalresults = page.evaluate(function() {
try {
table2csv('pgl_basic');
try {
ga('send','event','Tool','Action','CSV');
}
catch (e) {}
var list = document.querySelectorAll('#csv_pgl_basic');
var stats = [];
for (var i = 0; i < list.length; i++) {
stats.push(list[i].innerText);
}
return stats;
var closeresults = page.close();
} catch (e) {
console.log(e);
}
});
try {
fs.write("C:\\Users\\morgan\\Documents\\FantasyApp\\Data\\"+url+".txt", evalresults.join('\n'), 'w');
var closeresults = page.close();
} catch(e) {
console.log(e);
var closeresults = page.close();
}
}
done();
});
});
phantom.exit();
My symptoms are either the process memory increases until it reaches my Windows maximum and crashes, OR it finishes my list and the process hangs around forever.
I can implement a work around for either of these problems, but because they both happen, I am unable to put this script to work.
I am looking for assistance preventing the memory leak or simply closing my process when the script is finished. It is possible that these symptoms are from the same root cause.
If the page is not correctly garbage collected, you can try to use the same instance over and over again. The other thing is that you should call phantom.exit when the script actually finished e.g. in the callback of eachSeries.
var page = webpage.create();
async.eachSeries(urls, function (url, done) {
console.log(url)
page.open("http://"+url, function (status) {
if (status !== 'success') {
console.log('Unable to access network');
console.log(status)
} else {
var evalresults = page.evaluate(function() {
try {
table2csv('pgl_basic');
try {
ga('send','event','Tool','Action','CSV');
}
catch (e) {}
var list = document.querySelectorAll('#csv_pgl_basic');
var stats = [];
for (var i = 0; i < list.length; i++) {
stats.push(list[i].innerText);
}
return stats;
} catch (e) {
console.log(e);
}
});
try {
fs.write("C:\\Users\\morgan\\Documents\\FantasyApp\\Data\\"+url+".txt", evalresults.join('\n'), 'w');
} catch(e) {
console.log(e);
}
}
done();
});
}, function(err){
phantom.exit();
});
Some other issues:
page.close doesn't return anything, so closeresults will be undefined.
Any statement that comes after return cannot be executed.
page is not defined in the page context (inside page.evaluate) and therefore page.close(); produces an error which may break your code.
Please register to the onConsoleMessage and onError events to see if there are errors.

How to close a notification window automatically

Hi I am displaying a notification based on the output of a server file.The output of the server file is being checked for a period of 5 min interval.Now what I need is that when the output changes at any instant, notification should be automatically be closed at that moment.What I am trying to convey that if the output of the server file is 0,the notification is being displayed in every 5 min. If the output is 1, then notification will not be shown anymore.The problem of my code is that unless I close the notification manually it won't be closed automatically even if the output is 1. Anyone please help me to close my notification automatically.
here is my background.js
var myNotificationID = null;
var oldChromeVersion = !chrome.runtime;
function getGmailUrl() {
return "http://calpinemate.com/";
}
function isGmailUrl(url) {
return url.indexOf(getGmailUrl()) == 0;
}
chrome.browserAction.onClicked.addListener(function(tab) {
if(!localStorage.username){
chrome.windows.create({url : "userinfo.html",type: "popup", height: 200, width:300 , top :400 , left : 800});
}
else{
chrome.tabs.query({
url: "http://calpinemate.com/*",
currentWindow: true
},
function(tabs) {
if (tabs.length > 0) {
var tab = tabs[0];
console.log("Found (at least one) Gmail tab: " + tab.url);
console.log("Focusing and refreshing count...");
chrome.tabs.update(tab.id, { active: true });
updateIcon();
}
else {
console.log("Could not find Gmail tab. Creating one...");
chrome.tabs.create({ url: getGmailUrl() });
updateIcon();
}
});
}
});
function onInit() {
console.log('onInit');
updateIcon();
if (!oldChromeVersion) {
chrome.alarms.create('watchdog', {periodInMinutes:5});
}
}
function onAlarm(alarm) {
console.log('Got alarm', alarm);
if (alarm && alarm.name == 'watchdog') {
onWatchdog();
}
else {
updateIcon();
}
}
function onWatchdog() {
chrome.alarms.get('refresh', function(alarm) {
if (alarm) {
console.log('Refresh alarm exists. Yay.');
}
else {
console.log('Refresh alarm doesn\'t exist!? ' +
'Refreshing now and rescheduling.');
updateIcon();
}
});
}
if (oldChromeVersion) {
updateIcon();
onInit();
}
else {
chrome.runtime.onInstalled.addListener(onInit);
chrome.alarms.onAlarm.addListener(onAlarm);
}
function updateIcon(){
var urlPrefix = 'http://www.calpinemate.com/employees/attendanceStatus/';
var urlSuffix = '/2';
var req = new XMLHttpRequest();
req.addEventListener("readystatechange", function() {
if (req.readyState == 4) {
if (req.status == 200) {
var item=req.responseText;
if(item==1){
chrome.browserAction.setIcon({path:"calpine_logged_in.png"});
chrome.browserAction.setBadgeBackgroundColor({color:[190, 190, 190, 230]});
chrome.browserAction.setBadgeText({text:""});
chrome.notifications.clear(id1);//this is not working
}
else{
chrome.browserAction.setIcon({path:"calpine_not_logged_in.png"});
chrome.browserAction.setBadgeBackgroundColor({color:[190, 190, 190, 230]});
chrome.browserAction.setBadgeText({text:""});
chrome.notifications.create(
'id1',{
type: 'basic',
iconUrl: '/calpine_not_logged_in.png',
title: 'Warning : Attendance',
message: 'Please mark your Attendance !',
buttons: [{ title: 'Mark',
iconUrl: '/tick.jpg'
},{ title: 'Ignore',
iconUrl: '/cross.jpg'}],
priority: 0},
function(id) { myNotificationID = id;}
);
chrome.notifications.onButtonClicked.addListener(function(notifId, btnIdx) {
if (notifId === myNotificationID) {
if (btnIdx === 0) {
window.open("http://www.calpinemate.com/");
} else if (btnIdx === 1) {
notification.close();
}
}
});
chrome.notifications.onClosed.addListener(function() {
notification.close();
});
}
}
else {
// Handle the error
alert("ERROR: status code " + req.status);
}
}
});
var url = urlPrefix + encodeURIComponent(localStorage.username) + urlSuffix;
req.open("GET", url);
req.send(null);
}
Two possible problems:
You pass an undefined variable named id1 in chrome.notifications.clear(), when you actually mean the string 'id1'.
According to the docs on chrome.notifications.clear() method, the second argument (callback function) is not optional, yet you fail to supply one.
One possible solution:
// Replace that line:
chrome.notifications.clear(id1);//this is not working
// With this line:
chrome.notifications.clear('id1', function(){});//this is working perfectly
// Or with this line:
chrome.notifications.clear(myNotificationID, function(){});//this is working perfectly
BTW, you don't need to provide a notification ID yourself.
You can replace: chrome.notifications.create('id1',...
with: chrome.notifications.create('',...
(and then, of course, use: chrome.notifications.clear(myNotificationID,...)

Resources