Node PhantomJS script onResourceError path issue - node.js

Having some trouble using the webpage API in a phantomJS script I'm using for load testing.
I'm running the script in a child process, like so:
var path = require('path');
var childProcess = require('child_process');
var binPath = require('phantomjs').path;
var childArgs = [
path.join(__dirname, 'phantom-script.js')
];
var spawn = childProcess.spawn;
var child = spawn(binPath, childArgs);
child.stdout.on('data', function(data) {
const buf = Buffer.from(data);
console.log('stdout:', buf.toString());
});
child.stderr.on('data', function(data) {
const buf = Buffer.from(data);
console.log('stderr:', buf.toString());
});
And my simple phantomJS script:
var webPage = require('webpage');
var page = webPage.create();
page.onConsoleMessage = function (msg) {
console.log(msg);
};
page.onResourceError = function(resourceError) {
console.log(resourceError.errorCode + ':', resourceError.errorString);
};
function runScript() {
page.open('<webpage-url>', function(status) {
console.log('Status:', status);
if (status === 'success') {
page.evaluate(function() {
console.log('Title:', document.title);
});
}
});
}
runScript();
So to start the phantomJS script, if both of these files are in the test/ directory, and my current directory is up one from that: node test/child-process.js, which then spawns the child process and runs my phantomJS script.
So, this gets the script to run, but it always fails in page.open because of a resource error. Replacing my url with Google's, or really any website, works fine.
The error logged in onResourceError is stdout: 202: Cannot open file:///Users/<user>/path/to/local/current/directory: Path is a directory.
This is always the path from which I'm running this script. If I move down a directory into test/ and run it with node child-process.js, the error instead logs that directory.
As a headless browser, I assumed phantomJS would interface with a webpage like any client would, just without rendering the template--what does the current directory from which the script was run have anything to do with opening the webpage? Why would it be trying to load resources from my local directory when the webpage URL points to a public website, hosted at the IP and PORT specified in the first argument of page.open (e.g. xx.xxx.xx.xx:PORT)?
I'm at a bit of a loss here. The phantomJS path and all that is correct, since it runs the script fine. I just don't understand why page.open would attempt to open the directory from which the script was called--what does that have to do with its function, which is to open the URL and load it to the page?

Not sure if this is even worthy of answering--as opposed to just deleting.
I figured it out when I manually typed in the argument www.google.com, instead of copy/pasting from the browser, and and I got this as the path in the error: file:///Users/<user>/path/to/local/current/directory/www.google.com.
Now I know why I couldn't find a SO question for it. A stupid error on my part at any rate, it would've been a quick debug if the error had appended the IP address and PORT (my "url") to the end of the file path like it did for www.google.com, a clear indicator that it's not pinging a URL.
TL;DR: It's a URL, you need http(s)://...

Related

nodejs open chrome in window with arguments

Trying to write my own plugins for gulp. So I've written a gulp task that attempts opens the chrome browser in windows (i'll work on getting working for mac/linux later).
It seems to work except it's not passing in my arguments:
/*
* Open
*/
gulp.task('open', function (done) {
var uri = 'http://localhost:' + CONFIG.PORT,
CONFIG.PORT = 8080,
args = [
uri,
'--no-first-run',
'--no-default-browser-check',
'--disable-translate',
'--disable-default-apps',
'--disable-popup-blocking',
'--disable-zero-browsers-open-for-tests',
'--disable-web-security',
'--new-window',
'--user-data-dir="C:/temp-chrome-eng"'
]
cp.spawn('C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe', args);
})
How do i get it to accept my arguments it passed in? Am in providing it the wrong arguments?
I would recommend using a quite popular npm module, opener, instead which will solve both your issue with arguments and cross platform support.
Instead of finding the browser executable like you are doing, you can simply write:
var opener = require('opener')
opener('http://google.com')
If you however want to go with your current method, try capturing the output by naming your process and then listening on stderr and stdout:
var chrome = cp.spawn ...
chrome.stdout.on('data', function (data) {
console.log(data.toString())
})
chrome.stderr.on('data', function (data) {
console.error(data.toString())
})
It does work for me on linux if I replace your chrome path with chromium.

Meteor/Node writeFile crashes server

I have the following code:
Meteor.methods({
saveFile: function(blob, name, path, encoding) {
var path = cleanPath(path), fs = __meteor_bootstrap__.require('fs'),
name = cleanName(name || 'file'), encoding = encoding || 'binary',
chroot = Meteor.chroot || 'public';
// Clean up the path. Remove any initial and final '/' -we prefix them-,
// any sort of attempt to go to the parent directory '..' and any empty directories in
// between '/////' - which may happen after removing '..'
path = chroot + (path ? '/' + path + '/' : '/');
// TODO Add file existance checks, etc...
fs.writeFile(path + name, blob, encoding, function(err) {
if (err) {
throw (new Meteor.Error(500, 'Failed to save file.', err));
} else {
console.log('The file ' + name + ' (' + encoding + ') was saved to ' + path);
}
});
function cleanPath(str) {
if (str) {
return str.replace(/\.\./g,'').replace(/\/+/g,'').
replace(/^\/+/,'').replace(/\/+$/,'');
}
}
function cleanName(str) {
return str.replace(/\.\./g,'').replace(/\//g,'');
}
}
});
Which I took from this project
https://gist.github.com/dariocravero/3922137
The code works fine, and it saves the file, however it repeats the call several time and each time it causes meteor to reset using windows version 0.5.4. The F12 console ends up looking like this: . The meteor console loops over the startup code each time the 503 happens and repeats the console logs in the saveFile function.
Furthermore in the target directory the image thumbnail keeps displaying and then display as broken, then a valid thumbnail again, as if the fs is writing it multiple times.
Here is the code that calls the function:
"click .savePhoto":function(e, template){
e.preventDefault();
var MAX_WIDTH = 400;
var MAX_HEIGHT = 300;
var id = e.srcElement.id;
var item = Session.get("employeeItem");
var file = template.find('input[name='+id+']').files[0];
// $(template).append("Loading...");
var dataURL = '/.bgimages/'+file.name;
Meteor.saveFile(file, file.name, "/.bgimages/", function(){
if(id=="goodPhoto"){
EmployeeCollection.update(item._id, { $set: { good_photo: dataURL }});
}else{
EmployeeCollection.update(item._id, { $set: { bad_photo: dataURL }});
}
// Update an image on the page with the data
$(template.find('img.'+id)).delay(1000).attr('src', dataURL);
});
},
What's causing the server to reset?
My guess would be that since Meteor has a built-in "automatic directories scanning in search for file changes", in order to implement auto relaunching of the application to newest code-base, the file you are creating is actually causing the server reset.
Meteor doesn't scan directories beginning with a dot (so called "hidden" directories) such as .git for example, so you could use this behaviour to your advantage by setting the path of your files to a .directory of your own.
You should also consider using writeFileSync insofar as Meteor methods are intended to run synchronously (inside node fibers) contrary to the usual node way of asynchronous calls, in this code it's no big deal but for example you couldn't use any Meteor mechanics inside the writeFile callback.
asynchronousCall(function(error,result){
if(error){
// handle error
}
else{
// do something with result
Collection.update(id,result);// error ! Meteor code must run inside fiber
}
});
var result=synchronousCall();
Collection.update(id,result);// good to go !
Of course there is a way to turn any asynchronous call inside a synchronous one using fibers/future, but that's beyond the point of this question : I recommend reading this EventedMind episode on node future to understand this specific area.

NodeJS not spawning child process except in tests

I have the following NodeJS code:
var spawn = require('child_process').spawn;
var Unzipper = {
unzip: function(src, dest, callback) {
var self = this;
if (!fs.existsSync(dest)) {
fs.mkdir(dest);
}
var unzip = spawn('unzip', [ src, '-d', dest ]);
unzip.stdout.on('data', function (data) {
self.stdout(data);
});
unzip.stderr.on('data', function (data) {
self.stderr(data);
callback({message: "There was an error executing an unzip process"});
});
unzip.on('close', function() {
callback();
});
}
};
I have a NodeUnit test that executes successfully. Using phpStorm to debug the test the var unzip is assigned correctly
However if I run the same code as part of a web service, the spawn call doesn't return properly and the server crashes on trying to attach an on handler to the nonexistent stdout property of the unzip var.
I've tried running the program outside of phpStorm, however it crashes on the command line as well for the same reason. I'm suspecting it's a permissions issue that the tests don't have to deal with. A web server spawning processes could cause chaos in a production environment, therefore some extra permissions might be needed, but I haven't been able to find (or I've missed) documentation to support my hypothesis.
I'm running v0.10.3 on OSX Snow Leopard (via MacPorts).
Why can't I spawn the child process correctly?
UPDATES
For #jonathan-wiepert
I'm using Prototypical inheritance so when I create an "instance" of Unzipper I set stdout and stderr ie:
var unzipper = Unzipper.spawn({
stdout: function(data) { util.puts(data); },
stderr: function(data) { util.puts(data); }
});
This is similar to the concept of "constructor injection". As for your other points, thanks for the tips.
The error I'm getting is:
project/src/Unzipper.js:15
unzip.stdout.on('data', function (data) {
^
TypeError: Cannot call method 'on' of undefined
As per my debugging screenshots, the object that is returned from the spawn call is different under different circumstances. My test passes (it checks that a ZIP can be unzipped correctly) so the problem occurs when running this code as a web service.
The problem was that the spawn method created on the Object prototype (see this article on Protypical inheritance) was causing the child_process.spawn function to be replaced, so the wrong function was being called.
I saved child_process.spawn into a property on the Unzipper "class" before it gets clobbered and use that property instead.

Spawned phantomjs process hanging

I'm trying to create a node server that spawns phantomjs processes to create screenshots. The grab.js script works fine when executed and I've confirmed that it writes to stdout. Problem is the node code that spawns the process simply hangs. I've confirmed that phantomjs is in the path. Anyone know what might be happening here or how I might troubleshoot this?
Here's the phantomjs code (grab.js) that renders the page and writes the data to stdout:
var page = require('webpage').create(),
system = require('system'),
fs = require('fs');
var url = system.args[1] || 'google.com';
page.viewportSize = {
width: 1024,
height: 1200
};
page.open(url, function() {
var b64 = page.renderBase64('png');
fs.write('/dev/stdout', b64, 'w');
phantom.exit();
});
And here's the node code that spawns the phantom progress and prints the result (hangs):
var http = require('http'),
exec = require('child_process').exec,
fs = require('fs');
exec('phantomjs grab.js google.com', function(error, stdout, stderr) {
console.log(error, stdout, stderr);
});
I have had similar issues with exec and then switched to using spawn instead and it worked.
According to this article , Use spawn when you want the child process to return huge binary data to Node, use exec when you want the child process to return simple status messages.
hth
I had same problem, in my case it was not in nodejs, but in phantomjs (v2.1).
It's known problem when phantom`s open method hangs.
Also, found second link (I guess same person wrote) in which author points that requestAnimationFrame is not working well with tweenJs, which causes freezing. PhantomJS returns unixtimestamp but tweenjs expects it to be DOMHighResTimeStamp, and so on...
Trick is to inject request-animation-frame.js (which is also provided in that article)

How to Use CasperJS in node.js?

I would like to use CasperJS in node.js.
I have referred to the following URL's to use CasperJS in node.js:
https://github.com/sgentle/phantomjs-node
http://casperjs.org/index.html#faq-executable
With the help of the above URLs I have written the following code:
//DISPLAY=:0 node test2.js
var phantom = require('phantom');
console.log('Hello, world!');
phantom.create(function (ph) {
ph.casperPath = '/opt/libs/casperjs'
ph.injectJs('/opt/libs/casperjs/bin/bootstrap.js');
var casper = require('casper').create();
casper.start('http://google.fr/');
casper.thenEvaluate(function (term) {
document.querySelector('input[name="q"]').setAttribute('value', term);
document.querySelector('form[name="f"]').submit();
}, {
term: 'CasperJS'
});
casper.then(function () {
// Click on 1st result link
this.click('h3.r a');
});
casper.then(function () {
console.log('clicked ok, new location is ' + this.getCurrentUrl());
});
casper.run();
});
When I run this code, I got the following error:
ERROR MSG:
tz#tz-ubuntu:/opt/workspaces/TestPhantomjs$ DISPLAY=:0 node test2.js
Hello, world!
Error: Cannot find module 'casper'
at Function._resolveFilename (module.js:332:11)
at Function._load (module.js:279:25)
at Module.require (module.js:354:17)
at require (module.js:370:17)
at /opt/workspaces/TestPhantomjs/test2.js:6:14
at Object.<anonymous> (/opt/workspaces/TestPhantomjs/node_modules/phantom/phantom.js:82:43)
at EventEmitter.<anonymous> (/opt/workspaces/TestPhantomjs/node_modules/phantom/node_modules/dnode/index.js:215:30)
at EventEmitter.emit (events.js:67:17)
at handleMethods (/opt/workspaces/TestPhantomjs/node_modules/phantom/node_modules/dnode-protocol/index.js:138:14)
at EventEmitter.handle (/opt/workspaces/TestPhantomjs/node_modules/phantom/node_modules/dnode-protocol/index.js:98:13)
phantom stdout: Unable to load casper environment: Error: Failed to resolve module fs, tried fs
You can use SpookyJS to drive CasperJS from Node.
https://groups.google.com/group/casperjs/browse_thread/thread/641e9e6dff50fb0a/e67aaef5ab4ec918?hl=zh-CN#e67aaef5ab4ec918
Nicolas Perriault
2012/2/27 天猪 蓝虫. :
I wan to use casperjs in nodejs.
and refs to:
https://github.com/sgentle/phantomjs-node and
http://casperjs.org/index.html#faq-executable
You can't run CasperJS that way; QtWebKit and V8 don't share the same
js environment (and event loop), so your node.js app won't be able to
load and use a CasperJS module. You have to run your CasperJS script
separately using a subprocess call, like this one on github. I
don't plan to make CasperJS compatible with phantomjs-node because it
uses alert()-based dirty hacks I'm not easy with.
Cheers,
-- Nicolas Perriault
CasperJS includes a web server to talk to the outside world. Node (using request, superagent etc) can now talk to casper over HTTP.
In scraper.js:
#!/usr/bin/env casperjs
// I AM NOT NODEJS
// I AM CASPER JS
// I RUN IN QTWEBKIT, NOT V8
var casper = require('casper').create();
var server = require('webserver').create();
var ipAndPort = '127.0.0.1:8585';
server.listen(ipAndPort, function(request, response) {
casper.start('https://connect.data.com/login');
casper.userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36");
casper.then(function(){
// lots of code here, and a few more cassper.then()s
});
casper.run(function(){
console.log('\n\nFinished')
response.statusCode = 200;
var body = JSON.stringify({
phoneNumber: '1800-YOLO-SWAG'
})
response.write(body);
response.close();
});
});
You can now run scraper.js as a web server:
chmod +x scraper.js
./scraper.js
You should run it as a Linux service just like you would for a node app.
One solution (which worked for me) is to start and stop your server on a per-test basis. For example, I have a runtests.coffee which looks like:
http = require 'http'
glob = require 'glob'
spawn = require('child_process').spawn
db = require './db' # Contains all database stuff.
webapp = require './webapp' # Contains all of the Express stuff.
db.connect 'test' # Connects to the db server and creates an empty test db.
server = http.createServer webapp.makeApp()
server.listen 0, ->
port = server.address().port
process.env.URL = "http://localhost:#{ port }"
glob 'tests/*', (err, filenames) ->
child = spawn 'casperjs', ['test'].concat(filenames)
child.stdout.on 'data', (msg) -> process.stdout.write msg
child.stderr.on 'data', (msg) -> process.stderr.write msg
child.on 'exit', (code) ->
db.disconnect() # Drops the test db.
server.close()
process.exit code
And my CasperJS tests in tests/ look like:
URL = require('system').env.URL # Note, Casper code here, not Node.
casper.test.begin 'Test something', 1, (test) ->
casper.start "#{ URL }/welcome"
casper.then ->
test.assertHttpStatus 200
# ....
casper.run ->
test.done()
It basically means that your script can't find Casper; have you checked the path and made sure that
/opt/libs/casperjs
and:
/opt/libs/casperjs/bin/bootstrap.js
Are accessible by a website user ? considering the location it's probably not likely.
/opt is a unix path, but the website will be looking in {websiterootpath}/opt.
I'd create a subfolder 'casperjs' in the root folder of your website and copy the contents of
/opt/libs/casperjs
To there.
Then change your paths from
/opt/libs/casperjs
To
/casperjs
I tried to run casper by node cron job too,
here's my solution
in casper.js echo your response:
casper.then(function() {
var comments = this.evaluate(getComments);
this.echo(JSON.stringify(comments));
})
use node-cmd in node file casper_wrapper.js:
var cmd = require('node-cmd');
module.exports = function(url) {
return new Promise(function(resolve, reject) {
cmd.get(
'casperjs casper.js ' + url, // casper takes args to run the script
function(err, data, stderr){
if (err) {
reject(err);
return;
}
var obj = JSON.parse(data);
resolve(obj);
}
);
});
}

Resources