How to Use CasperJS in node.js? - node.js

Question

I would like to use CasperJS in node.js.
I have referred to the following URL's to use CasperJS in node.js:
https://github.com/sgentle/phantomjs-node
http://casperjs.org/index.html#faq-executable
With the help of the above URLs I have written the following code:
//DISPLAY=:0 node test2.js
var phantom = require('phantom');
console.log('Hello, world!');
phantom.create(function (ph) {
ph.casperPath = '/opt/libs/casperjs'
ph.injectJs('/opt/libs/casperjs/bin/bootstrap.js');
var casper = require('casper').create();
casper.start('http://google.fr/');
casper.thenEvaluate(function (term) {
document.querySelector('input[name="q"]').setAttribute('value', term);
document.querySelector('form[name="f"]').submit();
}, {
term: 'CasperJS'
});
casper.then(function () {
// Click on 1st result link
this.click('h3.r a');
});
casper.then(function () {
console.log('clicked ok, new location is ' + this.getCurrentUrl());
});
casper.run();
});
When I run this code, I got the following error:
ERROR MSG:
tz#tz-ubuntu:/opt/workspaces/TestPhantomjs$ DISPLAY=:0 node test2.js
Hello, world!
Error: Cannot find module 'casper'
at Function._resolveFilename (module.js:332:11)
at Function._load (module.js:279:25)
at Module.require (module.js:354:17)
at require (module.js:370:17)
at /opt/workspaces/TestPhantomjs/test2.js:6:14
at Object.<anonymous> (/opt/workspaces/TestPhantomjs/node_modules/phantom/phantom.js:82:43)
at EventEmitter.<anonymous> (/opt/workspaces/TestPhantomjs/node_modules/phantom/node_modules/dnode/index.js:215:30)
at EventEmitter.emit (events.js:67:17)
at handleMethods (/opt/workspaces/TestPhantomjs/node_modules/phantom/node_modules/dnode-protocol/index.js:138:14)
at EventEmitter.handle (/opt/workspaces/TestPhantomjs/node_modules/phantom/node_modules/dnode-protocol/index.js:98:13)
phantom stdout: Unable to load casper environment: Error: Failed to resolve module fs, tried fs

Answer 1

You can use SpookyJS to drive CasperJS from Node.

Answer 2

https://groups.google.com/group/casperjs/browse_thread/thread/641e9e6dff50fb0a/e67aaef5ab4ec918?hl=zh-CN#e67aaef5ab4ec918
Nicolas Perriault
2012/2/27 天猪 蓝虫. :
I wan to use casperjs in nodejs.
and refs to:
https://github.com/sgentle/phantomjs-node and
http://casperjs.org/index.html#faq-executable
You can't run CasperJS that way; QtWebKit and V8 don't share the same
js environment (and event loop), so your node.js app won't be able to
load and use a CasperJS module. You have to run your CasperJS script
separately using a subprocess call, like this one on github. I
don't plan to make CasperJS compatible with phantomjs-node because it
uses alert()-based dirty hacks I'm not easy with.
Cheers,
-- Nicolas Perriault

Answer 3

CasperJS includes a web server to talk to the outside world. Node (using request, superagent etc) can now talk to casper over HTTP.
In scraper.js:
#!/usr/bin/env casperjs
// I AM NOT NODEJS
// I AM CASPER JS
// I RUN IN QTWEBKIT, NOT V8
var casper = require('casper').create();
var server = require('webserver').create();
var ipAndPort = '127.0.0.1:8585';
server.listen(ipAndPort, function(request, response) {
casper.start('https://connect.data.com/login');
casper.userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36");
casper.then(function(){
// lots of code here, and a few more cassper.then()s
});
casper.run(function(){
console.log('\n\nFinished')
response.statusCode = 200;
var body = JSON.stringify({
phoneNumber: '1800-YOLO-SWAG'
})
response.write(body);
response.close();
});
});
You can now run scraper.js as a web server:
chmod +x scraper.js
./scraper.js
You should run it as a Linux service just like you would for a node app.

Answer 4

One solution (which worked for me) is to start and stop your server on a per-test basis. For example, I have a runtests.coffee which looks like:
http = require 'http'
glob = require 'glob'
spawn = require('child_process').spawn
db = require './db' # Contains all database stuff.
webapp = require './webapp' # Contains all of the Express stuff.
db.connect 'test' # Connects to the db server and creates an empty test db.
server = http.createServer webapp.makeApp()
server.listen 0, ->
port = server.address().port
process.env.URL = "http://localhost:#{ port }"
glob 'tests/*', (err, filenames) ->
child = spawn 'casperjs', ['test'].concat(filenames)
child.stdout.on 'data', (msg) -> process.stdout.write msg
child.stderr.on 'data', (msg) -> process.stderr.write msg
child.on 'exit', (code) ->
db.disconnect() # Drops the test db.
server.close()
process.exit code
And my CasperJS tests in tests/ look like:
URL = require('system').env.URL # Note, Casper code here, not Node.
casper.test.begin 'Test something', 1, (test) ->
casper.start "#{ URL }/welcome"
casper.then ->
test.assertHttpStatus 200
# ....
casper.run ->
test.done()

Answer 5

It basically means that your script can't find Casper; have you checked the path and made sure that
/opt/libs/casperjs
and:
/opt/libs/casperjs/bin/bootstrap.js
Are accessible by a website user ? considering the location it's probably not likely.
/opt is a unix path, but the website will be looking in {websiterootpath}/opt.
I'd create a subfolder 'casperjs' in the root folder of your website and copy the contents of
/opt/libs/casperjs
To there.
Then change your paths from
/opt/libs/casperjs
To
/casperjs

Answer 6

I tried to run casper by node cron job too,
here's my solution
in casper.js echo your response:
casper.then(function() {
var comments = this.evaluate(getComments);
this.echo(JSON.stringify(comments));
})
use node-cmd in node file casper_wrapper.js:
var cmd = require('node-cmd');
module.exports = function(url) {
return new Promise(function(resolve, reject) {
cmd.get(
'casperjs casper.js ' + url, // casper takes args to run the script
function(err, data, stderr){
if (err) {
reject(err);
return;
}
var obj = JSON.parse(data);
resolve(obj);
}
);
});
}