I have a requirement where I have to GET a page using a url in NodeJS.
The Problem
The url redirects to a Login page where we have to authenticate using credentials.
Once we provide the credentials, it redirects to the actual page I wanted to GET.
This is how it works in a browser.
Is there a way I can get the final page as output?
var https = require('follow-redirects').https;
var options = {
host: 'myURL',
port: 443,
path: 'myPath'
};
https.get(options, function(res) {
res.setEncoding('utf8');
res.on('data', function(chunk){
console.log(chunk);
});
});
I am getting the login page with a 200. Now, How do I automatically authenticate myself with the credentials I have and get to the page I actually want?
Finally achieved the requirement using casperJS. Thanks to the comment by #Dave Newton.
var casper = require('casper').create();
casper.on("remote.message", function (msg) {
console.log(msg);
});
casper.start("https://url.com/path", function () {
//Enter Credentials
this.evaluate(function () {
console.log("filling inputs");
var usernameInput = document.getElementById("accountname");
usernameInput.value = "username";
var passwordInput = document.getElementById("accountpassword");
passwordInput.value = "password";
});
this.click("#continueFieldbutton");
this.echo("login button was submitted");
});
casper.then(function () {
var js = this.evaluate(function() {
return document;
});
this.echo(js.all[0].outerHTML);
this.capture('example.png');
});
casper.run();
Related
I want to get the html of this page for parsing(click the link to understand what content i want to get).
750-bond list
Here's my code to request this page content
var https = require("https");
var fs = require("fs");
var options = {
hostname: "www.prizebond.net",
port: 443,
path: "/dlist.php?num=455",
method: "GET"
};
var response = "";
var req = https.request(options, function (res) {
res.setEncoding("UTF-8");
console.log(res.statusCode);
res.on("data", function (chunk) {
response += chunk;
});
res.on("end", function () {
fs.writeFile("750-bond.html", response, function (err) {
if (err) {
console.log(err.message);
}
console.log("File downloaded");
});
console.log("end");
});
});
req.end();
Now the problem is that in my 750-bont.html file, I am getting the weird the
result of "Checking your browser before accessing the prizebond.net" not the
original content. Here's the screenshot what I got when I open the 750-
bond.html file in browser.
What I am doing wrong? And how can I get the original content of this webpage?
You can't, unless you write something more sophisticated, but you probably shouldn't.
The purpose of Cloudflare-protection is to prevent what you are trying to realize unfortunately.
You could look into a possibility to access whatever you want to access by a public API or something that prizebond.net provides for example.
I'm trying to use recaptcha on my website. Nodejs server with express framework. The site isn't being hosted, I'm still working on it locally. On the homepage, after the user enters his info to create an account, and solves the recaptcha, I send the results
$("#g-recaptcha-response").val()
to the server. And on my server,
https.get("https://www.google.com/recaptcha/api/siteverify?secret=" + SECRET + "&response=" + key, function(res) {
var data = "";
res.on('data', function (chunk) {
data += chunk.toString();
});
res.on('end', function() {
try {
var parsedData = JSON.parse(data);
console.log(parsedData);
callback(parsedData.success);
} catch (e) {
callback(false);
}
});
});
where key is the response and SECRET is the secret key they give you. I declared
a variable SECRET and stored the secret key as a string in it.
Every single time, the for the
console.log(parsedData);
It's saying
{ success: false, 'error-codes': [ 'invalid-input-secret' ] }
I copied and pasted the secret key, how could it be invalid. It's only supposed to show this error if "The secret parameter is invalid or malformed" as it says on their website. I followed this tutorial.
I followed the tutorial too and then bumped into the same error that you have reported here. Looking closely at the screenshot in the tutorial it shows
Send a GET request with these parameters
And checking the Google reCaptcha website it says
Send a POST request with these parameters
I am curious whether Google changed their mind about POST instead of GET or the screenshot in the tutorial is from a different source.
Regardless, I have tweaked the version of code in the tutorial to make POST request (below code uses querystring module), see below:
var SECRET = "YourSecretHere";
// Helper function to make API call to recatpcha and check response
function verifyRecaptcha(key, callback) {
//declare above var querystring = require('querystring') on top
var post_data = querystring.stringify({
'secret' : SECRET,
'response': key
});
var post_options = {
host: 'www.google.com',
port: '443',
method: 'POST',
path: '/recaptcha/api/siteverify',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(post_data)
}
};
var req = https.request(post_options, function(res) {
var data = "";
res.on('data', function (chunk) {
data += chunk.toString();
});
res.on('end', function() {
try {
var parsedData = JSON.parse(data);
callback(parsedData.success);
} catch (e) {
callback(false);
}
});
});
req.write(post_data);
req.end();
req.on('error',function(err) {
console.error(err);
});
}
I also wanted to add that remoteip field is optional but, you can pass that value too if you want too. In order to do that, you need to retrieve the remoteIpAddress from connection object or simply enable trust proxy on your app as shown below:
app.enable('trust proxy');
and then pass the ip address to the verifyRecaptcha and the call would look like follow:
verifyRecaptcha(req.ip, req.body["g-recaptcha-response"], function(success) {
if(success) { ...} else { ... }
});
You then need to modify the post_params to include remoteip field as follow:
var post_data = querystring.stringify({
'secret' : SECRET,
'response': key,
'remoteip': ip
});
app.enable('trust proxy'); allows req.ip and req.ips which is an array of ip addresses. For more info on getting the ip address of request see this SO question.
If you are developing and you get fed up with all tricky famous and the most annoying Street Names reCaptcha, then I recommend that you use the test Site and Secret keys provided by Google to override captcha solving in order to speed up development. See here
This is really stupid, and I can't believe I wasted this much time on it but instead of using the variable SECRET, I just added my secret key to the url and it worked.
So I want to use this: (taken from their API site -> node.js documentation)
https://github.com/blockchain/api-v1-client-node
Recieving payments:
https://github.com/blockchain/api-v1-client-node/blob/master/docs/Receive.md
var blockchain = require('blockchain.info');
var identifier = 'myidentifier';
var password = 'mypassword';
var myWallet = new blockchain.MyWallet(identifier, password);
var myBTCadress = '14Q3ufL1BUHtWskBKtsshVDATRY65TaJMB';
Ok, so the recieving part:
var receive = new blockchain.Receive( [confirmations: 1], ? ); // What do I need to put here?
Documentation says:
callbackURL: the url to which the callback should be sent (string)
I don't understand what URL it should go to?!
The callback URL should be the one that redirects back to your site. So setup a callback url with blockchain like...
https://www.yoursite.com/callback/blockchain
Assuming you are using something like express in your app make a route like so.
app.get('/callback/blockchain', function (req, res) {
// Stuff here
});
you will prob need to include
var https = require('https');
That way then you can set up your logic inside for example...
// Stuff here
var options = {
host : 'api.blockchain.info',
path : '/some/path/',
port : 443,
method : 'GET'
}
var request = https.request(options, function(response){
var body = ""
response.on('data', function(data) {
body += data;
});
response.on('end', function() {
res.send(JSON.parse(body));
});
});
request.on('error', function(e) {
console.log('Problem with request: ' + e.message);
});
request.end();
That will for example output you request in json on whatever page you have your app.get('whateverurl') set to.
I am trying to make a WebAPI call from server script and i am getting authentication error:
This is how my server.js looks like:
var app = require('http').createServer()
, io = require('socket.io').listen(app)
, fs = require('fs')
, moment = require('moment')
, request = require('request'); //https://github.com/mikeal/request
app.listen(8000, function () {
console.log('server started');
doSomethingOnServerStart();
});
function doSomethingOnServerStart()
{
console.log('Getting something from server');
request.get({
url: 'http://localhost:63213/Api/MyAPI/GetSomething',
},
function (error, response, body) {
console.log(response.statusCode);
if (response.statusCode == 200) {
console.log('data received from server');
} else {
console.log('error: ' + response.statusCode);
console.log(body);
}
});
}
I am using Mikeal's Request library to make the WebAPI call. In the HTTP Authentication section (https://github.com/mikeal/request#http-authentication), it says pass username and password as "hash" but does not say how do i generate that hash.
I am kind of stuck and dont know how to proceed.
To address your initial question, judging from the documentation it looks like you need to make an additional chained function call:
request.get('http://some.server.com/').auth('username', 'password', false);
or like this
request.get('http://some.server.com/', {
'auth': {
'user': 'username',
'pass': 'password',
'sendImmediately': false
}
});
In the second case it mentions having to have auth as a hash (as it is above). It doesn't appear you're sending a username or password anywhere in your code...
I need to connect to a web page and return the status code of the page, which I've been able to achieve using http.request however the pages I need to request can take a long time, sometimes several minutes, so I'm always getting a socket hang up error.
I'm using the following code so far:
var reqPage = function(urlString, cb) {
// Resolve the URL
var path = url.parse(urlString);
var req = http.request({
host: path.hostname,
path: path.pathname,
port: 80,
method: 'GET'
});
req.on('end', function() {
cb.call(this, res);
});
req.on('error', function(e) {
winston.error(e.message);
});
};
What do I need to do to ensure that my application still attempts to connect to the page even if it's going to take a few minutes?
Use the request module and set the timeout option to an appropriate value (in milliseconds)
var request = require('request')
var url = 'http://www.google.com' // input your url here
// use a timeout value of 10 seconds
var timeoutInMilliseconds = 10*1000
var opts = {
url: url,
timeout: timeoutInMilliseconds
}
request(opts, function (err, res, body) {
if (err) {
console.dir(err)
return
}
var statusCode = res.statusCode
console.log('status code: ' + statusCode)
})
Add this if you don't want to use a higher level http client like request or superagent , then add this...
req.on("connection", function(socket){
socket.setTimeout((1000*60*5)); //5 mins
});