I use node.js to make requests on google.
I used puppeteer but he detected my robot.
so i used tor-request.
I can change my ip at each connection.
But google still hangs with the same error message.
I want to change the user-agent of the tor-expert bundle.
How to do ?
Our systems have detected unusual traffic from your computer network.
Please try your request again later. Why did this happen?
IP address: 109.70.100.** Time: 2019-08-03T08:47:17Z URL:
https://www.google.com/search?q=re&gbv=1&sei=6UlFXY6mKsSiab3EjbAF
var tr = require('tor-request');
tr.TorControlPort.password = "***";
io.sockets.on('connection', async(socket) => {
socket.on('key', function(value) {
function torIp() {
tr.request('https://www.google.com/search?q=re&gbv=1&sei=6UlFXY6mKsSiab3EjbAF', function(error, response, body) {
if (!error && response.statusCode == 200) {
console.log("Your public IP is: " + body);
}
socket.emit('google_web', body)
});
}
torIp();
tr.renewTorSession(function(error, msg) {
console.log(error);
console.log(msg);
if (msg) {
torIp();
}
});
})
})
To change the user agent, you can use the method described here, a tor-request issue:
tr.request({
url: 'https://www.google.com/search?q=re&gbv=1&sei=6UlFXY6mKsSiab3EjbAF',
headers: {
'user-agent': 'giraffe'
}
}, function(error, response, body) {
// ...
});
Just replace giraffe with whichever user agent you wish.
(But, as the comment noted, Google may well have blocked certain Tor exit nodes completely, in which case changing the user-agent won't help)
Related
Using Heroku, node.js, the npmjs request module, express and php, heroku log is reporting that it can not find a php file.
I am making a GET request from node.js in the index.js server that I made in the Heroku dyno web.1 with this url: "https://chatscroll-code2.herokuapp.com/login2.php" with the hope of receiving some json data back from the php file.
var requestOptions = {
url : "https://chatscroll-code2.herokuapp.com/login2.php",
json : {}
};
request(requestOptions , function(err, resp, body) {
if (err) {
console.log(err);
}
else if (resp.statusCode === 200) {
console.log(body);
} else {
console.log("returned status code="+resp.statusCode);
callback(body);
}
});
The error that I get back from Heroku says that it "Cannot GET /login2.php." The html presented error value from Heroku is contained in the returned body variable.
In the code, body should contain: "{\"result\":"."\"1\",\"reason\":"."\"Successful login\"}";
or
"{\"result\":"."\"0\",\"reason\":"."\"Unsuccessful login\"}";
welcome to StackOverflow !
I think the problem is that you should use the POST method to reach this PHP page. Check in the PHP code which method it is handling.
If it is handling the POST method, your code should look something like this:
var requestOptions = {
method: 'POST',
url: 'https://chatscroll-code2.herokuapp.com/login2.php',
form: {
username: 'something',
password: 'greatpassword' // encrypt with something like PBKDF2
}
}
request(requestOptions, function (err, resp, body) {
if (err) {
console.log(err)
} else if (resp.statusCode === 200) {
console.log(body)
} else {
console.log('returned status code=' + resp.statusCode)
callback(body)
}
})
I have a Discord server where we help each others, don't hesitate joining it :)
i'm trying to change Ip by using tor
According to Tor-Request Documentation I should be able to do this simply by using
newTorSession
But ip not changing. What is wrong in my code?
var tr = require('tor-request');
requestIP();
tr.setTorAddress('localhost', 9050);
tr.newTorSession( (err) =>
{
console.log (err);
requestIP();
return;
});
//console.log (tr.TorControlPort)
function requestIP() {
tr.request('https://api.ipify.org', function (err, res, body) {
if (!err && res.statusCode == 200) {
console.log("Your public (through Tor) IP is: " + body);
}
})
}
From the doc
You need to enable the Tor ControlPort if you want to programmatically
refresh the Tor session (i.e., get a new proxy IP address) without
restarting your Tor client.
so you need to follow these steps to enable ControlPort then pass that password to tor-request
tr.TorControlPort.password = 'PASSWORD'
I successfully generated an access token but now I can't use it. Sadly there is no official yandex node library for using the API. There are 2 unofficial npm module that will only work with a server but I want it localhost. As example with code like this, in this case I want to display all files from my disk. Of course I enabled all scopes for my app
request.get('https://cloud-api.yandex.net/v1/disk/resources/files/', {
'auth': {
'bearer': 'xxxxxxxxxxxxxxxxxxx'
}
}, function(err,httpResponse,body){ /* ... */
if(err) {
console.log('err: ' + err)
}
console.log('body: ' + body)
});
Also if I would use https://cloud-api.yandex.net/v1/disk/resources/files?oauth_token=xxxxxxxxxxxxxx&oauth_client_id=xxxxxxxxxxxxxxxxxxx
or https://cloud-api.yandex.net/v1/disk/resources/files?access_token=xxxxxxxxxxxxxx&client_id=xxxxxxxxxxxxxxxxxxx
in my browser I would get
{"message":"?? ???????????.","description":"Unauthorized","error":"UnauthorizedError"}
Somebody has working code or an idea why I´am getting this message?
Now I'm using yandex-disk package. It works enougth for my purposes.
That my steps:
Register app in Yandex Oath Service.
After that I can get info about my app
Realize the access method to get Oath Token by calling this.
npm i yandex-disk
Inserting the code into index.js
const YandexDisk = require('yandex-disk').YandexDisk;
const disk = new YandexDisk('YourOathTokenHere');
console.log('disk', disk);
disk.readdir('/', (err, response) => {
console.log('callback error', err);
console.log('callback response', response);
});
Just include authorization header like that:
axios.get(encodeURI(`https://cloud-api.yandex.net:443/v1/disk/resources?path=${path}`), {
headers: {
'Authorization': `OAuth ${token}`
}
}).then((res) => {
console.log(res);
}).catch((err) => {
console.log(err);
});
I'm crawling a lot of links with the request module in parallel with combination of the async module.
I'm noticing alot of ETIMEDOUT and ESOCKETTIMEDOUT errors although the links are reachable and respond fairly quickly using chrome.
I've limit the maxSockets to 2 and the timeout to 10000 in the request options.
I'm using async.filterLimit() with a limit of 2 to even cut down the parallelism to 2 request each time.
So I have 2 sockets, 2 request, and a timeout of 10 seconds to wait for headers response from the server yet I get these errors.
Here;s request configuration I use:
{
...
pool: {
maxSockets: 2
},
timeout: 10000
,
time: true
...
}
Here's the snippet of code I use to fecth links:
var self = this;
async.filterLimit(resources, 2, function(resource, callback) {
request({
uri: resource.uri
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
...
} else {
self.emit('error', resource, error);
}
callback(...);
})
}, function(result) {
callback(null, result);
});
I listened to the error event and I see whenever the error code is ETIMEDOUT the connect object is either true/false so sometimes it's a connection timeout and sometimes it's not (according to request docs)
UPDATE:
I decided to boost up the maxSockets to Infinity so no connection will be hangup due to lack of available sockets:
pool: {
maxSockets: Infinity
}
In-order to control the bandwidth I implemented a requestLoop method that handle the request with a maxAttemps and retryDelay parameters to control the requests:
async.filterLimit(resources, 10, function(resource, callback) {
self.requestLoop({
uri: resource.uri
}, 100, 5000, function (error, response, body) {
var fetched = false;
if (!error) {
...
} else {
....
}
callback(...);
});
}, function(result) {
callback(null, result);
});
Implementation of requestLoop:
requestLoop = function(options, attemptsLeft, retryDelay, callback, lastError) {
var self = this;
if (attemptsLeft <= 0) {
callback((lastError != null ? lastError : new Error('...')));
} else {
request(options, function (error, response, body) {
var recoverableErrors = ['ESOCKETTIMEDOUT', 'ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED'];
var e;
if ((error && _.contains(recoverableErrors, error.code)) || (response && (500 <= response.statusCode && response.statusCode < 600))) {
e = error ? new Error('...');
e.code = error ? error.code : response.statusCode;
setTimeout((function () {
self.requestLoop(options, --attemptsLeft, retryDelay, callback, e);
}), retryDelay);
} else if (!error && (200 <= response.statusCode && response.statusCode < 300)) {
callback(null, response, body);
} else if (error) {
e = new Error('...');
e.code = error.code;
callback(e);
} else {
e = new Error('...');
e.code = response.statusCode;
callback(e);
}
});
}
};
So this to sum it up:
- Boosted maxSockets to Infinity to try overcome timeout error of sockets connection
- Implemnted requestLoop method to control failed request and maxAttemps as well as retryDelay of such requests
- Also there's maxium number of concurrent request set by the number passed to async.filterLimit
I want to note that I've also played with the settings of everything here in-order to get errors free crawling but so far attempts failed as-well.
Still looking for help about solving this problem.
UPDATE2:
I've decided to drop async.filterLimit and make my own limit mechanism.
I just have 3 variables to help me achieve this:
pendingRequests - a request array which will hold all requests (will explain later)
activeRequests - number of active requests
maxConcurrentRequests - number of maximum allowed concurrent requests
into the pendingRequests array, i push a complex object containing a reference to the requestLoop function as well as arguments array containing the arguments to be passed to the loop function:
self.pendingRequests.push({
"arguments": [{
uri: resource.uri.toString()
}, self.maxAttempts, function (error, response, body) {
if (!error) {
if (self.policyChecker.isMimeTypeAllowed((response.headers['content-type'] || '').split(';')[0]) &&
self.policyChecker.isFileSizeAllowed(body)) {
self.totalBytesFetched += body.length;
resource.content = self.decodeBuffer(body, response.headers["content-type"] || '', resource);
callback(null, resource);
} else {
self.fetchedUris.splice(self.fetchedUris.indexOf(resource.uri.toString()), 1);
callback(new Error('Fetch failed because a mime-type is not allowed or file size is bigger than permited'));
}
} else {
self.fetchedUris.splice(self.fetchedUris.indexOf(resource.uri.toString()), 1);
callback(error);
}
self.activeRequests--;
self.runRequest();
}],
"function": self.requestLoop
});
self.runRequest();
You'' notice the call to runRequest() at the end.
This function job is to manage the requests and fire requests when it can while keeping the maximum activeRequests under the limit of maxConcurrentRequests:
var self = this;
process.nextTick(function() {
var next;
if (!self.pendingRequests.length || self.activeRequests >= self.maxConcurrentRequests) {
return;
}
self.activeRequests++;
next = self.pendingRequests.shift();
next["function"].apply(self, next["arguments"]);
self.runRequest();
});
This should solve any Timeouts errors, through my testings tho, I've still noticed some timeouts in specific websites I've tested this on. I can't be 100% sure about this, but I'm thinking it's due to the nature of the website backing http-server limiting a user requests to a maximum by doing an ip-checking and as a result returning some HTTP 400 messages to prevent a possible 'attack' on the server.
Edit: duplicate of https://stackoverflow.com/a/37946324/744276
By default, Node has 4 workers to resolve DNS queries. If your DNS query takes long-ish time, requests will block on the DNS phase, and the symptom is exactly ESOCKETTIMEDOUT or ETIMEDOUT.
Try increasing your uv thread pool size:
export UV_THREADPOOL_SIZE=128
node ...
or in index.js (or wherever your entry point is):
#!/usr/bin/env node
process.env.UV_THREADPOOL_SIZE = 128;
function main() {
...
}
Edit 1: I also wrote a blog post about it.
Edit 2: if queries are non-unique, you may want to use a cache, like nscd.
I found if there are too many async requests, then ESOCKETTIMEDOUT exception happens in linux. The workaround I've found is doing this:
setting this options to request():
agent: false, pool: {maxSockets: 100}
Notice that after that, the timeout can be lying so you might need to increase it.
I'm trying to use Azure Mobile Service to process / handle GET and POST requests on an empty data table. (really just using the mobile service as a pass through)
As part of this I'm trying to forward the request to another url and receive the response back and return it via mobile service. I've figured out the GET part shown below but I'm having trouble the POST part.
GET Part:(Which works)
function read(query, user, request)
{
var p = request.parameters;
var httpRequest = require('request');
var url = 'http://someURL/'+ p.ssoid;
httpRequest.get(url, function(err, response, body)
{
if (err)
{
request.respond(500, "INTERNAL SERVER ERROR");
}
else
{
request.respond(200,JSON.parse(body) );
}
});
}
Post Code:(Does not work)
function insert(item, user, request)
{
var p = request.parameters;
require('request').post({
uri:'http://someURL/',
headers:{'content-type': 'application/json'},
body:p.body
},function(err,res,body){
if (err)
{
request.respond(500, "INTERNAL SERVER ERROR");
}
else
{
request.respond(200,"Success");
}
});
}
I know the POST requires a body with the post information, but how to I get it to pass forward?
On an insert, the body of the request will be stored in the item argument (assuming you're passing a JSON object). So your function would look something like this:
function insert(item, user, request)
{
var p = request.parameters;
require('request').post({
uri : 'http://someURL/',
headers : {'Content-Type': 'application/json'},
body : item
}, function(err, res, body){
if (err)
{
request.respond(500, "INTERNAL SERVER ERROR");
}
else
{
request.respond(200,"Success");
}
});
}
On a related note, if you're using the mobile service as a simple pass-through, you can also consider using a custom API instead of a table, where you can also apply your logic without having any (empty) table behind it.
function insert(item, user, request)
{
var p = request.parameters;
require('request').post({
uri : 'http://someURL/',
headers : {'Content-Type': 'application/json'},
body : JSON.stringify(item)
}, function(err, res, body){
if (err)
{
request.respond(500, "INTERNAL SERVER ERROR");
}
else
{
request.respond(200,"Success");
}
});
}