I am using this package to get the videos from Tiktok tiktok-package.
It's working fine. But now after some time, I could not get any data maybe my IP blocked. So I used proxy for this which gives me error
Error
Exception thrown in tiktok scraper Error: tunneling socket could not be established, cause=getaddrinfo ENOTFOUND http
I run below lines in my project, but still, it's not working
npm config set proxy http://myproxy:port
npm config set https-proxy http://myproxy:port
Code
'use strict';
const TikTokScraper = require('tiktok-scraper');
var main = async(user, numOfVideos) => {
try {
var data = [];
var myPorxy = 'http://proxy_host:port' //my proxy and port
const posts = await TikTokScraper.user(user, { number: numOfVideos, proxy: myPorxy }
);
if (posts) {
posts.collector.map(post => {
data.push(post);
});
}
//return data;
console.log(data);
} catch (error) {
return error;
}
};
main('zachking', 5);
After taking a look at the Options for the packet your are using, it seems like you don't have to specify the protocol if you are using a http proxy:
// Set proxy {string[] | string default: ''}
// http proxy: 127.0.0.1:8080
// socks proxy: socks5://127.0.0.1:8080
// You can pass proxies as an array and scraper will randomly select a proxy from the array to execute the requests
proxy: '',
Assuming everything else is correct var myPorxy = 'proxy_host:port' should work.
Related
I have two socket.io servers listening to two different ports on my node.js project.
I was running them as a http servers like that.
my server file :
var server = require('http').createServer(app);
//server port are 5000
var io = require('socket.io').listen(server);
var io3 = require('socket.io').listen(5132);
and on my first client side page /usrlist
var socket = io.connect('http://localhost:5000',{
reconnection: true,
reconnectionDelay: 5000,
reconnectionAttempts: 12
});
and on my client sides page /playingroom
var socket = io.connect('http://localhost:5132');
then i decided to making my connection secure using https so i changed my code to this
my server file :
const cors = require('cors');
app.use(cors({credentials: true, origin: true}));
var httpsOptions = {
key: fs.readFileSync('./app/certsandkeys/nwcertandkey/my-prvkey.pem'),
cert: fs.readFileSync('./app/certsandkeys/nwcertandkey/my-pubcert.pem')
};
var sctio = require('https').createServer(httpsOptions, app);
sctio.listen(443);
var sctio3 = require('https').createServer(httpsOptions, app);
sctio3.listen(5132);
//using the default port 443 instead of the port 5000
var io = require('socket.io').listen(sctio);
var io3 = require('socket.io').listen(sctio3);
and on my first client side page /usrlist
var socket = io.connect('https://localhost',{
secure: true,
reconnection: true,
reconnectionDelay: 5000,
reconnectionAttempts: 12
});
and on my client sides page /playingroom
var socket = io.connect('https://localhost:5132',{secure: true});
then after running my code my javascript code start to giving me error that saying i have undefined var but my javascript code was working very good before the changes from http to https and i tested that hundreds of time and im sure all my included files urls edited from http to https and my page can reading it perfectly but i don't know why one of my var just giving me undefined with out changing anything except my socket.io code
UPDATE:
after runing my code agian today the code worked fine for one time then the error come back agian then the error start to came and go in random way ????
the error massage :
> jQuery.Deferred exception: $usercolor is not defined
> MakeMove#https://localhost/cssFiles/js/makemove.js:170:4
> CheckResult#https://localhost/cssFiles/js/guiMultiPlayer.js:62:9
> CheckAndSet#https://localhost/cssFiles/js/guiMultiPlayer.js:183:6
> NewGame#https://localhost/cssFiles/js/guiMultiPlayer.js:764:3
> #https://localhost/cssFiles/js/main.js:9:2
> e#https://localhost/js/jquery.min.js:2:29453
> l/</t<#https://localhost/js/jquery.min.js:2:29755 undefined
> jquery.min.js:2:31008
my post code part on server :
app.post('/getcolor', function (req, res, next) {
var lCol="";
if(req.body.MoveString !== null){
Move.setMoveUserId(req.user.id);
a.getColor(req.user.id,function(col){
res.json({"msg": col, "loggedin": "true"}); // <=== here you have a defined lCol
});
} else {
var output = {"msg": lCol, "loggedin": "true"}; // <=== here lCol always equals ""
res.json(output);
}
});
the javascript post code :
function UserColor() { // Is used to check which color the opponent has
var MoveString = BoardToFen();
$.ajax({
type:'POST',
url:'/getcolor',
data:{MoveString:MoveString},
dataType:'json',
cache: false,
success:function(data){
if (data.msg == "white"){
$usercolor=0;
console.log("usercolorusercolor",$usercolor);
} else{
console.log("thrusercolorusercolor",$usercolor);
$usercolor=1;
}
}
});
return $usercolor;
}
I hope from anyone who will answering to explain why my code was working ok before converting my connection from http socket.io to https secure connection but after converting i started to get this error ?
According to the text of the error, the problem is not in socket.io. The mistake is that you define variable $usercolor without var/let/const statement. By default js code executes in “strict mode”, so you need to use var/let/const statement
As already explained in the comments, you have a problem setting the variable $usercolor.
But if you say the issue appeared after switching from http to https, the problem is likely that the 'success' callback in your ajax call is not triggered anymore.
Ajax success event not working
Edit
The UserColor function with support of async and an 'error' callback in the ajax call to log network errors.
function UserColor(callback) { // Is used to check which color the opponent has
var MoveString = BoardToFen();
$.ajax({
type:'POST',
url:'/getcolor',
data:{MoveString:MoveString},
dataType:'json',
cache: false,
success:function(data){
if (data.msg == "white"){
$usercolor=0;
} else{
$usercolor=1;
}
console.log("usercolorusercolor",$usercolor);
callback(null,$usercolor);
},
error:function(err) {
console.error(err);
callback(err);
}
});
}
_http_outgoing.js:797
socket.cork();
^
TypeError: socket.cork is not a function
at ClientRequest._flushOutput (_http_outgoing.js:797:10)
at ClientRequest._flush (_http_outgoing.js:776:16)
....
Getting above error when trying to http.get() or request() using SOCKS proxy agent previously created with "proxysocket" library.
I am trying to create working agent to use it socket.io, or ws or http to make connections via SOCKS proxy. I tries "proxysocket" library and its agent gives me error above.
let agent = proxysocket.createAgent("127.0.0.1", 9050);
request("http://www.google.com", {agent: agent}, (err, res, body) => {
if (err){
console.log("Http request error: " + err);
} else{
console.log("Http request connection success!");
console.log('statusCode:', response && response.statusCode);
console.log(body)
}
});
Ok, here is the solution. I tried tons of proxy agents, and the only working one is "socks5-http-client".
Other agents were giving all kinds of errors when using SOCKS5 protocol.
const http = require("http");
const Agent = require("socks5-http-client/lib/Agent"); // Constructor
let agent = new Agent({
socksHost: 'localhost', // Defaults to 'localhost'.
socksPort: 9050 // Defaults to 1080.
});
http.get({
hostname: "www.google.com", // Can be also onion address if used behind TOR
port: 80,
agent: agent,
method: 'GET'
}, (res)=>{
console.log("Connected");
res.pipe(process.stdout);
//Process res
}).on('error', (err) => {
console.error(`Got error: ${err.message}`);
});
Such agent can be passed into any kinds of libraries that use http.Agent such as socket.io, ws, etc.
I am trying to use axios with a proxy server to make an https call:
const url = "https://walmart.com/ip/50676589"
var config = { proxy: { host: proxy.ip, port: proxy.port } }
axios.get(url, config)
.then(result => {})
.catch(error => {console.log(error)})
The proxy servers I am using are all in the United States, highly anonymous, with support for HTTP and HTTPS.
I am receiving this error:
{ Error: write EPROTO 140736580649920:error:140770FC:SSL
routines:SSL23_GET_SERVER_HELLO:unknown
protocol:../deps/openssl/openssl/ssl/s23_clnt.c:794:
In order to ensure that the problem is with axios and NOT the proxy, I tried this:
curl -x 52.8.172.72:4444 -L 'https://www.walmart.com/ip/50676589'
This totally works just fine.
How do I configure axios to work with proxies and https URL's?
Axios https proxy support is borked if using https proxies. Try passing the proxy through httpsProxyAgent using http.
const axios = require('axios');
const httpsProxyAgent = require('https-proxy-agent');
const httpsAgent = new httpsProxyAgent('http://username:pass#myproxy:port');
// or const httpsAgent = new httpsProxyAgent({ host: 'myproxy', port: 9999 });
const config = {
url: 'https://google.com',
httpsAgent
}
axios.request(config).then((res) => console.log(res)).catch(err => console.log(err))
Alternatively there is a fork of Axios that incorporates this: axios-https-proxy-fix but I'd recommend the first method to ensure latest Axios changes.
Try this. That work for me.
First
npm install axios-https-proxy-fix
Then
import axios from 'axios-https-proxy-fix';
const proxy = {
host: 'some_ip',
port: some_port_number,
auth: {
username: 'some_login',
password: 'some_pass'
}
};
async someMethod() {
const result = await axios.get('some_https_link', {proxy});
}
You can solve this problem looking this issue
At this solution instead use the proxy interface, use the http(s)Agent.
For it the solution use the native node module https-proxy-agent.
var ProxyAgent = require('https-proxy-agent');
var axios = require('axios');
const agent = ProxyAgent('http://username:pass#myproxy:port')
var config = {
url: 'https://google.com',
proxy: false,
httpsAgent: agent
};
For it works the proxy property must be equal to false.
The https-proxy-agent and node-tunnel solutions did work for me, but both of them doesn't support conditional proxying using NO_PROXY.
I found global-agent as the best solution in my case as it modifies the core http and https objects and will be applied automatically to any library that makes use of them, including axios, got, request, etc.
The usage is very simple.
npm i global-agent
npm i -D #types/global-agent
Add import 'global-agent/bootstrap'; to the entrypoint (index.ts) of the server.
Run with these env vars and make sure HTTP_PROXY, HTTPS_PROXY are NOT in the env.
export GLOBAL_AGENT_NO_PROXY='*.foo.com,baz.com'
export GLOBAL_AGENT_HTTP_PROXY=http://127.0.0.1:8080
This is how I finally ended up using it.
import { bootstrap } from 'global-agent';
const proxy = process.env.EXTERNAL_PROXY;
if (proxy) {
process.env.GLOBAL_AGENT_HTTP_PROXY = proxy;
process.env.GLOBAL_AGENT_NO_PROXY = process.env.NO_PROXY;
process.env.GLOBAL_AGENT_FORCE_GLOBAL_AGENT = 'false';
bootstrap();
logger.info(`External proxy ${proxy} set`);
}
I know this is an old post, but I hope this solution saves time for anyone facing an SSL issue with Axios.
You can use an HTTP agent, I suggest using hpagent
const axios = require("axios");
const { HttpProxyAgent, HttpsProxyAgent } = require("hpagent");
async function testProxy() {
try {
const proxy = "http://username:password#myproxy:port";
// hpagent configuration
let agentConfig = {
proxy: proxy,
// keepAlive: true,
// keepAliveMsecs: 2000,
// maxSockets: 256,
// maxFreeSockets: 256,
};
axios.defaults.httpAgent = new HttpProxyAgent(agentConfig);
axios.defaults.httpsAgent = new HttpsProxyAgent(agentConfig);
// Make a simple request to check for the IP address.
let res = await axios.get("https://api.ipify.org/?format=json");
console.log(res.data);
} catch (error) {
console.error(error);
}
}
testProxy();
Try to explicitly specify the port in the URL:
const url = "https://walmart.com:443/ip/50676589"
If you also need an HTTPS-over-HTTP tunnel, you'll find a solution in this article.
Hope this helps,
Jan
This error is because axios is trying to proxy your request via https (it takes it from your url), there is this ticket tracking it: https://github.com/axios/axios/issues/925
I lost a day of work when I updated my dependencies last week (Feb. 2020) trying to figure out why services were stalling. axios-https-proxy-fix will cause Axios to hang indefinitely without timeout or throwing an error conflicting with other libraries in npm. Using node-tunnel (https://github.com/koichik/node-tunnel) to create an agent also works.
const tunnel = require('tunnel');
class ApiService {
get proxyRequest() {
const agent = tunnel.httpsOverHttp({
proxy: {
host: 'http://proxy.example.com',
port: 22225,
proxyAuth: `username:password`,
},
});
return axios.create({
agent,
})
}
}
I have created a http server which is creating proxy to send requests to target system.
function sendErrorResponse(client_req,client_res){
var options = {
:
};
var proxy = http.request(options, function (res) {
res.pipe(client_res, {
end: true
});
});
client_req.pipe(proxy, {
end: true
});
}
or using http-proxy npm package
function sendErrorResponse(client_req,client_res){
proxy.web(client_req, client_res, { target: 'http://localhost:7777' });
}
Now when client request to my server, I proxy some of the requests to other server and send to client whtever I get from the target server. I want that if target server gives some error (DNS resolution, network errors etc.) I can send same to the client instead of responding client with 500 with some error message.
How can I do that?
Regular client initiated requests to the node server are captured fine in Fiddler. However, requests sent from node to a web service are not captured. It did not help to pass in config for proxy (127.0.0.1:8888) to the request method. How can I route the request messages through Fiddler?
var http = require('http');
var request = require('request');
request.get(webserviceURL, { "auth" : {"user": "user", "pass" = "pass", sendImmediately: true },
"proxy" : { "host" : "127.0.0.1", "port" : 8888 }},
function (error, response) { console.log( "response received" );
});
Request repo: https://github.com/mikeal/request
I just tried to do this myself (using Fiddler and the request library from npm). Here's how I got it working:
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'; // Ignore 'UNABLE_TO_VERIFY_LEAF_SIGNATURE' authorization error
// Issue the request
request(
{
method: "GET",
uri: "https://secure.somewebsite.com/",
proxy: "http://127.0.0.1:8888" // Note the fully-qualified path to Fiddler proxy. No "https" is required, even for https connections to outside.
},
function(err, response, body) {
console.log("done");
});
This is with Fiddler2 using the default port and proxy options (and no proxy authentication).
Fiddler works by setting your "Internet Options" (from start menu) "Connections" > "LAN Settings" > "Proxy Server" to its port, thus making all HTTP traffic (clients which obey this setting) go through it.
You should point your node.js client lib to use a proxy, the settings are written in that options dialog after you start Fiddler.
The proxy option should be a full url, like this:
proxy : "http://127.0.0.1:8888"
To do this on an ad-hoc basis, without changing your code, you can use environment variables.
Request respects:
HTTP_PROXY
HTTPS_PROXY
NO_PROXY
So, to proxy just set these in your console before running your process.
For example, to setup http and https proxy use:
set HTTP_PROXY="http://127.0.0.1:8888"
set HTTPS_PROXY="http://127.0.0.1:8888"
set NODE_TLS_REJECT_UNAUTHORIZED=0
The latter line stops issues with SSL through the fiddler proxy.
I've been wanting the same... an equivalent of the Network tab in chrome DevTools, only for Nodejs. Unfortunately, it doesn't appear as though one exists. I don't have Fiddler on macos, so this is how I went about stubbing the require('http') methods to log and pass though. Leaving this here in case I need it again or someone else finds it helpful. You can turn it on by attaching a debugger and require('filename')() the file containing this script.
module.exports = () => {
const http = require('http');
http._request = http.request;
global.DO_LOG_AJAX = true;
const log = str => {
if (global.DO_LOG_AJAX) {
console.debug(str);
}
};
const flushLog = (requestLines, responseLines) => {
if (global.DO_LOG_AJAX) {
log([
'----------------Begin Request-----------------------------------',
...requestLines,
'----------------End Request / Begin Response--------------------',
...responseLines,
'----------------End Reponse-------------------------------------',
].join('\n'));
}
};
let write;
let end;
http.request = (...requestParams) => {
const req = http._request(...requestParams);
const { method, path, headers, host, port } = requestParams[0];
const requestLogLines = [];
requestLogLines.push(`${method} ${path}`);
requestLogLines.push(`Host: ${host}:${port}`);
for (const header of Object.keys(headers)) {
requestLogLines.push(`${header}: ${headers[header]}`);
}
write = write || req.write;
end = end || req.end;
req.on('error', err => {
log({ err });
});
req._write = write;
req._end = end;
const requestBody = [];
req.write = (...writeParams) => {
requestBody.push(writeParams[0].toString());
return req._write(...writeParams);
};
req.end = (...endParams) => {
if (endParams[0]) {
requestBody.push(endParams[0].toString());
}
requestLogLines.push('');
requestLogLines.push(requestBody.join(''));
return req._end(...endParams);
};
const responseLogLines = [];
req.once('response', response => {
const responseBody = [];
responseLogLines.push(`${response.statusCode} ${response.statusMessage}`);
for (const header of Object.keys(response.headers)) {
responseLogLines.push(`${header}: ${response.headers[header]}`);
}
const onData = chunk => {
responseBody.push(chunk.toString());
};
const onClose = err => {
responseLogLines.push('');
responseLogLines.push(responseBody.join(''));
responseLogLines.push('');
responseLogLines.push(`--- ERROR --- ${err.toString()}`);
flushLog(requestLogLines, responseLogLines);
req.removeListener('data', onData);
};
const onEnd = () => {
responseLogLines.push('');
responseLogLines.push(responseBody.join(''));
flushLog(requestLogLines, responseLogLines);
req.removeListener('data', onData);
};
response.on('data', onData);
response.once('close', onClose);
response.once('end', onEnd);
});
return req;
};
};