I am trying to convert a curl request to a nodeJS script, but somehow it fails on me., by failing means, that the url I setup with cloudflare triggers a captcha whilst it doesn't when I use the curl request I copied from dev tools > network tab.
here is the curl
curl -q 'https://foo.bar/path' -H 'cookie: somecookie' -H 'origin: https://foo.bar' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9,pt;q=0.8' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36' -H 'content-type: application/x-www-form-urlencoded; charset=UTF-8' -H 'accept: */*' -H 'referer: https://foo.bar/path' -H 'authority: www.foo.bar' -H 'x-requested-with: XMLHttpRequest' --data "foo=bazz" --compressed
and here is the nodeJS code
var request = require('request');
var url = 'https://foo.bar/path';
var cookie = 'somecookie';
var ua = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36';
var baseRequest = request.defaults({
headers: {
'Cookie': cookie,
'User-Agent': ua,
'origin': 'https://foo.bar',
'referer': 'https://foo.bar/path'
}
});
baseRequest.post({ url: url, formData: {
foo: 'bazz'
}}, function(err, response, body) {
console.log(body);
});
There are some online tools that doing it for you like https://curl.trillworks.com/#node
by the way I did for you and the result is :
var request = require('request');
var headers = {
'cookie': 'somecookie',
'origin': 'https://foo.bar',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'en-US,en;q=0.9,pt;q=0.8',
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'accept': '*/*',
'referer': 'https://foo.bar/path',
'authority': 'www.foo.bar',
'x-requested-with': 'XMLHttpRequest'
};
var dataString = 'foo=bazz';
var options = {
url: 'https://foo.bar/path',
method: 'POST',
headers: headers,
body: dataString
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}
}
request(options, callback);
exec(`curl -q 'https://foo.bar/path' -H 'cookie: somecookie' -H 'origin: https://foo.bar' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9,pt;q=0.8' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36' -H 'content-type: application/x-www-form-urlencoded; charset=UTF-8' -H 'accept: */*' -H 'referer: https://foo.bar/path' -H 'authority: www.foo.bar' -H 'x-requested-with: XMLHttpRequest' --data "foo=bazz" --compressed`
, (error, result, metadata) => {
console.log(result);
});
You can do direct CURL using exec
From ES6, you can make use of the async/await to ececute a child process with curl. Here's a simple example:
const { exec: execAsync } = require('child-process-async');
const { result, stderr } = await execAsync(`curl -s https://api.ipdata.co/country_name?api-key=test`);
console.log(result);
Related
So I was trying to fetch data by reverse engineering Twitter's API. Fetching the data using cURL or even in other languages and runtimes such as .NET (using RestSharp), python (using requests) works fine and returns the tweets as expected.
When I do the same in NodeJS, irrespective of the library (axios, https, requests), it always returns Error 404
The request I'm using is:
cURL Code
curl --location --request GET 'https://api.twitter.com/2/search/adaptive.json?q=(from%3Aelonmusk)&query_source=typed_query&count=40' --header 'sec-ch-ua: "Not_A Brand";v="99", "Microsoft Edge";v="109", "Chromium";v="109"' --header 'x-twitter-client-language: en' --header 'x-csrf-token: <csrf_token>' --header 'sec-ch-ua-mobile: ?0' --header 'authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA' --header 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.61' --header 'x-twitter-auth-type: OAuth2Session' --header 'x-twitter-active-user: yes' --header 'sec-ch-ua-platform: "Windows"' --header 'Accept: */*' --header 'host: api.twitter.com' --header 'Cookie: <cookie>'
NodeJS Axios Code
var axios = require('axios');
var config = {
method: 'get',
maxBodyLength: Infinity,
url: 'https://api.twitter.com/2/search/adaptive.json?q=(from%3Aelonmusk)&query_source=typed_query&count=40',
headers: {
'sec-ch-ua': '"Not_A Brand";v="99", "Microsoft Edge";v="109", "Chromium";v="109"',
'x-twitter-client-language': 'en',
'x-csrf-token': '<csrf_token>',
'sec-ch-ua-mobile': '?0',
'authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.61',
'x-twitter-auth-type': 'OAuth2Session',
'x-twitter-active-user': 'yes',
'sec-ch-ua-platform': '"Windows"',
'Accept': '*/*',
'host': 'api.twitter.com',
'Cookie': '<cookie>'
}
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
The response I'm getting is:
AxiosError: Request failed with status code 404
If I use node-libcurl library to fetch data instead of axios or https, and disable ssl verification while using node-libcurl library, the data is fetched as intended and I am getting the list of tweets.
Also, executing the same request in Postman works alright too.
The problem only occurs in NodeJS
Bearer token has expiry, after expired time, it will not works.
So you need to call get token first with API key/secret,
Detail for get access-token in here
Then search API will remove time dependency.
detail about search in here
This code will works
const axios = require("axios")
const getToken = async () => {
try {
const API_KEY = '<your API Key>'
const API_KEY_SECRET = '<your API SECRET>'
const response = await axios.post(
'https://api.twitter.com/oauth2/token',
'',
{
params: {
'grant_type': 'client_credentials'
},
auth: {
username: API_KEY,
password: API_KEY_SECRET
}
}
);
return Promise.resolve(response.data.access_token);
} catch (error) {
return Promise.reject(error);
}
}
const searchTweet = async (token, query, limit) => {
try {
const siteUrl = `https://api.twitter.com/2/tweets/search/recent?tweet.fields=created_at,geo&query=${query}&max_results=${limit}`
const response = await axios.get(siteUrl, {
headers: {
'Accept-Encoding': 'application/json',
Authorization: `Bearer ${token}`
}
});
return Promise.resolve(response.data);
} catch (error) {
return Promise.reject(error);
}
}
getToken().then(token => {
// console.log("access token: " + token)
searchTweet(token, 'elonmusk', 10).then(tweets => {
for (tweet of tweets.data) {
console.log(JSON.stringify(tweet))
}
}).catch(error => console.log(error));
}).catch(error => console.log(error));
Result
$ node get-tweet.js
{"created_at":"2023-02-01T16:29:26.000Z","edit_history_tweet_ids":["1620821639058563072"],"id":"1620821639058563072","text":"RT #ThierryBreton: I take note of the path that Twitter is committed to take in Europe to comply with #DSA rules.\n\nNext few months will be…"}
{"created_at":"2023-02-01T16:29:26.000Z","edit_history_tweet_ids":["1620821638102278145"],"id":"1620821638102278145","text":"#Tessa430 #elonmusk He explained that. He wants to see how the code effects his account personally"}
{"created_at":"2023-02-01T16:29:26.000Z","edit_history_tweet_ids":["1620821637305352192"],"id":"1620821637305352192","text":"#kucoincom It's OK to have your eggs in one basket as long as you control what happens to that basket” ~ #ElonMusk; Listen to him and invest in #TechTrees #TTC $TTC"}
{"created_at":"2023-02-01T16:29:26.000Z","edit_history_tweet_ids":["1620821636370014210"],"id":"1620821636370014210","text":"#Hauwah_omar0 #elonmusk plz remove this kind of bitchsss😪"}
{"created_at":"2023-02-01T16:29:26.000Z","edit_history_tweet_ids":["1620821636332290049"],"id":"1620821636332290049","text":"RT #GermanosGhassan: #GermanosPeter #Twitter After acquiring twitter, Elon musk promised to block spams and fakes. Twitter for Lebanese pol…"}
{"created_at":"2023-02-01T16:29:25.000Z","edit_history_tweet_ids":["1620821635262464000"],"id":"1620821635262464000","text":"#elonmusk #TaraBull808 Don’t worry, twitter is dropping faster than a nuclear bomb."}
{"created_at":"2023-02-01T16:29:25.000Z","edit_history_tweet_ids":["1620821634511949828"],"id":"1620821634511949828","text":"#elonmusk #ElijahSchaffer AKA hard-coded."}
{"created_at":"2023-02-01T16:29:25.000Z","edit_history_tweet_ids":["1620821634436464640"],"id":"1620821634436464640","text":"RT #SidneyPowell1: Thank you #elonmusk \nI'm back!! Freedom of speech is crucial to our survival as the beacon of freedom for the world. I &…"}
{"created_at":"2023-02-01T16:29:25.000Z","edit_history_tweet_ids":["1620821632918118401"],"id":"1620821632918118401","text":"#elonmusk 👏👏👏👏"}
{"created_at":"2023-02-01T16:29:25.000Z","edit_history_tweet_ids":["1620821632615940096"],"id":"1620821632615940096","text":"#libsoftiktok #elonmusk How do we change to private?"}
I get a response like
config: {
url: '',
method: 'post',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
'accept-language': 'de-DE,de;q=0.8',
cookie: '**WANT THIS AS A STRING**',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.72 Safari/537.36'
},
},
now I would like to get the cookie as a string to post it in the next request.
let cookies = response.config.headers.cookie;
So scenario is like this I
From Local system - able to call api via curl and via nodejs code.
mitmproxy screenshots of connection details are in screenshot.
From server hosted on linode - able to call api via curl but same code working on local gets stuck.
mitmproxy screenshots of connection details are in screenshot.
From what i am guessing its most likely related to ssl handshake being stuck somewhere. What is bothering me since last few days is request and headers are same from curl and nodejs then why different behaviour
sample curl curl 'https://www.nseindia.com/api/liveEquity-derivatives?index=stock_fut' -H 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:79.0) Gecko/20100101 Firefox/79.0' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Connection: keep-alive' -H 'Cookie: _ga=GA1.2.1464035869.1595747215; RT="z=1&dm=nseindia.com&si=d2d25dd8-0b59-4941-a83c-9e3fd2ff9f63&ss=kdllvs49&sl=0&tt=0&bcn=%2F%2F684d0d37.akstat.io%2F&ul=36pqu&hd=36pwb"; ak_bmsc=32ED2E364503B6429496808D1A081032DFC42B6C6C2B0000013A315F40BDEA22~pl0PkvDhq66QJbIByCPofydTlTrNC70ggGqe1TUq9qks8cAr8R3eZemC9KyklftDSOsn0lHOiNNrEUdQWVD7K9oyJ4mGjON/9EjkvUCiHECyXvRkGJmEOaZcBzDtdvX+B8wg8spNZ3RI3jWLg7w+Asr66XwLm8l04X6nfSo9UkF+iOv49hjP4Lfhb4tC6p74oLmL/TQsX5a1gQlb9MGZznGPrXhhxFu1Rd1m3XD9/PToM=' -H 'Upgrade-Insecure-Requests: 1' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'TE: Trailers'
sample nodejs code -NODE_DEBUG=* node mytest.js
var request = require('request');
var https = require('https')
var fetch = require('node-fetch');
const httpsAgent = new https.Agent({
rejectUnauthorized: false,
secureProtocol: "TLSv1_2_method"
});
var headers = {
'Host': 'www.nseindia.com',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:79.0) Gecko/20100101 Firefox/79.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
'TE': 'Trailers',
'Cookie': '_ga=GA1.2.1464035869.1595747215; RT=z=1&dm=nseindia.com&si=d2d25dd8-0b59-4941-a83c-9e3fd2ff9f63&ss=kdllvs49&sl=0&tt=0&bcn=%2F%2F684d0d37.akstat.io%2F&ul=36pqu&hd=36pwb; ak_bmsc=32ED2E364503B6429496808D1A081032DFC42B6C6C2B0000013A315F40BDEA22~pl0PkvDhq66QJbIByCPofydTlTrNC70ggGqe1TUq9qks8cAr8R3eZemC9KyklftDSOsn0lHOiNNrEUdQWVD7K9oyJ4mGjON/9EjkvUCiHECyXvRkGJmEOaZcBzDtdvX+B8wg8spNZ3RI3jWLg7w+Asr66XwLm8l04X6nfSo9UkF+iOv49hjP4Lfhb4tC6p74oLmL/TQsX5a1gQlb9MGZznGPrXhhxFu1Rd1m3XD9/PToM='
};
var options = {
url: 'https://www.nseindia.com/api/liveEquity-derivatives?index=stock_fut',
headers: headers
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}
}
(async function(){
if(require.main==module){
console.log('before')
let data = await fetch(options.url, {
headers: {
Connection: 'keep-alive',
'Cache-Control': 'no-cache',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0'
},
agent:httpsAgent
}).then(res => res.json())
console.log(data)
}
})();
Here is the code which works,
curl -v --output test.gz -H User-Agent: -H Accept: -H "Connection: keep-alive" -H "Accept-Encoding: gzip, deflate, br" "https://www.nseindia.com/api/liveEquity-derivatives?index=stock_fut"
the output is stored in test.gz
Edit: (Some more explaination)
I think that the server accept some specific encoding only, which I have specified using -H "Accept-Encoding: gzip, deflate, br" and need to keep the connection alive. The remaining parameters are pretty self explanatory.
I am using the request npm module.I want to retrieve an image from a url. The request.get(url) function is returning me a '400 Bad Request', whereas the image is accessible from the browser.
The url i am hitting is : http://indiatribune.com/wp-content/uploads/2017/09/health.jpg
You could try to add some headers:
const request = require('request');
request.get({
url: 'http://indiatribune.com/wp-content/uploads/2017/09/health.jpg',
headers: {
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'en-GB,en;q=0.8,en-US;q=0.6,hu;q=0.4',
'Cache-Control': 'max-age=0',
Connection: 'keep-alive',
Host: 'indiatribune.com',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
},
}, (err, response, data) => {
console.log(response, data);
});
The User-Agent seems to be enough.
Use download module . It's pretty simple.
const fs = require('fs');
const download = require('download');
download('http://indiatribune.com/wp-content/uploads/2017/09/health.jpg').pipe(fs.createWriteStream('foo.jpg'));
Here's the cURL request which it works:
curl -H
'X-New-ID: weR1RRzRw3R3R3Rz1'
-H 'Brand: 1'
-H 'X-Device-Version: 4.02'
-H 'X-Device-Source: 6'
-H 'Accept-Language: en-US'
-H 'Content-Type: application/json'
-H 'User-Agent: Dalv1k/2.1.0 (Linux; U; Andr0id 5.1; Go0gle Nexus 10 - 5.1.0 - API 22 - 2560x1600_1 Build/LM227D)'
-H 'Host: api.autoigs.com'
--data-binary '{"areaId":10,"cityId":1,"countryId":1,"kickId":0}' --compressed 'https://api.autoigs.com/apiAndroid/v1/kicks'
Unfortunately, I am not able to figure that out as well (like curl from cli) Though I would like to use nodejs to send this request. How do i do this?
Using the request package, you can do it like this:
const request = require('request');
request.post({
url: 'https://api.autoigs.com/apiAndroid/v1/kicks',
form: {"areaId":10,"cityId":1,"countryId":1,"kickId":0},
json: true,
headers: {
'X-New-ID': 'weR1RRzRw3R3R3Rz1',
'Brand': '1',
'X-Device-Version': '4.02',
'X-Device-Source': '6',
'Accept-Language': 'en-US',
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.1; Google Nexus 10 - 5.1.0 - API 22 - 2560x1600_1 Build/LMY47D)',
},
}, function(err, response, body) {
if(err) {
console.error('error', err);
// handle error!
return;
}
// body contains response body
console.log(body);
});