How to make a post request in NodeJS - node.js

Here's the cURL request which it works:
curl -H
'X-New-ID: weR1RRzRw3R3R3Rz1'
-H 'Brand: 1'
-H 'X-Device-Version: 4.02'
-H 'X-Device-Source: 6'
-H 'Accept-Language: en-US'
-H 'Content-Type: application/json'
-H 'User-Agent: Dalv1k/2.1.0 (Linux; U; Andr0id 5.1; Go0gle Nexus 10 - 5.1.0 - API 22 - 2560x1600_1 Build/LM227D)'
-H 'Host: api.autoigs.com'
--data-binary '{"areaId":10,"cityId":1,"countryId":1,"kickId":0}' --compressed 'https://api.autoigs.com/apiAndroid/v1/kicks'
Unfortunately, I am not able to figure that out as well (like curl from cli) Though I would like to use nodejs to send this request. How do i do this?

Using the request package, you can do it like this:
const request = require('request');
request.post({
url: 'https://api.autoigs.com/apiAndroid/v1/kicks',
form: {"areaId":10,"cityId":1,"countryId":1,"kickId":0},
json: true,
headers: {
'X-New-ID': 'weR1RRzRw3R3R3Rz1',
'Brand': '1',
'X-Device-Version': '4.02',
'X-Device-Source': '6',
'Accept-Language': 'en-US',
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 5.1; Google Nexus 10 - 5.1.0 - API 22 - 2560x1600_1 Build/LMY47D)',
},
}, function(err, response, body) {
if(err) {
console.error('error', err);
// handle error!
return;
}
// body contains response body
console.log(body);
});

Related

Fetching data using other runtimes (python, .NET, Java) as well as cURL works fine, but on NodeJS, returns Error 404

So I was trying to fetch data by reverse engineering Twitter's API. Fetching the data using cURL or even in other languages and runtimes such as .NET (using RestSharp), python (using requests) works fine and returns the tweets as expected.
When I do the same in NodeJS, irrespective of the library (axios, https, requests), it always returns Error 404
The request I'm using is:
cURL Code
curl --location --request GET 'https://api.twitter.com/2/search/adaptive.json?q=(from%3Aelonmusk)&query_source=typed_query&count=40' --header 'sec-ch-ua: "Not_A Brand";v="99", "Microsoft Edge";v="109", "Chromium";v="109"' --header 'x-twitter-client-language: en' --header 'x-csrf-token: <csrf_token>' --header 'sec-ch-ua-mobile: ?0' --header 'authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA' --header 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.61' --header 'x-twitter-auth-type: OAuth2Session' --header 'x-twitter-active-user: yes' --header 'sec-ch-ua-platform: "Windows"' --header 'Accept: */*' --header 'host: api.twitter.com' --header 'Cookie: <cookie>'
NodeJS Axios Code
var axios = require('axios');
var config = {
method: 'get',
maxBodyLength: Infinity,
url: 'https://api.twitter.com/2/search/adaptive.json?q=(from%3Aelonmusk)&query_source=typed_query&count=40',
headers: {
'sec-ch-ua': '"Not_A Brand";v="99", "Microsoft Edge";v="109", "Chromium";v="109"',
'x-twitter-client-language': 'en',
'x-csrf-token': '<csrf_token>',
'sec-ch-ua-mobile': '?0',
'authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.61',
'x-twitter-auth-type': 'OAuth2Session',
'x-twitter-active-user': 'yes',
'sec-ch-ua-platform': '"Windows"',
'Accept': '*/*',
'host': 'api.twitter.com',
'Cookie': '<cookie>'
}
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
The response I'm getting is:
AxiosError: Request failed with status code 404
If I use node-libcurl library to fetch data instead of axios or https, and disable ssl verification while using node-libcurl library, the data is fetched as intended and I am getting the list of tweets.
Also, executing the same request in Postman works alright too.
The problem only occurs in NodeJS
Bearer token has expiry, after expired time, it will not works.
So you need to call get token first with API key/secret,
Detail for get access-token in here
Then search API will remove time dependency.
detail about search in here
This code will works
const axios = require("axios")
const getToken = async () => {
try {
const API_KEY = '<your API Key>'
const API_KEY_SECRET = '<your API SECRET>'
const response = await axios.post(
'https://api.twitter.com/oauth2/token',
'',
{
params: {
'grant_type': 'client_credentials'
},
auth: {
username: API_KEY,
password: API_KEY_SECRET
}
}
);
return Promise.resolve(response.data.access_token);
} catch (error) {
return Promise.reject(error);
}
}
const searchTweet = async (token, query, limit) => {
try {
const siteUrl = `https://api.twitter.com/2/tweets/search/recent?tweet.fields=created_at,geo&query=${query}&max_results=${limit}`
const response = await axios.get(siteUrl, {
headers: {
'Accept-Encoding': 'application/json',
Authorization: `Bearer ${token}`
}
});
return Promise.resolve(response.data);
} catch (error) {
return Promise.reject(error);
}
}
getToken().then(token => {
// console.log("access token: " + token)
searchTweet(token, 'elonmusk', 10).then(tweets => {
for (tweet of tweets.data) {
console.log(JSON.stringify(tweet))
}
}).catch(error => console.log(error));
}).catch(error => console.log(error));
Result
$ node get-tweet.js
{"created_at":"2023-02-01T16:29:26.000Z","edit_history_tweet_ids":["1620821639058563072"],"id":"1620821639058563072","text":"RT #ThierryBreton: I take note of the path that Twitter is committed to take in Europe to comply with #DSA rules.\n\nNext few months will be…"}
{"created_at":"2023-02-01T16:29:26.000Z","edit_history_tweet_ids":["1620821638102278145"],"id":"1620821638102278145","text":"#Tessa430 #elonmusk He explained that. He wants to see how the code effects his account personally"}
{"created_at":"2023-02-01T16:29:26.000Z","edit_history_tweet_ids":["1620821637305352192"],"id":"1620821637305352192","text":"#kucoincom It's OK to have your eggs in one basket as long as you control what happens to that basket” ~ #ElonMusk; Listen to him and invest in #TechTrees #TTC $TTC"}
{"created_at":"2023-02-01T16:29:26.000Z","edit_history_tweet_ids":["1620821636370014210"],"id":"1620821636370014210","text":"#Hauwah_omar0 #elonmusk plz remove this kind of bitchsss😪"}
{"created_at":"2023-02-01T16:29:26.000Z","edit_history_tweet_ids":["1620821636332290049"],"id":"1620821636332290049","text":"RT #GermanosGhassan: #GermanosPeter #Twitter After acquiring twitter, Elon musk promised to block spams and fakes. Twitter for Lebanese pol…"}
{"created_at":"2023-02-01T16:29:25.000Z","edit_history_tweet_ids":["1620821635262464000"],"id":"1620821635262464000","text":"#elonmusk #TaraBull808 Don’t worry, twitter is dropping faster than a nuclear bomb."}
{"created_at":"2023-02-01T16:29:25.000Z","edit_history_tweet_ids":["1620821634511949828"],"id":"1620821634511949828","text":"#elonmusk #ElijahSchaffer AKA hard-coded."}
{"created_at":"2023-02-01T16:29:25.000Z","edit_history_tweet_ids":["1620821634436464640"],"id":"1620821634436464640","text":"RT #SidneyPowell1: Thank you #elonmusk \nI'm back!! Freedom of speech is crucial to our survival as the beacon of freedom for the world. I &…"}
{"created_at":"2023-02-01T16:29:25.000Z","edit_history_tweet_ids":["1620821632918118401"],"id":"1620821632918118401","text":"#elonmusk 👏👏👏👏"}
{"created_at":"2023-02-01T16:29:25.000Z","edit_history_tweet_ids":["1620821632615940096"],"id":"1620821632615940096","text":"#libsoftiktok #elonmusk How do we change to private?"}

able to call an api via curl but same api hangs when called from nodejs

So scenario is like this I
From Local system - able to call api via curl and via nodejs code.
mitmproxy screenshots of connection details are in screenshot.
From server hosted on linode - able to call api via curl but same code working on local gets stuck.
mitmproxy screenshots of connection details are in screenshot.
From what i am guessing its most likely related to ssl handshake being stuck somewhere. What is bothering me since last few days is request and headers are same from curl and nodejs then why different behaviour
sample curl curl 'https://www.nseindia.com/api/liveEquity-derivatives?index=stock_fut' -H 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:79.0) Gecko/20100101 Firefox/79.0' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Connection: keep-alive' -H 'Cookie: _ga=GA1.2.1464035869.1595747215; RT="z=1&dm=nseindia.com&si=d2d25dd8-0b59-4941-a83c-9e3fd2ff9f63&ss=kdllvs49&sl=0&tt=0&bcn=%2F%2F684d0d37.akstat.io%2F&ul=36pqu&hd=36pwb"; ak_bmsc=32ED2E364503B6429496808D1A081032DFC42B6C6C2B0000013A315F40BDEA22~pl0PkvDhq66QJbIByCPofydTlTrNC70ggGqe1TUq9qks8cAr8R3eZemC9KyklftDSOsn0lHOiNNrEUdQWVD7K9oyJ4mGjON/9EjkvUCiHECyXvRkGJmEOaZcBzDtdvX+B8wg8spNZ3RI3jWLg7w+Asr66XwLm8l04X6nfSo9UkF+iOv49hjP4Lfhb4tC6p74oLmL/TQsX5a1gQlb9MGZznGPrXhhxFu1Rd1m3XD9/PToM=' -H 'Upgrade-Insecure-Requests: 1' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'TE: Trailers'
sample nodejs code -NODE_DEBUG=* node mytest.js
var request = require('request');
var https = require('https')
var fetch = require('node-fetch');
const httpsAgent = new https.Agent({
rejectUnauthorized: false,
secureProtocol: "TLSv1_2_method"
});
var headers = {
'Host': 'www.nseindia.com',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:79.0) Gecko/20100101 Firefox/79.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
'TE': 'Trailers',
'Cookie': '_ga=GA1.2.1464035869.1595747215; RT=z=1&dm=nseindia.com&si=d2d25dd8-0b59-4941-a83c-9e3fd2ff9f63&ss=kdllvs49&sl=0&tt=0&bcn=%2F%2F684d0d37.akstat.io%2F&ul=36pqu&hd=36pwb; ak_bmsc=32ED2E364503B6429496808D1A081032DFC42B6C6C2B0000013A315F40BDEA22~pl0PkvDhq66QJbIByCPofydTlTrNC70ggGqe1TUq9qks8cAr8R3eZemC9KyklftDSOsn0lHOiNNrEUdQWVD7K9oyJ4mGjON/9EjkvUCiHECyXvRkGJmEOaZcBzDtdvX+B8wg8spNZ3RI3jWLg7w+Asr66XwLm8l04X6nfSo9UkF+iOv49hjP4Lfhb4tC6p74oLmL/TQsX5a1gQlb9MGZznGPrXhhxFu1Rd1m3XD9/PToM='
};
var options = {
url: 'https://www.nseindia.com/api/liveEquity-derivatives?index=stock_fut',
headers: headers
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}
}
(async function(){
if(require.main==module){
console.log('before')
let data = await fetch(options.url, {
headers: {
Connection: 'keep-alive',
'Cache-Control': 'no-cache',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0'
},
agent:httpsAgent
}).then(res => res.json())
console.log(data)
}
})();
Here is the code which works,
curl -v --output test.gz -H User-Agent: -H Accept: -H "Connection: keep-alive" -H "Accept-Encoding: gzip, deflate, br" "https://www.nseindia.com/api/liveEquity-derivatives?index=stock_fut"
the output is stored in test.gz
Edit: (Some more explaination)
I think that the server accept some specific encoding only, which I have specified using -H "Accept-Encoding: gzip, deflate, br" and need to keep the connection alive. The remaining parameters are pretty self explanatory.

How to convert this cURL request into node request call?

I have tried every way I can think of to format this cURL request into something I can do server side on node and I keep getting back an invalid credentials response. I know the creds are valid so I must come to the conclusion that the code is making the wrong type of request.
cURL request:
curl --request GET \
--url https://api.timekit.io/v2/bookings\
--header 'Content-Type: application/json' \
--user : api_key_dfagafrgaegrfareVhROys9V1bUJ1z7
my format:
var options = {
url: 'https://api.timekit.io/v2/bookings',
headers:{
"Content-Type": "application/json",
'user': APP_KEY
},
method:'GET'
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}
}
request(options, callback);
I think you mean --user user:password in your curl command.
When you pass --user to curl, it sends it to the server as an basic authorization header (with the user:password encoded using base64). You can try this by running it against httpbin.org:
curl --request GET --url http://httpbin.org/headers --header 'Content-Type: application/json' --user foo:bar
{
"headers": {
"Accept": "*/*",
"Authorization": "Basic Zm9vOmJhcg==",
"Connection": "close",
"Content-Type": "application/json",
"Host": "httpbin.org",
"User-Agent": "curl/7.61.1"
}
}
As you can see, they received the header "Authorization": "Basic Zm9vOmJhcg==" where Zm9vOmJhcg== is the base64 encode of foo:bar

NodeJS equivalent of a curl request

I am trying to convert a curl request to a nodeJS script, but somehow it fails on me., by failing means, that the url I setup with cloudflare triggers a captcha whilst it doesn't when I use the curl request I copied from dev tools > network tab.
here is the curl
curl -q 'https://foo.bar/path' -H 'cookie: somecookie' -H 'origin: https://foo.bar' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9,pt;q=0.8' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36' -H 'content-type: application/x-www-form-urlencoded; charset=UTF-8' -H 'accept: */*' -H 'referer: https://foo.bar/path' -H 'authority: www.foo.bar' -H 'x-requested-with: XMLHttpRequest' --data "foo=bazz" --compressed
and here is the nodeJS code
var request = require('request');
var url = 'https://foo.bar/path';
var cookie = 'somecookie';
var ua = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36';
var baseRequest = request.defaults({
headers: {
'Cookie': cookie,
'User-Agent': ua,
'origin': 'https://foo.bar',
'referer': 'https://foo.bar/path'
}
});
baseRequest.post({ url: url, formData: {
foo: 'bazz'
}}, function(err, response, body) {
console.log(body);
});
There are some online tools that doing it for you like https://curl.trillworks.com/#node
by the way I did for you and the result is :
var request = require('request');
var headers = {
'cookie': 'somecookie',
'origin': 'https://foo.bar',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'en-US,en;q=0.9,pt;q=0.8',
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'accept': '*/*',
'referer': 'https://foo.bar/path',
'authority': 'www.foo.bar',
'x-requested-with': 'XMLHttpRequest'
};
var dataString = 'foo=bazz';
var options = {
url: 'https://foo.bar/path',
method: 'POST',
headers: headers,
body: dataString
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}
}
request(options, callback);
exec(`curl -q 'https://foo.bar/path' -H 'cookie: somecookie' -H 'origin: https://foo.bar' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: en-US,en;q=0.9,pt;q=0.8' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36' -H 'content-type: application/x-www-form-urlencoded; charset=UTF-8' -H 'accept: */*' -H 'referer: https://foo.bar/path' -H 'authority: www.foo.bar' -H 'x-requested-with: XMLHttpRequest' --data "foo=bazz" --compressed`
, (error, result, metadata) => {
console.log(result);
});
You can do direct CURL using exec
From ES6, you can make use of the async/await to ececute a child process with curl. Here's a simple example:
const { exec: execAsync } = require('child-process-async');
const { result, stderr } = await execAsync(`curl -s https://api.ipdata.co/country_name?api-key=test`);
console.log(result);

Got "Bad request " HTTPS Post request in NodeJS

I am trying to send Https post request using NodeJS, but getting back 'Bad request'. At the same time when I send the same request via curl everything is fine. Can you help to fix the Node code:
var options = {
host: 'api.wit.ai',
port: 443,
path: '/converse?v=20170611&session_id=125abc&q=Hi',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': 'Bearer <token>'
}
};
var req = https.request(options, function(res) {...}
The curl query:
curl -XPOST 'https://api.wit.ai/converse?v=20170611&session_id=125abc&q=Hi' \
-H "Content-Type: application/json" \
-H "Accept: application/json" \
-H 'Authorization: Bearer <token>'
With http.request() your post data is not to be sent in the query string. If you include a query string, it is sent as a query string. Your post data needs to be sent with req.write(data). See the code example in the doc.
Probably your server is returning that error because there is no data in the body of the POST.

Resources