Sending http2 get requests with cookies in nodejs - node.js

I'm trying to send an HTTP/2 GET request using nodejs http2 module but I'm having trouble with sending cookies.
I have to read the cookies from a JSON file and send them with the request.
I tried this:
const http2 = require("http2");
const client = http2.connect("http://192.168.1.50");
const fs = require('fs');
var COOKIE = fs.readFileSync('cookies.json', 'utf8');
const req = client.request( {
':path': '/cookies.php',
':method': 'GET',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0',
'Cookie' : COOKIE,
} )
let data = "";
req.on("response", (headers, flags) => {
for (const name in headers) {
console.log(`${name}: ${headers[name]}`);
}
});
req.on("data", chunk => {
data += chunk;
});
req.on("end", () => {
console.log(data);
client.close();
});
req.end();
cookies.json:
[{"name":"test","value":"12345","domain":"192.168.1.50","path":"/"}]
But it doesn't work, because I made a quick php page that prints $_COOKIE but it prints an empty array
I'm not an expert with HTTP headers, how can I do this?

Related

use Lightstreamer Server with Node WebSocket only as Client?

Hello for some reasons i don't like to use the libs from Lightstreamer for example debugging with Charles Webproxy or sending raw text commands.
I get a connection to the server, but i fail after sending any command.
I used this https://demos.lightstreamer.com/HelloWorld/ and try make me a Node/TS version of it.
I send from the example the first string which is "wsok" and all what i get from server is (1011) Cannot continue due to an unexpected error.
My problem must be related to sending the message because without the connection is work.
http://192.168.185.24:8888 is my Charles Webdebugging.
import * as WebSocket from 'ws';
var url = require('url');
var HttpsProxyAgent = require('https-proxy-agent');
class sg{
ws:WebSocket;
constructor(){
var headers = {};
headers["Host"] ="push.lightstreamer.com";
headers["User-Agent"] ="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0";
headers["Accept"] ="*/*";
headers["Accept-Language"] ="de,en-US;q=0.7,en;q=0.3";
headers["Accept-Encoding"] ="gzip, deflate, br";
headers["Sec-WebSocket-Version"] ="13";
headers["Origin"] ="https://demos.lightstreamer.com";
headers["Sec-WebSocket-Protocol"] ="TLCP-2.2.0.lightstreamer.com";
headers["Sec-WebSocket-Extensions"] ="permessage-deflate";
headers["Sec-WebSocket-Key"] ="a key==";
headers["Connection"] ="keep-alive, Upgrade";
headers["Sec-Fetch-Dest"] ="websocket";
headers["Sec-Fetch-Mode"] ="websocket";
headers["Sec-Fetch-Site"] ="same-site";
headers["Pragma"] ="no-cache";
headers["Cache-Control"] ="no-cache";
headers["Upgrade"] ="websocket";
var options = url.parse('http://192.168.185.24:8888');
var agent = new HttpsProxyAgent(options);
this.ws = new WebSocket("wss://push.lightstreamer.com/lightstreamer", ["js.lightstreamer.com", "websocket"], {
headers, agent: agent, rejectUnauthorized: false
});
this.ws.onopen = (e) => {
console.log("open")
setTimeout(() => {
this.ws.send(`wsok`);
}, 1500);
};
this.ws.onmessage = (e) => {
console.log(e)
}
this.ws.onerror = error => {
console.log(`WebSocket error: ${error}`)
}
}
}
let xsg = new sg();
You are not using the object WebSocket correctly.
Try this instead:
ws = new WebSocket("wss://push.lightstreamer.com/lightstreamer", "TLCP-2.2.0.lightstreamer.com")
ws.onopen = (e) => {
console.log("open")
ws.send("wsok")
}
For the rest, you must look at the TLCP Specification document, the section Hands on in particular.

I am having issues properly sending headers to a SOAP endpoint

I have a SOAP endpoint I need to gather data from and I have a working prototype using python3 however I would like this to be a node JS project to diversify my portfolio however when sending Headers to the endpoint I have noticed an error (Media Type not supported) so I looked to the headers and noticed something odd with them, some of the keys are in quotes and other not and i believe this may be the source of the Issue, any help would be appreciated,
Headers when request is made
{
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36',
'Content-Type': 'text/xml; charset=utf-8',
SOAPAction: 'http://api.neatoscan.com/RequestOrdersReport',
Host: 'desktopbridge.neatoscan.com',
'Content-Length': '486',
Expect: '100-continue',
Connection: 'Keep-Alive'
}
Request Code (All vars in header are pulled from a config file to keep authentication separate from main file)
import {Client_User_Name, Token,start_date_time,end_date_time, SOAP_Header_CONTENT_TYPE,SOAP_Header_Host_Config,USER_AGENT,SOAP_actions,
SOAP_Content_len,SOAP_urls} from './config.js';
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
const axios = require('axios');
const soapRequest = require('easy-soap-request');
var fs = require('fs'),parseString = require("xml2js").parseString,xml2js = require("xml2js");
var Request_Book_Order_report_XML = '/home/malachi/Desktop/Projects/Daily_Reports_Goodwill/NeatoScan/xml_files/Request_Book_Order_Report.xml'
var Report_Status_Books_XML = '/home/malachi/Desktop/Projects/Daily_Reports_Goodwill/NeatoScan/xml_files/Request_Report_Status.xml'
const url = SOAP_urls['Books_Url'];
function req(xml, headers) {
axios.post(url,xml, headers
).then(response => {console.log(response.data)}
).catch(err => {console.log(err)});
}
var SA = 'Book_Report_Request'
var CL = 'Book_Report_Request_Content_Len'
var url_Headers = {
"User-Agent": USER_AGENT,'Content-Type': SOAP_Header_CONTENT_TYPE['Books_Content_Type'],'SOAPAction': SOAP_actions[SA] ,
'Host': SOAP_Header_Host_Config['Books_Host'], 'Content-Length':SOAP_Content_len[CL], 'Expect':'100-continue', 'Connection': 'Keep-Alive'};
//
fs.readFile(Request_Book_Order_report_XML, "utf-8", function(err, data) {
parseString(data, function(err, result) {
var json = result;
var auth_filter = json['soap:Envelope']['soap:Body'][0]['RequestOrdersReport'][0];
var auth_Client_User = auth_filter['username'] = Client_User_Name;
var auth_token = auth_filter['token'] = Token;
var date_start_filter = auth_filter['startDate'] = start_date_time;
var date_end_filter = auth_filter['endDate'] = end_date_time;
var builder = new xml2js.Builder();
var xml = builder.buildObject(json);
console.log(xml);
console.log(url_Headers);
// req(xml, url_Headers)
});
});

NodeJS GET request working in local machine but not inside remote VM

I am trying to create a simple tool which will hit a specific Facebook page and extract some data. I am using NodeJS for this. I am using inbuilt https module to the GET request. The thing in my local machine it is working fine but inside the VM it is not able to get the response, it is failing every time.
I tried with axios also but it is not working either.
Previously I thought that could be firewall issue but I enabled every thing inside the firewall but then also it is not working. Very strange issue I am facing with this.
What can be the possible reason for this?
Is it Facebook blocking my request? No right? I am sending User-Agent header also but it is not working.
My code looks like below :
const options = {
hostname: 'www.facebook.com',
// port: 443,
path: `/${username}`,
method: 'GET',
headers: {
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
'Accept-Language': 'en-GB,en-US;q=0.8,en;q=0.6',
},
};
const request = https.request(options, (response) => {
let body = '';
console.log('response : \n' + response);
response.setEncoding('utf8');
response.on('data', (chunk) => {
body += chunk;
console.log('response on data chunk : \n' + chunk);
});
response.on('end', () => {
console.log('body : \n' + body);
const arrMatches = body.match(rePattern);
if (arrMatches && arrMatches.length > 1) {
resolve(arrMatches[1]);
} else {
reject(new Error('Facebook user not found'));
}
});
});
request.on('error', (err) => {
console.log('request error : \n' + err);
reject(err);
});
request.end();

How to do a get request and get the same results that a browser would with Nodejs?

I'm trying to do a get request for image search, and I'm not getting the same result that I am in my browser. Is there a way to get the same result using node.js?
Here's the code I'm using:
var keyword = "Photographie"
keyword = keyword.replace(/[^a-zA-Z0-9éàèùâêîôûçëïü]/g, "+")
var httpOptions = { hostname: 'yandex.com',
path: '/images/search?text=' + keyword, //path does not accept spaces or dashes
headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'user-agent': 'Mozilla/5.0'}}
console.log(httpOptions.hostname + httpOptions.path +postTitle)
https.get(httpOptions, (httpResponse) => {
console.log(`STATUS: ${httpResponse.statusCode}`);
httpResponse.setEncoding('utf8');
httpResponse.on('data', (htmlBody) => {
console.log(`BODY: ${htmlBody}`);
});
});
By switching to the request-promise library and using the proper capitalization of the User-Agent header name and an actual user agent string from the Chrome browser, this code works for me:
const rp = require('request-promise');
let keyword = "Photographie"
let options = { url: 'http://yandex.com/images/search?text=' + keyword,
headers: {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'}
};
rp(options).then(response => {
console.log(response);
}).catch(err => {
console.log(err);
});
When I try to run your actual code, I get a 302 redirect and a cookie set. I'm guessing that they are expecting you to follow the redirect and retain the cookie. But, you can apparently just switch to the above code and it appears to work for me. I don't know exactly what makes my code work, but it could be that is has a more recognizable user agent.

What is Postman "Interception Mode" equivalent to in Node.js?

I used to send requests using Postman Interceptor. This is how I handled the headers and body of the request:
You can try it by yourself. You can see that once you turn on "interception mode", you get a different response than "without" it.
Now, I want to send the same request, but by using 'HTTPS module in Node.js.
I followed the following pattern:
var https = require('https');
var querystring = require('querystring');
var post_data = querystring.stringify({
hid_last: "SMITH",
hid_first: "JOHN",
__RequestVerificationToken: "EiO369xBXRY9sHV/x26RNwlMzWjM9sR/mNlO9p9tor0PcY0j3dRItKH8XeljXmTfFWT0vQ1DYBzlGpLtnBBqEcOB51E9lh6wrEQbtMLUNOXpKKR3RzFqGc9inDP+OBIyD7s9fh9aMAypCHFCNFatUkx666nf7NOMHHKfiJKhfxc=",
hid_max_rows: 20,
hid_page: 1,
hid_SearchType: 'PARTYNAME'
});
// An object of options to indicate where to post to
var post_options = {
host: 'a836-acris.nyc.gov',
path: '/DS/DocumentSearch/PartyNameResult',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Origin': 'https://a836-acris.nyc.gov',
'Referer': "https://a836-acris.nyc.gov/DS/DocumentSearch/PartyName",
'Upgrade-Insecure-Requests': 1,
'User-Agent': "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",
'Content-Length': Buffer.byteLength(post_data),
'Cookie': '_ga=GA1.2.1526584332.1483281720; WT_FPC=id=2fb6833e-6ae6-4529-b84a-4a1c61f24978:lv=1483256520738:ss=1483256520738',
}
};
// Set up the request
var post_req = https.request(post_options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('Response: ' + chunk);
});
});
// post the data
post_req.write(post_data);
post_req.end();
The only thing missing is the "interceptor" issue. When I use this code now, I get the same response I used to get without using 'interceptor' mode in Postman.
My question is how to "convert" the "interceptor mode" in Postman to 'HTTPS module in node.js?

Resources