I have a SOAP endpoint I need to gather data from and I have a working prototype using python3 however I would like this to be a node JS project to diversify my portfolio however when sending Headers to the endpoint I have noticed an error (Media Type not supported) so I looked to the headers and noticed something odd with them, some of the keys are in quotes and other not and i believe this may be the source of the Issue, any help would be appreciated,
Headers when request is made
{
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36',
'Content-Type': 'text/xml; charset=utf-8',
SOAPAction: 'http://api.neatoscan.com/RequestOrdersReport',
Host: 'desktopbridge.neatoscan.com',
'Content-Length': '486',
Expect: '100-continue',
Connection: 'Keep-Alive'
}
Request Code (All vars in header are pulled from a config file to keep authentication separate from main file)
import {Client_User_Name, Token,start_date_time,end_date_time, SOAP_Header_CONTENT_TYPE,SOAP_Header_Host_Config,USER_AGENT,SOAP_actions,
SOAP_Content_len,SOAP_urls} from './config.js';
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
const axios = require('axios');
const soapRequest = require('easy-soap-request');
var fs = require('fs'),parseString = require("xml2js").parseString,xml2js = require("xml2js");
var Request_Book_Order_report_XML = '/home/malachi/Desktop/Projects/Daily_Reports_Goodwill/NeatoScan/xml_files/Request_Book_Order_Report.xml'
var Report_Status_Books_XML = '/home/malachi/Desktop/Projects/Daily_Reports_Goodwill/NeatoScan/xml_files/Request_Report_Status.xml'
const url = SOAP_urls['Books_Url'];
function req(xml, headers) {
axios.post(url,xml, headers
).then(response => {console.log(response.data)}
).catch(err => {console.log(err)});
}
var SA = 'Book_Report_Request'
var CL = 'Book_Report_Request_Content_Len'
var url_Headers = {
"User-Agent": USER_AGENT,'Content-Type': SOAP_Header_CONTENT_TYPE['Books_Content_Type'],'SOAPAction': SOAP_actions[SA] ,
'Host': SOAP_Header_Host_Config['Books_Host'], 'Content-Length':SOAP_Content_len[CL], 'Expect':'100-continue', 'Connection': 'Keep-Alive'};
//
fs.readFile(Request_Book_Order_report_XML, "utf-8", function(err, data) {
parseString(data, function(err, result) {
var json = result;
var auth_filter = json['soap:Envelope']['soap:Body'][0]['RequestOrdersReport'][0];
var auth_Client_User = auth_filter['username'] = Client_User_Name;
var auth_token = auth_filter['token'] = Token;
var date_start_filter = auth_filter['startDate'] = start_date_time;
var date_end_filter = auth_filter['endDate'] = end_date_time;
var builder = new xml2js.Builder();
var xml = builder.buildObject(json);
console.log(xml);
console.log(url_Headers);
// req(xml, url_Headers)
});
});
Related
I'm trying to send an HTTP/2 GET request using nodejs http2 module but I'm having trouble with sending cookies.
I have to read the cookies from a JSON file and send them with the request.
I tried this:
const http2 = require("http2");
const client = http2.connect("http://192.168.1.50");
const fs = require('fs');
var COOKIE = fs.readFileSync('cookies.json', 'utf8');
const req = client.request( {
':path': '/cookies.php',
':method': 'GET',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0',
'Cookie' : COOKIE,
} )
let data = "";
req.on("response", (headers, flags) => {
for (const name in headers) {
console.log(`${name}: ${headers[name]}`);
}
});
req.on("data", chunk => {
data += chunk;
});
req.on("end", () => {
console.log(data);
client.close();
});
req.end();
cookies.json:
[{"name":"test","value":"12345","domain":"192.168.1.50","path":"/"}]
But it doesn't work, because I made a quick php page that prints $_COOKIE but it prints an empty array
I'm not an expert with HTTP headers, how can I do this?
Hello for some reasons i don't like to use the libs from Lightstreamer for example debugging with Charles Webproxy or sending raw text commands.
I get a connection to the server, but i fail after sending any command.
I used this https://demos.lightstreamer.com/HelloWorld/ and try make me a Node/TS version of it.
I send from the example the first string which is "wsok" and all what i get from server is (1011) Cannot continue due to an unexpected error.
My problem must be related to sending the message because without the connection is work.
http://192.168.185.24:8888 is my Charles Webdebugging.
import * as WebSocket from 'ws';
var url = require('url');
var HttpsProxyAgent = require('https-proxy-agent');
class sg{
ws:WebSocket;
constructor(){
var headers = {};
headers["Host"] ="push.lightstreamer.com";
headers["User-Agent"] ="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0";
headers["Accept"] ="*/*";
headers["Accept-Language"] ="de,en-US;q=0.7,en;q=0.3";
headers["Accept-Encoding"] ="gzip, deflate, br";
headers["Sec-WebSocket-Version"] ="13";
headers["Origin"] ="https://demos.lightstreamer.com";
headers["Sec-WebSocket-Protocol"] ="TLCP-2.2.0.lightstreamer.com";
headers["Sec-WebSocket-Extensions"] ="permessage-deflate";
headers["Sec-WebSocket-Key"] ="a key==";
headers["Connection"] ="keep-alive, Upgrade";
headers["Sec-Fetch-Dest"] ="websocket";
headers["Sec-Fetch-Mode"] ="websocket";
headers["Sec-Fetch-Site"] ="same-site";
headers["Pragma"] ="no-cache";
headers["Cache-Control"] ="no-cache";
headers["Upgrade"] ="websocket";
var options = url.parse('http://192.168.185.24:8888');
var agent = new HttpsProxyAgent(options);
this.ws = new WebSocket("wss://push.lightstreamer.com/lightstreamer", ["js.lightstreamer.com", "websocket"], {
headers, agent: agent, rejectUnauthorized: false
});
this.ws.onopen = (e) => {
console.log("open")
setTimeout(() => {
this.ws.send(`wsok`);
}, 1500);
};
this.ws.onmessage = (e) => {
console.log(e)
}
this.ws.onerror = error => {
console.log(`WebSocket error: ${error}`)
}
}
}
let xsg = new sg();
You are not using the object WebSocket correctly.
Try this instead:
ws = new WebSocket("wss://push.lightstreamer.com/lightstreamer", "TLCP-2.2.0.lightstreamer.com")
ws.onopen = (e) => {
console.log("open")
ws.send("wsok")
}
For the rest, you must look at the TLCP Specification document, the section Hands on in particular.
Nodejs project that scraps information from a phone web address is returning empty results. When it does the requestHTTP the result that comes off of console.log is something like this>
\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0000\u000b�Wmo�H\u0010�|��\u001f����t���&����q���F�ɇ~�6�B�d��6�z��f�\u0002&\n�U�r\b�=���̳�b[���"\u0013\u000b����6���\f%\u0005/�ze��Z�˔��RQ�;��6�D�D���q��l+\u0015/�-�g�Y^\nX�"\u0016�L���B)س��\u0003D�Z†\u000bR�2�0�E\u0016�\rK���!�)\u0007�&��\u0000*\u0005T�T{;�\u0005�eʎ#�\u001d϶��j*j�:B���\u000f�_�8��q�\u000f��#\u0001�!\n��G�.���{c\u001f&\u0001\u0019�\u001d,I�\\u0000\u001a\u0014�\u0016+�C�W+\u0001ņǂ��mC��m"��Q[�\�FlOH#\u001e�dJ\tL\u0016T��]�h�\u001c3'�#��\u001dS�\r� \u0003_��o\u001b�\u0018\u0017 \u0016%�\u001c�h\u001a�~�?
(it also beeps)
The rawdata became unreadable gibberish, so the rest of the thing doesn't work as it can't pick up the information(I think?)
this is the code causing the problem (I believe)
'use strict';
var userConfig = require("../setup_config/setup_config");
const requestHTTP = require("request-promise");
const requestPath = "https://mob.processotelematico.giustizia.it/proxy/index_mobile.php?"
var requestParameters = {
"version": "2.0.15",
"platform": "Android 8.1.0",
"devicename": "USER_UNIQUE_DEVICE_NAME_GOES_HERE",
"devicewidth": "USER_UNIQUE_DEVICE_WIDTH_GOES_HERE",
"deviceheight": "USER_UNIQUE_DEVICE_HEIGHT_GOES_HERE",
"token": "USER_UNIQUE_DEVICE_TOKEN_GOES_HERE",
"azione": "direttarg_sicid_mobile",
"registro": "CC",
"idufficio": "0580910098",
"numproc": "ID_OF_PROCCESS (CASE_NUMBER)",
"aaproc": "YEAR",
"tipoufficio": "1",
"_": "CURRENT_TIME_IN_MS_GOES_HERE"
};
//get case info from Italy's Giustizia Civile's Server
async function getCase(caseNumber, caseYear){
let userInfo = userConfig.getUserInfo();
if(!(userInfo && Array.isArray(userInfo) && userInfo.length == 5)) return {};
var userUUID = userInfo[0];
var userToken = userInfo[1];
var userDeviceName = userInfo[2];
var userDeviceWidth = userInfo[3];
var userDeviceHeight = userInfo[4];
console.log(userInfo);
//populate user related parameters
requestParameters.uuid = userUUID
requestParameters.token = userToken
requestParameters.devicename = userDeviceName;
requestParameters.devicewidth = userDeviceWidth;
requestParameters.deviceheight = userDeviceHeight;
requestParameters.numproc = caseNumber
requestParameters.aaproc = caseYear
requestParameters._ = new Date().getTime()
//prepare parameters to send
var fullURL = requestPath + Object.entries(requestParameters).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join('&');
const options = {
url: fullURL,
method: 'GET',
headers: {
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 8.1.0; Moto G (5) Build/OPP28.85-19-4-2)',
'Host': 'mob.processotelematico.giustizia.it',
'path': '/',
'Accept': '*/*',
'Accept-Language': 'en-US',
'Connection': 'keep-alive',
'Accept-Encoding': 'gzip, deflate',
'X-Requested-With': 'it.giustizia.civile'
}
};
//make request (get)
var result = await requestHTTP(options);
//console.log(result);
//check if it returned any error signs
if(checkForErrors(result) == false) return null;
return result;
}
//check if response contains any signs of errors found
function checkForErrors(response){
//console.log(response)
if(response.includes("<Errore>") && response.includes("</Errore>")){
return false; //found error
}
return true;
}
module.exports = {
getCase
};
It seems to me that the part that is causing this problem is in //prepare parameters to send/ where it deals with encodeURIComponent (?)
Any ideas, if this is the problem and how to fix this? Thank you.
I fixed it by erasing 'Accept-Encoding': 'gzip, deflate'
Apparently by adding this header it disables the built in decompressor of request-promise (something like that).
i am trying to scrape today's matches on betfair and wanna get:
home team
away team
x odd
draw odd
y odd
problem is i keep getting multiple spaces, i have tried alot and cannot fix it, the problem is not with trim but with the execution flow that causes empty lines
Can somebody tell me what im doing wrong?
My code:
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
var url = 'https://www.betfair.com/sport/football';
var customHeaderRequest = request.defaults({
headers: {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'}
})
customHeaderRequest.get(url, function(err, resp, body){
$ = cheerio.load(body);
links = $('.section-list .section:nth-child(2) .event-list li');
$(links).each(function(i, link){
var home = $(link).find('.event-information div:nth-child(3) div a div span.team-name:nth-child(1)');
var h = home.text();
if(h != null || h!=''){
fs.appendFile('message.txt', h+'\n', function (err) {});
}
});
});
You shouldn't be calling fs.appendFile() in a loop like this and you may need a better test for an empty line than just what you were using. fs.appendFile() is an asynchronous operation and you're essentially calling a whole bunch of fs.appendFile() operations one after another without waiting for the prior ones to finish.
You can either use a stream or you have to wait until the previous fs.appendFile() is done before calling the next one.
And, if you want to make sure you have no blank-looking results, you need a better filter for results that have only whitespace in them (I added .trim() to my code below).
Here's one way to do that:
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
const util = require('util');
const appendFile = util.promisify(fs.appendFile);
var url = 'https://www.betfair.com/sport/football';
var customHeaderRequest = request.defaults({
headers: {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'}
})
customHeaderRequest.get(url, async function(err, resp, body){
try {
let $ = cheerio.load(body);
let links = $('.section-list .section:nth-child(2) .event-list li').toArray();
for (let link of links) {
const home = $(link).find('.event-information div:nth-child(3) div a div span.team-name:nth-child(1)').text().trim();
if (home) {
await appendFile('message.txt', home +'\n');
}
}
} catch(e) {
// error writing to the file, handle that error here
}
});
Other notes: You should also always declare all local variables you are using so they are never allowed to be implicit globals.
I'm trying to use the Unirest Jar feature to make session authenticated requests however I cant seem to get my name console logged. The following is sample code Im using. Any ideas on how I can get my name console logged?
var cheerio = require('cheerio');
var unirest = require('unirest');
var CookieJar = unirest.jar();
var twitterLogin = 'https://twitter.com/sessions';
var twitterUsername = 'TWITTERUSERNAME';
var twitterPassword = 'TWITTERPASSWORD!'
var Request = unirest.get('https://twitter.com/login').jar(CookieJar);
Request.end(function(response) {
var $ = cheerio.load(response.body);
var authToken = $('input[name="authenticity_token"]').val();
console.log(authToken)
unirest.post(twitterLogin)
.jar(CookieJar)
.followRedirect(true)
.header('Referer', 'https://twitter.com/login')
.header('content-type', 'application/x-www-form-urlencoded')
.header('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.124 Safari/537.36')
.field('authenticity_token', authToken)
.field('session[username_or_email]', twitterUsername)
.field('session[password]', twitterPassword)
.end(function(response) {
console.log(response.statusCode)
console.log(response.body)
unirest.get('https://twitter.com')
.jar(CookieJar)
.followRedirect(true)
.end(function(response) {
var $ = cheerio.load(response.body)
console.log($('.DashboardProfileCard-name').text());
})
})
});
From the docs:
Request.jar(Boolean) or Request.jar(Jar)
Sets jar, cookie container, on Request.options. When set to true it stores cookies for future usage.
You should call unirest.jar() with true as argument:
var CookieJar = unirest.jar(true);