use Lightstreamer Server with Node WebSocket only as Client? - node.js

Hello for some reasons i don't like to use the libs from Lightstreamer for example debugging with Charles Webproxy or sending raw text commands.
I get a connection to the server, but i fail after sending any command.
I used this https://demos.lightstreamer.com/HelloWorld/ and try make me a Node/TS version of it.
I send from the example the first string which is "wsok" and all what i get from server is (1011) Cannot continue due to an unexpected error.
My problem must be related to sending the message because without the connection is work.
http://192.168.185.24:8888 is my Charles Webdebugging.
import * as WebSocket from 'ws';
var url = require('url');
var HttpsProxyAgent = require('https-proxy-agent');
class sg{
ws:WebSocket;
constructor(){
var headers = {};
headers["Host"] ="push.lightstreamer.com";
headers["User-Agent"] ="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0";
headers["Accept"] ="*/*";
headers["Accept-Language"] ="de,en-US;q=0.7,en;q=0.3";
headers["Accept-Encoding"] ="gzip, deflate, br";
headers["Sec-WebSocket-Version"] ="13";
headers["Origin"] ="https://demos.lightstreamer.com";
headers["Sec-WebSocket-Protocol"] ="TLCP-2.2.0.lightstreamer.com";
headers["Sec-WebSocket-Extensions"] ="permessage-deflate";
headers["Sec-WebSocket-Key"] ="a key==";
headers["Connection"] ="keep-alive, Upgrade";
headers["Sec-Fetch-Dest"] ="websocket";
headers["Sec-Fetch-Mode"] ="websocket";
headers["Sec-Fetch-Site"] ="same-site";
headers["Pragma"] ="no-cache";
headers["Cache-Control"] ="no-cache";
headers["Upgrade"] ="websocket";
var options = url.parse('http://192.168.185.24:8888');
var agent = new HttpsProxyAgent(options);
this.ws = new WebSocket("wss://push.lightstreamer.com/lightstreamer", ["js.lightstreamer.com", "websocket"], {
headers, agent: agent, rejectUnauthorized: false
});
this.ws.onopen = (e) => {
console.log("open")
setTimeout(() => {
this.ws.send(`wsok`);
}, 1500);
};
this.ws.onmessage = (e) => {
console.log(e)
}
this.ws.onerror = error => {
console.log(`WebSocket error: ${error}`)
}
}
}
let xsg = new sg();

You are not using the object WebSocket correctly.
Try this instead:
ws = new WebSocket("wss://push.lightstreamer.com/lightstreamer", "TLCP-2.2.0.lightstreamer.com")
ws.onopen = (e) => {
console.log("open")
ws.send("wsok")
}
For the rest, you must look at the TLCP Specification document, the section Hands on in particular.

Related

Sending http2 get requests with cookies in nodejs

I'm trying to send an HTTP/2 GET request using nodejs http2 module but I'm having trouble with sending cookies.
I have to read the cookies from a JSON file and send them with the request.
I tried this:
const http2 = require("http2");
const client = http2.connect("http://192.168.1.50");
const fs = require('fs');
var COOKIE = fs.readFileSync('cookies.json', 'utf8');
const req = client.request( {
':path': '/cookies.php',
':method': 'GET',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0',
'Cookie' : COOKIE,
} )
let data = "";
req.on("response", (headers, flags) => {
for (const name in headers) {
console.log(`${name}: ${headers[name]}`);
}
});
req.on("data", chunk => {
data += chunk;
});
req.on("end", () => {
console.log(data);
client.close();
});
req.end();
cookies.json:
[{"name":"test","value":"12345","domain":"192.168.1.50","path":"/"}]
But it doesn't work, because I made a quick php page that prints $_COOKIE but it prints an empty array
I'm not an expert with HTTP headers, how can I do this?

I am having issues properly sending headers to a SOAP endpoint

I have a SOAP endpoint I need to gather data from and I have a working prototype using python3 however I would like this to be a node JS project to diversify my portfolio however when sending Headers to the endpoint I have noticed an error (Media Type not supported) so I looked to the headers and noticed something odd with them, some of the keys are in quotes and other not and i believe this may be the source of the Issue, any help would be appreciated,
Headers when request is made
{
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36',
'Content-Type': 'text/xml; charset=utf-8',
SOAPAction: 'http://api.neatoscan.com/RequestOrdersReport',
Host: 'desktopbridge.neatoscan.com',
'Content-Length': '486',
Expect: '100-continue',
Connection: 'Keep-Alive'
}
Request Code (All vars in header are pulled from a config file to keep authentication separate from main file)
import {Client_User_Name, Token,start_date_time,end_date_time, SOAP_Header_CONTENT_TYPE,SOAP_Header_Host_Config,USER_AGENT,SOAP_actions,
SOAP_Content_len,SOAP_urls} from './config.js';
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
const axios = require('axios');
const soapRequest = require('easy-soap-request');
var fs = require('fs'),parseString = require("xml2js").parseString,xml2js = require("xml2js");
var Request_Book_Order_report_XML = '/home/malachi/Desktop/Projects/Daily_Reports_Goodwill/NeatoScan/xml_files/Request_Book_Order_Report.xml'
var Report_Status_Books_XML = '/home/malachi/Desktop/Projects/Daily_Reports_Goodwill/NeatoScan/xml_files/Request_Report_Status.xml'
const url = SOAP_urls['Books_Url'];
function req(xml, headers) {
axios.post(url,xml, headers
).then(response => {console.log(response.data)}
).catch(err => {console.log(err)});
}
var SA = 'Book_Report_Request'
var CL = 'Book_Report_Request_Content_Len'
var url_Headers = {
"User-Agent": USER_AGENT,'Content-Type': SOAP_Header_CONTENT_TYPE['Books_Content_Type'],'SOAPAction': SOAP_actions[SA] ,
'Host': SOAP_Header_Host_Config['Books_Host'], 'Content-Length':SOAP_Content_len[CL], 'Expect':'100-continue', 'Connection': 'Keep-Alive'};
//
fs.readFile(Request_Book_Order_report_XML, "utf-8", function(err, data) {
parseString(data, function(err, result) {
var json = result;
var auth_filter = json['soap:Envelope']['soap:Body'][0]['RequestOrdersReport'][0];
var auth_Client_User = auth_filter['username'] = Client_User_Name;
var auth_token = auth_filter['token'] = Token;
var date_start_filter = auth_filter['startDate'] = start_date_time;
var date_end_filter = auth_filter['endDate'] = end_date_time;
var builder = new xml2js.Builder();
var xml = builder.buildObject(json);
console.log(xml);
console.log(url_Headers);
// req(xml, url_Headers)
});
});

how to receive files using nodejs express

im trying to make my own api and one part of it is that I want it to be able to receive text and files from a discord webhook, the text part works fine but when I send the file I can't seem to figure out how to receive it. I looked around for a bit and saw that req.files will return the file but I just get undefined. Here is my api code:
const express = require('express'),
bodyParser = require('body-parser'),
base32 = require('base32'),
const pass = "5'g5?cDzy}\\p5zAwvT[DJ/SeD"
const port = 8080
function denied(res) {
res.status(403);
res.send('no');
}
const api = express();
api.use(bodyParser.json());
api.listen(port, () => { console.log(`api running on port ${port}`) });
api.post('/webhook', (req, res) => {
if (!req.headers) {
denied(res);
}
var auth = req.headers['authorization'];
if (auth && base32.decode(auth) === pass) {
console.log(req.files) //undefined
res.send('done');
} else {
denied(res);
}
});
and this is the code I use to send the file and text:
import requests
key = '6mkped9zcd27mybxbhr3ayj1exv58pu498qn6ta4'
header = {
"Content-Type": "application/json",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
'Authorization': key
}
info = {
"avatar_url":"https://cdn.discordapp.com/attachments/901070985434918965/904813984753000469/nedladdning_14.jpg",
"name":"test",
"embeds": [...]
}
r = requests.post('http://localhost:8080/webhook', headers=header, json=info)
r2 = requests.post('http://localhost:8080/webhook', headers={'Authorization': key}, files={'file': open('test.zip','rb')})
print(r.text, r.status_code)
print(r2.text, r2.status_code)
Upload files (e.g. how requests does it) are sent as multipart/form-data payloads.
As per body-parser's npm page,
[...] does not handle multipart bodies, due to their complex and typically large nature. For multipart bodies, you may be interested in the following modules...
You'll need one of those modules, and you'll then need to read its documentation to see if it provides eg. the req.files you've seen somewhere. Otherwise it will indeed be undefined.

nodejs http request using async/await syntax [duplicate]

In my program I make async call for my function from another API module:
var info = await api.MyRequest(value);
Module code:
var request = require("request")
module.exports.MyRequest = async function MyRequest(value) {
var options = {
uri: "http://some_url",
method: "GET",
qs: { // Query string like ?key=value&...
key : value
},
json: true
}
try {
var result = await request(options);
return result;
} catch (err) {
console.error(err);
}
}
Execution returns immediately, however result and therefore info contains request object and request body - info.body like key=value&..., not required response body.
What I'm doing wrong? How to fix? What is proper request usage with async, or it only works correctly with promises like mentioned here: Why await is not working for node request module? Following article mentioned it is possible: Mastering Async Await in Node.js.
You need to use the request-promise module, not the request module or http.request().
await works on functions that return a promise, not on functions that return the request object and expect you to use callbacks or event listeners to know when things are done.
The request-promise module supports the same features as the request module, but asynchronous functions in it return promises so you can use either .then() or await with them rather than the callbacks that the request module expects.
So, install the request-promise module and then change this:
var request = require("request");
to this:
const request = require("request-promise");
Then, you can do:
var result = await request(options);
EDIT Jan, 2020 - request() module in maintenance mode
FYI, the request module and its derivatives like request-promise are now in maintenance mode and will not be actively developed to add new features. You can read more about the reasoning here. There is a list of alternatives in this table with some discussion of each one.
I have been using got() myself and it's built from the beginning to use promises, supports many of the same options as the request() module and is simple to program.
Pretty sure you can also do the following. If what you need does not return Promise by default you can provide it via new Promise method. Above answer is less verbose though.
async function getBody(url) {
const options = {
url: url,
method: 'GET',
};
// Return new promise
return new Promise(function(resolve, reject) {
// Do async job
request.get(options, function(err, resp, body) {
if (err) {
reject(err);
} else {
resolve(body);
}
})
})
}
I just managed to get it to work with async/await. I wrapped it inside a function promisifiedRequest to return a promise that runs the request callback and resolves or rejects it based on error and response.
const request = require('request');
const promisifiedRequest = function(options) {
return new Promise((resolve,reject) => {
request(options, (error, response, body) => {
if (response) {
return resolve(response);
}
if (error) {
return reject(error);
}
});
});
};
(async function() {
const options = {
url: 'https://www.google.com',
method: 'GET',
gzip: true,
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36'
}
};
let response = await promisifiedRequest(options);
console.log(response.headers);
console.log(response.body);
})();
Since request-promise has been deprecated, here are other options that don't depend on the NPM request package. got has been mentioned already, but it depends on 11 other packages. axios, in contrast, only has 1 dependency (for redirects). Everything else is natively implemented and built on top of the native NodeJS packages.
Here is the same example using axios:
const axios = require('axios')
const response = await axios.get(url)
const result = response.data
or, as a one-liner in JavaScript
const result = (await axios.get(url)).data
One-liner in TypeScript:
const {data} = await axios.get(url)
For simple cases where you don't need advanced features like cookies, following redirects and retrying, you can use native http/https module to make requests:
const https = require('https')
async function fetch(url) {
return new Promise((resolve, reject) => {
const request = https.get(url, { timeout: 1000 }, (res) => {
if (res.statusCode < 200 || res.statusCode > 299) {
return reject(new Error(`HTTP status code ${res.statusCode}`))
}
const body = []
res.on('data', (chunk) => body.push(chunk))
res.on('end', () => {
const resString = Buffer.concat(body).toString()
resolve(resString)
})
})
request.on('error', (err) => {
reject(err)
})
request.on('timeout', () => {
request.destroy()
reject(new Error('timed out'))
})
})
}
const res = await fetch('https://...')

Scrape and store Shopify ecommerce websites using Node.js

I wrote a code to scrape an array of Shopify ecommerce websites using website-scraper npm module in node.js but it is showing 403 error but the same code is working for other websites.
How can we get around this problem?
My scraperTest.js file is :
var scrape = require('website-scraper');
let test = require('./test')
let urls = [];
urlList = ['1500.academy'];
urlList.forEach(url =>{
test.checkRedirect(url)
.then(domain =>{
urls.push('https://' + domain);
console.log(urls);
var options = {
urls: urls,
directory: './autochat/',
'User-Agent': 'request',
};
// with promise
scrape(options).then((result) => {
/* some code here */
}).catch((err) => {
/* some code here */
});
// or with callback
scrape(options, (error, result) => {
/* some code here */
});
})
})
and test.js file is
const request = require('request');
const extractDomain = require('extract-domain');
//var link = 'oneplustwocase.com';
function checkRedirect(link) {
return new Promise((resolve, reject) => {
var url = "http://" + link;
var options = {
url: url,
headers: {
'User-Agent': 'request'
}
};
request(options, function (error, response, body) {
let redirectedDomain = extractDomain(response.request.uri.href);
if(response !== undefined){
extractDomain(response.request.uri.href);
if (response.statusCode === 200 && link !== redirectedDomain) {
resolve(redirectedDomain);
} else {
resolve(link);
}
} else {
resolve(link);
}
});
});
}
module.exports.checkRedirect = checkRedirect;
I got the solution.
We are able to fetch the html data of the domain using request();
The response.body contains the html data
the solution I got by using the following code :
const request = require('request');
const extractDomain = require('extract-domain');
let fs = require('fs');
function checkRedirect(link) {
var url = "http://" + link;
var options = {
url: url,
headers: {
'User-Agent': 'request'
}
};
request(options, function (error, response, body) {
if(response !== undefined){
let redirectedDomain = extractDomain(response.request.uri.href);
let writeStream = fs.createWriteStream(redirectedDomain + '.html');
writeStream.write(response.body)
writeStream.end();
});
}
module.exports.checkRedirect = checkRedirect;
//checkRedirect('oneplustwocase.com')
/*
var r = request(url, function (e, resp) {
r.uri
resp.request.uri
})*/
Since you are interested in data, save yourself the headache of scraping and simply download the site XML file. It contains all the products and interesting information, just like Google or any other search engine.
It seems that website http://1500.academy returns 403 if it doesn't like user-agent header. I suggest to try user-agent which looks like browser
According to website-scraper documentation https://www.npmjs.com/package/website-scraper#request you should pass headers for request in request property, not on root level
So options should be like:
const options = {
urls:[{url: 'http://1500.academy/'}],
directory: './autochat/',
request: {
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
}
}
}
By the way website-scraper follows redirects by default, so you can skip checking redirects

Resources