Https proxy decode encoded data node js - node.js

I am trying to setup a proxy for http and https. Here is my code,
const http = require('http');
var https = require('https');
const fs = require('fs');
var url = require('url');
var net = require('net');
const config = require('./config');
let proxify = function (req, res) {
var urlObj = url.parse(req.url);
var target = urlObj.protocol + '//' + urlObj.host;
if (!req.headers['x-target']) req.headers['x-target'] = target;
req.headers['x-proxy-username'] = config.username;
req.headers['x-proxy-password'] = config.password;
console.log(target);
console.log('Proxy HTTP request for:', target);
var proxy = httpProxy.createProxyServer({});
proxy.on('error', function (err, req, res) {
console.log('proxy error', err);
res.end();
});
proxy.web(req, res, { target: config.server, changeOrigin: true });
};
var httpserver = http.createServer(proxify).listen(2890); //this is the port your clients will connect to
const httpsserver = https
.createServer(
{
cert: fs.readFileSync('./ssl_cert/cert.pem'),
key: fs.readFileSync('./ssl_cert/key.pem'),
},
proxify
)
.listen(2891);
var regex_hostport = /^([^:]+)(:([0-9]+))?$/;
var getHostPortFromString = function (hostString, defaultPort) {
var host = hostString;
var port = defaultPort;
var result = regex_hostport.exec(hostString);
if (result != null) {
host = result[1];
if (result[2] != null) {
port = result[3];
}
}
return [host, port];
};
httpserver.addListener('connect', function (req, socket, bodyhead) {
var hostPort = getHostPortFromString(req.url, 443);
var hostDomain = hostPort[0];
var port = parseInt(hostPort[1]);
console.log('Proxying HTTPS request for:', hostDomain, port);
req.headers['x-target'] = 'http://' + hostDomain + ':' + port;
req.headers['x-proxy-username'] = config.username;
req.headers['x-proxy-password'] = config.password;
var proxyHost = new URL(config.server);
var proxySocket = new net.Socket();
proxySocket.connect(
{ port: proxyHost.port, host: proxyHost.hostname },
function () {
console.log('bodyhead', bodyhead.toString()); //debug
proxySocket.write(bodyhead);
socket.write(
'HTTP/' + req.httpVersion + ' 200 Connection established\r\n\r\n'
);
}
);
proxySocket.on('data', function (chunk) {
console.log('proxy data chunk', chunk.toString()); // debug
socket.write(chunk);
});
proxySocket.on('end', function () {
socket.end();
});
proxySocket.on('error', function () {
socket.write('HTTP/' + req.httpVersion + ' 500 Connection error\r\n\r\n');
socket.end();
});
socket.on('data', function (chunk) {
console.log('data chunk', chunk.toString('utf8')); // debug
proxySocket.write(chunk);
});
socket.on('end', function () {
proxySocket.end();
});
socket.on('error', function () {
proxySocket.end();
});
});
Don't judge me too hard, just trying to get it working first.
When proxying http with windows 10 proxy settings, it works fine. But when I am trying to proxy https, it logs encoded data like `↕►♦♦♦☺♣♣♣♠♠☺↕3+)/1.1♣♣☺
☺↔ \s☻�t�DQ��g}T�c\‼sO��♦��U��ޝ∟-☻☺☺+♂
→→♥♦♥♥♥☻♥☺♥☻☻j☺§�` and gives a 400 bad request.I don't know if its the encoding of https response or something else, I have no idea what i am doing at this point and need help.

it is because https uses tls/ssl to encrypt the data.

Related

Express + MQTT from POST request to publish in topic

I need a Node.js web service that accept a specific POST request that will trigger a publishing in a specific MQTT topic.
I use Express to listen for the request and this part of the script works fine.
The problem is when I it should trigger the publish in topic to perform a downlink.
The code without the Express part works fine. So It seems like Express interfere with MQTT. But the connection to the Broker works fine. Only the publish procedure doesn't work.
var express = require('express');
var bodyParser = require('body-parser');
var mqtt = require('mqtt')
var fs = require('fs')
var path = require('path')
const util = require('util')
var app = express();
var CERT = //certificate
var PORT = //port
var HOST = // host
var options = {
port: PORT,
host: HOST,
clientId: //client_id
username: //uname
password: //pswd
cert: CERT,
rejectUnauthorized: false,
protocol: 'mqtts'
}
var client;
var jsonParser = bodyParser.json();
var port = process.env.PORT || 8080;
app.use(express.static('public'));
app.get('/', function (req, res) {
res.render(__dirname + "/public/index.html");
})
client = mqtt.connect(options);
client.on("connect", () => {
console.log("MQTT connected");
})
client.on('message', function (topic, message) {
var msg = JSON.parse(message)
console.log("topic: " + topic + " msg:" + util.inspect(msg))
});
app.post('/', jsonParser, function (req, res) {
// Prepare output in JSON format
data = {
dev_id: req.body.dev_id,
pswd: req.body.password,
tx_cycle: req.body.tx_cycle
};
if (data.pswd != "password") {
console.log("Wrong password")
}
else {
console.log(data);
var topic = 'publish_topic';
var tx_cy = data.tx_cycle;
var msg = '{"port":"222","payload":"' + tx_cy + '","confirmed": false,"window":"BOTH","priority":0}';
console.log('Try to send downlink message, for ' + data.dev_id + ' set to ' + data.tx_cycle + ' min -> hex ' + tx_cy);
client.subscribe('reply/+/id/+');
client.publish(topic, msg);
res.status(200).send(msg + " sent to broker");
}
});
var server = app.listen(port, function () {
var host = server.address().address
var port = server.address().port
console.log("App listening at http://%s:%s", host, port)
})
I solve the problem. The client id was refused by the broker because was not unique. Changing it solved the issue.

listening https traffic in nodejs

I need to listening https traffic, which my code I Can listening all http traffic, but which https I can't capture. How can I do it?
This is my code
var http = require('http');
var url = require('url');
var fs = require('fs');
http.createServer(function(request, response) {
var url_part = url.parse(request.url,true);
var originUrl = url_part.href;
var options = url.parse(originUrl);
var proxy = http.request(options);
proxy.addListener('response', function (proxy_response) {
proxy_response.addListener('data', function(chunk) {
response.write(chunk, 'binary');
});
proxy_response.addListener('end', function() {
response.end();
});
response.writeHead(proxy_response.statusCode, proxy_response.headers);
});
request.addListener('data', function(chunk) {
proxy.write(chunk, 'application/json');
});
request.addListener('end', function() {
proxy.end();
});
}).listen(3128);

Nodejs basic auth with https

I have a nodjs app running that uses basic auth with the password details in a file generated by htpasswd. This is working fine.
What I'd like to do is run it over https. I can get it working with https fine, but can't seem to work out how to get the two working together, https + basic auth.
This is for a browser going to the server and sending a variable in the url. I've found a few solutions but many seem to focus on the server doing a outbound call to something else with basic auth.
This is the basic auth one.
// Authentication module.
const auth = require('http-auth');
const basic = auth.basic({
realm: "REALM.",
file: __dirname + "/users.htpasswd"
});
const http = require('http');
const fs = require('fs');
var url = require('url');
const hostname = '0.0.0.0';
const port = 80;
const server = http.createServer(basic, (req, res) => {
var url_parts = url.parse(req.url, true);
var key = url_parts.query["name"];
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
fs.readFile(__dirname + '/basic.json', function (err, data) {
if (err) {
throw err;
}
table = JSON.parse(data.toString());
var value = table[key];
res.end(JSON.stringify({"group" : value}));
});
});
server.listen(port, hostname, () => {
console.log(`Server running at http://${hostname}:${port}/`);
});
And this is the https bit without the basic auth.
const https = require('https');
const fs = require('fs');
var url = require('url');
const hostname = '0.0.0.0';
const port = 443;
const options = {
key: fs.readFileSync('star.pkey'),
cert: fs.readFileSync('star.pem'),
};
const server = https.createServer(options, (req, res) => {
var url_parts = url.parse(req.url, true);
var key = url_parts.query["name"];
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
fs.readFile(__dirname + '/basic.json', function (err, data) {
if (err) {
throw err;
}
table = JSON.parse(data.toString());
var value = table[key];
res.end(JSON.stringify({"group" : value}));
});
});
server.listen(port, hostname, () => {
console.log(`Server running at http://${hostname}:${port}/`);
});
Any help appreciated.

api route not working over https

I have a node server running on port 3000 to serve api request.
this works fine...
http://www.skoolaide.com:3000/api/accounts
this does not.
https://www.skoolaide.com:3000/api/accounts
Can someone tell me why? If you go to https://www.skoolaide.com, the ssl cert is configured correctly, but do I need to do something on the node side?
this is my server js file. Please note: is only runs the middleware(api) on port 3000. For some reason I cannot access the middleware via https...
'use strict';
var express = require('express');
var router = express.Router();
var app = express();
var http = require('http');
var open = require('open');
var cors = require('cors');
var path = require('path');
var morgan = require('morgan');
var errorhandler = require('errorhandler');
var bodyParser = require('body-parser');
var mongoose = require('mongoose');
var config = require('./config/config');
var jwt = require('jsonwebtoken');
var compression = require('compression');
var runMiddleware = require('run-middleware')(app);
var fs = require('fs');
var readline = require('readline');
var google = require('googleapis');
var googleAuth = require('google-auth-library');
var multer = require('multer');
var node_xj = require("xls-to-json");
var moment = require('moment');
var async = require('async');
var btoa = require('btoa');
var sharp = require('sharp');
var students = require("./middleware/students.api");
var accounts = require("./middleware/accounts.api");
var messages = require("./middleware/messages.api");
var advocates = require("./middleware/advocates.api");
var authenticate = require("./middleware/authenticate.api");
var email = require("./middleware/email.api");
var text = require("./middleware/text.api");
var colleges = require("./middleware/colleges.api");
var amazon = require("./middleware/amazon.api");
var rewards = require("./middleware/rewards.api");
var files = require("./middleware/files.api");
var validations = require("./middleware/validations.api");
var points = require("./middleware/points.api");
var notifications = require("./middleware/notifications.api");
var notificationsMap = require("./middleware/notificationsMap.api");
var trivia = require("./middleware/trivia.api");
var tasks = require("./middleware/rewardgoals.api");
var classes = require("./middleware/classes.api");
var connections = require("./middleware/connections.api");
var badges = require("./middleware/badges.api");
var fixpasswords = require("./middleware/fixpasswords.api");
var Files = require('./models/files');
mongoose.connect(config.database);
process.on('SIGINT', function() {
mongoose.connection.close(function () {
console.log('Mongoose disconnected on app termination');
process.exit(0);
});
});
// use body parser so we can get info from POST and/or URL parameters
app.use(bodyParser.json({
limit: '50mb'
}));
app.use(bodyParser.urlencoded({
limit: '50mb',
extended: true
}));
app.set('uploads', path.join(__dirname, 'uploads'));
app.get("/uploads/*", function (req, res, next) {
res.sendFile(__dirname + req.url);
});
var whitelist = ['https://www.skoolaide.com', 'https://skoolaide.com'];
var corsOptionsDelegate = function (req, callback) {
var corsOptions;
if (whitelist.indexOf(req.headers['origin']) !== -1) {
corsOptions = { origin: true, credentials: true } // reflect (enable) the requested origin in the CORS response
} else {
corsOptions = { origin: false, credentials: false } // disable CORS for this request
}
callback(null, corsOptions) // callback expects two parameters: error and options
}
app.use(cors(corsOptionsDelegate));
//this is used because the body parser removes undefined values from the database.
app.set('json replacer', function (key, value) {
// undefined values are set to `null`
if (typeof value === "undefined") {
return null;
}
return value;
});
app.use(multer({
dest: './uploads/',
rename: function (fieldname, filename) {
return filename.replace(/\W+/g, '-').toLowerCase() + Date.now()
},
onFileUploadStart: function (file) {
//console.log(file.fieldname + ' is starting ...')
},
onFileUploadData: function (file, data) {
//console.log(data.length + ' of ' + file.fieldname + ' arrived')
},
onFileUploadComplete: function (file) {
//console.log(file.fieldname + ' uploaded to ' + file.path)
}
}).any());
// "files" should be the same name as what's coming from the field name on the client side.
app.post("/api/upload", function (req, res) {
//console.log(req.files[0].mimetype)
fs.readFile(req.files[0].path, function (err, data) {
var newPath = __dirname + "/uploads/" + req.headers["account_id"] + '_' + moment().format('MM_DD_YYYY_HH-mm-ss') + '_' + req.files[0].originalname.replace(/[^a-zA-Z0-9.]/g, '_');
var readPath = "/uploads/" + req.headers["account_id"] + '_' + moment().format('MM_DD_YYYY_HH-mm-ss') + '_' + req.files[0].originalname.replace(/[^a-zA-Z0-9.]/g, '_');
if(req.files[0].mimetype.indexOf('image') > -1){
sharp(data)
.rotate()
.resize(800, 800)
.max()
.toFile(newPath, function(err, info){
var file = new Files();
file.owner = req.headers.account_id || null;
file.classId = req.body.classId || null;
file.rewardGoalId = req.body.rewardGoalId || null;
file.avatarId = req.body.avatarId || null;
file.mimeType = req.files[0].mimetype || null;
file.originalName = req.files[0].originalname || null;
file.path = readPath;
file.save(function (err, newFile) {
if (err) {
res.send(err);
} else {
res.send({ success: true, data: newFile }).end();
}
});
});
} else{
//not an image file...
var newPath = __dirname + "/uploads/" + req.headers["account_id"] + '_' + moment().format('MM_DD_YYYY_HH-mm-ss') + '_' + req.files[0].originalname.replace(/[^a-zA-Z0-9.]/g, '_');
//console.log('Writing file: ', newPath);
fs.writeFile(newPath, data, function (err) {
var readPath = "/uploads/" + req.headers["account_id"] + '_' + moment().format('MM_DD_YYYY_HH-mm-ss') + '_' + req.files[0].originalname.replace(/[^a-zA-Z0-9.]/g, '_');
//console.log(readPath)
var file = new Files();
file.owner = req.headers.account_id || null;
file.classId = req.body.classId || null;
file.rewardGoalId = req.body.rewardGoalId || null;
file.profileId = req.body.profileId || null;
file.mimeType = req.files[0].mimetype || null;
file.originalName = req.files[0].originalname || null;
file.path = readPath;
file.save(function (err, newFile) {
if (err) {
res.send(err);
} else {
res.send({ success: true, data: newFile }).end();
}
});
});
}
});
});
app.use(express.static('www'))
var a = ['/'];
//all get requests resolve to index.
app.get(a, function (req, res) {
console.log('This is a test', req.originalUrl, req.url);
res.setHeader('Cache-Control', 'private, no-cache, no-store, must-revalidate');
res.setHeader('Expires', '-1');
res.setHeader('Pragma', 'no-cache');
res.setHeader('X-UA-Compatible', 'IE=Edge,chrome=1');
res.setHeader('Judson', 'Rocks!');
if (req.originalUrl == '/') {
res.sendFile('www/index.html', {root: __dirname});
}
});
app.set("superSecret", config.secret)
//var upload = require("./middleware/upload.api");
app.use('/api', router);
app.use('/api/students', students);
app.use('/api/accounts', accounts);
app.use('/api/messages', messages);
app.use('/api/advocates', advocates);
app.use('/api/authenticate', authenticate);
app.use('/api/email', email);
app.use('/api/text', text);
app.use('/api/colleges', colleges);
app.use('/api/amazon', amazon);
app.use('/api/rewards', rewards);
app.use('/api/files', files);
app.use('/api/validate', validations);
app.use('/api/points', points);
app.use('/api/notifications', notifications);
app.use('/api/notificationsMap', notificationsMap);
app.use('/api/trivia', trivia);
app.use('/api/rewardgoals', tasks);
app.use('/api/classes', classes);
app.use('/api/badges', badges);
app.use('/api/connections', connections);
app.use('/api/fixpasswords', fixpasswords);
/**SERVER*************************************/
// all environments
app.set('port', process.env.PORT || 3000);
app.engine('html', require('ejs').renderFile);
// express/connect middleware
app.use(morgan('dev'));
app.use(compression()); //use compression
// development only
if ('development' === app.get('env')) {
app.use(errorhandler());
}
/**END SERVER*************************************/
var cluster = require('cluster');
if (cluster.isMaster) {
var numWorkers = require('os').cpus().length;
console.log('Master cluster setting up ' + numWorkers + ' workers...');
for (var i = 0; i < numWorkers; i++) {
cluster.fork();
}
cluster.on('online', function (worker) {
console.log('Worker ' + worker.process.pid + ' is online');
});
cluster.on('exit', function (worker, code, signal) {
console.log('Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal);
console.log('Starting a new worker');
cluster.fork();
});
} else {
app.listen(app.get('port'),
function () {
console.log('myApp server listening on port ' + app.get('port'));
});
}
Your code only starts an HTTP server. This line:
app.listen(app.get('port'))
starts only an http server. If you want support for HTTPS, you have to also start an HTTPS server and you have to start it on a different port. The express doc for app.listen() shows you how to do that and it's basicaly like this:
var express = require('express');
var https = require('https');
var http = require('http');
var app = express();
http.createServer(app).listen(80);
https.createServer(options, app).listen(443);
where options contains your SSL certificates and where you fill in the desired port numbers for your http server and your https server.
If you were operating under the illusion that you don't need a port number in the browser in order to access both an http and https server, that's because the browser fills in a default port number for you. If you just type an http URL, it uses port 80. If you type an https URL with no port, it uses port 443. If you're not on those two specific ports, you have to specify the desired port in the browser URL. In either case, you need a separate server on separate ports for both http and https.
Use var https = require("https"); not
Var http = require ("http");

Https server with websockets on node.js

I'm trying to create simple https server with websockets support. I had some functional code for https server and websockets, but both standalone. Don't know how to combine it. I prefer to use free to use libraries (MIT,..)
I want to be able to serve http request and also websockets.
Https example with upgrade, but don't know how to handle upgraded websocket connection as my second example bellow this.
connection. process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
var https = require('https');
var websocket = require('websocket');
var fs = require('fs');
var options = {
key:fs.readFileSync('./cert/server.key'),
cert:fs.readFileSync('./cert/server.crt')
};
var server = https.createServer(
options
, function(req,res) {
console.log('req');
res.writeHeader(200, { 'Content-Type': 'text/plain' });
res.write('test');
res.end();
}
);
server.on('upgrade', (req, socket, head) => {
console.log('upgrade ');
socket.write('HTTP/1.1 101 Web Socket Protocol Handshake\r\n' +
'Upgrade: WebSocket\r\n' +
'Connection: Upgrade\r\n' +
'\r\n');
socket.pipe(socket);
});
server.listen(8080, '127.0.0.1', () => {
// make a request
const options = {
port: 8080,
hostname: '127.0.0.1',
headers: {
'Connection': 'Upgrade',
'Upgrade': 'websocket'
}
};
const req = https.request(options);
req.end();
req.on('upgrade', (res, socket, upgradeHead) => {
console.log('got upgraded!');
*** so what to do here ***
/*
socket.end();
process.exit(0);
*/
});
});
This is functional websockets but without possibility to handle simple http(s) requests.
var https = require('https');
var ws = require('websocket').server;
var fs = require('fs');
var options = {
key:fs.readFileSync('./cert/server.key'),
cert:fs.readFileSync('./cert/server.crt')
};
var server = https.createServer(
options
, function(req,res) {
res.writeHeader(200);
res.end();
}
);
server.listen(8080);
var wss = new ws({httpServer:server});
var connectionNumber=0;
console.log('start ');
wss.on('request',function(req){
req.on('requestAccepted',function(conn){
conn.on('message',function(msg){
conn.send('test');
});
conn.on('close',function(msg){
});
});
req.accept(null,req.origin);
});
I was looking for functional example of combined solution but haven't any luck.
You can to have wss-connection trouble with self-signed certificate (code 1006). In this case you must add your certificate to root CA (Firefox, Chrome).
// app.js
'use strict'
const https = require('https');
const fs = require('fs');
const ws = require('ws');
const options = {
key: fs.readFileSync('key.pem'),
cert: fs.readFileSync('cert.pem')
};
const index = fs.readFileSync('./index.html');
let server = https.createServer(options, (req, res) => {
res.writeHead(200);
res.end(index);
});
server.addListener('upgrade', (req, res, head) => console.log('UPGRADE:', req.url));
server.on('error', (err) => console.error(err));
server.listen(8000, () => console.log('Https running on port 8000'));
const wss = new ws.Server({server, path: '/echo'});
wss.on('connection', function connection(ws) {
ws.send('Hello');
ws.on('message', (data) => ws.send('Receive: ' + data));
});
// index.html
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
</head>
<body>
<script>
var socket = new WebSocket('wss://127.0.0.1:8000/echo');
socket.onopen = () => console.log('Connected') || setInterval(() => socket.send(new Date().toLocaleString()), 1000);
socket.onclose = (event) => console.log((event.wasClean) ? 'Disconnected' : 'Connection break: ' + (event.reason || event.code));
socket.onmessage = (event) => console.log('DATA', event.data);
socket.onerror = (err) => console.error(err.message);
</script>
Press F12 to open console...
</body>
</html>
Since you are using Node, I suggest taking a look at Socket.io. It is easier to use than doing all the manual websocket connection logic yourself.

Resources