I have a nodejs express application which is behind an nginx reverse proxy. Everything works as it should, except that when I try to set cookies on a response, nginx returns a 502 page.
Here is the relevant route code:
officeAuth.getToken(req.query.code).then((data) => {
const key = jwt.sign({access_token: data.access_token}, process.env.JWT_PRIVATE_KEY);
const refreshKey = jwt.sign({refresh_token: data.refresh_token}, process.env.JWT_PRIVATE_KEY);
res.cookie('token', key, {maxAge: data.expires_in * 24000, httpOnly: true});
res.cookie('refresh', refreshKey, {maxAge: data.expires_in * 24000, httpOnly: true});
res.redirect(process.env.APP_HOME_PAGE);
}, (err) => {
res.status(500).send(err);
});
With this code, the nodejs log does not show any errors, and in-fact shows this request as returning a 302 as it should. However in the browser I get Nginx's 502 page.
When I remove the res.cookie statements from the code above, the redirect works fine.
Nginx config:
server {
listen 443 ssl;
server_name my.server.com;
ssl_certificate /my/ssl/cert;
ssl_certificate_key /my/ssl/key;
location / {
proxy_pass http://localhost:3001;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header Host $http_host;
}
}
Turns out my cookies were too large for nginx to handle, so I just increased the header size limit by adding:
proxy_buffers 8 16k;
proxy_buffer_size 16k;
to the location / block.
Related
Impossible to increase buffer width to avoid dropping frames
OR
Unable to manage WS fragmentation correctly
Summary
My goal:
A very simple thing: have websocket tunnel to transfer at least 2/3 MB of data per tunnel. I need to send directory structure, therefore the data can be very many
The problem:
Sending WebSocket messages over 17KB, from A to B, cause a "communication lost" or packet drop/loss; the connection/tunnel remains up with the inability to send new messages over the same tunnel, from A to B; conversely, from B to A continues to work.
I must restart the tunnel to get functionality back.
It could also be an idea, the management of the packet heap that restarts the tunnel when the threshold is reached, but it is clear that I need to send more than the threshold at one time.
The "signal path":
GoLang app(Client) ---> :443 NGINX Proxy(Debian) ---> :8050 NodeJS WS Server
The tests:
Sending X messages/chunks of 1000 byte each | messages are received up to the 17th chunk, the following ones are not received (see below)
The analyses:
Wireshark on Go app shows the flow of all packets
tcpdump, on Debian machine, set to listen on eth (public), shows the flow of all packets
tcpdump, on Debian machine, set to listen on lo interface (for rev proxy scanning), shows the flow of all packets
NodeJS/fastify-websocket ws.on('message', (msg)=>{console.log(msg)}) shows up to the 17th chunk
Code & Config:
GoLang app relevant part
websocket.DefaultDialer = &websocket.Dialer{
Proxy: http.ProxyFromEnvironment,
HandshakeTimeout: 45 * time.Second,
WriteBufferSize: 1000, //also tried with 2000, 5000, 10000, 11000
}
c, _, err := websocket.DefaultDialer.Dial(u.String(), nil)
wsConn = c
bufferChunk := 1000
bufferSample := ""
for j := 7; j <= bufferChunk; j++ {
bufferSample = bufferSample + "0"
}
i := 1
for {
sendingBytes := i * bufferChunk
fmt.Println(strconv.Itoa(sendingBytes) + " bytes sent")
wsConn.WriteMessage(websocket.TextMessage, []byte(bufferSample))
i++
time.Sleep(1000 * time.Millisecond)
}
NGINX conf:
upstream backend {
server 127.0.0.1:8050;
}
server {
server_name my.domain.com;
large_client_header_buffers 8 32k;
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-NginX-Proxy true;
proxy_buffers 8 2m;
proxy_buffer_size 10m;
proxy_busy_buffers_size 10m;
proxy_pass http://backend;
proxy_redirect off;
#proxy_buffering off; ### ON/OFF IT'S THE SAME
# enables WS support
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade"; ### "upgrade" it's the same
}
listen 443 ssl; # managed by Certbot
ssl_certificate /etc/letsencrypt/live/my.domain.com/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/my.domain.com/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
server {
if ($host = my.domain.com) {
return 301 https://$host$request_uri;
} # managed by Certbot
server_name my.domain.com;
listen 80;
return 404; # managed by Certbot
}
NodeJS code:
//index.js
const config = require("./config.js");
const fastify = require('fastify')();
const WsController = require("./controller");
fastify.register(require('fastify-websocket'), {
/*these options are the same as the native nodeJS WS*/
options :{
maxPayload: 10 * 1024 * 1024,
maxReceivedFrameSize: 131072,
maxReceivedMessageSize: 10 * 1024 * 1024,
autoAcceptConnections: false
}
});
fastify.ready(err => {
if (err) throw err
console.log("Server started")
fastify.websocketServer
.on("connection", WsController)
})
//controller.js
module.exports = (ws, req) => {
ws.on("message", (msg) => {
log("msg received"); //it is shown as long as the tunnel does not "fill" up to 17KB
})
})
SOLVED
Updating fastify and fastify-websocket the problem disappeared. What a shame!
I came up with this solution by creating a new cloud instance and installing everything from scratch.
Just npm update.
Thank you all for your support
I have my own VPS server and I have script in Node.js which display the visitor's IP but always when I visit website I get in console local IP address (127.0.0.1). I use Nginx.
Any idea?
Node.js script:
#!/usr/bin/env nodejs
const http = require('http');
const host = '127.0.0.1';
const port = 8080;
const server = http.createServer((req, res) => {
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
var ip = (req.headers['x-forwarded-for'] || '').split(',').pop().trim() ||
req.connection.remoteAddress || req.socket.remoteAddress || req.connection.socket.remoteAddress;
console.log(`IP = ${ip}`);
});
server.listen(port, host);
Nginx proxy/headers configuration:
server {
listen 80 default_server;
listen [::]:80 default_server;
root /var/www/html;
index index.html index.htm index.nginx-debian.html;
server_name _;
location / {
try_files $uri $uri/ =404;
proxy_pass http://127.0.0.1:8080;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
}
You need to rig your nginx reverse proxy to pass along the requester's IP address. Adding these two settings to nginx.conf does the trick.
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
Put these lines in your location{...} stanza of nginx.conf along with your proxy-pass and the rest.
With these changes, nginx inserts two http headers into each request: X-Forwarded-For and X-Real-IP. (There's a new standard Forwarded: header, but nginx doesn't handle it easily as of mid-2020.)
Then, use app.set() to add proxy server support to your nodejs program to interpret those headers. Put this line in your www or http-server.js Javascript program shortly after your const app = express() line.
app.set( 'trust proxy', 'loopback' )
Express will then muck around with the X-Forwarded-For header for you and put the appropriate IP address in req.ip.
I've linked to some documentation. You would be wise to read it.
I have an application using socket.io with Node and Express. I'm also using AWS EC2 and Nginx.
I'm getting a timeout with socket io.
The error is:
GET https://vusgroup.com/socket.io/?EIO=3&transport=polling&t=MnUHunS 504 (Gateway Time-out)
Express file:
var port = 8090;
host = 'https://18.237.109.96'
var app = express(host);
var webServer = http.createServer(app);
...
// Start Socket.io so it attaches itself to Express server
var socketServer = socketIo.listen(webServer, {"log level":1});
//listen on port
webServer.listen(port, function () {
console.log('listening on http://localhost:' + port);
});
Nginx file:
server {
listen 80 default_server;
listen [::]:80 default_server;
location / {
# First attempt to serve request as file, then
# as directory, then fall back to displaying a 404.
proxy_pass http://18.237.109.96:8090/;
}
}
server {
server_name vusgroup.com www.vusgroup.com; # managed by Certbot
listen [::]:443 ssl ipv6only=on; # managed by Certbot
listen 443 ssl; # managed by Certbot
...
ssl stuff
...
location / {
# First attempt to serve request as file, then
# as directory, then fall back to displaying a 404.
proxy_pass http://18.237.109.96:8090/;
}
location /socket.io/ {
proxy_pass http://18.237.109.96:3000;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
I've tried chaning the proxypass for socket.io to http://18.237.109.96:8090; but that gave me a 400 error.
I'm using nginx for web-facing traffic and proxying my node.js connections, as well as handling my SSL.
The connection IS successfully established--io.on('connection') does trigger a console log server side, but then I get a 400 (Bad Request) on the client (in both Firefox and Chrome) and then the connection resets over and over (and continues throwing the same error).
The error is as follows (from Chrome):
polling-xhr.js:264 GET https://192.168.56.101/socket.io/?EIO=3&transport=polling&t=M54C3iW&sid=byqOIkctI9uWOAU2AAAA 400 (Bad Request)
i.create # polling-xhr.js:264
i # polling-xhr.js:165
o.request # polling-xhr.js:92
o.doPoll # polling-xhr.js:122
n.poll # polling.js:118
n.onData # polling.js:157
(anonymous) # polling-xhr.js:125
n.emit # index.js:133
i.onData # polling-xhr.js:299
i.onLoad # polling-xhr.js:366
hasXDR.r.onreadystatechange # polling-xhr.js:252
XMLHttpRequest.send (async)
i.create # polling-xhr.js:264
i # polling-xhr.js:165
o.request # polling-xhr.js:92
o.doPoll # polling-xhr.js:122
n.poll # polling.js:118
n.doOpen # polling.js:63
n.open # transport.js:80
n.open # socket.js:245
n # socket.js:119
n # socket.js:28
n.open.n.connect # manager.js:226
n # manager.js:69
n # manager.js:37
n # index.js:60
(anonymous) # control.js:6
192.168.56.101/:1 WebSocket connection to 'wss://192.168.56.101/socket.io/?EIO=3&transport=websocket&sid=byqOIkctI9uWOAU2AAAA' failed: WebSocket is closed before the connection is established.
Nginx logs (at info level) show the following:
2018/01/29 19:37:10 [info] 28262#28262: *18403 client closed connection while waiting for request, client: 192.168.56.1, server: 192.168.56.101:443
My nginx config is as follows
(I HAVE tried this both with and without the "location /socket.io/ " block, and get exactly the same results.):
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
upstream altairServer {
server 192.168.56.101:8000;
}
server {
listen 192.168.56.101:443;
server_name altair.e6diaspora.com;
ssl on;
ssl_certificate /home/e6serv/crypto/domain.pem;
ssl_certificate_key /home/e6serv/crypto/server.key;
access_log /home/e6serv/logs/nginx/host.access.log;
error_log /home/e6serv/logs/nginx/host.error.log;
root /home/e6serv/e6Code/e6GS1/public;
location / {
try_files maintain.html $uri $uri/index.html #node;
}
location /socket.io/ {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-NginX-Proxy true;
proxy_pass http://altairServer;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
location #node {
proxy_pass http://altairServer;
proxy_http_version 1.1;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-NginX-Proxy true;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_max_temp_file_size 0;
proxy_redirect off;
proxy_read_timeout 240s;
}
}
The relevant server side code is as follows:
const app = express();
app.set('port', 8000);
app.engine('html', require('ejs').renderFile);
app.use(methodOverride());
app.use(session({
secret: SITE_SECRET,
store: redisSesStore,
cookie: {maxAge: 604800000},
resave: false,
saveUninitialized: false
}));
app.use(parseCookie());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
app.use('/',router);
const httpServer = http.createServer(app)
const io = socketIo.listen(httpServer);
io.use(passportSocketIo.authorize({
key: 'connect.sid',
secret: SITE_SECRET,
store: redisSesStore,
passport: passport,
cookieParser: parseCookie
}));
httpServer.listen(app.get('port'), '192.168.56.101', function(){
log.warn('Worker Started HTTP Server')
});
io.on('connection', function(socket) {
log.debug(socket.request.user)
var event = { type:'userConnect',data:'Hello Client'};
process.send(event);
}
My client side code is as follows:
control.socket = io.connect('https://'+hostname);
console.log("Should be connected")
//NOTE: This final line does not work--the console.log never fires:
control.socket.on('userConnect',function (data) {console.log(data)})
I've discovered the source of the problem... The extra element that was in here that I didn't know to talk about was node.js's Cluster.
https://github.com/socketio/socket.io/issues/1942
https://socket.io/docs/using-multiple-nodes/
Socket.io defaults to polling, which requires a sticky load balancing between the various workers. The solution was as found in the socket.io multiple node documentation.
I added something like the following to my nginx config:
upstream io_nodes {
ip_hash;
server 127.0.0.1:6001;
server 127.0.0.1:6002;
server 127.0.0.1:6003;
server 127.0.0.1:6004;
}
(Also note, specific workers must be set up to listen on specific ports.)
I have set up a digital ocean droplet that is a reverse proxy server using nginx and node. I used this tutorial from digital ocean as a starting point
https://www.digitalocean.com/community/tutorials/how-to-set-up-a-node-js-application-for-production-on-ubuntu-14-04.
I have also set up ssl with lets encrypt. The issue I am currently having is that I am unable to make cross domain ajax calls to the server. I am getting a error of No 'Access-Control-Allow-Origin' header is present. I have set up the appropriate header response in my node app and have attempted to follow the few examples I could find for nginx with no luck. Below is my code.
nginx with my attempts at headers removed
server {
listen 443 ssl;
server_name lefthookservices.com www.lefthookservices.com;
ssl_certificate /etc/letsencrypt/live/lefthookservices.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/lefthookservices.com/privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/ssl/certs/dhparam.pem;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-$
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
location / {
proxy_pass http://127.0.0.1:8080;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
location ~ /.well-known {
allow all;
}
}
server {
listen 80;
server_name lefthookservices.com www.lefthookservices.com;
return 301 https://$host$request_uri;
}
Here is my app.js script using express
'use strict';
var colors = require('colors/safe');
var express = require('express');
var knack = require('./knack_call.js');
var bodyParser = require('body-parser');
var cors = require('cors');
colors.setTheme({
custom: ['blue', 'bgWhite']
});
var app = express();
app.use(bodyParser.json());
// allow for cross domain ajax
app.get('/', function(request, response){
response.send('hello\n');
});
app.post('/', function(request, response){
response.header("Access-Control-Allow-Origin", "*");
response.header("Access-Control-Allow-Headers", "X-Requested-With");
response.header("Access-Control-Allow-Methods', 'GET,POST");
knack.getData(request, response);
});
app.listen(8080, '127.0.0.1', function(m){
console.log(colors.custom("Captin the server is at full strength"));
});
Any suggestion that could help me set the correct headers to allow CORS would be greatly appreciated. Thank you in advance.
As a result of Tristans answer below my Nginx code now looks like this.
server {
listen 443 ssl;
server_name lefthookservices.com www.lefthookservices.com;
ssl_certificate /etc/letsencrypt/live/lefthookservices.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/lefthookservices.com/privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/ssl/certs/dhparam.pem;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES$
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
location / {
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
if ($http_origin ~*(https?://.*\exponential.singularityu\.org(:[0-9]+)?$)){
set $cors "1";
}
if ($request_method = 'OPTIONS') {
set $cors "${cors}o";
}
if ($cors = "1") {
more_set_headers 'Access-Control-Allow-Origin: $http_origin';
more_set_headers 'Access-Control-Allow-Credentials: true';
proxy_pass http://127.0.0.1:8080;
}
if ($cors = "1o") {
more_set_headers 'Access-Control-Allow-Origin: $http_origin';
more_set_headers 'Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE';
more_set_headers 'Access-Control-Allow-Credentials: true';
more_set_headers 'Access-Control-Allow-Headers: Origin,Content-Type,Accept';
add_header Content-Length 0;
add_header Content-Type text/plain;
return 204;
}
proxy_pass http://127.0.0.1:8080;
}
}
location ~ /.well-known {
allow all;
}
}
Sadly this is still not working.
server {
listen 80;
server_name lefthookservices.com www.lefthookservices.com;
return 301 https://$host$request_uri;
}
It turns out the error message I was getting was inaccurate. The issue was not header setting. It turned out that I needed to make the request with jsonp and I needed to handle the incoming data differently. An error in the function called by app.js was erroring and causing the connection to time out. This resulted in the appropriate headers not being returned to the browser which caused the error message.
For anyone hoping to find an NGINX config that worked this is mine.
proxy_pass http://127.0.0.1:8080;
# proxy_http_version 1.1;
proxy_set_header Access-Control-Allow-Origin *;
# proxy_set_header Upgrade $http_upgrade;
# proxy_set_header Connection '';
proxy_set_header Host $host;
# proxy_cache_bypass $http_upgrade;
Thanks you for the suggestions.
Pull Nginx out of this equation. It doesn't have anything to do with your CORs problem if your setup is as similar to mine as I believe it is. I see that you're using the cors module, but you're not actually using it that I can see.
Your settings are simply enough that you might be able to get away with the defaults so, right below app.use(bodyParser.json());, update your app.js with:
app.use(cors());
That might work right out of the box. If it doesn't, you can pass a set of options. Mine looks something like this:
app.use(cors({
origin: myorigin.tld,
allowedHeaders: [ 'Accept-Version', 'Authorization', 'Credentials', 'Content-Type' ]
}));
Other config options are available in the docs.
You're almost there.
You have to think of the proxy as an external server as well as your Node.js application.
So, in short, you need to add a header to your nginx configuration.
Take a look at this link,
https://gist.github.com/pauloricardomg/7084524
In case this ever gets deleted:
#
# Acts as a nginx HTTPS proxy server
# enabling CORS only to domains matched by regex
# /https?://.*\.mckinsey\.com(:[0-9]+)?)/
#
# Based on:
# * http://blog.themillhousegroup.com/2013/05/nginx-as-cors-enabled-https-proxy.html
# * http://enable-cors.org/server_nginx.html
#
server {
listen 443 default_server ssl;
server_name localhost;
# Fake certs - fine for development purposes :-)
ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem;
ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
ssl_session_timeout 5m;
location / {
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# Nginx doesn't support nested If statements, so we
# concatenate compound conditions on the $cors variable
# and process later
# If request comes from allowed subdomain
# (*.mckinsey.com) then we enable CORS
if ($http_origin ~* (https?://.*\.mckinsey\.com(:[0-9]+)?$)) {
set $cors "1";
}
# OPTIONS indicates a CORS pre-flight request
if ($request_method = 'OPTIONS') {
set $cors "${cors}o";
}
# Append CORS headers to any request from
# allowed CORS domain, except OPTIONS
if ($cors = "1") {
more_set_headers 'Access-Control-Allow-Origin: $http_origin';
more_set_headers 'Access-Control-Allow-Credentials: true';
proxy_pass http://serverIP:serverPort;
}
# OPTIONS (pre-flight) request from allowed
# CORS domain. return response directly
if ($cors = "1o") {
more_set_headers 'Access-Control-Allow-Origin: $http_origin';
more_set_headers 'Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE';
more_set_headers 'Access-Control-Allow-Credentials: true';
more_set_headers 'Access-Control-Allow-Headers: Origin,Content-Type,Accept';
add_header Content-Length 0;
add_header Content-Type text/plain;
return 204;
}
# Requests from non-allowed CORS domains
proxy_pass http://serverIP:serverPort;
}
}