Issue with uploading 1 GB file via nginx - node.js

We are using nginx as used as load balancer behind sails server is running. We have to upload 1 Gb file to the server. We are using angular js as front end. When we upload 1 GB file into the server it shows upload progress as 99% then after that it shows 400 error code.
While looking in to the error logs it show client prematurely closed connection
We tried client_body_in_file_only and upload module. But fails
How can we solve this issue?
Our Nginx conf is
load_module modules/ngx_http_upload_module.so;
user www-data;
worker_processes auto;
pid /run/nginx.pid;
events {
worker_connections 768;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 360;
types_hash_max_size 2048;
proxy_ignore_client_abort on;
proxy_buffering off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log info;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
upstream api{
ip_hash;
server 127.0.0.1:1337;
}
server {
client_max_body_size 2048M;
client_body_buffer_size 2048M;
client_body_timeout 50m;
client_header_timeout 50m;
send_timeout 50m;
listen 80;
# location /test/upload {
# add_header 'Access-Control-Allow-Origin' *;
# #add_header 'Access-Control-Allow-Headers' 'Content-Type';
# add_header 'Access-Control-Allow-Credentials' 'true' always;
# add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
# add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always;
# # required to be able to read Authorization header in frontend
# add_header 'Access-Control-Expose-Headers' 'Authorization' always;
# # Pass altered request body to this location
# upload_pass #upload;
#
# # Store files to this directory
# # The directory is hashed, subdirectories 0 1 2 3 4 5 6 7 8 9 should exist
# upload_store /tmp;
#
# # Allow uploaded files to be read only by user
# upload_store_access user:r;
#
# # Set specified fields in request body
# upload_set_form_field $upload_field_name.name "$upload_file_name";
# upload_set_form_field $upload_field_name.content_type "$upload_content_type";
# upload_set_form_field $upload_field_name.path "$upload_tmp_path";
#
# # Inform backend about hash and size of a file
# upload_aggregate_form_field "$upload_field_name.md5" "$upload_file_md5";
# upload_aggregate_form_field "$upload_field_name.size" "$upload_file_size";
#
# upload_pass_form_field "^submit$|^description$|^account$";
#
# upload_cleanup 400 404 499 500-505;
# }
location #upload {
proxy_send_timeout 50m;
proxy_read_timeout 50m;
proxy_pass http://api;
proxy_http_version 1.1;
proxy_redirect off;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
}
location / {
proxy_send_timeout 50m;
proxy_read_timeout 50m;
proxy_pass http://api;
proxy_http_version 1.1;
proxy_redirect off;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
}
location /test/upload {
proxy_send_timeout 50m;
proxy_read_timeout 50m;
proxy_pass http://api;
proxy_http_version 1.1;
proxy_redirect off;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
client_body_in_file_only clean;
client_body_buffer_size 16K;
client_body_temp_path /tmp;
}
}
}

Related

How do i enable auth_basic on nginx for a domain and multiple subdomains with only one password?

I successfully added the auth_basic directive to my server block with the following config:
proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=proxy_cache_path_global:10m loader_threshold=300 loader_files=200 max_size=2g inactive=60m use_temp_path=off;
upstream backend {
server localhost:8080;
keepalive 128;
}
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name example.com www.example.com app.example.com;
auth_basic "Administrator’s Area";
auth_basic_user_file /etc/nginx/.htpasswd;
location / {
proxy_pass http://backend;
proxy_cache_methods GET HEAD;
proxy_cache proxy_cache_path_global;
proxy_cache_key $host$request_uri$cookie_user$slice_range;
proxy_cache_bypass $cookie_nocache $arg_nocache$arg_comment;
proxy_cache_min_uses 3;
proxy_cache_valid any 1m;
proxy_cache_revalidate on;
proxy_cache_background_update on;
proxy_cache_lock on;
proxy_cache_lock_age 10s;
proxy_cache_lock_timeout 3s;
proxy_redirect off;
proxy_http_version 1.1;
proxy_read_timeout 300;
proxy_connect_timeout 300;
slice 1m;
add_header X-Cache-Status $upstream_cache_status;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_set_header Range $slice_range;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
}
}
When i visit my website, 2 login prompts appear after each other; one for example.com, and another one for app.example.com.
How do i configure this, so that i only have to type in my credentials once?
I already tried looking for a solution by myself unsuccessfully. Any hints would be appreciated.

Node.js on nginx + Cent OS7 shows blank page or ERR_TOO_MANY_REDIRECTS

Symptom
I think I've messed up my domain/nginx settings. The server shows up properly for 70~60% of time but sometimes it shows blank page(no html at all) or ERR_TOO_MANY_REDIRECTS page.
the url is: sungryeol.com
I've checked both Stackoverflow and Node.js + CentOS 7 + nginx setup tutorials on Google(which leads to digital ocean mostly)
Settings
Node.js / Express.js
OS: Cent OS 7(Vultr VPS)
Reverse proxy: nginx
DNS: Namecheap
Namecheap domain setting
Type------------------Host---Value-------------------------------TTL
A Record--------------#------45.32.9.128-------------------------5 min
CNAME Record----------www----sungryeol.com.----------------------5 min
URL Redirect Record---www----http://www.sungryeol.com Unmasked---
the dot on sungryeol.com'.' is automatically added by Namecheap UI.
nginx setting
nginx -t shows the configuration has no error. The internal setting has no problem I assume. It works fine with IP address but still shows blank page or redirect error from the URL : (www).sungryeol.com.
/etc/nginx/conf.d
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log /var/log/nginx/host.access.log main;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
...
}
/etc/nginx/nginx.conf
user nginx;
worker_processes 1;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
#include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-available/*.conf;
}
/etc/nginx/sites-available/portfolio.conf
upstream portfolio {
server 127.0.0.1:3001 max_fails=0 fail_timeout=10s weight=1;
ip_hash;
keepalive 512;
}
server
{
listen 80;
listen [::]:80 ipv6only=on default_server;
server_name sungryeol.com www.sungryeol.com;
keepalive_timeout 10;
# client_max_body_size 50M;
index index.html;
location /
{
proxy_pass http://portfolio;
include /etc/nginx/proxy_params;
}
}
/etc/nginx/proxy_params
proxy_buffers 16 32k;
proxy_buffer_size 64k;
proxy_busy_buffers_size 128k;
proxy_cache_bypass $http_pragma $http_authorization;
proxy_connect_timeout 59s;
proxy_hide_header X-Powered-By;
proxy_http_version 1.1;
proxy_ignore_headers Cache-Control Expires;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 htt$
proxy_no_cache $http_pragma $http_authorization;
proxy_pass_header Set-Cookie;
proxy_read_timeout 600;
proxy_redirect off;
proxy_send_timeout 600;
proxy_temp_file_write_size 64k;
proxy_set_header Accept-Encoding '';
proxy_set_header Cookie $http_cookie;
proxy_set_header Host $host;
proxy_set_header Proxy '';
proxy_set_header Referer $http_referer;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Server $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Original-Request $request_uri;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";

connect() failed (111: Connection refused) while connecting upstream

[error] 7697#7697: *100335 connect() failed (111: Connection refused) while connecting to upstream, client: XXX.XXX.XXX.XXX, server: v4.domain.com, request: "GET /socket.io/?__sails_io_sdk_version=0.13.8&__sails_io_sdk_platform=browser&__sails_io_sdk_language=javascript&EIO=3&transport=polling&t=Luvcibs HTTP/1.1", upstream: "http://127.0.0.1:1338/socket.io/?__sails_io_sdk_version=0.13.8&__sails_io_sdk_platform=browser&__sails_io_sdk_language=javascript&EIO=3&transport=polling&t=Luvcibs", host: "v4.domain.com", referrer: "http://v4.domain.com/?ct=t(Flash_Sals_Videotoolz_copy_05_12_29_2016)&mc_cid=404a630ab2&mc_eid=c44f7937fe"
[error] 7700#7700: *101735 connect() failed (111: Connection refused) while connecting to upstream, client: XX.XX.XX.XX, server: v4.domain.com, request: "GET /socket.io/?__sails_io_sdk_version=0.13.8&__sails_io_sdk_platform=browser&__sails_io_sdk_language=javascript&EIO=3&transport=polling&t=LuvciVy HTTP/1.1", upstream: "http://127.0.0.1:1338/socket.io/?__sails_io_sdk_version=0.13.8&__sails_io_sdk_platform=browser&__sails_io_sdk_language=javascript&EIO=3&transport=polling&t=LuvciVy", host: "v4.domain.com", referrer: "http://v4.domain.com/"
Recently i have configured my sails js application on Ubuntu 16.04 VPS Server which have nginx as reverse server. below is my nginx conf for site
Site runs fine but all of a sudden site break and shows 502 bad gateway.
Tried almost everything whatever i can.
Please help me to get it sorted.
server {
listen 80 default_server;
listen [::]:80 default_server;
# SSL configuration
#
listen 443 ssl default_server;
listen [::]:443 ssl default_server;
#
# Note: You should disable gzip for SSL traffic.
# See: https://bugs.debian.org/773332
#
# Read up on ssl_ciphers to ensure a secure configuration.
# See: https://bugs.debian.org/765782
#
# Self signed certs generated by the ssl-cert package
# Don't use them in a production server!
#
# include snippets/snakeoil.conf;
root /var/www/html/php;
# Add index.php to the list if you are using PHP
index index.html index.php index.htm index.nginx-debian.html;
server_name domain.com www.domain.com;
ssl_certificate /etc/letsencrypt/live/domain.com/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/domain.com/privkey.pem; # managed by Certbot
ssl_dhparam /etc/ssl/certs/dhparam.pem;
location / {
# First attempt to serve request as file, then
# as directory, then fall back to displaying a 404.
#try_files $uri $uri/ =404;
#try_files $uri $uri/ /index.php?q=$uri&$args;
try_files $uri $uri/ /index.php?$args;
}
location ~ \.php$ {
try_files not-existing-file #php;
}
location #php {
#fastcgi_pass 127.0.0.1:9000;
fastcgi_read_timeout 300;
fastcgi_pass unix:/var/run/php/php7.0-fpm.sock;
include snippets/fastcgi-php.conf;
}
location ~* \.(css|js|png|jpg|jpeg|gif|ico)$ {
expires 1d;
}
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}
upstream sails_server {
server 127.0.0.1:1338; # fail_timeout=0;
# keepalive 64;
}
server {
listen 80;
listen [::]:80;
server_name v4.domain.com;
root /root/domain/;
#Logging
error_log /root/domain/log/error.log notice;
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_read_timeout 300;
proxy_pass http://sails_server;
proxy_redirect off;
# proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
# proxy_set_header Connection "";
# proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-NginX-Proxy true;
# proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=one:10ms;
# proxy_cache one;
# proxy_cache_key sfs$request_uri$scheme;
# proxy_pass_request_headers on;
}
location /socket.io/ {
proxy_pass http://sails_server/socket.io/;
proxy_http_version 1.1;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header Port $server_port;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Nginx-Proxy true;
proxy_pass_request_headers on;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_buffers 8 32k;
proxy_buffer_size 64k;
}
}
My PM2 config file for my sails application is below
{
"apps": [
{
"name": "dj",
"script" : "./app.js",
"watch": false,
"ignore_watch" : ["node_modules", ".tmp"],
"watch_options": {
"followSymlinks": false
},
"env" : {
"PORT": 1338,
"NODE_ENV": "production"
}
}
]
}

Cashing packages for npm registry with NGINX

My project is javascript app.
I have lots of dependencies.
I/O for npm registry takes major portion of CI execution.
So my idea is to setup NGINX in front of npm registry and cache tgz files downloads.
Im running Ubuntu 14.04.
NGINX version is 1.4.6
This is my nginx configuration script
user www-data;
worker_processes 4;
pid /run/nginx.pid;
events {
worker_connections 768;
# multi_accept on;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# Logging Settings
##
# access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
include /etc/nginx/conf.d/*.conf;
# include /etc/nginx/sites-enabled/*;
# HTTP 1.1 support
proxy_http_version 1.1;
proxy_buffering off;
proxy_set_header Host $http_host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $proxy_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl;
proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port;
# If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the
# scheme used to connect to this server
map $http_x_forwarded_proto $proxy_x_forwarded_proto {
default $http_x_forwarded_proto;
'' $scheme;
}
# If we receive X-Forwarded-Port, pass it through; otherwise, pass along the
# server port the client connected to
map $http_x_forwarded_port $proxy_x_forwarded_port {
default $http_x_forwarded_port;
'' $server_port;
}
# If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any
# Connection header that may have been passed to this server
map $http_upgrade $proxy_connection {
default upgrade;
'' close;
}
# Set appropriate X-Forwarded-Ssl header
map $scheme $proxy_x_forwarded_ssl {
default off;
https on;
}
server {
listen 80 default_server;
location / {
access_log /var/log/nginx/root.log;
root /var/tmp/nginx/npm;
try_files $request_uri #fetch;
}
location #fetch {
internal;
proxy_pass http://nmregistry:4873$request_uri;
proxy_store /var/tmp/nginx/npm$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_store_access user:rw group:rw all:r;
}
}
}
It works, i can install packages but they are not cached on NGINX machine.
Cant see any tgz files in /var/tmp/nginx/npm
What am i doing wrong here?
This is final version of configuration file.
The main problem was proxy_buffering on;
user www-data;
worker_processes 4;
pid /run/nginx.pid;
events {
worker_connections 768;
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# Logging Settings
##
error_log /var/log/nginx/error.logi debug;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
include /etc/nginx/conf.d/*.conf;
# include /etc/nginx/sites-enabled/*;
# HTTP 1.1 support
proxy_http_version 1.1;
proxy_buffering on;
proxy_set_header Host $http_host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $proxy_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl;
proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port;
# If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the
# scheme used to connect to this server
map $http_x_forwarded_proto $proxy_x_forwarded_proto {
default $http_x_forwarded_proto;
'' $scheme;
}
# If we receive X-Forwarded-Port, pass it through; otherwise, pass along the
# server port the client connected to
map $http_x_forwarded_port $proxy_x_forwarded_port {
default $http_x_forwarded_port;
'' $server_port;
}
# If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any
# Connection header that may have been passed to this server
map $http_upgrade $proxy_connection {
default upgrade;
'' close;
}
# Set appropriate X-Forwarded-Ssl header
map $scheme $proxy_x_forwarded_ssl {
default off;
https on;
}
server {
listen 80 default_server;
location / {
access_log /var/log/nginx/cache_root.log;
proxy_pass http://nmregistry:4873;
}
location ~* .+/-/.+$ {
root /var/tmp/nginx/npm;
expires max;
try_files $uri #fetch;
}
location #fetch {
internal;
proxy_pass http://nmregistry:4873$request_uri;
proxy_store on;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_store_access user:rw group:rw all:rw;
proxy_temp_path /var/tmp/nginx/npm 1 2;
root /var/tmp/nginx/npm;
}
}
}

Nginx - Redirect HTTPS back to HTTP when leaving a secured page

I have Nginx in front of a Node.js app. I have it set up so that if the url has /account in it, it'll redirect to HTTPS. My question is - how do I set it up so that if the user leaves the /account url (clicks a link to go to the home page), it'll get sent back to HTTP?
Here's my ngnix.conf:
worker_processes 1;
error_log logs/error.log;
pid logs/nginx.pid;
events {
worker_connections 128;
}
http {
include mime.types;
default_type application/octet-stream;
sendfile on;
server_tokens off;
#keepalive_timeout 0;
keepalive_timeout 65;
tcp_nodelay on;
proxy_buffer_size 128k;
proxy_buffers 4 256k;
proxy_busy_buffers_size 256k;
gzip on;
server {
listen 80;
server_name localhost;
location / {
proxy_set_header x-path $uri;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-NginX-Proxy true;
proxy_pass http://127.0.0.1:3000;
proxy_redirect off;
}
location /account {
rewrite ^(.*) https://$host$1 permanent; #redirect to https
}
error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
server {
listen 443;
ssl on;
ssl_certificate ssl/server.crt;
ssl_certificate_key ssl/server.key;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-M-Secure "true";
proxy_redirect off;
proxy_max_temp_file_size 0;
proxy_pass http://127.0.0.1:3000;
}
}
}
Thanks in advance for any assistance.
This is untested.
server {
listen 443;
ssl on;
ssl_certificate ssl/server.crt;
ssl_certificate_key ssl/server.key;
location /account/ {
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-M-Secure "true";
proxy_redirect off;
proxy_max_temp_file_size 0;
proxy_pass http://127.0.0.1:3000;
}
location / {
rewrite ^(.*) http://$host$1 permanent; # redirect to http
}
}

Resources