I get 502 and 504 error from nginx in node.js - node.js

I changed the config file to increase the time and uploading size and doesn't found the php-fsm file. whenever I try to click after some time it stuck and gives 504 error and then 502 error every time.
This is my nginx.conf file.. which is located on etc/nginx/nginx.conf Please give me the solution for this issue as my project is on hold because of these things and i need to launch my web app as soon as possible.
user www-data;
worker_processes auto;
pid /run/nginx.pid;
events {
worker_connections 768;
# multi_accept on;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 600;
types_hash_max_size 2048;
# set client body size to 500M #
proxy_connect_timeout 600;
proxy_send_timeout 600;
proxy_read_timeout 600;
send_timeout 600;
client_header_timeout 3000;
client_body_timeout 3000;
fastcgi_read_timeout 3000;
client_max_body_size 800m;
fastcgi_buffers 8 128k;
fastcgi_buffer_size 128k;
#client_max_body_size 800M;
#client_header_timeout 3000;
#client_body_timeout 3000;
#fastcgi_read_timeout 3000;
#client_max_body_size 800m;
#fastcgi_buffers 8 128k;
#fastcgi_buffer_size 128k;
#proxy_connect_timeout 600;
#proxy_send_timeout 600;
#proxy_read_timeout 600;
#send_timeout 600;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
#mail {
# # See sample authentication script at:
# # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
#
# # auth_http localhost/auth.php;
# # pop3_capabilities "TOP" "USER";
# # imap_capabilities "IMAP4rev1" "UIDPLUS";
#
# server {
# listen localhost:110;
# protocol pop3;
# proxy on;
# }
#
# server {
# listen localhost:143;
# protocol imap;
# proxy on;
# }
#}

Related

seek function mp4 nginx

The current configuration allows us to reverse proxy + cache a certain server with video content (mp4).
The problem we have is in the seek function (go back and forth through the player on the video), this in fact takes 1-2 minutes before resuming the stream.
How can we solve this problem?
nginx.conf
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
pcre_jit on;
events {
worker_connections 8096;
use epoll;
accept_mutex on;
multi_accept on;
}
thread_pool mp4Cache threads=4;
http {
##
# Basic Settings
##
sendfile on;
sendfile_max_chunk 512k;
tcp_nopush on;
tcp_nodelay on;
reset_timedout_connection on;
keepalive_timeout 15;
types_hash_max_size 2048;
open_file_cache max=30000 inactive=5m;
open_file_cache_valid 2m;
open_file_cache_min_uses 1;
open_file_cache_errors on;
gzip on;
gzip_comp_level 5;
gzip_min_length 256;
gzip_disable "msie6";
gzip_vary on;
gzip_types application/json application/vnd.apple.mpegurl;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Caching & Proxy
##
proxy_cache_path /etc/nginx/cache levels=1:2 keys_zone=mp4Cache:500m inactive=48h max_size=1800G use_temp_path=off;
proxy_cache_valid 200 500m;
proxy_set_header Range $slice_range;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_ignore_headers X-Accel-Expires X-Accel-Buffering Cache-Control Set-Cookie;
proxy_cache mp4Cache;
proxy_set_header X-Real-IP $http_cf_connecting_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
default conf
server {
server_name XXX;
set_real_ip_from 103.21.244.0/22;
set_real_ip_from 103.22.200.0/22;
set_real_ip_from 103.31.4.0/22;
set_real_ip_from 104.16.0.0/12;
set_real_ip_from 108.162.192.0/18;
set_real_ip_from 131.0.72.0/22;
set_real_ip_from 141.101.64.0/18;
set_real_ip_from 162.158.0.0/15;
set_real_ip_from 172.64.0.0/13;
set_real_ip_from 173.245.48.0/20;
set_real_ip_from 188.114.96.0/20;
set_real_ip_from 190.93.240.0/20;
set_real_ip_from 197.234.240.0/22;
set_real_ip_from 198.41.128.0/17;
set_real_ip_from 2400:cb00::/32;
set_real_ip_from 2606:4700::/32;
set_real_ip_from 2803:f800::/32;
set_real_ip_from 2405:b500::/32;
set_real_ip_from 2405:8100::/32;
set_real_ip_from 2c0f:f248::/32;
set_real_ip_from 2a06:98c0::/29;
real_ip_header CF-Connecting-IP;
location /secure {
secure_link $arg_s;
secure_link_md5 "secret $remote_addr";
if ( $secure_link = "" ) { return 403; }
rewrite ^/secure/(.*)$ /$1;
}
location / {
internal;
mp4;
slice 20m;
aio threads=mp4Cache;
aio_write on;
#mp4;
proxy_pass https://XX.XX.XX.XX$uri;
proxy_set_header Host $host;
proxy_buffering on;
proxy_pass_request_headers on;
proxy_set_header X-Real-IP $http_cf_connecting_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
##
# Caching
##
#proxy_cache mp4Cache;
#proxy_cache_lock on;
proxy_cache_key $uri;
#proxy_cache_lock_age 24h;
#proxy_cache_use_stale updating;
##
# Disabling Cookies
##
proxy_ignore_headers "Set-Cookie";
proxy_hide_header "Set-Cookie";
}
##
# SSL
##
listen 443 ssl; # managed by Certbot
ssl_certificate /etc/letsencrypt/live/XXX/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/XXX/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
server {
if ($host = XXX) {
return 301 https://$host$request_uri;
} # managed by Certbot
listen 80;
server_name XXX;
return 404; # managed by Certbot
}

Cherrypy NGINX error: 403 directory index of /some/path is forbidden

I am running NGINX on an Ubuntu 18.04 x64 Digital Ocean server. I have a Cherrypy app running directly on the Ubuntu server. I am trying to use NGINX to proxy_pass to my Cherrypy app for a specific route. The proxy_pass appears to be working, but I am getting a 403 Forbidden error when I try to POST to the route. The Cherrypy route works with the Python requests POST request when I test it locally, but doesn't work when I send the request through NGINX with the proxy_pass.
Cherrypy is in a Pipenv virtual environment. To run it I run Python3 app.py.
Here's the error in the NGINX error log:
2019/10/17 20:51:50 [error] 29574#29574: *51 directory index of "/mnt/media_storage/media_root/media/monday/monday-file-upload/" is forbidden, client: 73.14.140.118, server: media.bscs.org, request: "GET /monday/monday-file-upload/ HTTP/1.1", host: "media.bscs.org"
Here's my NGINX config:
# Microcaching
proxy_cache_path /tmp/cache keys_zone=cache:10m levels=1:2 inactive=600s max_size=100m;
# Cache in browser
# Expires map
map $sent_http_content_type $expires {
default off;
text/html epoch;
text/css 30d;
application/javascript 30d;
~image/ 30d;
}
upstream apps {
server 127.0.0.1:8080;
}
server {
listen 80;
listen [::]:80;
server_name media.bscs.org;
rewrite ^/(.*) https://media.bscs.org/$1 permanent;
}
server {
listen *:443 ssl http2;
listen [::]:443 ssl http2;
server_name media.bscs.org;
root /mnt/media_storage/media_root/media;
charset utf-8;
client_max_body_size 1000M;
# Gzip/compress text-based assets
gzip on;
gzip_http_version 1.0;
gzip_vary on;
gzip_comp_level 6;
gzip_proxied any;
gzip_types text/plain text/html text/css application/json application/x-javascript text/xml application/xml text/javascript application/javascript image/svg+xml;
gzip_disable "MSIE [1-6]\.";
# make sure gzip does not lose large gzipped js or css files
# see http://blog.leetsoft.com/2007/7/25/nginx-gzip-ssl
gzip_buffers 16 8k;
# Microcaching
proxy_cache cache;
proxy_cache_valid 200 1s;
# Cache in browser
expires $expires;
ssl on;
ssl_ciphers "my-cipher";
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_session_cache shared:SSL:10m;
add_header Strict-Transport-Security "max-age=63072000; includeSubdomains; preload";
add_header X-Content-Type-Options nosniff;
add_header 'Access-Control-Allow-Origin' '*';
ssl_session_tickets off;
ssl_stapling on;
ssl_stapling_verify on;
resolver_timeout 5s;
ssl_certificate /etc/nginx/ssl/cert_chain.crt;
ssl_certificate_key /etc/nginx/ssl/STAR.bscs.org.key;
ssl_dhparam /etc/nginx/ssl/dhparam.pem;
location = /favicon.ico {
access_log off;
log_not_found off;
sendfile on;
sendfile_max_chunk 1m;
}
location ~* \.(gif|jpg|jpeg|png|js|css)$ {
log_not_found off;
access_log off;
sendfile on;
sendfile_max_chunk 1m;
}
location /media/ {
alias /mnt/media_storage/media_root/media/;
location /media/monday/monday-file-upload/ {
alias /mnt/media_storage/media_root/media/monday/monday-file-upload/;
proxy_pass http://apps/;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
}
}
# Redirects
rewrite ^/tech-report/2018-1/2018-1.html$ https://bscs.org/resources/reports/designing-citizen-science-for-both-science-and-education-a-workshop-report/$1 permanent;
}
Here's my post request in with Python requests:
def uploadFileToMediaServer(uploaded_file_local_path):
with open(uploaded_file_local_path, 'rb') as f:
files = {'uploaded_file': f}
r = requests.post('https://media.bscs.org/monday/monday-file-upload', files=files)
print(r.request.url, file=sys.stderr)
print(r.request.headers, file=sys.stderr)
return r
Here's my Cherrypy app:
import cherrypy
from cherrypy.process.plugins import Daemonizer
config = {
'global': {
'server.socket_host': '127.0.0.1',
'server.socket_port': 8080,
'server.thread_pool': 8,
'server.max_request_body_size': 0,
'server.socket_timeout': 60
}
}
class App:
#cherrypy.expose
def index(self, uploaded_file):
try:
with open('../uploads/{}'.format(uploaded_file.filename), 'wb') as f:
while True:
data = uploaded_file.file.read(8192)
if not data:
return {'message': 'File failed to upload'}
f.write(data)
return {'message': 'File uploaded successfully'}
except Exception:
cherrypy.log(Exception, traceback=True)
if __name__ == '__main__':
d = Daemonizer(cherrypy.engine)
d.subscribe()
cherrypy.tree.mount(App(), "/", config)
cherrypy.engine.start()
cherrypy.engine.block()

modsecurity does not work if no required SSL certificate was sent

I have a lot of rules in modsecurity but none works if the host is numeric in SSL https://SERVER_IP, i get this response:
400 Bad Request No required SSL certificate was sent
My SSL is only valid to my domain name, but should not modsecurity work anyways? Because any request pass thru modsecurity before go to the application or something like that.
Question:
1 - How can i fix it?
2 - Why modsecurity does not work, and am i vunerable if i don't fix it?
This is my nginx.conf:
load_module modules/ngx_http_modsecurity_module.so;
user nobody;
worker_processes 1;
error_log /var/log/nginx/error.log error;
pid /var/run/nginx.pid;
events {
worker_connections 5000;
use epoll;
multi_accept on;
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
modsecurity on;
modsecurity_rules_file /etc/nginx/modsec/main.conf;
client_header_timeout 20s;
client_body_timeout 20s;
client_max_body_size 20m;
client_header_buffer_size 6k;
client_body_buffer_size 128k;
large_client_header_buffers 2 2k;
send_timeout 10s;
keepalive_timeout 30 30;
reset_timedout_connection on;
server_names_hash_max_size 1024;
server_names_hash_bucket_size 1024;
ignore_invalid_headers on;
connection_pool_size 256;
request_pool_size 4k;
output_buffers 4 32k;
postpone_output 1460;
include mime.types;
default_type application/octet-stream;
# SSL Settings
ssl_certificate /etc/nginx/ssl/cf_cert.pem;
ssl_certificate_key /etc/nginx/ssl/cf_key.pem;
ssl_client_certificate /etc/nginx/ssl/origin-pull-ca.pem;
ssl_verify_client on;
ssl_verify_depth 5;
ssl_session_cache shared:SSL:10m;
ssl_session_timeout 1h;
ssl_protocols TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers "EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH+aRSA!RC4:EECDH:!RC4:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS";
ssl_session_tickets on;
ssl_session_ticket_key /etc/nginx/ssl/ticket.key;
ssl_dhparam /etc/nginx/ssl/dhparam.pem;
ssl_ecdh_curve secp384r1;
ssl_buffer_size 4k;
# Logs
log_format main '$remote_addr - $remote_user [$time_local] $request '
'"$status" $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
log_format bytes '$body_bytes_sent';
access_log off;
# Cache bypass
map $http_cookie $no_cache {
default 0;
~SESS 1;
~wordpress_logged_in 1;
}
etag off;
server_tokens off;
# Headers
add_header X-XSS-Protection "1; mode=block" always;
add_header X-Frame-Options deny always;
server {
listen 443 ssl http2;
server_name domain.com;
root /home/user/public_html;
index index.php index.html;
access_log /var/log/domain/domain.com.bytes bytes;
access_log /var/log/domain/domain.com.log combined;
error_log /var/log/domain/domain.com.error.log warn;
location / {
location ~.*\.(jpeg|jpg|png|gif|bmp|ico|svg|css|js)$ {
expires max;
}
location ~ [^/]\.php(/|$) {
try_files $uri =404;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_pass unix:/opt/alt/php-fpm73/usr/var/sockets/user.sock;
fastcgi_index index.php;
include /etc/nginx/fastcgi_params;
}
}
}
}
In short: This is unrelated to modsecurity.
Your server configuration requires the client to send client certificate. The TLS handshake will fail, if the client does not send such certificate - and this is the error you see.
modsecurity only analyzes the application data at the HTTP level. With HTTPS the TLS handshake first needs to be successfully done before the any application data gets exchanged. Since in this case the TLS handshake fails due to no certificate send by the client, the connection gets closed before any HTTP data gets exchanged and thus before modsecurity is used.

NGINX Browser Caching Not Working - Node JS EC2

Went through several articles but cannot figure out why the browser caching isnt working. I am using prerender.io as well as SSL:
gzip on;
gzip_min_length 500;
gzip_proxied any;
gzip_comp_level 4;
gzip_types text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml;
gzip_vary on;
gzip_disable "msie6";
server {
listen 8080 default_server;
listen [::]:8080 default_server;
server_name <servername> www.<servername>.com;
return 301 https://$server_name$request_uri;
}
server {
listen 443 ssl http2 default_server;
server_name <servername> www.<servername>.com;
ssl_certificate /etc/pki/tls/private/<servername>.com.chained.crt;
ssl_certificate_key /etc/pki/tls/private/private.key;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers <ssl_ciphers_code>
ssl_session_cache shared:SSL:5m;
ssl_session_timeout 1h;
add_header Strict-Transport-Security "max-age=15768000" always;
root /var/app/current;
location / {
proxy_set_header X-Prerender-Token iKJwgCElYIfxtt9u99Zg;
set $prerender 0;
if ($http_user_agent ~* "baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest|slackbot|vkShare|W3C_Validator") {
set $prerender 1;
}
if ($args ~ "_escaped_fragment_") {
set $prerender 1;
}
if ($http_user_agent ~ "Prerender") {
set $prerender 0;
}
if ($uri ~* "\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)") {
set $prerender 0;
}
#resolve using Google's DNS server to force DNS resolution and prevent caching of IPs
resolver 8.8.8.8;
if ($prerender = 1) {
#setting prerender as a variable forces DNS resolution since nginx caches IPs and doesnt play well with load balancing
set $prerender "service.prerender.io";
rewrite .* /$scheme://$host$request_uri? break;
proxy_pass http://$prerender;
}
# Proxy_pass configuration
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-NginX-Proxy true;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_max_temp_file_size 0;
proxy_pass http://0.0.0.0:3000;
proxy_redirect off;
proxy_read_timeout 240s;
}
location ~* \.(ico|css|js|gif|jpeg|jpg|png|woff|ttf|otf|svg|woff2|eot)$ {
root /var/app/current/app/dist/client/; #if i comment this out it, my css and js files are not found...
expires 30d;
access_log off;
log_not_found off;
add_header Pragma "public";
add_header Cache-Control "public";
}
# Increase http2 max sizes
http2_max_field_size 64k;
http2_max_header_size 64k;
client_max_body_size 4G;
keepalive_timeout 10;
}
My assets dir is as follows:
JS: /var/app/current/app/dist/client/js
CSS: /var/app/current/app/dist/client/assets/css
Images: /var/app/current/app/dist/client/assets/graphics
Fonts: /var/app/current/app/dist/client/assets/fonts
Videos: /var/app/current/app/dist/client/assets/videos
UPDATED CONFIG:
gzip on;
gzip_min_length 500;
gzip_proxied any;
gzip_comp_level 4;
gzip_types text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml;
gzip_vary on;
gzip_disable "msie6";
server {
listen 8080 default_server;
listen [::]:8080 default_server;
server_name <servername.com> <www.servername.com>;
return 301 https://$server_name$request_uri;
}
server {
listen 443 ssl http2 default_server;
server_name <servername.com> <www.servername.com>;
ssl_certificate /etc/pki/tls/private/<servername>.com.chained.crt;
ssl_certificate_key /etc/pki/tls/private/private.key;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers <ciphers>;
ssl_session_cache shared:SSL:5m;
ssl_session_timeout 1h;
add_header Strict-Transport-Security "max-age=15768000" always;
root /var/app/current;
location / {
proxy_set_header X-Prerender-Token <token> ;
set $prerender 0;
if ($http_user_agent ~* "developers\.google\.com|googlebot|gigabot|yeti|yandex|ia_archiver|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest|slackbot|vkShare|W3C_Validator") {
set $prerender 1;
}
if ($args ~ "_escaped_fragment_") {
set $prerender 1;
}
if ($http_user_agent ~ "Prerender") {
set $prerender 0;
}
if ($uri ~* "\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)") {
set $prerender 0;
}
#resolve using Google's DNS server to force DNS resolution and prevent caching of IPs
resolver 8.8.8.8;
if ($prerender = 1) {
#setting prerender as a variable forces DNS resolution since nginx caches IPs and doesnt play well with load balancing
set $prerender "service.prerender.io";
rewrite .* /$scheme://$host$request_uri? break;
proxy_pass http://$prerender;
}
# Proxy_pass configuration
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-NginX-Proxy true;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_max_temp_file_size 0;
proxy_pass http://0.0.0.0:3000;
proxy_redirect off;
proxy_read_timeout 240s;
}
location ~* \.(ico|css|js|gif|jpeg|jpg|png|woff|ttf|otf|svg|woff2|eot)$ {
root /var/app/current/app/dist/client/; #if i comment this out it, my css and js files are not found...
expires 30d;
access_log off;
log_not_found off;
add_header Pragma "public";
add_header Cache-Control "public";
}
location /assets/graphics/ {
proxy_ignore_headers Cache-Control;
proxy_cache_valid any 30d;
}
# Increase http2 max sizes
proxy_buffers 8 16k;
proxy_buffer_size 32k;
http2_max_field_size 64k;
http2_max_header_size 64k;
client_max_body_size 4G;
keepalive_timeout 10;
}
In your NGINX configuration, you set the cache expiration to 30 days for your images with the line:
expires 30d;
However looking at your images coming from your server, the max-age of the images is set to 0 which is causing your browser to re-pull the images on a refresh (below image was after a refresh):
I suspect that NGINX is acting as a proxy to an origin server that is part of your solution. This origin server is setting the max-age to 0 in the cache-control header and NGINX is respecting that setting.
Per the NGINX caching guide:
By default, NGINX respects the Cache-Control headers from origin servers. It does not cache responses with Cache-Control set to Private, No-Cache, or No-Store or with Set-Cookie in the response header. NGINX only caches GET and HEAD client requests.
To override the cache-control set at the origin server and set the max-age to 30d, use the NGINX proxy_ignore_headers and proxy_cache_valid directive like so:
...
location /assets/graphics/ {
proxy_ignore_headers Cache-Control;
proxy_cache_valid any 30d;
...
}
...
The code in my solution is taken directly from the NGINX caching guide and modified to your configuration.
Or determine how to change the cache control headers at the origin server.
UPDATE
After you updated your NGINX config, your images in the /assets/graphics/ directory are pulled from local browser memory and have expiration of 30 days (2595200) as seen below. Yesterday, they were all being pulled from your server and not being cached. This solution solves your problem. For the rest of the assets that you want cached, you need to further change your config to also cache these according to your requirements.

API endpoint performance

I am attempting to benchmark the perf of my rest API endpoint. It's a node/mongdb endpoint with a pretty nasty aggregate query - a few unwraps, groups and a lookup. I am using Apache Bench (ab) to test it perf under 100 concurrency 1000 times. My issue: I am getting different results based on where I run the ab test:
localhost:3000/api_endpoint/ using ngrok to tunnel it = I am
getting around 50 requests per second.
Running the test on the
production app server directly where the app is hosed I am seeing
6000 requests per second.
Running it from my web server/reverse proxy I am seeing similarly 5000reqs per second -if I use a relative path to the API.
NOW: If I run the same test on the web server
but this time use the domain name instead or run it from local dev
machine but hitting the domain, now it drops to 9 requests per
second.
What can I deduce this? To me, It seems the query must not be that nasty if I am getting such high perf when I hit it directly. It seems I either have something configured wrong in my nginx.conf file for handing my domain requests or there is a DNS resolution problem? Its an exotic TLD (.ly) could that be it? Where should I look?
Edit: Adding my nginx.conf file for analysis:
user www-data;
worker_processes 1;
#error_log /log/nginx/error.log;
pid /run/nginx.pid;
# Load dynamic modules. See /usr/share/nginx/README.dynamic.
#include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
http {
#optimizations per digitalOcean
client_body_buffer_size 10K;
client_header_buffer_size 1k;
client_max_body_size 8m;
large_client_header_buffers 2 1k;
client_body_timeout 12;
client_header_timeout 12;
keepalive_timeout 15;
send_timeout 10;
#compression
gzip on;
gzip_disable "msie6";
gzip_proxied any;
gzip_comp_level 6;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_min_length 256;
gzip_proxied expired no-cache no-store private auth;
gzip_types text/plain application/x-javascript text/xml text/css application/xml application/json text/xml application/xml+rss application/javascript text/javascript application/vnd.ms-fontobject application/x-font-ttf font/opentype image/svg+xml image/x-icon;
access_log off;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
#access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
# Load modular configuration files from the /etc/nginx/conf.d directory.
# See http://nginx.org/en/docs/ngx_core_module.html#include
# for more information.
include /etc/nginx/conf.d/*.conf;
#redirect to https
server {
server_name www.staq.ly staq.ly getstackrank.com www.getstackrank.com;
return 301 https://staq.ly;
}
server {
listen 80;
listen 443 ssl;
server_name staq.ly;
ssl_certificate /etc/nginx/ssl/staqly.crt;
ssl_certificate_key /etc/nginx/ssl/staqly.key;
root /var/local;
location ~ ^/(sitemap.xml) {
}
location ~ ^/(robots.txt) {
}
location ~ ^/(googlee828ea2f1ef594b3.html) {
}
location / {
try_files $uri #prerender;
}
location #prerender {
proxy_set_header X-Prerender-Token XXXXXXXX;
set $prerender 0;
if ($http_user_agent ~* "baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest|slackbot|vkShare|W3C_Validator") {
set $prerender 1;
}
if ($args ~ "_escaped_fragment_") {
set $prerender 1;
}
if ($http_user_agent ~ "Prerender") {
set $prerender 0;
}
if ($uri ~* "\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)") {
set $prerender 0;
}
#resolve using Google's DNS server to force DNS resolution and prevent caching of IPs
resolver 8.8.8.8;
if ($prerender = 1) {
#setting prerender as a variable forces DNS resolution since nginx caches IPs and doesnt play well with load balancing
set $prerender "service.prerender.io";
rewrite .* /$scheme://$host$request_uri? break;
proxy_pass http://$prerender;
}
if ($prerender = 0) {
# rewrite .* /index.ejs break; # Throw away the path because this is a single page web app with only an index.html
proxy_pass http://10.132.126.36:3000;
}
}
}
}

Resources