Varnish http to https - varnish

I am new to varnish or programming in general. I want to setup http to https redirect for my domain.
Here's what I have so far
sub vcl_recv {
if (req.http.X-Forwarded-Proto != "https" || req.http.Scheme != "https"){
error 750 "http to https";
}
sub vcl_error {
if (obj.status == 750){
set obj.status = 302;
set obj.http.location = "https://" + req.http.Host + req.url;
}
return(deliver);
}
However, this seems to trigger for https requests instead of http. What am I doing wrong?

You seem to be using syntax from an extremely old version of Varnish.
See https://www.varnish-software.com/developers/tutorials/redirect/ for an official tutorial on HTTP to HTTPS redirection.
Here's a code snippet from that tutorial that allows you to redirect HTTP to HTTPS:
vcl 4.1;
import proxy;
backend default {
.host = "127.0.0.1";
.port = 8080;
}
sub vcl_recv {
if ((req.http.X-Forwarded-Proto && req.http.X-Forwarded-Proto != "https") ||
(req.http.Scheme && req.http.Scheme != "https")) {
return (synth(750));
} elseif (!req.http.X-Forwarded-Proto && !req.http.Scheme && !proxy.is_ssl()) {
return (synth(750));
}
}
sub vcl_synth {
if (resp.status == 750) {
set resp.status = 301;
set resp.http.location = "https://" + req.http.Host + req.url;
set resp.reason = "Moved";
return (deliver);
}
}

Related

Varnish Request Url was not the same with apache request

I am sending a request to Varnish but the request was changed so that the output is wrong
original request:
admin/catalog/category/grid/key/d53fe1b4a969a873d1e0b2174848cef5b10797c35dce4c51be04d470db648e8c/id/2/filter//page/2/?ajax=true&isAjax=true
request which was pushed to varnish (varnishlog)
admin/catalog/category/grid/key/b5fbeb78a5bd6bedcb698beac083361a9891a04484bb59541b69e9420fb691aa/id/2/filter/page/3/?ajax=true&isAjax=true
a slightly different between the 2 requests is the filter/ and filter// part.
When coming with an original request: the site takes filter as param and with value ""
But with Request pushed to varnish: it becomes filter as param and with value page, then param page was not found and made the code go wrongs.
Adding my vcl content:
vcl 4.0;
import std;
# The minimal Varnish version is 4.0
# For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https'
backend default {
.host = "web";
.port = "80";
}
acl purge {
"web";
"localhost";
"127.0.0.1";
}
sub vcl_recv {
if (req.method == "PURGE") {
if (client.ip !~ purge) {
return (synth(405, "Method not allowed"));
}
if (!req.http.X-Magento-Tags-Pattern) {
return (purge);
}
if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
}
if (req.http.host && req.http.host != "") {
ban("obj.http.X-Host ~ " + req.http.host + " && obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
} else {
ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
}
# If all Tags should be purged clear
# ban everything to catch assets as well
if (req.http.X-Magento-Tags-Pattern == ".*") {
ban("req.url ~ .*");
}
return (synth(200, "Purged"));
}
if (req.method != "GET" &&
req.method != "HEAD" &&
req.method != "PUT" &&
req.method != "POST" &&
req.method != "TRACE" &&
req.method != "OPTIONS" &&
req.method != "DELETE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
# We only deal with GET and HEAD by default
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}
# Bypass shopping cart, checkout
if (req.url ~ "/checkout") {
return (pass);
}
# normalize url in case of leading HTTP scheme and domain
set req.url = regsub(req.url, "^http[s]?://", "");
# collect all cookies
std.collect(req.http.Cookie);
# Remove Google gclid parameters to minimize the cache objects
set req.url = regsuball(req.url,"\?gclid=[^&]+$",""); # strips when QS = "?gclid=AAA"
set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"
# static files are always cacheable. remove SSL flag and cookie
if (req.url ~ "^/(pub/)?(media|static)/.*\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)$") {
unset req.http.Https;
unset req.http.X-Forwarded-Proto;
unset req.http.Cookie;
}
return (hash);
}
sub vcl_hash {
if (req.http.cookie ~ "X-Magento-Vary=") {
hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
}
# For multi site configurations to not cache each other's content
if (req.http.host) {
hash_data(req.http.host);
} else {
hash_data(server.ip);
}
# To make sure http users don't see ssl warning
if (req.http.X-Forwarded-Proto) {
hash_data(req.http.X-Forwarded-Proto);
}
}
sub vcl_backend_response {
set beresp.http.X-Host = bereq.http.host;
if (beresp.http.content-type ~ "text") {
set beresp.do_esi = true;
}
if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
set beresp.do_gzip = true;
}
# cache only successfully responses and 404s
if (beresp.status != 200 && beresp.status != 404) {
set beresp.ttl = 0s;
set beresp.uncacheable = true;
return (deliver);
} elsif (beresp.http.Cache-Control ~ "private") {
set beresp.uncacheable = true;
set beresp.ttl = 86400s;
return (deliver);
}
if (beresp.http.X-Magento-Debug) {
set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
}
# validate if we need to cache it and prevent from setting cookie
# images, css and js are cacheable by default so we have to remove cookie also
if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
unset beresp.http.set-cookie;
if (bereq.url !~ "\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)(\?|$)") {
set beresp.http.Pragma = "no-cache";
set beresp.http.Expires = "-1";
set beresp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
set beresp.grace = 1m;
}
}
# "Microcache" for search
if (bereq.url ~ "/catalogsearch") {
set beresp.ttl = 30m;
}
# If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
if (beresp.ttl <= 0s ||
beresp.http.Surrogate-control ~ "no-store" ||
(!beresp.http.Surrogate-Control && beresp.http.Vary == "*")) {
# Mark as Hit-For-Pass for the next 2 minutes
set beresp.ttl = 120s;
set beresp.uncacheable = true;
}
return (deliver);
}
sub vcl_deliver {
if (resp.http.X-Magento-Debug) {
if (resp.http.x-varnish ~ " ") {
set resp.http.X-Magento-Cache-Debug = "HIT";
} else {
set resp.http.X-Magento-Cache-Debug = "MISS";
}
} else {
unset resp.http.Age;
}
unset resp.http.X-Magento-Debug;
unset resp.http.X-Magento-Tags;
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.Link;
}

Varnish add more slash in URL

My varnish cache is accelerator mode and it cause adding more slash after URL.
I check with gtmetrix:
Avoid landing page redirects for the following chain of redirected URLs.
https://www.example2.com/
https://www.example2.com//
https://www.example2.com/
and with google pagespeed test:
Reduce server response time
In our test, your server responded in 1.6 seconds. There are many factors that can slow down your server response time.
Here is my config:
vcl 4.0;
backend default {
.host = "localhost";
.port = "8080";
}
acl purger {
"localhost";
"XXX.XXX.XXX.XXX";
}
sub vcl_recv {
if (client.ip != "127.0.0.1" && req.http.host ~ "XXX.com") {
set req.http.x-redir = "https://www.example.com/" + req.url;
return(synth(850, ""));
}
if (client.ip != "127.0.0.1" && req.http.host ~ "YYY.com") {
set req.http.x-redir = "https://www.example2.com/" + req.url;
return(synth(850, ""));
}
if (req.method == "PURGE") {
if (!client.ip ~ purger) {
return(synth(405, "This IP is not allowed to send PURGE requests."));
}
return (purge);
}
if (req.restarts == 0) {
if (req.http.X-Forwarded-For) {
set req.http.X-Forwarded-For = client.ip;
}
}
if (req.http.Authorization || req.method == "POST") {
return (pass);
}
if (req.url ~ "/feed") {
return (pass);
}
set req.http.cookie = regsuball(req.http.cookie, "wp-settings-\d+=[^;]+(; )?", "");
set req.http.cookie = regsuball(req.http.cookie, "wp-settings-time-\d+=[^;]+(; )?", "");
if (req.http.cookie == "") {
unset req.http.cookie;
}
}
sub vcl_synth {
if (resp.status == 850) {
set resp.http.Location = req.http.x-redir;
set resp.status = 302;
return (deliver);
}
}
sub vcl_purge {
set req.method = "GET";
set req.http.X-Purger = "Purged";
return (restart);
}
sub vcl_backend_response {
set beresp.ttl = 24h;
set beresp.grace = 1h;
}
sub vcl_deliver {
if (req.http.X-Purger) {
set resp.http.X-Purger = req.http.X-Purger;
}
}
Somebody know the reason why varnish do that ?
req.url always includes a / at the beginning.
so correct would be:
set req.http.x-redir = "https://www.example2.com" + req.url;
your question does not correspond to the example1/example2 as it includes XXX and YYY
if you can post a less redacted VCL - i may can help you a bit more
you may need a != "/" on req.url before redirecting. as this would result in a //

Varnish (v-4.1.2) HTTP Live Streaming got Halt

Few days back i have deployed Varnish 4.1.2 on Debian 8.4 64bit...Service configuration is OK but while i am starting Streaming by it, the streaming is freezes for 60s and then again automatically started running for next 2 or 3 minutes and again freezes for another 60s..
While i dont face this problem in Varnish 3...
here is my varnish 4.1.2 configuration...
default.conf
> vcl 4.0;
import std; import directors; import querystring;
backend server1 { .host = "192.168.4.1"; .probe = {
.url = "/hls/newtv/index.m3u8";
.interval = 5s;
.timeout = 1s;
.window = 5;
.threshold = 3; } }
sub vcl_init { new vdir = directors.round_robin(); vdir.add_backend(server1); }
sub vcl_recv { set req.backend_hint = vdir.backend(); # send all traffic to the vdir director set req.http.Host = regsub(req.http.Host, ":[0-9]+", ""); set req.url = std.querysort(req.url);
if (req.url ~ "^[^?]*\.(m3u8)(\?.*)?$") {
unset req.http.Cookie;
return (hash); }
if (req.http.Authorization) {
# Not cacheable by default
return (pass); }
return (hash); } sub vcl_pipe { if (req.http.upgrade) {
set bereq.http.upgrade = req.http.upgrade; }
return (pipe); }
sub vcl_hash {
if (req.method == "GET" || req.method == "HEAD") {
hash_data(querystring.remove(req.url));
}
else {
hash_data(req.url);
}
hash_data(req.http.host);
return (lookup);
}
sub vcl_hit {
if (obj.ttl >= 0s) {
return (deliver); }
if (std.healthy(req.backend_hint)) {
if (obj.ttl + 10s > 0s) {
return (deliver);
} else {
return(fetch);
} } else {
if (obj.ttl + obj.grace > 0s) {
return (deliver);
} else {
return (fetch);
} } return (fetch); # Dead code, keep as a safeguard }
sub vcl_miss { return (fetch); }
sub vcl_backend_response { if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
unset beresp.http.Surrogate-Control;
set beresp.do_esi = true; }
if (bereq.url ~ "^[^?]*\.(m3u8)(\?.*)?$") {
unset beresp.http.set-cookie; }
if (bereq.url ~ "^[^?]*\.(m3u8)(\?.*)?$") {
unset beresp.http.set-cookie;
set beresp.do_stream = true;
set beresp.do_gzip = false; }
if (beresp.status == 301 || beresp.status == 302) {
set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", ""); }
if (beresp.status == 500 || beresp.status == 502 || beresp.status == 503 || beresp.status == 504) {
return (abandon); }
set beresp.grace = 6h;
return (deliver); }
sub vcl_deliver {
if (obj.hits > 0) { # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed
set resp.http.X-Cache = "HIT"; } else { set resp.http.X-Cache = "MISS"; }
set resp.http.X-Cache-Hits = obj.hits;
return (deliver); }
sub vcl_purge { if (req.method != "PURGE") {
# restart request
set req.http.X-Purge = "Yes";
return(restart); } }
sub vcl_synth { if (resp.status == 720) {
set resp.http.Location = resp.reason;
set resp.status = 301;
return (deliver); } elseif (resp.status == 721) {
set resp.http.Location = resp.reason;
set resp.status = 302;
return (deliver); }
return (deliver); }
sub vcl_fini {
return (ok); }
And /etc/default/varnish
START=yes
NFILES=131072
MEMLOCK=82000
DAEMON_OPTS="-a 192.168.2.1:80 \
-T localhost:6082 \
-f /etc/varnish/default.vcl \
-S /etc/varnish/secret \
-s malloc,256m"
Can anyone help me through out the problem !!!
thanks in advance...
I don't think this is a configuration problem. It sounds like a Varnish bug and I suggest you file it in the official bugtracker.
If you are in a hurry, I'd recommend trying set beresp.do_stream = False but I'm not convinced that will help.

using varnish for load balancing

i write a varnish vcl rules for my server and i don't know this is sufficient for my intent. i want run my project on 2 ports of server and on another server such that if one server gone down ,requests forward to other server. moreover i want to serve static files such that don't working apache web server and request for them directly sent to tornado. this is my vcl rules , can u help me ?
import directors;
probe healthcheck {
.url = "/";
.interval = 30s;
.timeout = 1 s;
.window = 5;
.threshold = 2;
.initial = 2;
.expected_response = 200;
}
backend server1{
.host="127.0.0.1"
.port = 8004
.probe = healtcheck;
}
backend server1-2{
.host="127.0.0.1"
.port = 8005
.probe = healtcheck;
}
backend server2{
.host="192.168.1.1"
.port = 8004
.probe = healtcheck;
}
sub vcl_init {
new vdir = directors.round_robin();
vdir.add_backend(server1);
vdir.add_backend(server1-2);
vdir.add_backend(server2);
}
sub vcl_recv{
set req.grace = 600s;
### always cache these items:
if (req.request == "GET" && req.url ~ "\.(js)") {
lookup;
}
## images
if (req.request == "GET" && req.url ~ "\.(gif|jpg|jpeg|bmp|png|tiff|tif|ico|img|tga|wmf)$") {
lookup;
}
## various other content pages
if (req.request == "GET" && req.url ~ "\.(css|html)$") {
lookup;
}
## multimedia
if (req.request == "GET" && req.url ~ "\.(svg|swf|ico|mp3|mp4|m4a|ogg|mov|avi|wmv)$") {
lookup;
}
## xml
if (req.request == "GET" && req.url ~ "\.(xml)$") {
lookup;
}
### do not cache these rules:
if (req.request != "GET" && req.request != "HEAD") {
pipe;
}
### if it passes all these tests, do a lookup anyway;
lookup;
}
sub vcl_fetch {
if (req.url ~ "\.(gif|jpg|jpeg|swf|css|js|flv|mp3|mp4|pdf|ico|png)(\?.*|)$") {
set beresp.ttl = 365d;
}
}
Depending on if you require sessions to work for users you should switch from round_robin to hash. This will ensure the same users will be directed to the same backend.

Varnish frontend server, Litespeed backend server

I wonder if someone could advise what I'm doing wrong.
I have Varnish set up on a frontend server, and this is working. I can put the IP address for several public websites into this (including CNN) into the configuration, and they are cached by Varnish.
When I put one of my site IP address' into Varnish, I get a 404 error back from the Litespeed server - this shows it's connecting to the server, but obviously not picking up the correct site to show Varnish. I have several domains on the Litespeed server, each with it's own IP address. The same happens with the direct URL.
This is I guess a problem with the Litespeed configuration - but any assistance would be appreciated.
For reference, I'm using the VCL configuration file for PageCache, which is a Magento module for Varnish.
Justin
# default backend definition. Set this to point to your content server.
backend default {
.host = "xxx.xxx.xxx.xxx";
.port = "80";
}
# admin backend with longer timeout.
backend admin {
.host = "xxx.xxx.xxx.xxx";
.port = "80";
.first_byte_timeout = 18000s;
.between_bytes_timeout = 18000s;
}
# add your Magento server IP to allow purges from the backend
acl purge {
"localhost";
"127.0.0.1";
}
sub vcl_recv {
if (req.restarts == 0) {
if (req.http.x-forwarded-for) {
set req.http.X-Forwarded-For =
req.http.X-Forwarded-For + ", " + client.ip;
} else {
set req.http.X-Forwarded-For = client.ip;
}
}
if (req.request != "GET" &&
req.request != "HEAD" &&
req.request != "PUT" &&
req.request != "POST" &&
req.request != "TRACE" &&
req.request != "OPTIONS" &&
req.request != "DELETE" &&
req.request != "PURGE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
# purge request
if (req.request == "PURGE") {
if (!client.ip ~ purge) {
error 405 "Not allowed.";
}
ban("obj.http.X-Purge-Host ~ " + req.http.X-Purge-Host + " && obj.http.X-Purge-URL ~ " + req.http.X-Purge-Regex + " && obj.http.Content-Type ~ " + req.http.X-Purge-Content-Type);
error 200 "Purged.";
}
# switch to admin backend configuration
if (req.http.cookie ~ "adminhtml=") {
set req.backend = admin;
}
# we only deal with GET and HEAD by default
if (req.request != "GET" && req.request != "HEAD") {
return (pass);
}
# normalize url in case of leading HTTP scheme and domain
set req.url = regsub(req.url, "^http[s]?://[^/]+", "");
# static files are always cacheable. remove SSL flag and cookie
if (req.url ~ "^/(media|js|skin)/.*\.(png|jpg|jpeg|gif|css|js|swf|ico)$") {
unset req.http.Https;
unset req.http.Cookie;
}
# not cacheable by default
if (req.http.Authorization || req.http.Https) {
return (pass);
}
# do not cache any page from
# - index files
# - ...
if (req.url ~ "^/(index)") {
return (pass);
}
# as soon as we have a NO_CACHE cookie pass request
if (req.http.cookie ~ "NO_CACHE=") {
return (pass);
}
# normalize Aceept-Encoding header
# http://varnish.projects.linpro.no/wiki/FAQ/Compression
if (req.http.Accept-Encoding) {
if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
# No point in compressing these
remove req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
set req.http.Accept-Encoding = "deflate";
} else {
# unkown algorithm
remove req.http.Accept-Encoding;
}
}
# remove Google gclid parameters
set req.url = regsuball(req.url,"\?gclid=[^&]+$",""); # strips when QS = "?gclid=AAA"
set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"
return (lookup);
}
sub vcl_hash {
hash_data(req.url);
if (req.http.host) {
hash_data(req.http.host);
} else {
hash_data(server.ip);
}
if (!(req.url ~ "^/(media|js|skin)/.*\.(png|jpg|jpeg|gif|css|js|swf|ico)$")) {
call design_exception;
}
return (hash);
}
sub vcl_fetch {
if (beresp.status == 500) {
set beresp.saintmode = 10s;
return (restart);
}
set beresp.grace = 5m;
# add ban-lurker tags to object
set beresp.http.X-Purge-URL = req.url;
set beresp.http.X-Purge-Host = req.http.host;
if (beresp.status == 200 || beresp.status == 301 || beresp.status == 404) {
if (beresp.http.Content-Type ~ "text/html" || beresp.http.Content-Type ~ "text/xml") {
if ((beresp.http.Set-Cookie ~ "NO_CACHE=") || (beresp.ttl < 1s)) {
set beresp.ttl = 0s;
return (hit_for_pass);
}
# marker for vcl_deliver to reset Age:
set beresp.http.magicmarker = "1";
# Don't cache cookies
unset beresp.http.set-cookie;
} else {
# set default TTL value for static content
set beresp.ttl = 4h;
}
return (deliver);
}
return (hit_for_pass);
}
sub vcl_deliver {
# debug info
if (resp.http.X-Cache-Debug) {
if (obj.hits > 0) {
set resp.http.X-Cache = "HIT";
set resp.http.X-Cache-Hits = obj.hits;
} else {
set resp.http.X-Cache = "MISS";
}
set resp.http.X-Cache-Expires = resp.http.Expires;
} else {
# remove Varnish/proxy header
remove resp.http.X-Varnish;
remove resp.http.Via;
remove resp.http.Age;
remove resp.http.X-Purge-URL;
remove resp.http.X-Purge-Host;
}
if (resp.http.magicmarker) {
# Remove the magic marker
unset resp.http.magicmarker;
set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0";
set resp.http.Pragma = "no-cache";
set resp.http.Expires = "Mon, 31 Mar 2008 10:00:00 GMT";
set resp.http.Age = "0";
}
}
sub design_exception {
}
And...
START=yes
NFILES=131072
MEMLOCK=82000
INSTANCE=$(uname -n)
## Alternative 2, Configuration with VCL
#
# Listen on port 6080, administration on localhost:6082, and forward to
# one content server selected by the vcl file, based on the request. Use a 1GB
# fixed-size cache file.
DAEMON_OPTS="-a :80 \
-T localhost:6082 \
-f /etc/varnish/default.vcl \
-S /etc/varnish/secret \
-s malloc,256m"
To what port is your Varnish server bound? And what port do you use for Litespeed?
It would be useful if you could post your config files here to give you further advices.
This is what I needed! Configure multiple sites with Varnish
I needed to put the virtual host names into the configuration file. Followed this guide and had it work in moments!

Resources