Related
I'm trying to manage a Xkey soft-purge on objects stored from different hosts, using Xkey as a sort of tag to purge all objects that match the Xkey tag disregarding the hashing.
First of all, is it possible? Or is hashing in the loop in any case?
In vcl_recv I manually set a Xkey key with set req.http.xkey = req.url;
Then during PURGE call I use set req.http.x-purges = xkey.purge(req.url); to find objects, here non objects are found.
Full VCL:
vcl 4.0;
import std;
import directors;
import purge;
import xkey;
backend server1 {
.host = "ecommerce-node1-prod";
.port = "80";
.probe = {
.url = "/admin";
.timeout = 1s;
.interval = 5s;
.window = 5;
.threshold = 3;
}
}
backend server2 {
.host = "ecommerce-node2-prod";
.port = "80";
.probe = {
.url = "/admin";
.timeout = 1s;
.interval = 5s;
.window = 5;
.threshold = 3;
}
}
sub vcl_init {
new bar = directors.round_robin();
bar.add_backend(server1);
bar.add_backend(server2);
}
# ACL for purgers IP. (This needs to contain app server ips)
acl purgers {
"127.0.0.1";
"localhost";
"::1";
"ecommerce-node1-prod";
"ecommerce-node2-prod";
}
sub vcl_recv {
# Mitigate httpoxy application vulnerability, see: https://httpoxy.org/
unset req.http.Proxy;
set req.http.xkey = req.url;
# Strip query strings only needed by browser javascript. Customize to used tags.
if (req.url ~ "(\?|&)(pk_campaign|piwik_campaign|pk_kwd|piwik_kwd|pk_keyword|pixelId|kwid|kw|adid|chl|dv|nk|pa|camid|adgid|cx|ie|cof|siteurl|utm_[a-z]+|_ga|gclid)=") {
# see rfc3986#section-2.3 "Unreserved Characters" for regex
set req.url = regsuball(req.url, "(pk_campaign|piwik_campaign|pk_kwd|piwik_kwd|pk_keyword|pixelId|kwid|kw|adid|chl|dv|nk|pa|camid|adgid|cx|ie|cof|siteurl|utm_[a-z]+|_ga|gclid)=[A-Za-z0-9\-\_\.\~]+&?", "");
}
set req.url = regsub(req.url, "(\?|\?&|&)$", "");
# Normalize query arguments
set req.url = std.querysort(req.url);
# Normalize Accept-Encoding header
# straight from the manual: https://www.varnish-cache.org/docs/3.0/tutorial/vary.html
if (req.http.Accept-Encoding) {
if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
# No point in compressing these
unset req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";
} else {
# unkown algorithm
unset req.http.Accept-Encoding;
}
}
# Handle PURGE
if (req.method == "PURGE") {
if (!client.ip ~ purgers) {
return (synth(405, "Method not allowed"));
}
#set req.http.n-gone = xkey.softpurge(req.http.xkey);
set req.http.x-purges = xkey.purge(req.url);
return (synth(200, "Invalidated "+ req.http.x-purges +" objects"));
}
if (req.method != "GET" &&
req.method != "HEAD" &&
req.method != "PUT" &&
req.method != "POST" &&
req.method != "TRACE" &&
req.method != "OPTIONS" &&
req.method != "PATCH" &&
req.method != "DELETE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
# We only deal with GET and HEAD by default
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}
#pass media to backend (already cached by CloudFront)
if (req.url ~ "^\/(media|bundles|css|fonts|js|theme|thumbnail)(\/.*)?$") {
return (pass);
}
# Don't cache Authenticate & Authorization
if (req.http.Authenticate || req.http.Authorization) {
return (pass);
}
# Always pass these paths directly to php without caching
# Note: virtual URLs might bypass this rule (e.g. /en/checkout)
if (req.url ~ "^/(checkout|account|admin|api)(/.*)?$") {
return (pass);
}
return (hash);
}
sub vcl_hash {
# Consider Shopware http cache cookies
if (req.http.cookie ~ "sw-cache-hash=") {
hash_data("+context=" + regsub(req.http.cookie, "^.*?sw-cache-hash=([^;]*);*.*$", "\1"));
} elseif (req.http.cookie ~ "sw-currency=") {
hash_data("+currency=" + regsub(req.http.cookie, "^.*?sw-currency=([^;]*);*.*$", "\1"));
}
#return(lookup);
}
sub vcl_hit {
# Consider client states for response headers
if (req.http.cookie ~ "sw-states=") {
set req.http.states = regsub(req.http.cookie, "^.*?sw-states=([^;]*);*.*$", "\1");
if (req.http.states ~ "logged-in" && obj.http.sw-invalidation-states ~ "logged-in" ) {
return (pass);
}
if (req.http.states ~ "cart-filled" && obj.http.sw-invalidation-states ~ "cart-filled" ) {
return (pass);
}
}
}
sub vcl_backend_response {
# Fix Vary Header in some cases
if (beresp.http.Vary ~ "User-Agent") {
set beresp.http.Vary = regsub(beresp.http.Vary, ",? *User-Agent *", "");
set beresp.http.Vary = regsub(beresp.http.Vary, "^, *", "");
if (beresp.http.Vary == "") {
unset beresp.http.Vary;
}
}
# Respect the Cache-Control=private header from the backend
if (
beresp.http.Pragma ~ "no-cache" ||
beresp.http.Cache-Control ~ "no-cache" ||
beresp.http.Cache-Control ~ "private"
) {
set beresp.ttl = 0s;
set beresp.http.X-Cacheable = "NO:Cache-Control=private";
set beresp.uncacheable = true;
return (deliver);
}
# strip the cookie before the image is inserted into cache.
if (bereq.url ~ "\.(png|gif|jpg|swf|css|js|webp)$") {
unset beresp.http.set-cookie;
}
# Allow items to be stale if needed.
set beresp.grace = 60s;
# Save the bereq.url so purges work efficiently
set beresp.http.x-url = bereq.url;
set beresp.http.X-Cacheable = "YES";
# Remove the exact PHP Version from the response for more security
unset beresp.http.x-powered-by;
return (deliver);
}
sub vcl_deliver {
## we don't want the client to cache
set resp.http.Cache-Control = "max-age=0, private";
# remove link header, if session is already started to save client resources
if (req.http.cookie ~ "session-") {
unset resp.http.Link;
}
# Set a cache header to allow us to inspect the response headers during testing
if (obj.hits > 0) {
unset resp.http.set-cookie;
set resp.http.X-Cache = "HIT";
} else {
set resp.http.X-Cache = "MISS";
}
# Remove the exact PHP Version from the response for more security (e.g. 404 pages)
unset resp.http.x-powered-by;
# invalidation headers are only for internal use
unset resp.http.sw-invalidation-states;
set resp.http.X-Cache-Hits = obj.hits;
}
Request where xkey is added
* << Request >> 32796
- Begin req 32795 rxreq
- Timestamp Start: 1664372821.615382 0.000000 0.000000
- Timestamp Req: 1664372821.615382 0.000000 0.000000
- VCL_use boot
- ReqStart 172.16.0.80 20244 a0
- ReqMethod GET
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- ReqProtocol HTTP/1.1
- ReqHeader X-Forwarded-For: 84.247.245.84, 130.176.89.143
- ReqHeader X-Forwarded-Proto: https
- ReqHeader X-Forwarded-Port: 443
- ReqHeader Host: test-prod.luminalpark.com
- ReqHeader X-Amzn-Trace-Id: Root=1-63345055-76f876fd27d8613e28b7c9db
- ReqHeader User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36 Edg/105.0.1343.53
- ReqHeader X-Amz-Cf-Id: JA7XrQuv5k-jhdhb3uKmlf88YjlJVvcDcw6udobH5vL-YkSMDT2flw==
- ReqHeader Via: 2.0 6c61cea6f371b1744d3b5315a0029062.cloudfront.net (CloudFront)
- ReqHeader Cookie: _dc_gtm_UA-830149-18=1; _ga=GA1.1.1015042543.1664369859; _ga_499EG6CXZD=GS1.1.1664371930.2.1.1664372814.0.0.0; _gcl_au=1.1.1616057317.1664369858; _gid=GA1.2.1284203691.1664369859; lp-platform=it-it; lp-state=IT; newUser=1; poll_manager=1; session
- ReqHeader Accept-Language: it,it-IT;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6,fr;q=0.5
- ReqHeader Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
- ReqHeader Accept-Encoding: gzip, deflate, br
- ReqHeader cache-control: max-age=0
- ReqHeader sec-ch-ua: "Microsoft Edge";v="105", "Not)A;Brand";v="8", "Chromium";v="105"
- ReqHeader sec-ch-ua-mobile: ?0
- ReqHeader sec-ch-ua-platform: "Windows"
- ReqHeader dnt: 1
- ReqHeader upgrade-insecure-requests: 1
- ReqHeader sec-fetch-site: none
- ReqHeader sec-fetch-mode: navigate
- ReqHeader sec-fetch-user: ?1
- ReqHeader sec-fetch-dest: document
- ReqHeader CloudFront-Viewer-HTTP-Version: 2.0
- ReqHeader CloudFront-Forwarded-Proto: https
- ReqHeader CloudFront-Viewer-Address: 84.247.245.84:59146
- ReqHeader CloudFront-Viewer-TLS: TLSv1.3:TLS_AES_128_GCM_SHA256:connectionReused
- ReqHeader X-Cloudfront-Origin: VC5ZNQ588QNE3S
- ReqUnset X-Forwarded-For: 84.247.245.84, 130.176.89.143
- ReqHeader X-Forwarded-For: 84.247.245.84, 130.176.89.143, 172.16.0.80
- ReqUnset Via: 2.0 6c61cea6f371b1744d3b5315a0029062.cloudfront.net (CloudFront)
- ReqHeader Via: 2.0 6c61cea6f371b1744d3b5315a0029062.cloudfront.net (CloudFront), 1.1 ip-172-16-3-107 (Varnish/trunk)
- VCL_call RECV
- ReqHeader xkey: /it-it/prodotti/catene/catene-di-luci/
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- ReqUnset Accept-Encoding: gzip, deflate, br
- ReqHeader Accept-Encoding: gzip
- VCL_return hash
- VCL_call HASH
- VCL_return lookup
- Hit 32771 7190.679091 60.000000 0.000000
- VCL_call HIT
- VCL_return deliver
- RespProtocol HTTP/1.1
- RespStatus 200
- RespReason OK
- RespHeader Server: nginx/1.18.0 (Ubuntu)
- RespHeader Content-Type: text/html; charset=UTF-8
- RespHeader Cache-Control: must-revalidate, public, s-maxage=7200
- RespHeader Date: Wed, 28 Sep 2022 13:46:52 GMT
- RespHeader Strict-Transport-Security: max-age=31536000; includeSubDomains
- RespHeader X-Frame-Options: deny
- RespHeader X-Content-Type-Options: nosniff
- RespHeader Referrer-Policy: strict-origin-when-cross-origin
- RespHeader sw-invalidation-states:
- RespHeader Set-Cookie: sw-states=deleted; expires=Tue, 28-Sep-2021 13:46:51 GMT; Max-Age=0; path=/; secure; httponly; samesite=lax
- RespHeader Set-Cookie: sw-cache-hash=deleted; expires=Tue, 28-Sep-2021 13:46:51 GMT; Max-Age=0; path=/; httponly
- RespHeader Content-Encoding: gzip
- RespHeader Vary: Accept-Encoding
- RespHeader x-url: /it-it/prodotti/catene/catene-di-luci/
- RespHeader X-Cacheable: YES
- RespHeader X-Varnish: 32796 32771
- RespHeader Age: 9
- RespHeader Via: 1.1 ip-172-16-3-107 (Varnish/trunk)
- RespHeader Accept-Ranges: bytes
- VCL_call DELIVER
- RespUnset Cache-Control: must-revalidate, public, s-maxage=7200
- RespHeader Cache-Control: max-age=0, private
- RespUnset Set-Cookie: sw-states=deleted; expires=Tue, 28-Sep-2021 13:46:51 GMT; Max-Age=0; path=/; secure; httponly; samesite=lax
- RespUnset Set-Cookie: sw-cache-hash=deleted; expires=Tue, 28-Sep-2021 13:46:51 GMT; Max-Age=0; path=/; httponly
- RespHeader X-Cache: HIT
- RespUnset sw-invalidation-states:
- RespHeader X-Cache-Hits: 1
- VCL_return deliver
- Timestamp Process: 1664372821.615457 0.000074 0.000074
- Filters
- RespHeader Content-Length: 110820
- RespHeader Connection: keep-alive
- Timestamp Resp: 1664372821.615548 0.000166 0.000091
- ReqAcct 1598 0 1598 619 110820 111439
- End
Purge request log, showing no objects found:
* << Request >> 1277962
- Begin req 1277960 rxreq
- Timestamp Start: 1664372901.809064 0.000000 0.000000
- Timestamp Req: 1664372901.809064 0.000000 0.000000
- VCL_use boot
- ReqStart 172.16.2.136 51214 a0
- ReqMethod PURGE
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- ReqProtocol HTTP/1.1
- ReqHeader Host: 172.16.3.107
- ReqHeader User-Agent: GuzzleHttp/7
- ReqHeader X-Forwarded-For: 172.16.2.136
- ReqHeader Via: 1.1 ip-172-16-3-107 (Varnish/trunk)
- VCL_call RECV
- ReqHeader xkey: /it-it/prodotti/catene/catene-di-luci/
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- ReqHeader x-purges: 0
- VCL_return synth
- VCL_call HASH
- VCL_Log hashing for BAN request
- VCL_return lookup
- RespProtocol HTTP/1.1
- RespStatus 200
- RespReason Invalidated 0 objects
- RespHeader Date: Wed, 28 Sep 2022 13:48:21 GMT
- RespHeader Server: Varnish
- RespHeader X-Varnish: 1277962
- VCL_call SYNTH
- RespHeader Content-Type: text/html; charset=utf-8
- RespHeader Retry-After: 5
- VCL_return deliver
- Timestamp Process: 1664372901.809126 0.000061 0.000061
- RespHeader Content-Length: 287
- Storage malloc Transient
- Filters
- RespHeader Connection: keep-alive
- Timestamp Resp: 1664372901.809155 0.000090 0.000029
- ReqAcct 101 0 101 213 287 500
- End
It's perfectly possible, but you need to add the header to beresp object (in vcl_backend_response) to have the objects properly tagged
Setting keys
vmod_xkey doesn't offer you a lot of flexibility when it comes to setting keys. The only way to do this is by adding one or more xkey headers to your response containing the tags you want to assign to this object:
Here's an example where we set 3 tags:
HTTP/1.1 200 OK
xkey: 8155054
xkey: 166412
xkey: 234323
It's also possible to set multiple keys in a single xkey header as illustrated below:
HTTP/1.1 200 OK
xkey: 8155054 166412 234323
Unlike Varnish Enterprise's vmod_ykey, vmod_xkey doesn't allow you to set tags in vcl_backend_response. vmod_ykey also allows you to choose the header you want to use to tag content. vmod_xkey only allows the xkey header.
Invalidating keys
Once you have tagged the content, you can remove content based on those tags. Here's a quick example of a tag that was passed to Varnish through the xkey request header:
sub vcl_recv {
if (req.method == "PURGE") {
if (client.ip !~ purgers) {
return (synth(403, "Forbidden"));
}
if (req.http.xkey) {
set req.http.n-gone = xkey.softpurge(req.http.xkey)
return (synth(200, "Invalidated "+req.http.n-gone+" objects"));
} else {
return (purge);
}
}
}
The purgers ACL isn't included in the VCL example, but should be defined in your VCL code and contain the allowed IP addresses, hostnames or subnets.
Here's an example of how you would execute the PURGE using cURL :
curl -XPURGE -H "xkey: 8155054" http://test-prod.luminalpark.com/
I'm trying to do a soft purge only for certain req.url values, all other invalidations are managed with a ban.
While ban is working, purge.soft(0s,30s) does not modify anythig in the cache, TTL remains the standard (7200s) and the cache remains active.
What am I doing wrong?
Full VCL code:
https://pastebin.com/QLmBh0hw
GET request log:
* << Request >> 524366
- Begin req 524315 rxreq
- Timestamp Start: 1664176259.698910 0.000000 0.000000
- Timestamp Req: 1664176259.698910 0.000000 0.000000
- ReqStart 172.16.1.194 23952 a0
- ReqMethod GET
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- ReqProtocol HTTP/1.1
- ReqHeader X-Forwarded-For: 84.247.245.84, 130.176.221.197
- ReqHeader X-Forwarded-Proto: https
- ReqHeader X-Forwarded-Port: 443
- ReqHeader Host: test-prod.luminalpark.com
- ReqHeader X-Amzn-Trace-Id: Root=1-63315083-50faab2c2fbacda82eba1f9c
- ReqHeader User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36
- ReqHeader X-Amz-Cf-Id: ToerPgpur_0Y4SvPsU3hkSSzfK-PywWzlX-nlhnQasaji9IGXsFb5g==
- ReqHeader Via: 2.0 0f03c98743d9ffe79330c1f694241fc2.cloudfront.net (CloudFront)
- ReqHeader Cookie: _dc_gtm_UA-830149-18=1; _fbp=fb.1.1663062717375.1186628968; _ga=GA1.1.2073680112.1662976397; _ga_499EG6CXZD=GS1.1.1664175973.166.1.1664176257.0.0.0; _gcl_au=1.1.1770949614.1662976397; _gid=GA1.2.1336671677.1662976397; _hjSessionUser_21623=eyJpZCI
- ReqHeader Accept-Language: it-IT,it;q=0.9,en-US;q=0.8,en;q=0.7,de;q=0.6
- ReqHeader Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
- ReqHeader Referer: https://test-prod.luminalpark.com/it-it/
- ReqHeader Accept-Encoding: gzip, deflate, br
- ReqHeader cache-control: max-age=0
- ReqHeader sec-ch-ua: "Google Chrome";v="105", "Not)A;Brand";v="8", "Chromium";v="105"
- ReqHeader sec-ch-ua-mobile: ?0
- ReqHeader sec-ch-ua-platform: "Windows"
- ReqHeader dnt: 1
- ReqHeader upgrade-insecure-requests: 1
- ReqHeader sec-fetch-site: same-origin
- ReqHeader sec-fetch-mode: navigate
- ReqHeader sec-fetch-user: ?1
- ReqHeader sec-fetch-dest: document
- ReqHeader CloudFront-Viewer-HTTP-Version: 2.0
- ReqHeader CloudFront-Forwarded-Proto: https
- ReqHeader CloudFront-Viewer-Address: 84.247.245.84:61581
- ReqHeader CloudFront-Viewer-TLS: TLSv1.3:TLS_AES_128_GCM_SHA256:connectionReused
- ReqHeader X-Cloudfront-Origin: VC5ZNQ588QNE3S
- ReqUnset X-Forwarded-For: 84.247.245.84, 130.176.221.197
- ReqHeader X-Forwarded-For: 84.247.245.84, 130.176.221.197, 172.16.1.194
- VCL_call RECV
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- ReqUnset Accept-Encoding: gzip, deflate, br
- ReqHeader Accept-Encoding: gzip
- VCL_return hash
- VCL_call HASH
- VCL_return lookup
- Hit 884747 7171.150022 60.000000 0.000000
- VCL_call HIT
- VCL_return deliver
- RespProtocol HTTP/1.1
- RespStatus 200
- RespReason OK
- RespHeader Server: nginx/1.18.0 (Ubuntu)
- RespHeader Content-Type: text/html; charset=UTF-8
- RespHeader Cache-Control: must-revalidate, public, s-maxage=7200
- RespHeader Date: Mon, 26 Sep 2022 07:10:30 GMT
- RespHeader Strict-Transport-Security: max-age=31536000; includeSubDomains
- RespHeader X-Frame-Options: deny
- RespHeader X-Content-Type-Options: nosniff
- RespHeader Referrer-Policy: strict-origin-when-cross-origin
- RespHeader sw-invalidation-states:
- RespHeader Set-Cookie: sw-states=deleted; expires=Sun, 26-Sep-2021 07:10:29 GMT; Max-Age=0; path=/; secure; httponly; samesite=lax
- RespHeader Set-Cookie: sw-cache-hash=deleted; expires=Sun, 26-Sep-2021 07:10:29 GMT; Max-Age=0; path=/; httponly
- RespHeader Content-Encoding: gzip
- RespHeader Vary: Accept-Encoding
- RespHeader x-url: /it-it/prodotti/catene/catene-di-luci/
- RespHeader X-Cacheable: YES
- RespHeader X-Varnish: 524366 884747
- RespHeader Age: 28
- RespHeader Via: 1.1 varnish (Varnish/6.0)
- VCL_call DELIVER
- RespUnset Cache-Control: must-revalidate, public, s-maxage=7200
- RespHeader Cache-Control: max-age=0, private
- RespUnset Set-Cookie: sw-states=deleted; expires=Sun, 26-Sep-2021 07:10:29 GMT; Max-Age=0; path=/; secure; httponly; samesite=lax
- RespUnset Set-Cookie: sw-cache-hash=deleted; expires=Sun, 26-Sep-2021 07:10:29 GMT; Max-Age=0; path=/; httponly
- RespHeader X-Cache: HIT
- RespUnset sw-invalidation-states:
- RespHeader X-Cache-Hits: 2
- VCL_return deliver
- Timestamp Process: 1664176259.698985 0.000076 0.000076
- RespHeader Accept-Ranges: bytes
- RespHeader Content-Length: 110181
- RespHeader Connection: keep-alive
- Timestamp Resp: 1664176259.699106 0.000196 0.000120
- ReqAcct 3158 0 3158 612 110181 110793
- End
and here's varnishlog during PURGE
curl -XPURGE http://<varnish-host>/it-it/prodotti/catene/catene-di-luci/
* << Request >> 622680
- Begin req 622679 rxreq
- Timestamp Start: 1664176453.712271 0.000000 0.000000
- Timestamp Req: 1664176453.712271 0.000000 0.000000
- ReqStart 172.16.2.136 47728 a0
- ReqMethod PURGE
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- ReqProtocol HTTP/1.1
- ReqHeader Host: 172.16.3.37
- ReqHeader User-Agent: curl/7.68.0
- ReqHeader Accept: */*
- ReqHeader X-Forwarded-For: 172.16.2.136
- VCL_call RECV
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- ReqURL /it-it/prodotti/catene/catene-di-luci/
- VCL_acl MATCH purgers "ecommerce-node1-prod"
- VCL_return hash
- VCL_call HASH
- VCL_return lookup
- VCL_call MISS
- ReqHeader purged: 0
- VCL_return synth
- Timestamp Process: 1664176453.712326 0.000055 0.000055
- RespProtocol HTTP/1.1
- RespStatus 404
- RespReason Not Found
- RespReason Not Found
- RespHeader Date: Mon, 26 Sep 2022 07:14:13 GMT
- RespHeader Server: Varnish
- RespHeader X-Varnish: 622680
- VCL_call SYNTH
- RespHeader purged: 0
- VCL_return deliver
- RespHeader Content-Length: 0
- Storage malloc Transient
- RespHeader Connection: keep-alive
- Timestamp Resp: 1664176453.712364 0.000093 0.000038
- ReqAcct 114 0 114 153 0 153
- End
I've tested it myself and I'm not experiencing any issues.
The VCL code
Here's the standard vmod_purge example VCL code I took from https://varnish-cache.org/docs/6.0/reference/vmod_generated.html#example and adapted:
I removed the ACL check
I converted purge.hard() to purge.soft(0s,30s)
sub vcl_recv {
if (req.method == "PURGE") {
return (hash);
}
}
sub my_purge {
set req.http.purged = purge.soft(0s,30s);
if (req.http.purged == "0") {
return (synth(404));
}
else {
return (synth(200));
}
}
sub vcl_hit {
if (req.method == "PURGE") {
call my_purge;
}
}
sub vcl_miss {
if (req.method == "PURGE") {
call my_purge;
}
}
sub vcl_synth {
if (req.method == "PURGE") {
if (req.http.purged) {
set resp.http.purged = req.http.purged;
}
return (deliver);
}
}
What soft purging does
The way you've configured soft purging will ensure the object is considered stale, however because you set the grace time to 30s, async revalidation may still happen until 30 seconds past the lifetime of the object.
Example HTTP calls
Here are some example calls that illustrate what happens when soft purging takes place.
Step 1: call the endpoint and receive a fresh object
➜ ~ curl -I localhost
HTTP/1.1 200 OK
Content-Type: application/json; charset=utf-8
Content-Length: 595
ETag: W/"253-BCiA67uc9pD4vCc07ppQaevMbj0"
Date: Thu, 22 Sep 2022 07:51:22 GMT
X-Varnish: 65546 12
Age: 18
Via: 1.1 varnish (Varnish/6.6)
Accept-Ranges: bytes
Connection: keep-alive
As you can see we're calling the http://localhost endpoint and get a result with an Age: 18 header. This means the object has been cached for 18 seconds.
Step 2: purge
➜ ~ curl -XPURGE -I localhost
HTTP/1.1 200 OK
Date: Thu, 22 Sep 2022 07:51:42 GMT
Server: Varnish
X-Varnish: 32784
purged: 1
Content-Length: 0
Accept-Ranges: bytes
Connection: keep-alive
In step 2 we're performing the soft purge. The purged: 1 header implies that 1 object was purged.
Step 3: grace mode kicks in
➜ ~ curl -I localhost
HTTP/1.1 200 OK
Content-Type: application/json; charset=utf-8
Content-Length: 595
ETag: W/"253-BCiA67uc9pD4vCc07ppQaevMbj0"
Date: Thu, 22 Sep 2022 07:51:22 GMT
X-Varnish: 65552 12
Age: 26
Via: 1.1 varnish (Varnish/6.6)
Accept-Ranges: bytes
Connection: keep-alive
After the purge the we're still seeing cached content being served because the Age: 26 header that implies the object has been cached for 26 seconds.
But the output from varnishlog -g request show that the stale content is served while an asynchronous fetch happens for the new content. This is a direct result of calling purge.soft(0s, 30s):
* << Request >> 65552
- Begin req 65551 rxreq
- Timestamp Start: 1663833108.524685 0.000000 0.000000
- Timestamp Req: 1663833108.524685 0.000000 0.000000
- VCL_use boot
- ReqStart 172.24.0.1 58300 http
- ReqMethod HEAD
- ReqURL /
- ReqProtocol HTTP/1.1
- ReqHeader Host: localhost
- ReqHeader User-Agent: curl/7.79.1
- ReqHeader Accept: */*
- ReqHeader X-Forwarded-For: 172.24.0.1
- VCL_call RECV
- VCL_return hash
- VCL_call HASH
- VCL_return lookup
- Hit 12 -1.375188 30.000000 0.000000
- VCL_call HIT
- VCL_return deliver
- Link bereq 65553 bgfetch
- Timestamp Fetch: 1663833108.524864 0.000179 0.000179
- RespProtocol HTTP/1.1
- RespStatus 200
- RespReason OK
- RespHeader Content-Type: application/json; charset=utf-8
- RespHeader Content-Length: 595
- RespHeader ETag: W/"253-BCiA67uc9pD4vCc07ppQaevMbj0"
- RespHeader Date: Thu, 22 Sep 2022 07:51:22 GMT
- RespHeader X-Varnish: 65552 12
- RespHeader Age: 26
- RespHeader Via: 1.1 varnish (Varnish/6.6)
- VCL_call DELIVER
- VCL_return deliver
- Timestamp Process: 1663833108.524876 0.000191 0.000011
- Filters
- RespHeader Accept-Ranges: bytes
- RespHeader Connection: keep-alive
- Timestamp Resp: 1663833108.524937 0.000252 0.000061
- ReqAcct 74 0 74 275 0 275
** << BeReq >> 65553
-- Begin bereq 65552 bgfetch
-- VCL_use boot
-- Timestamp Start: 1663833108.524815 0.000000 0.000000
-- BereqMethod HEAD
-- BereqURL /
-- BereqProtocol HTTP/1.1
-- BereqHeader Host: localhost
-- BereqHeader User-Agent: curl/7.79.1
-- BereqHeader Accept: */*
-- BereqHeader X-Forwarded-For: 172.24.0.1
-- BereqMethod GET
-- BereqHeader Accept-Encoding: gzip
-- BereqHeader If-None-Match: W/"253-BCiA67uc9pD4vCc07ppQaevMbj0"
-- BereqHeader X-Varnish: 65553
-- VCL_call BACKEND_FETCH
-- VCL_return fetch
-- Timestamp Fetch: 1663833108.524843 0.000027 0.000027
-- Timestamp Connected: 1663833108.525006 0.000191 0.000163
-- BackendOpen 31 default 172.24.2.11 80 172.24.2.14 52026 connect
-- Timestamp Bereq: 1663833108.525047 0.000231 0.000040
-- Timestamp Beresp: 1663833108.530071 0.005256 0.005024
-- BerespProtocol HTTP/1.1
-- BerespStatus 200
-- BerespReason OK
-- BerespHeader Content-Type: application/json; charset=utf-8
-- BerespHeader Content-Length: 598
-- BerespHeader ETag: W/"256-1rfBjX0LanGZOnmzdwpQfzIjU30"
-- BerespHeader Date: Thu, 22 Sep 2022 07:51:48 GMT
-- BerespHeader Connection: keep-alive
-- BerespHeader Keep-Alive: timeout=5
-- TTL RFC 120 10 0 1663833109 1663833109 1663833108 0 0 cacheable
-- VCL_call BACKEND_RESPONSE
-- VCL_return deliver
-- Timestamp Process: 1663833108.530118 0.005302 0.000046
-- Filters
-- Storage malloc s0
-- Fetch_Body 3 length stream
-- BackendClose 31 default recycle
-- Timestamp BerespBody: 1663833108.530294 0.005479 0.000176
-- Length 598
-- BereqAcct 195 0 195 214 598 812
-- End
The Hit 12 -1.375188 30.000000 0.000000 line from the logs shows that the object has a remaining lifetime of -1.375188 seconds which is still more than -30 which is the grace limit.
The Link bereq 65553 bgfetch log line proves that a background fetch is made to the backend to store the latest version of the content. The transaction id in the Link tag matches the BeReq transaction that is part of the logs.
So while the new object is being fetched in transaction 65553, the response that is returned is still the old one with Age: 26 to prove it.
Step 4: next the next request gets the fresh content
Because grace mode will trigger an async fetch if there's grace time left, the current user doesn't see the impact of that fetch. However, the next user will get fresh content. The following cURL output illustrates this:
➜ ~ curl -I localhost
HTTP/1.1 200 OK
Content-Type: application/json; charset=utf-8
Content-Length: 598
ETag: W/"256-1rfBjX0LanGZOnmzdwpQfzIjU30"
Date: Thu, 22 Sep 2022 07:51:48 GMT
X-Varnish: 32790 65553
Age: 0
Via: 1.1 varnish (Varnish/6.6)
Accept-Ranges: bytes
Connection: keep-alive
Because Age: 0 is set, this is a brand new object. However as time progresses, the age counter will increase.
Conclusion
Soft purging with a non-zero grace value will not immediately remove an object from the cache. Instead it marks the object as expired and gives it grace time.
This ensures that the first visitor after the soft purge doesn't have to wait for the backend fetch to be completed.
It's a tradeoff between serving fresh content immediately and letting users benefit from the performance of Varnish will fetching content asynchronously and serving stale objects while that happens.
UPDATE: examining the your VCL & logs
After having examined your VCL & the VSL logs you provided, I'm assuming the creation of the caching hash differs from your GET call and your PURGE call.
Just for reference, the hash in vcl_hash is created using the URL value and the value of the Host header.
The logs indicate that you perform the GET call with the following Host header:
Host: test-prod.luminalpark.com
When you do the purge, you use the following Host header:
Host: 172.16.3.37
As this values differ, the hash will also differ. That's why the PURGE call results in a 404.
Conclusion: please use the right Host header to invalidate your content.
I have a haproxy-varnish setup for my ASPNET Core Web application and my application sending request via haproxy.conf file to the varnish. I can download all downloadable files except one url /Payroll/Export/EmployeeAllDataExport?.... File format is .xlsx. Tranfer-encoding header added manually by someone else so maybe that change refers the problem but i couldn't resolve until now.
I am sharing varnishlog -g request -q "RespStatus == 503" output, default.vcl file and releated lines in haproxy.cfg
P.S: If i disable varnish request from haproxy i can download the file but i need to do it with varnish and there is a problem for only one url not in an entire application.
varnishlog output:
* << Request >> 1384717
- Begin req 1384716 rxreq
- Timestamp Start: 1658919403.185251 0.000000 0.000000
- Timestamp Req: 1658919403.185251 0.000000 0.000000
- VCL_use boot
- ReqStart $IP1 43974 a0
- ReqMethod GET
- ReqURL /Payroll/Report/EmployeeAllDataExport?1658919403106
- ReqProtocol HTTP/1.1
- ReqHeader Host: $URL1
- ReqHeader sec-ch-ua: ".Not/A)Brand";v="99", "Google Chrome";v="103", "Chromium";v="103"
- ReqHeader sec-ch-ua-mobile: ?0
- ReqHeader sec-ch-ua-platform: "Windows"
- ReqHeader Upgrade-Insecure-Requests: 1
- ReqHeader User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36
- ReqHeader Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
- ReqHeader Sec-Fetch-Site: same-origin
- ReqHeader Sec-Fetch-Mode: navigate
- ReqHeader Sec-Fetch-User: ?1
- ReqHeader Sec-Fetch-Dest: document
- ReqHeader Referer: https://$URL1/Payroll/Report/EmployeeAllData?sitemap=63f8e1f5-a873-4cb5-8a5c-16b79863da33
- ReqHeader Accept-Encoding: gzip, deflate, br
- ReqHeader Accept-Language: tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7
- ReqHeader Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
- ReqHeader X-Forwarded-Proto: https
- ReqHeader X-Forwarded-Port: 443
- ReqHeader X-Forwarded-Proto: https
- ReqHeader X-Forwarded-For: $IP2
- ReqHeader Connection: close
- ReqUnset X-Forwarded-For: $IP2
- ReqHeader X-Forwarded-For: $IP2, $IP1
- VCL_call RECV
- ReqUnset Host: $URL1
- ReqHeader Host: $URL1
- ReqURL /Payroll/Report/EmployeeAllDataExport?1658919403106
- ReqUnset Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
- ReqHeader Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
- ReqUnset Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
- ReqHeader Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
- ReqUnset Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
- ReqHeader Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
- ReqUnset Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
- ReqHeader Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
- ReqUnset Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
- ReqHeader Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
- ReqHeader Surrogate-Capability: key=ESI/1.0
- VCL_return hash
- ReqUnset Accept-Encoding: gzip, deflate, br
- ReqHeader Accept-Encoding: gzip
- VCL_call HASH
- VCL_return lookup
- VCL_call MISS
- VCL_return fetch
- Link bereq 1384718 fetch
- Timestamp Fetch: 1658919404.932211 1.746959 1.746959
- RespProtocol HTTP/1.1
- RespStatus 503
- RespReason Backend fetch failed
- RespHeader Date: Wed, 27 Jul 2022 10:56:44 GMT
- RespHeader Server: Varnish
- RespHeader Content-Type: text/html; charset=utf-8
- RespHeader Retry-After: 5
- RespHeader X-Varnish: 1384717
- RespHeader Age: 0
- RespHeader Via: 1.1 varnish (Varnish/6.2)
- VCL_call DELIVER
- RespHeader X-Cache: MISS
- RespHeader X-Cache-Hits: 0
- RespUnset Server: Varnish
- RespUnset X-Varnish: 1384717
- RespUnset Via: 1.1 varnish (Varnish/6.2)
- VCL_return deliver
- Timestamp Process: 1658919404.932258 1.747006 0.000047
- Filters
- RespHeader Content-Length: 284
- RespHeader Connection: close
- Timestamp Resp: 1658919404.932344 1.747092 0.000086
- ReqAcct 5695 0 5695 210 284 494
- End
** << BeReq >> 1384718
-- Begin bereq 1384717 fetch
-- VCL_use boot
-- Timestamp Start: 1658919403.185407 0.000000 0.000000
-- BereqMethod GET
-- BereqURL /Payroll/Report/EmployeeAllDataExport?1658919403106
-- BereqProtocol HTTP/1.1
-- BereqHeader sec-ch-ua: ".Not/A)Brand";v="99", "Google Chrome";v="103", "Chromium";v="103"
-- BereqHeader sec-ch-ua-mobile: ?0
-- BereqHeader sec-ch-ua-platform: "Windows"
-- BereqHeader Upgrade-Insecure-Requests: 1
-- BereqHeader User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36
-- BereqHeader Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
-- BereqHeader Sec-Fetch-Site: same-origin
-- BereqHeader Sec-Fetch-Mode: navigate
-- BereqHeader Sec-Fetch-User: ?1
-- BereqHeader Sec-Fetch-Dest: document
-- BereqHeader Referer: https://$URL1/Payroll/Report/EmployeeAllData?sitemap=63f8e1f5-a873-4cb5-8a5c-16b79863da33
-- BereqHeader Accept-Language: tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7
-- BereqHeader X-Forwarded-Proto: https
-- BereqHeader X-Forwarded-Port: 443
-- BereqHeader X-Forwarded-Proto: https
-- BereqHeader X-Forwarded-For: $IP1, $IP2
-- BereqHeader Host: $URL1
-- BereqHeader Cookie: CurrentLanguage=tr; .AspNetCore.Antiforgery.9TtSrW0hzOs=CfDJ8E7lWAHTLKBOsOUJGWpm85n139G92-PEfuGVynC-Lh4dd-5XOYoILAhHGAD6DW-my0vgWvin4bBKCLT-GFQonAQCvkTZV8EuOB8s6RPOunsRFPpgucKPwKQSmF7Syx0tg8TubZs_Xw6qTxh9gVq03lo; Hcm=CfDJ8E7lWAHTLKBOsOUJGWpm85lmo
-- BereqHeader Surrogate-Capability: key=ESI/1.0
-- BereqHeader Accept-Encoding: gzip
-- BereqHeader X-Varnish: 1384718
-- VCL_call BACKEND_FETCH
-- VCL_return fetch
-- BackendOpen 31 haproxy $IP1 8585 $IP3 52722
-- BackendStart $IP1 8585
-- Timestamp Bereq: 1658919403.185494 0.000087 0.000087
-- Timestamp Beresp: 1658919404.931989 1.746582 1.746495
-- BerespProtocol HTTP/1.1
-- BerespStatus 200
-- BerespReason OK
-- BerespHeader Connection: close
-- BerespHeader Date: Wed, 27 Jul 2022 10:56:44 GMT
-- BerespHeader Content-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
-- BerespHeader Server: Kestrel
-- BerespHeader Transfer-Encoding: identity
-- BerespHeader Strict-Transport-Security: max-age=2592000
-- BerespHeader content-disposition: attachment; filename="$Filename.xlsx"
-- BackendClose 31 haproxy
-- Error Body cannot be fetched
-- Timestamp Error: 1658919404.932043 1.746636 0.000054
-- BerespProtocol HTTP/1.1
-- BerespStatus 503
-- BerespReason Service Unavailable
-- BerespReason Backend fetch failed
-- BerespHeader Date: Wed, 27 Jul 2022 10:56:44 GMT
-- BerespHeader Server: Varnish
-- VCL_call BACKEND_ERROR
-- BerespHeader Content-Type: text/html; charset=utf-8
-- BerespHeader Retry-After: 5
-- VCL_return deliver
-- Storage malloc Transient
-- Length 284
-- BereqAcct 5731 0 5731 327 0 327
-- End
default.vcl
vcl 4.1;
# Based on: https://github.com/mattiasgeniar/varnish-6.0-configuration-templates/blob/master/default.vcl
import std;
import directors;
backend haproxy { # Define one backend
.host = "10.10.0.117"; # IP or Hostname of backend
.port = "8585"; # Port Apache or whatever is listening
.max_connections = 300; # That's it
.first_byte_timeout = 300s; # How long to wait before we receive a first byte from our backend?
.connect_timeout = 300s; # How long to wait for a backend connection?
.between_bytes_timeout = 300s; # How long to wait between bytes received from our backend?
}
acl purge {
"localhost";
"127.0.0.1";
"::1";
}
sub vcl_init {
new vdir = directors.round_robin();
vdir.add_backend(haproxy);
}
sub vcl_recv {
#if (req.url ~ "^/Payroll/Report") {
# return(pipe);
#}
set req.backend_hint = vdir.backend(); # send all traffic to the vdir director
if (req.http.Host) {
set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
}
unset req.http.proxy;
set req.url = std.querysort(req.url);
if (req.method == "PURGE") {
if (!client.ip ~ purge) {
return (synth(405, "This IP is not allowed to send PURGE requests."));
}
return (purge);
}
# Only deal with "normal" types
if (req.method != "GET" &&
req.method != "HEAD" &&
req.method != "PUT" &&
req.method != "POST" &&
req.method != "TRACE" &&
req.method != "OPTIONS" &&
req.method != "PATCH" &&
req.method != "DELETE" ){
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
# Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html)
if (req.http.Upgrade ~ "(?i)websocket") {
return (pipe);
}
# Only cache GET or HEAD requests. This makes sure the POST requests are always passed.
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}
if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=") {
set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
set req.url = regsub(req.url, "\?&", "?");
set req.url = regsub(req.url, "\?$", "");
}
# Strip hash, server doesn't need it.
if (req.url ~ "\#") {
set req.url = regsub(req.url, "\#.*$", "");
}
# Strip a trailing ? if it exists
if (req.url ~ "\?$") {
set req.url = regsub(req.url, "\?$", "");
}
set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "__gads=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "__atuv.=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");
if (req.http.cookie ~ "^\s*$") {
unset req.http.cookie;
}
if (req.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(\?.*)?$") {
unset req.http.Cookie;
return (hash);
}
# Remove all cookies for static files
# A valid discussion could be held on this line: do you really need to cache static files that don't cause load? Only if you have memory left.
# Sure, there's disk I/O, but chances are your OS will already have these files in their buffers (thus memory).
# Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/
if (req.url ~ "^[^?]*\.(7z|apng|webp|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") {
unset req.http.Cookie;
return (hash);
}
#if (req.url ~ "^/Payroll/Report") {
# unset req.http.Cookie;
# return (hash);
#}
# Send Surrogate-Capability headers to announce ESI support to backend
set req.http.Surrogate-Capability = "key=ESI/1.0";
if (req.http.Authorization) {
# Not cacheable by default
return (pass);
}
if (req.url ~ "(jpg|jpeg|png|gif|tiff|bmp|apng|webp)$") {
return (pipe);
}
return (hash);
}
sub vcl_pipe {
# Called upon entering pipe mode.
# In this mode, the request is passed on to the backend, and any further data from both the client
# and backend is passed on unaltered until either end closes the connection. Basically, Varnish will
# degrade into a simple TCP proxy, shuffling bytes back and forth. For a connection in pipe mode,
# no other VCL subroutine will ever get called after vcl_pipe.
# Note that only the first request to the backend will have
# X-Forwarded-For set. If you use X-Forwarded-For and want to
# have it set for all requests, make sure to have:
# set bereq.http.connection = "close";
# here. It is not set by default as it might break some broken web
# applications, like IIS with NTLM authentication.
# set bereq.http.Connection = "Close";
# Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html)
if (req.http.upgrade) {
set bereq.http.upgrade = req.http.upgrade;
}
return (pipe);
}
sub vcl_pass {
# Called upon entering pass mode. In this mode, the request is passed on to the backend, and the
# backend's response is passed on to the client, but is not entered into the cache. Subsequent
# requests submitted over the same client connection are handled normally.
# return (pass);
}
# The data on which the hashing will take place
sub vcl_hash {
# Called after vcl_recv to create a hash value for the request. This is used as a key
# to look up the object in Varnish.
hash_data(req.url);
if (req.http.host) {
hash_data(req.http.host);
} else {
hash_data(server.ip);
}
# hash cookies for requests that have them
if (req.http.Cookie) {
hash_data(req.http.Cookie);
}
# Cache the HTTP vs HTTPs separately
if (req.http.X-Forwarded-Proto) {
hash_data(req.http.X-Forwarded-Proto);
}
}
sub vcl_hit {
# Called when a cache lookup is successful.
if (obj.ttl >= 0s) {
# A pure unadultered hit, deliver it
return (deliver);
}
# https://www.varnish-cache.org/docs/trunk/users-guide/vcl-grace.html
# When several clients are requesting the same page Varnish will send one request to the backend and place the others
# on hold while fetching one copy from the backend. In some products this is called request coalescing and Varnish does
# this automatically.
# If you are serving thousands of hits per second the queue of waiting requests can get huge. There are two potential
# problems - one is a thundering herd problem - suddenly releasing a thousand threads to serve content might send the
# load sky high. Secondly - nobody likes to wait. To deal with this we can instruct Varnish to keep the objects in cache
# beyond their TTL and to serve the waiting requests somewhat stale content.
# if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) {
# return (deliver);
# } else {
# return (miss);
# }
# We have no fresh fish. Lets look at the stale ones.
if (std.healthy(req.backend_hint)) {
# Backend is healthy. Limit age to 10s.
if (obj.ttl + 10s > 0s) {
#set req.http.grace = "normal(limited)";
return (deliver);
}
} else {
# backend is sick - use full grace
if (obj.ttl + obj.grace > 0s) {
#set req.http.grace = "full";
return (deliver);
}
}
}
sub vcl_miss {
# Called after a cache lookup if the requested document was not found in the cache. Its purpose
# is to decide whether or not to attempt to retrieve the document from the backend, and which
# backend to use.
return (fetch);
}
# Handle the HTTP request coming from our backend
sub vcl_backend_response {
# Called after the response headers has been successfully retrieved from the backend.
# Pause ESI request and remove Surrogate-Control header
if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
unset beresp.http.Surrogate-Control;
set beresp.do_esi = true;
}
# Enable cache for all static files
# The same argument as the static caches from above: monitor your cache size, if you get data nuked out of it, consider giving up the static file cache.
# Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/
if (bereq.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") {
unset beresp.http.set-cookie;
}
#if (bereq.url ~ "^/Payroll/Report") {
# unset beresp.http.set-cookie;
#}
# Large static files are delivered directly to the end-user without
# waiting for Varnish to fully read the file first.
# Varnish 4 fully supports Streaming, so use streaming here to avoid locking.
if (bereq.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(\?.*)?$") {
unset beresp.http.set-cookie;
set beresp.do_stream = true; # Check memory usage it'll grow in fetch_chunksize blocks (128k by default) if the backend doesn't send a Content-Length header, so only enable it for big objects
}
# Sometimes, a 301 or 302 redirect formed via Apache's mod_rewrite can mess with the HTTP port that is being passed along.
# This often happens with simple rewrite rules in a scenario where Varnish runs on :80 and Apache on :8080 on the same box.
# A redirect can then often redirect the end-user to a URL on :8080, where it should be :80.
# This may need finetuning on your setup.
#
# To prevent accidental replace, we only filter the 301/302 redirects for now.
if (beresp.status == 301 || beresp.status == 302) {
set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", "");
}
# Set 2min cache if unset for static files
if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") {
set beresp.ttl = 120s; # Important, you shouldn't rely on this, SET YOUR HEADERS in the backend
set beresp.uncacheable = true;
return (deliver);
}
# Don't cache 50x responses
if (beresp.status == 500 || beresp.status == 502 || beresp.status == 503 || beresp.status == 504) {
return (abandon);
}
# Allow stale content, in case the backend goes down.
# make Varnish keep all objects for 6 hours beyond their TTL
set beresp.grace = 6h;
return (deliver);
}
# The routine when we deliver the HTTP request to the user
# Last chance to modify headers that are sent to the client
sub vcl_deliver {
# Called before a cached object is delivered to the client.
if (obj.hits > 0) { # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed
set resp.http.X-Cache = "HIT";
} else {
set resp.http.X-Cache = "MISS";
}
# Please note that obj.hits behaviour changed in 4.0, now it counts per objecthead, not per object
# and obj.hits may not be reset in some cases where bans are in use. See bug 1492 for details.
# So take hits with a grain of salt
set resp.http.X-Cache-Hits = obj.hits;
# Remove some headers: PHP version
unset resp.http.X-Powered-By;
# Remove some headers: Apache version & OS
unset resp.http.Server;
unset resp.http.X-Drupal-Cache;
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.Link;
unset resp.http.X-Generator;
return (deliver);
}
sub vcl_purge {
# Only handle actual PURGE HTTP methods, everything else is discarded
if (req.method == "PURGE") {
# restart request
set req.http.X-Purge = "Yes";
return(restart);
}
}
sub vcl_synth {
if (resp.status == 720) {
# We use this special error status 720 to force redirects with 301 (permanent) redirects
# To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
set resp.http.Location = resp.reason;
set resp.status = 301;
return (deliver);
} elseif (resp.status == 721) {
# And we use error status 721 to force redirects with a 302 (temporary) redirect
# To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
set resp.http.Location = resp.reason;
set resp.status = 302;
return (deliver);
}
return (deliver);
}
sub vcl_fini {
# Called when VCL is discarded only after all requests have exited the VCL.
# Typically used to clean up VMODs.
return (ok);
}
P.S: please consider i tried if(req.url ~ $URL){return (pass);} or pipe and if(bereq.url ~ $URL){unset beresp.http.set-cookie;}. Those lines which i tried now command line at the default.vcl file.
haproxy.cfg
frontend HTTP
bind *:80
redirect scheme https if { hdr(Host) -i $URL1 } !{ ssl_fc }
frontend returning-varnish # returning traffic from varnish
bind :8585
#acl $Backend1 hdr(host) -i $URL1
#use_backend $Backend1 if $Backend1
frontend HTTPS
bind *:443 ssl crt $Directory1 crt $Directory2
# http-request lua.cors
# http-response lua.cors "GET,PUT,POST" "localhost:1907"
# http-response lua.cors "GET,PUT,POST" "*"
http-response add-header Access-Control-Allow-Origin %[capture.req.hdr(0)] if { capture.req.hdr(0) -m found }
rspadd Content-Security-Policy:\ upgrade-insecure-requests
reqadd X-Forwarded-Proto:\ https
#rspadd Strict-Transport-Security:\ max-age=31536000
http-response set-header Strict-Transport-Security "max-age=63072000; includeSubDomains; always;"
maxconn 100000
redirect prefix https://$URL2 if { hdr(host) -i $URL4 }
redirect prefix https://$URL3 if { hdr(host) -i $URL4 }
#use_backend Varnish if $Backend1
use_backend $Backend1 if $Backend1
backend Varnish
http-request set-header X-Forwarded-Port %[dst_port]
http-request add-header X-Forwarded-Proto https if { ssl_fc }
server varnish1 $IP2:6081 check
backend $Backend1
mode http
balance leastconn
server $Server $Application_Ip:31717 check inter 1s rise 2 fall 2
P.S: Because of the varnish is not working while I want to download file, I made those lines as a command line at haproxy.cfg file as well.
Based on the logs I'm assuming the response headers of your backend response are not properly formatted.
The VSL error is the following:
-- Error Body cannot be fetched
Your backend response doesn't include a Content-Length header, which is fine. But if you don't return a buffered response, Varnish looks at the Transfer-Encoding header to see whether or not you'r using Chunked Transfer Encoding.
This is the header your application is returning:
-- BerespHeader Transfer-Encoding: identity
When we take a look at the code, specifically this line https://github.com/varnishcache/varnish-cache/blob/55753f98b1b392766bd4baed7eb684c006225f57/bin/varnishd/http1/cache_http1_proto.c#L319, you can see that Varnish throws the error because it doesn't recognize the transfer encoding.
Please set a Transfer-Encoding: chunked header and Varnish will have no issue with this response and will stream the content through based on how your backend flushes the various chunks of content.
I'm trying to use Varnish to cache my site. But I hope Varnish bypass some requests with a named Cookies(RSS).
Varnish ver: 6.3
Here is a part of my default.vcl
sub vcl_recv {
set req.http.X-Varnish-PHP_SID = req.http.Cookie;
set req.http.X-Varnish-PHP_SID =
regsuball(req.http.X-Varnish-PHP_SID, ";? ?RSS=([a-zA-Z0-9]+)( |;| ;).*","\1");
set req.http.Cookie = req.http.X-Varnish-PHP_SID;
unset req.http.X-Varnish-PHP_SID;
}
Here is logs from Varnish
* << Request >> 452
- Begin req 411 rxreq
- Timestamp Start: 1591947637.399591 0.000000 0.000000
- Timestamp Req: 1591947637.400066 0.000475 0.000475
- VCL_use boot
- ReqStart 42.88.136.35 10072 a0
- ReqMethod POST
- ReqURL /~YeV.js
- ReqProtocol HTTP/1.1
- ReqHeader Host: www.ttester.com:8000
- ReqHeader Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
- ReqHeader Xiaytvhcnb: liacddpfckbjzpovdxrvjhibipwqevuxivzelicudjgcngcgdvgddndfghzrtvoelxkblzkiwqewvjleubxycuitwojhdztogomqoqtyufkqqqecrfoytklnxztnjytcrfptiplkuidfrbtxgighjpoppdkrjmngknujsgeunfgdcghjejprujdcyqmgkmxwgfdamxofmzjneajjymvqrixfaypudahokxrydpoeuynewshpzq
- ReqHeader Accept-Encoding: gzip, deflate
- ReqHeader Accept-Language: en-us
- ReqHeader Content-Type: application/x-www-form-urlencoded
- ReqHeader Origin: http://www.ttester.com:8000
- ReqHeader User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)
- ReqHeader Connection: keep-alive
- ReqHeader Upgrade-Insecure-Requests: 1
- ReqHeader Referer: http://www.ttester.com:8000/~YeV.js
- ReqHeader Content-Length: 8
- ReqHeader Cookie:
- ReqHeader Cookie: BAIDU-STFS=6352be5e1748a73bfe4c898d9805215c56d6b8a9e6d30b2549168ae853c76078; RSS=1; Hm_lpvt_12e92bc53a305b423a5f20bd34b0a75e=1591947600; Hm_lvt_12e92bc53a305b423a5f20bd34b0a75e=1591936468; Hm_lpvt_12e92bc53a305b423a5f20bd34b0a75e=1591944261; Hm_l
- ReqHeader X-Forwarded-For: 42.88.136.35
- VCL_call RECV
- ReqHeader X-Varnish-PHP_SID:
- ReqUnset X-Varnish-PHP_SID:
- ReqHeader X-Varnish-PHP_SID:
- ReqUnset Cookie:
- ReqUnset Cookie: BAIDU-STFS=6352be5e1748a73bfe4c898d9805215c56d6b8a9e6d30b2549168ae853c76078; RSS=1; Hm_lpvt_12e92bc53a305b423a5f20bd34b0a75e=1591947600; Hm_lvt_12e92bc53a305b423a5f20bd34b0a75e=1591936468; Hm_lpvt_12e92bc53a305b423a5f20bd34b0a75e=1591944261; Hm_l
- ReqHeader Cookie:
- ReqUnset X-Varnish-PHP_SID:
- ReqUnset Host: www.ttester.com:8000
- ReqHeader host: www.ttester.com:8000
- VCL_return pass
- VCL_call HASH
- VCL_return lookup
- VCL_call PASS
- VCL_return fetch
- Link bereq 453 pass
- Storage malloc Transient
- Timestamp ReqBody: 1591947637.400221 0.000629 0.000154
- Timestamp Fetch: 1591947637.407862 0.008270 0.007641
- RespProtocol HTTP/1.1
- RespStatus 200
- RespReason OK
- RespHeader Date: Fri, 12 Jun 2020 07:40:37 GMT
- RespHeader Server: Apache/2.4.25 (Debian)
- RespHeader X-Powered-By: PHP/7.2.6
- RespHeader URL-Record-File: www.ttester.com/~YeV.js
- RespHeader URL-Cluster: cn5
- RespHeader cache-control: no-store, no-cache, must-revalidate
- RespHeader Set-Cookie: BAIDU-STFS=6352be5e1748a73bfe4c898d9805215c56d6b8a9e6d30b2549168ae853c76078; Expires=Friday, 12-Jun-2020 15:40:39 CST; Max-Age=2; path=/
- RespHeader Set-Cookie: RSS=1; Expires=Friday, 12-Jun-2020 15:40:39 CST; Max-Age=2; path=/
- RespHeader Vary: Accept-Encoding
- RespHeader Content-Encoding: gzip
- RespHeader Content-Length: 626
- RespHeader Content-Type: text/html; charset=utf-8
- RespHeader V-Cache-TTL: 0.000
- RespHeader V-Cache-Grace: 10.000
- RespHeader X-Varnish: 452
- RespHeader Age: 0
- RespHeader Via: 1.1 varnish (Varnish/6.3)
- VCL_call DELIVER
- RespHeader V-Cache: MISS
- RespUnset Server: Apache/2.4.25 (Debian)
- RespHeader Server: MFPad
- RespUnset X-Powered-By: PHP/7.2.6
- RespUnset Via: 1.1 varnish (Varnish/6.3)
- RespUnset X-Varnish: 452
- VCL_return deliver
- Timestamp Process: 1591947637.407883 0.008292 0.000021
- Filters
- RespHeader Accept-Ranges: bytes
- RespHeader Connection: keep-alive
- Timestamp Resp: 1591947637.407919 0.008328 0.000036
- ReqAcct 2056 8 2064 633 626 1259
- End
I also tried another version of VCL as below and doesn't work as well.
sub vcl_recv {
if (req.http.Cookie && !(req.http.Cookie ~ "RSS=")) {
unset req.http.Cookie;
}
}
I just want Varnish can bypass a request with a RSS Cookie.
If I understand correctly, you want all pages to be cached, except the ones that contain the RSS cookie.
Here's the old school ugly way of doing it:
sub vcl_recv {
if (req.http.Cookie) {
set req.http.Cookie = ";" + req.http.Cookie;
set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
set req.http.Cookie = regsuball(req.http.Cookie, ";(RSS)=", "; \1=");
set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
if (req.http.cookie ~ "^\s*$") {
unset req.http.cookie;
}
}
}
However, if you were to upgrade to Varnish Cache 6.4, you'll have the vmod_cookie packaged with that release, which makes cookie manipulation a lot easier.
Here's how that would work using vmod_cookie:
import cookie;
sub vcl_recv {
cookie.parse(req.http.cookie);
cookie.keep("RSS");
set req.http.cookie = cookie.get_string();
}
Both implementations rely on the default Varnish behavior. Meaning that if after the substitution of the cookies, there is still a cookie left (in our case RSS), Varnish will not cache and perform a return(pass) in the vcl_recv hook.
Is this what you're looking for?
I want to use Varnish as a "smart" proxy and it almost works. The idea is that some requests should be passed through Varnish, hit the backend and return, all other requests should return a "synt" message that the specific response contains no result.
This works apart from the fact that Varnish returns a 301 redirect to the backend instead of just the response from the actual backend.
Backend and Cache are not located on the same host (or not even on the same network in this case).
Backend is ALSO running a separate Varnish instance and this request is always passed through that.
// VCL.SHOW 0 1820 input
#
# This is an example VCL file for Varnish.
#
# It does not do anything by default, delegating control to the
# builtin VCL. The builtin VCL is called when there is no explicit
# return statement.
#
# See the VCL chapters in the Users Guide at https://www.varnish-cache.org/docs/
# and http://varnish-cache.org/trac/wiki/VCLExamples for more examples.
# Marker to tell the VCL compiler that this VCL has been adapted to the
# new 4.0 format.
vcl 4.0;
### Here starts my part of the VCL
# Default backend definition. Set this to point to your content server.
backend default {
.host = "myhost.mydomain";
.port = "80";
}
sub vcl_recv {
# Happens before we check if we have this in cache already.
#
# Typically you clean up the request here, removing cookies you don't need,
# rewriting the request, etc.
if (req.url ~ "^\/cgi-bin\/wspd_cgi\.sh/apiFlightSearch.p\?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27")
{
return (hash);
}
return (synth(750));
}
sub vcl_synth {
if (resp.status == 750) {
# Set a status the client will understand
set resp.status = 200;
# Create our synthetic response
set resp.http.content-type = "text/xml";
synthetic("<flights xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'><status><status>No flights</status></status></flights>");
return(deliver);
}
}
sub vcl_backend_response {
# Happens after we have read the response headers from the backend.
#
# Here you clean the response headers, removing silly Set-Cookie headers
# and other mistakes your backend does.
set beresp.ttl = 10 s;
}
sub vcl_deliver {
# Happens when we have all the pieces we need, and are about to send the
# response to the client.
#
# You can do accounting or modifying the final object here.
}
### Here ends my part of the VCL. The rest I guess is built in.
// VCL.SHOW 1 5479 Builtin
/*-
* Copyright (c) 2006 Verdens Gang AS
* Copyright (c) 2006-2014 Varnish Software AS
* All rights reserved.
*
* Author: Poul-Henning Kamp <phk#phk.freebsd.dk>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*
* The built-in (previously called default) VCL code.
*
* NB! You do NOT need to copy & paste all of these functions into your
* own vcl code, if you do not provide a definition of one of these
* functions, the compiler will automatically fall back to the default
* code from this file.
*
* This code will be prefixed with a backend declaration built from the
* -b argument.
*/
vcl 4.0;
#######################################################################
# Client side
sub vcl_recv {
if (req.method == "PRI") {
/* We do not support SPDY or HTTP/2.0 */
return (synth(405));
}
if (req.method != "GET" &&
req.method != "HEAD" &&
req.method != "PUT" &&
req.method != "POST" &&
req.method != "TRACE" &&
req.method != "OPTIONS" &&
req.method != "DELETE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
if (req.method != "GET" && req.method != "HEAD") {
/* We only deal with GET and HEAD by default */
return (pass);
}
if (req.http.Authorization || req.http.Cookie) {
/* Not cacheable by default */
return (pass);
}
return (hash);
}
sub vcl_pipe {
# By default Connection: close is set on all piped requests, to stop
# connection reuse from sending future requests directly to the
# (potentially) wrong backend. If you do want this to happen, you can undo
# it here.
# unset bereq.http.connection;
return (pipe);
}
sub vcl_pass {
return (fetch);
}
sub vcl_hash {
hash_data(req.url);
if (req.http.host) {
hash_data(req.http.host);
} else {
hash_data(server.ip);
}
return (lookup);
}
sub vcl_purge {
return (synth(200, "Purged"));
}
sub vcl_hit {
if (obj.ttl >= 0s) {
// A pure unadultered hit, deliver it
return (deliver);
}
if (obj.ttl + obj.grace > 0s) {
// Object is in grace, deliver it
// Automatically triggers a background fetch
return (deliver);
}
// fetch & deliver once we get the result
return (fetch);
}
sub vcl_miss {
return (fetch);
}
sub vcl_deliver {
return (deliver);
}
/*
* We can come here "invisibly" with the following errors: 413, 417 & 503
*/
sub vcl_synth {
set resp.http.Content-Type = "text/html; charset=utf-8";
set resp.http.Retry-After = "5";
synthetic( {"<!DOCTYPE html>
<html>
<head>
<title>"} + resp.status + " " + resp.reason + {"</title>
</head>
<body>
<h1>Error "} + resp.status + " " + resp.reason + {"</h1>
<p>"} + resp.reason + {"</p>
<h3>Guru Meditation:</h3>
<p>XID: "} + req.xid + {"</p>
<hr>
<p>Varnish cache server</p>
</body>
</html>
"} );
return (deliver);
}
#######################################################################
# Backend Fetch
sub vcl_backend_fetch {
return (fetch);
}
sub vcl_backend_response {
if (beresp.ttl <= 0s ||
beresp.http.Set-Cookie ||
beresp.http.Surrogate-control ~ "no-store" ||
(!beresp.http.Surrogate-Control &&
beresp.http.Cache-Control ~ "no-cache|no-store|private") ||
beresp.http.Vary == "*") {
/*
* Mark as "Hit-For-Pass" for the next 2 minutes
*/
set beresp.ttl = 120s;
set beresp.uncacheable = true;
}
return (deliver);
}
sub vcl_backend_error {
set beresp.http.Content-Type = "text/html; charset=utf-8";
set beresp.http.Retry-After = "5";
synthetic( {"<!DOCTYPE html>
<html>
<head>
<title>"} + beresp.status + " " + beresp.reason + {"</title>
</head>
<body>
<h1>Error "} + beresp.status + " " + beresp.reason + {"</h1>
<p>"} + beresp.reason + {"</p>
<h3>Guru Meditation:</h3>
<p>XID: "} + bereq.xid + {"</p>
<hr>
<p>Varnish cache server</p>
</body>
</html>
"} );
return (deliver);
}
#######################################################################
# Housekeeping
sub vcl_init {
return (ok);
}
sub vcl_fini {
return (ok);
}
Output from curl:
$ curl "thisandthatip.compute.amazonaws.com/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0" -i
HTTP/1.1 301 Moved Permanently
Date: Wed, 15 Mar 2017 07:14:11 GMT
Server: Apache/2.2.15 (Red Hat)
Location: http://myserver.mydomain/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
Content-Length: 514
Content-Type: text/html; charset=iso-8859-1
X-Varnish: 529144137
Via: 1.1 varnish-v4
X-Varnish: 98309 11
Age: 2
Via: 1.1 varnish-v4
Connection: keep-alive
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>301 Moved Permanently</title>
</head><body>
<h1>Moved Permanently</h1>
<p>The document has moved here.</p>
<hr>
<address>Apache/2.2.15 (Red Hat) Server at thisandthatip.eu-central-1.compute.amazonaws.com Port 80</address>
</body></html>
Backend apache access log
127.0.0.1 - - [15/Mar/2017:08:09:49 +0100] "GET /cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=arn&to=aok&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0 HTTP/1.1" 200 994 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"
Sending the request from the AWS instance to the backend renders no 301 redirection:
$ curl "myserver.mydomain/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=arn&to=aok&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0" -i
HTTP/1.1 200 OK
Date: Wed, 15 Mar 2017 08:54:14 GMT
Server: Apache/2.2.15 (Red Hat)
Cache-Control: max-age=1
Expires: Wed, 15 Mar 2017 08:54:15 GMT
Content-Type: text/xml
X-Varnish: 527559784
Age: 0
Via: 1.1 varnish-v4
Transfer-Encoding: chunked
Connection: keep-alive
Accept-Ranges: bytes
... Response body here ...
Complete varnishlog output of a single request
* << BeReq >> 98314
- Begin bereq 98313 fetch
- Timestamp Start: 1489568144.701450 0.000000 0.000000
- BereqMethod GET
- BereqURL /cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
- BereqProtocol HTTP/1.1
- BereqHeader Host: thisandthatip.compute.amazonaws.com
- BereqHeader Upgrade-Insecure-Requests: 1
- BereqHeader User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36
- BereqHeader Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
- BereqHeader Accept-Language: sv-SE,sv;q=0.8,en-US;q=0.6,en;q=0.4
- BereqHeader X-Forwarded-For: ip.ip.ip.ip
- BereqHeader Accept-Encoding: gzip
- BereqHeader X-Varnish: 98314
- VCL_call BACKEND_FETCH
- VCL_return fetch
- BackendClose 17 default(ip.ip.ip.ip,,80) toolate
- BackendOpen 17 default(ip.ip.ip.ip,,80) 172.31.31.195 42868
- Backend 17 default default(ip.ip.ip.ip,,80)
- Timestamp Bereq: 1489568144.730329 0.028878 0.028878
- Timestamp Beresp: 1489568144.759773 0.058322 0.029444
- BerespProtocol HTTP/1.1
- BerespStatus 301
- BerespReason Moved Permanently
- BerespHeader Date: Wed, 15 Mar 2017 08:55:44 GMT
- BerespHeader Server: Apache/2.2.15 (Red Hat)
- BerespHeader Location: http://myserver.mydomain/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
- BerespHeader Content-Length: 514
- BerespHeader Content-Type: text/html; charset=iso-8859-1
- BerespHeader X-Varnish: 526644873
- BerespHeader Age: 0
- BerespHeader Via: 1.1 varnish-v4
- BerespHeader Connection: keep-alive
- TTL RFC 120 -1 -1 1489568145 1489568145 1489568144 0 0
- VCL_call BACKEND_RESPONSE
- TTL VCL 10 10 0 1489568145
- VCL_return deliver
- Storage malloc s0
- ObjProtocol HTTP/1.1
- ObjStatus 301
- ObjReason Moved Permanently
- ObjHeader Date: Wed, 15 Mar 2017 08:55:44 GMT
- ObjHeader Server: Apache/2.2.15 (Red Hat)
- ObjHeader Location: http://myserver.mydomain/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
- ObjHeader Content-Length: 514
- ObjHeader Content-Type: text/html; charset=iso-8859-1
- ObjHeader X-Varnish: 526644873
- ObjHeader Via: 1.1 varnish-v4
- Fetch_Body 3 length stream
- BackendReuse 17 default(ip.ip.ip.ip,,80)
- Timestamp BerespBody: 1489568144.759849 0.058398 0.000076
- Length 514
- BereqAcct 578 0 578 415 514 929
- End
* << Request >> 98313
- Begin req 98312 rxreq
- Timestamp Start: 1489568144.701372 0.000000 0.000000
- Timestamp Req: 1489568144.701372 0.000000 0.000000
- ReqStart ip.ip.ip.ip 63485
- ReqMethod GET
- ReqURL /cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
- ReqProtocol HTTP/1.1
- ReqHeader Host: thisandthatip.compute.amazonaws.com
- ReqHeader Connection: keep-alive
- ReqHeader Upgrade-Insecure-Requests: 1
- ReqHeader User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36
- ReqHeader Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
- ReqHeader Accept-Encoding: gzip, deflate, sdch
- ReqHeader Accept-Language: sv-SE,sv;q=0.8,en-US;q=0.6,en;q=0.4
- ReqHeader X-Forwarded-For: ip.ip.ip.ip
- VCL_call RECV
- VCL_return hash
- ReqUnset Accept-Encoding: gzip, deflate, sdch
- ReqHeader Accept-Encoding: gzip
- VCL_call HASH
- VCL_return lookup
- Debug "XXXX MISS"
- VCL_call MISS
- VCL_return fetch
- Link bereq 98314 fetch
- Timestamp Fetch: 1489568144.759883 0.058511 0.058511
- RespProtocol HTTP/1.1
- RespStatus 301
- RespReason Moved Permanently
- RespHeader Date: Wed, 15 Mar 2017 08:55:44 GMT
- RespHeader Server: Apache/2.2.15 (Red Hat)
- RespHeader Location: http://myserver.mydomain/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
- RespHeader Content-Length: 514
- RespHeader Content-Type: text/html; charset=iso-8859-1
- RespHeader X-Varnish: 526644873
- RespHeader Via: 1.1 varnish-v4
- RespHeader X-Varnish: 98313
- RespHeader Age: 0
- RespHeader Via: 1.1 varnish-v4
- VCL_call DELIVER
- VCL_return deliver
- Timestamp Process: 1489568144.759907 0.058535 0.000024
- Debug "RES_MODE 2"
- RespHeader Connection: keep-alive
- Timestamp Resp: 1489568144.759933 0.058561 0.000026
- Debug "XXX REF 2"
- ReqAcct 566 0 566 454 514 968
- End
Varnish 4.0.4 running on AWS Amazon Linux.
The 301 redirect is not done by your varnish. It is done by an apache server probably your backend. It can be seen by the X-Server header in your curl.
What varnish does is proxify the request and forward it to the backend you declare myhost.mydomain. In fact Varnish will resolve the dns at startup and forward the request to the ip it got.
I see two things to check here :
ban your request from your varnish cache (it may result in a 301 at some time during your test and still be cached, that does not serm to be the case but better start from fresh cache)
Make a curl to your backend to see if you get a 301 or a 200.
If that does not work I would restart your varnish service to refresh the dns resolution.
The Host header entry sent to the backend matched that of the AWS instance. That triggered a redirect in the backend, not in the Varnish cache.
Overriding the http.resp.host value in Varnish solved the problem:
sub vcl_recv {
# Happens before we check if we have this in cache already.
#
# Typically you clean up the request here, removing cookies you don't need,
# rewriting the request, etc.
# Set req.http.host (Host header) to www.airtours.se otherwise a redirect will be triggered
set req.http.host = "myserver.mydomain";
... More setting goes here
}