From ae7300ac428b266912d13c484932f7e8c4174432 Mon Sep 17 00:00:00 2001 From: Nils Goroll Date: Tue, 25 Feb 2025 17:13:25 +0100 Subject: [PATCH] cache_ban: Add obj.last_hit This commit adds bans by last LRU time, basically equivalent to the existing obj.age from b06cdbbab4099a351e188106add56b5e1eb0b927 (the code was trivial to add). The primary use case is to remove objects from cache which have not been accessed for a long time, and, in particular, to get rid of request bans by removing all objects which have not been touched since the request ban. --- bin/varnishd/cache/cache_ban.c | 8 ++++- bin/varnishd/cache/cache_ban.h | 5 +-- bin/varnishtest/tests/c00059.vtc | 52 +++++++++++++++++++++++++++++--- include/tbl/ban_arg_oper.h | 1 + include/tbl/ban_vars.h | 3 ++ vmod/vmod_std.vcc | 22 ++++++++++++-- 6 files changed, 81 insertions(+), 10 deletions(-) diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index d611445d60..072f3e21ac 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -498,7 +498,7 @@ ban_evaluate(struct worker *wrk, const uint8_t *bsarg, struct objcore *oc, int rv; /* - * for ttl and age, fix the point in time such that banning refers to + * for ttl, age and lru, fix the point in time such that banning refers to * the same point in time when the ban is evaluated * * for grace/keep, we assume that the absolute values are pola and that @@ -546,6 +546,12 @@ ban_evaluate(struct worker *wrk, const uint8_t *bsarg, struct objcore *oc, darg1 = oc->keep; darg2 = bt.arg2_double; break; + case BANS_ARG_OBJLASTHIT: + if (isnan(oc->last_lru)) + return (0); + darg1 = 0.0 - oc->last_lru; + darg2 = 0.0 - (ban_time(bsarg) - bt.arg2_double); + break; default: WRONG("Wrong BAN_ARG code"); } diff --git a/bin/varnishd/cache/cache_ban.h b/bin/varnishd/cache/cache_ban.h index d3339eef02..468c54747c 100644 --- a/bin/varnishd/cache/cache_ban.h +++ b/bin/varnishd/cache/cache_ban.h @@ -97,7 +97,8 @@ #define BANS_ARG_OBJAGE 0x1d #define BANS_ARG_OBJGRACE 0x1e #define BANS_ARG_OBJKEEP 0x1f -#define BANS_ARG_LIM (BANS_ARG_OBJKEEP + 1) +#define BANS_ARG_OBJLASTHIT 0x20 +#define BANS_ARG_LIM (BANS_ARG_OBJLASTHIT + 1) #define BAN_ARGIDX(x) ((x) - BANS_ARG_OFF_) #define BAN_ARGARRSZ (BANS_ARG_LIM - BANS_ARG_OFF_) @@ -116,7 +117,7 @@ // has an arg2_double (BANS_FLAG_DURATION at build time) #define BANS_HAS_ARG2_DOUBLE(arg) \ ((arg) >= BANS_ARG_OBJTTL && \ - (arg) <= BANS_ARG_OBJKEEP) + (arg) <= BANS_ARG_OBJLASTHIT) /*--------------------------------------------------------------------*/ diff --git a/bin/varnishtest/tests/c00059.vtc b/bin/varnishtest/tests/c00059.vtc index 410e794a4a..4bee69addc 100644 --- a/bin/varnishtest/tests/c00059.vtc +++ b/bin/varnishtest/tests/c00059.vtc @@ -3,6 +3,9 @@ varnishtest "test ban obj.* except obj.http.*" # see c00021.vtc for obj.http.* tests server s1 { + rxreq + expect req.url == "/old" + txresp -status 204 rxreq txresp -bodylen 1 rxreq @@ -15,11 +18,25 @@ server s1 { txresp -bodylen 5 rxreq txresp -bodylen 6 + rxreq + txresp -bodylen 7 } -start -varnish v1 -vcl+backend {} -start +varnish v1 -vcl+backend { + sub vcl_deliver { + set resp.http.hits = obj.hits; + } +} -start client c1 { + txreq -url "/old" + rxresp + expect resp.status == 204 + + txreq -url "/old" + rxresp + expect resp.status == 204 + txreq rxresp expect resp.bodylen == 1 @@ -55,7 +72,7 @@ client c1 { expect resp.bodylen == 2 } -run -varnish v1 -cliok "ban obj.ttl <= 2m" +varnish v1 -cliok "ban obj.status == 200 && obj.ttl <= 2m" client c1 { txreq @@ -75,7 +92,7 @@ client c1 { expect resp.bodylen == 3 } -run -varnish v1 -cliok "ban obj.age < 1m" +varnish v1 -cliok "ban obj.status == 200 && obj.age < 1m" client c1 { txreq @@ -95,7 +112,7 @@ client c1 { expect resp.bodylen == 4 } -run -varnish v1 -cliok "ban obj.grace == 10s" +varnish v1 -cliok "ban obj.status == 200 && obj.grace == 10s" client c1 { txreq @@ -115,12 +132,37 @@ client c1 { expect resp.bodylen == 5 } -run -varnish v1 -cliok "ban obj.keep == 0s" +varnish v1 -cli "param.set ban_lurker_age 0.1" +varnish v1 -cliok "ban obj.status == 200 && obj.keep == 0s" +delay 1 client c1 { txreq rxresp expect resp.bodylen == 6 + + txreq + rxresp + expect resp.bodylen == 6 + expect resp.http.hits == 1 +} -run + +# now we should have two objects, /old from the beginning and the len==6 object +varnish v1 -cliexpect { 2 C} "ban.list" +varnish v1 -cli "param.set ban_lurker_age 600" +varnish v1 -cliok "ban obj.last_hit < 1s" + +# /old survives, but len==6 gets removed +client c1 { + txreq -url "/old" + rxresp + expect resp.http.age > 0 + expect resp.http.hits == 2 + + txreq + rxresp + expect resp.bodylen == 7 + expect resp.http.hits == 0 } -run # duration formatting - 0s is being tested above diff --git a/include/tbl/ban_arg_oper.h b/include/tbl/ban_arg_oper.h index d139e08a63..b118d697cf 100644 --- a/include/tbl/ban_arg_oper.h +++ b/include/tbl/ban_arg_oper.h @@ -52,6 +52,7 @@ ARGOPER(BANS_ARG_OBJTTL, BANS_OPER_DURATION) ARGOPER(BANS_ARG_OBJAGE, BANS_OPER_DURATION) ARGOPER(BANS_ARG_OBJGRACE, BANS_OPER_DURATION) ARGOPER(BANS_ARG_OBJKEEP, BANS_OPER_DURATION) +ARGOPER(BANS_ARG_OBJLASTHIT, BANS_OPER_DURATION) #undef ARGOPER #undef BANS_OPER_STRING diff --git a/include/tbl/ban_vars.h b/include/tbl/ban_vars.h index 015cef8175..efafff8ecc 100644 --- a/include/tbl/ban_vars.h +++ b/include/tbl/ban_vars.h @@ -57,6 +57,9 @@ PVAR("obj.grace", PVAR("obj.keep", BANS_FLAG_OBJ | BANS_FLAG_DURATION, BANS_ARG_OBJKEEP) +PVAR("obj.last_hit", + BANS_FLAG_OBJ | BANS_FLAG_DURATION | BANS_FLAG_NODEDUP, + BANS_ARG_OBJLASTHIT) #undef PVAR /*lint -restore */ diff --git a/vmod/vmod_std.vcc b/vmod/vmod_std.vcc index b5ab2a8aaa..9320147e88 100644 --- a/vmod/vmod_std.vcc +++ b/vmod/vmod_std.vcc @@ -612,10 +612,15 @@ The format of *STRING* is:: * duration fields: - * ``obj.ttl``: Remaining ttl at the time the ban is issued - * ``obj.age``: Object age at the time the ban is issued + * ``obj.ttl``: Remaining ttl + * ``obj.age``: Object age * ``obj.grace``: The grace time of the object * ``obj.keep``: The keep time of the object + * ``obj.last_hit``: Time since the last hit + + ``obj.ttl``, ``obj.age`` and ``obj.last_hit`` are relative to the submission + time of the ban, such that they represent a fixed point in time despite + being specified as a duration. * **: @@ -662,6 +667,19 @@ non-existing header, the operators ``==`` and ``~`` always evaluate as false, while the operators ``!=`` and ``!~`` always evaluate as true, respectively, for any value of **. +``obj.last_hit`` can be used almost as a "last accessed" time, so, for example, +``ban obj.last_hit > 1d`` removes all objects which were last accessed more than +one day ago. Also, it can be used to remove objects stuck at request bans by +issuing ``ban obj.last_hit > X``, with X being slightly less than the time since +the request ban. + +``obj.last_hit`` is based on an internal last LRU time, which might not be +implemented by all storage engines. Where it is not available, ban expressions +using ``obj.last_hit`` evaluate to ``false``, which means that the respective +ban behaves as if it was not present. Where implemented, the last LRU time might +only get updated by the ``lru_interval`` parameter, which therefore is the +maximum precision of ``obj.last_hit`` bans. + $Function STRING ban_error() Returns a textual error description of the last `std.ban()`_ call from