Skip to content

Commit 226cf36

Browse files
XeJasonLovesDoggo
Xe
and
JasonLovesDoggo
authored
feat(config): custom weight thresholds via CEL (#688)
* feat(config): add Thresholds to the top level config file Signed-off-by: Xe Iaso <[email protected]> * chore(config): make String() on ExpressionOrList join the component expressions Signed-off-by: Xe Iaso <[email protected]> * test(config): ensure unparseable json fails Signed-off-by: Xe Iaso <[email protected]> * fix(config): if no thresholds are set, use the default thresholds Signed-off-by: Xe Iaso <[email protected]> * feat(policy): half implement thresholds Signed-off-by: Xe Iaso <[email protected]> * chore(policy): continue wiring things up Signed-off-by: Xe Iaso <[email protected]> * feat(lib): wire up thresholds Signed-off-by: Xe Iaso <[email protected]> * test(lib): handle behavior from legacy configurations Signed-off-by: Xe Iaso <[email protected]> * docs: document thresholds Signed-off-by: Xe Iaso <[email protected]> * docs: update CHANGELOG, refer to threshold configuration Signed-off-by: Xe Iaso <[email protected]> * fix(lib): fix build Signed-off-by: Xe Iaso <[email protected]> * chore(lib): fix U1000 Signed-off-by: Xe Iaso <[email protected]> --------- Signed-off-by: Xe Iaso <[email protected]> Signed-off-by: Jason Cameron <[email protected]> Co-authored-by: Jason Cameron <[email protected]>
1 parent 1d5fa49 commit 226cf36

22 files changed

+683
-305
lines changed

‎data/botPolicies.yaml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,57 @@ dnsbl: false
9191
status_codes:
9292
CHALLENGE: 200
9393
DENY: 200
94+
95+
# The weight thresholds for when to trigger individual challenges. Any
96+
# CHALLENGE will take precedence over this.
97+
#
98+
# A threshold has four configuration options:
99+
#
100+
# - name: the name that is reported down the stack and used for metrics
101+
# - expression: A CEL expression with the request weight in the variable
102+
# weight
103+
# - action: the Anubis action to apply, similar to in a bot policy
104+
# - challenge: which challenge to send to the user, similar to in a bot policy
105+
#
106+
# See https://anubis.techaro.lol/docs/admin/configuration/thresholds for more
107+
# information.
108+
thresholds:
109+
# By default Anubis ships with the following thresholds:
110+
- name: minimal-suspicion # This client is likely fine, its soul is lighter than a feather
111+
expression: weight < 0 # a feather weighs zero units
112+
action: ALLOW # Allow the traffic through
113+
# For clients that had some weight reduced through custom rules, give them a
114+
# lightweight challenge.
115+
- name: mild-suspicion
116+
expression:
117+
all:
118+
- weight >= 0
119+
- weight < 10
120+
action: CHALLENGE
121+
challenge:
122+
# https://anubis.techaro.lol/docs/admin/configuration/challenges/metarefresh
123+
algorithm: metarefresh
124+
difficulty: 1
125+
report_as: 1
126+
# For clients that are browser-like but have either gained points from custom rules or
127+
# report as a standard browser.
128+
- name: moderate-suspicion
129+
expression:
130+
all:
131+
- weight >= 10
132+
- weight < 20
133+
action: CHALLENGE
134+
challenge:
135+
# https://anubis.techaro.lol/docs/admin/configuration/challenges/proof-of-work
136+
algorithm: fast
137+
difficulty: 2 # two leading zeros, very fast for most clients
138+
report_as: 2
139+
# For clients that are browser like and have gained many points from custom rules
140+
- name: extreme-suspicion
141+
expression: weight >= 20
142+
action: CHALLENGE
143+
challenge:
144+
# https://anubis.techaro.lol/docs/admin/configuration/challenges/proof-of-work
145+
algorithm: fast
146+
difficulty: 4
147+
report_as: 4

‎docs/docs/CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
2727
- Add `robots2policy` CLI utility to convert robots.txt files to Anubis challenge policies using CEL expressions ([#409](https://github.com/TecharoHQ/anubis/issues/409))
2828
- Implement GeoIP and ASN based checks via [Thoth](https://anubis.techaro.lol/docs/admin/thoth) ([#206](https://github.com/TecharoHQ/anubis/issues/206))
2929
- Replace internal SHA256 hashing with xxhash for 4-6x performance improvement in policy evaluation and cache operations
30+
- Add [custom weight thresholds](./admin/configuration/thresholds.mdx) via CEL ([#688](https://github.com/TecharoHQ/anubis/pull/688))
3031

3132
## v1.19.1: Jenomis cen Lexentale - Echo 1
3233

‎docs/docs/admin/configuration/thresholds.mdx‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
# Weight Threshold Configuration
2+
3+
Anubis offers the ability to assign "weight" to requests. This is a custom level of suspicion that rules can add to or remove from. For example, here's how you assign 10 weight points to anything that might be a browser:
4+
5+
```yaml
6+
# botPolicies.yaml
7+
8+
bots:
9+
- name: generic-browser
10+
user_agent_regex: >-
11+
Mozilla|Opera
12+
action: WEIGH
13+
weight:
14+
adjust: 10
15+
```
16+
17+
Thresholds let you take this per-request weight value and take actions in response to it. Thresholds are defined alongside your bot configuration in `botPolicies.yaml`.
18+
19+
:::note
20+
21+
Thresholds DO NOT apply when a request matches a bot rule with the CHALLENGE action. Thresholds only apply when requests don't match any terminal bot rules.
22+
23+
:::
24+
25+
```yaml
26+
# botPolicies.yaml
27+
28+
bots: ...
29+
30+
thresholds:
31+
- name: minimal-suspicion
32+
expression: weight < 0
33+
action: ALLOW
34+
35+
- name: mild-suspicion
36+
expression:
37+
all:
38+
- weight >= 0
39+
- weight < 10
40+
action: CHALLENGE
41+
challenge:
42+
algorithm: metarefresh
43+
difficulty: 1
44+
report_as: 1
45+
46+
- name: moderate-suspicion
47+
expression:
48+
all:
49+
- weight >= 10
50+
- weight < 20
51+
action: CHALLENGE
52+
challenge:
53+
algorithm: fast
54+
difficulty: 2
55+
report_as: 2
56+
57+
- name: extreme-suspicion
58+
expression: weight >= 20
59+
action: CHALLENGE
60+
challenge:
61+
algorithm: fast
62+
difficulty: 4
63+
report_as: 4
64+
```
65+
66+
This defines a suite of 4 thresholds:
67+
68+
1. If the request weight is less than zero, allow it through.
69+
2. If the request weight is greater than or equal to zero, but less than ten: give it [a very lightweight challenge](./challenges/metarefresh.mdx).
70+
3. If the request weight is greater than or equal to ten, but less than twenty: give it [a slightly heavier challenge](./challenges/proof-of-work.mdx).
71+
4. Otherwise, give it [the heaviest challenge](./challenges/proof-of-work.mdx).
72+
73+
Thresholds can be configured with the following options:
74+
75+
<table>
76+
<thead>
77+
<tr>
78+
<th>Name</th>
79+
<th>Description</th>
80+
<th>Example</th>
81+
</tr>
82+
</thead>
83+
<tbody>
84+
<tr>
85+
<td>`name`</td>
86+
<td>The human-readable name for this threshold.</td>
87+
<td>
88+
89+
```yaml
90+
name: extreme-suspicion
91+
```
92+
93+
</td>
94+
</tr>
95+
<tr>
96+
<td>`expression`</td>
97+
<td>A [CEL](https://cel.dev/) expression taking the request weight and returning true or false</td>
98+
<td>
99+
100+
To check if the request weight is less than zero:
101+
102+
```yaml
103+
expression: weight < 0
104+
```
105+
106+
To check if it's between 0 and 10 (inclusive):
107+
108+
```yaml
109+
expression:
110+
all:
111+
- weight >= 0
112+
- weight < 10
113+
```
114+
115+
</td>
116+
</tr>
117+
<tr>
118+
<td>`action`</td>
119+
<td>The Anubis action to apply: `ALLOW`, `CHALLENGE`, or `DENY`</td>
120+
<td>
121+
122+
```yaml
123+
action: ALLOW
124+
```
125+
126+
If you set the CHALLENGE action, you must set challenge details:
127+
128+
```yaml
129+
action: CHALLENGE
130+
challenge:
131+
algorithm: metarefresh
132+
difficulty: 1
133+
report_as: 1
134+
```
135+
136+
</td>
137+
</tr>
138+
139+
</tbody>
140+
</table>

‎docs/docs/admin/policies.mdx‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -261,17 +261,11 @@ Anubis rules can also add or remove "weight" from requests, allowing administrat
261261
adjust: -5
262262
```
263263

264-
This would remove five weight points from the request, making Anubis present the [Meta Refresh challenge](./configuration/challenges/metarefresh.mdx).
264+
This would remove five weight points from the request, which would make Anubis present the [Meta Refresh challenge](./configuration/challenges/metarefresh.mdx) in the default configuration.
265265

266266
### Weight Thresholds
267267

268-
Weight thresholds and challenge associations will be configurable with CEL expressions in the configuration file in an upcoming patch, for now here's how Anubis configures the weight thresholds:
269-
270-
| Weight Expression | Action |
271-
| -----------------------------------------------------: | :------------------------------------------------------------------------------------------------------------------------------------- |
272-
| `weight < 0` (weight is less than 0) | Allow the request through. |
273-
| `weight < 10` (weight is less than 10) | Challenge the client with the [Meta Refresh challenge](./configuration/challenges/metarefresh.mdx) at the default difficulty level. |
274-
| `weight >= 10` (weight is greater than or equal to 10) | Challenge the client with the [Proof of Work challenge](./configuration/challenges/proof-of-work.mdx) at the default difficulty level. |
268+
For more information on configuring weight thresholds, see [Weight Threshold Configuration](./configuration/thresholds.mdx)
275269

276270
### Advice
277271

‎lib/anubis.go‎

Lines changed: 20 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"time"
1616

1717
"github.com/golang-jwt/jwt/v5"
18+
"github.com/google/cel-go/common/types"
1819

1920
"github.com/prometheus/client_golang/prometheus"
2021
"github.com/prometheus/client_golang/prometheus/promauto"
@@ -411,12 +412,6 @@ func cr(name string, rule config.Rule, weight int) policy.CheckResult {
411412
}
412413
}
413414

414-
var (
415-
weightOkayStatic = policy.NewStaticHashChecker("weight/okay")
416-
weightMildSusStatic = policy.NewStaticHashChecker("weight/mild-suspicion")
417-
weightVerySusStatic = policy.NewStaticHashChecker("weight/extreme-suspicion")
418-
)
419-
420415
// Check evaluates the list of rules, and returns the result
421416
func (s *Server) check(r *http.Request) (policy.CheckResult, *policy.Bot, error) {
422417
host := r.Header.Get("X-Real-Ip")
@@ -448,34 +443,25 @@ func (s *Server) check(r *http.Request) (policy.CheckResult, *policy.Bot, error)
448443
}
449444
}
450445

451-
switch {
452-
case weight <= 0:
453-
return cr("weight/okay", config.RuleAllow, weight), &policy.Bot{
454-
Challenge: &config.ChallengeRules{
455-
Difficulty: s.policy.DefaultDifficulty,
456-
ReportAs: s.policy.DefaultDifficulty,
457-
Algorithm: config.DefaultAlgorithm,
458-
},
459-
Rules: weightOkayStatic,
460-
}, nil
461-
case weight > 0 && weight < 10:
462-
return cr("weight/mild-suspicion", config.RuleChallenge, weight), &policy.Bot{
463-
Challenge: &config.ChallengeRules{
464-
Difficulty: s.policy.DefaultDifficulty,
465-
ReportAs: s.policy.DefaultDifficulty,
466-
Algorithm: "metarefresh",
467-
},
468-
Rules: weightMildSusStatic,
469-
}, nil
470-
case weight >= 10:
471-
return cr("weight/extreme-suspicion", config.RuleChallenge, weight), &policy.Bot{
472-
Challenge: &config.ChallengeRules{
473-
Difficulty: s.policy.DefaultDifficulty,
474-
ReportAs: s.policy.DefaultDifficulty,
475-
Algorithm: "fast",
476-
},
477-
Rules: weightVerySusStatic,
478-
}, nil
446+
for _, t := range s.policy.Thresholds {
447+
result, _, err := t.Program.ContextEval(r.Context(), &policy.ThresholdRequest{Weight: weight})
448+
if err != nil {
449+
slog.Error("error when evaluating threshold expression", "expression", t.Expression.String(), "err", err)
450+
continue
451+
}
452+
453+
var matches bool
454+
455+
if val, ok := result.(types.Bool); ok {
456+
matches = bool(val)
457+
}
458+
459+
if matches {
460+
return cr("threshold/"+t.Name, t.Action, weight), &policy.Bot{
461+
Challenge: t.Challenge,
462+
Rules: &checker.List{},
463+
}, nil
464+
}
479465
}
480466

481467
return cr("default/allow", config.RuleAllow, weight), &policy.Bot{

‎lib/policy/celchecker.go‎

Lines changed: 3 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -17,47 +17,18 @@ type CELChecker struct {
1717
}
1818

1919
func NewCELChecker(cfg *config.ExpressionOrList) (*CELChecker, error) {
20-
env, err := expressions.NewEnvironment()
20+
env, err := expressions.BotEnvironment()
2121
if err != nil {
2222
return nil, err
2323
}
2424

25-
var src string
26-
var ast *cel.Ast
27-
28-
if cfg.Expression != "" {
29-
src = cfg.Expression
30-
var iss *cel.Issues
31-
intermediate, iss := env.Compile(src)
32-
if iss != nil {
33-
return nil, iss.Err()
34-
}
35-
36-
ast, iss = env.Check(intermediate)
37-
if iss != nil {
38-
return nil, iss.Err()
39-
}
40-
}
41-
42-
if len(cfg.All) != 0 {
43-
ast, err = expressions.Join(env, expressions.JoinAnd, cfg.All...)
44-
}
45-
46-
if len(cfg.Any) != 0 {
47-
ast, err = expressions.Join(env, expressions.JoinOr, cfg.Any...)
48-
}
49-
50-
if err != nil {
51-
return nil, err
52-
}
53-
54-
program, err := expressions.Compile(env, ast)
25+
program, err := expressions.Compile(env, cfg.String())
5526
if err != nil {
5627
return nil, fmt.Errorf("can't compile CEL program: %w", err)
5728
}
5829

5930
return &CELChecker{
60-
src: src,
31+
src: cfg.String(),
6132
program: program,
6233
}, nil
6334
}

0 commit comments

Comments
 (0)