metrics: Added prometheus metrics for rules and challenges

This commit is contained in:
WeebDataHoarder
2025-04-25 11:26:44 +02:00
parent 6f3d81618c
commit 47f9f6fee6
11 changed files with 156 additions and 16 deletions

View File

@@ -304,7 +304,8 @@ However, a few points are left before go-away can be called v1.0.0:
* [ ] Merge all rules and conditions into one large AST for higher performance.
* [ ] Explore exposing a module for direct Caddy usage.
* [ ] More defined way of picking HTTP/HTTP(s) listeners and certificates.
* [ ] Expose metrics for gathering common network ranges, challenge solve rates and acting on them.
* [x] Expose metrics for challenge solve rates and acting on them.
* [ ] Metrics for common network ranges / AS / useragent
## Setup

View File

@@ -13,6 +13,7 @@ import (
"git.gammaspectra.live/git/go-away/lib/settings"
"git.gammaspectra.live/git/go-away/utils"
"github.com/goccy/go-yaml"
"github.com/prometheus/client_golang/prometheus/promhttp"
"log"
"log/slog"
"net/http"
@@ -58,9 +59,10 @@ func main() {
flag.StringVar(&opt.Bind.Network, "bind-network", opt.Bind.Network, "network family to bind HTTP to, e.g. unix, tcp")
flag.BoolVar(&opt.Bind.Proxy, "bind-proxy", opt.Bind.Proxy, "use PROXY protocol in front of the listener")
flag.StringVar(&opt.Bind.SocketMode, "socket-mode", opt.Bind.SocketMode, "socket mode (permissions) for unix domain sockets.")
flag.StringVar(&opt.BindMetrics, "metrics-bind", opt.BindMetrics, "network address to bind metrics on")
flag.StringVar(&opt.BindDebug, "debug-bind", opt.BindDebug, "network address to bind debug on")
slogLevel := flag.String("slog-level", "WARN", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
debugMode := flag.Bool("debug", false, "debug mode with logs and server timings")
flag.BoolVar(&opt.Bind.Passthrough, "passthrough", opt.Bind.Passthrough, "passthrough mode sends all requests to matching backends until state is loaded")
check := flag.Bool("check", false, "check configuration and policies, then exit")
flag.StringVar(&opt.Bind.TLSAcmeAutoCert, "acme-autocert", opt.Bind.TLSAcmeAutoCert, "enables HTTP(s) mode and uses the provided ACME server URL or available service (available: letsencrypt)")
@@ -87,6 +89,10 @@ func main() {
flag.Parse()
if *backendIpHeader == "" {
*backendIpHeader = *clientIpHeader
}
var err error
{
@@ -100,8 +106,16 @@ func main() {
leveler.Set(programLevel)
h := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{
AddSource: *debugMode,
AddSource: programLevel <= slog.LevelDebug,
Level: leveler,
ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
if a.Key == "source" {
if src, ok := a.Value.Any().(*slog.Source); ok {
return slog.String(a.Key, fmt.Sprintf("%s:%d", src.File, src.Line))
}
}
return a
},
})
slog.SetDefault(slog.New(h))
}
@@ -166,11 +180,17 @@ func main() {
// make no-settings, default backend
opt.Backends[parts[0]] = settings.Backend{
URL: parts[1],
URL: parts[1],
IpHeader: *backendIpHeader,
}
}
for k, v := range opt.Backends {
if v.IpHeader == "" {
//set default value
v.IpHeader = *backendIpHeader
}
backend, err := v.Create()
if err != nil {
log.Fatal(fmt.Errorf("backend %s: failed to make reverse proxy: %w", k, err))
@@ -300,7 +320,7 @@ func main() {
}
slog.Warn(
"listening metrics",
"listening debug",
"bind", opt.BindDebug,
)
if err = debugServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
@@ -309,6 +329,25 @@ func main() {
}()
}
if opt.BindMetrics != "" {
go func() {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
metricsServer := http.Server{
Addr: opt.BindMetrics,
Handler: mux,
}
slog.Warn(
"listening metrics",
"bind", opt.BindMetrics,
)
if err = metricsServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
log.Fatal(err)
}
}()
}
if server.TLSConfig != nil {
if err := server.ServeTLS(listener, "", ""); !errors.Is(err, http.ErrServerClosed) {
log.Fatal(err)

8
go.mod
View File

@@ -12,6 +12,7 @@ require (
github.com/google/cel-go v0.25.0
github.com/itchyny/gojq v0.12.17
github.com/pires/go-proxyproto v0.8.0
github.com/prometheus/client_golang v1.22.0
github.com/tetratelabs/wazero v1.9.0
github.com/yl2chen/cidranger v1.0.2
golang.org/x/crypto v0.37.0
@@ -20,11 +21,18 @@ require (
require (
cel.dev/expr v0.23.1 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/itchyny/timefmt-go v0.1.6 // indirect
github.com/kevinpollet/nego v0.0.0-20211010160919-a65cd48cee43 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/stoewer/go-strcase v1.3.0 // indirect
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
golang.org/x/net v0.39.0 // indirect
golang.org/x/sys v0.32.0 // indirect
golang.org/x/text v0.24.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f // indirect

26
go.sum
View File

@@ -6,6 +6,10 @@ github.com/alphadose/haxmap v1.4.1 h1:VtD6VCxUkjNIfJk/aWdYFfOzrRddDFjmvmRmILg7x8
github.com/alphadose/haxmap v1.4.1/go.mod h1:rjHw1IAqbxm0S3U5tD16GoKsiAd8FWx5BJ2IYqXwgmM=
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -13,8 +17,6 @@ github.com/go-jose/go-jose/v4 v4.1.0 h1:cYSYxd3pw5zd2FSXk2vGdn9igQU2PS8MuxrCOCl0
github.com/go-jose/go-jose/v4 v4.1.0/go.mod h1:GG/vqmYm3Von2nYiB2vGTXzdoNKE5tix5tuc6iAd+sw=
github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY=
github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/google/cel-go v0.24.1 h1:jsBCtxG8mM5wiUJDSGUqU0K7Mtr3w7Eyv00rw4DiZxI=
github.com/google/cel-go v0.24.1/go.mod h1:Hdf9TqOaTNSFQA1ybQaRqATVoK7m/zcf7IMhGXP5zI8=
github.com/google/cel-go v0.25.0 h1:jsFw9Fhn+3y2kBbltZR4VEz5xKkcIFRPDnuEzAGv5GY=
github.com/google/cel-go v0.25.0/go.mod h1:hjEb6r5SuOSlhCHmFoLzu8HGCERvIsDAbxDAyNU/MmI=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@@ -25,10 +27,24 @@ github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/my
github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg=
github.com/kevinpollet/nego v0.0.0-20211010160919-a65cd48cee43 h1:Pdirg1gwhEcGjMLyuSxGn9664p+P8J9SrfMgpFwrDyg=
github.com/kevinpollet/nego v0.0.0-20211010160919-a65cd48cee43/go.mod h1:ahLMuLCUyDdXqtqGyuwGev7/PGtO7r7ocvdwDuEN/3E=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pires/go-proxyproto v0.8.0 h1:5unRmEAPbHXHuLjDg01CxJWf91cw3lKHc/0xzKpXEe0=
github.com/pires/go-proxyproto v0.8.0/go.mod h1:iknsfgnH8EkjrMeMyvfKByp9TiBZCKZM0jx2xmKqnVY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -50,14 +66,12 @@ golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5Z
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e h1:UdXH7Kzbj+Vzastr5nVfccbmFsmYNygVLSPk1pEfDoY=
google.golang.org/genproto/googleapis/api v0.0.0-20250414145226-207652e42e2e/go.mod h1:085qFyf2+XaZlRdCgKNCIZ3afY2p4HHZdoIRpId8F4A=
google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f h1:tjZsroqekhC63+WMqzmWyW5Twj/ZfR5HAlpd5YQ1Vs0=
google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f/go.mod h1:Cd8IzgPo5Akum2c9R6FsXNaZbH3Jpa2gpHlW89FqlyQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250414145226-207652e42e2e h1:ztQaXfzEXTmCBvbtWYRhJxW+0iJcz2qXfd38/e9l7bA=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250414145226-207652e42e2e/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f h1:N/PrbTw4kdkqNRzVfWPrBekzLuarFREcbFOiOLkXon4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=

View File

@@ -112,6 +112,9 @@ func (a Challenge) Handle(logger *slog.Logger, w http.ResponseWriter, r *http.Re
data := challenge.RequestDataFromContext(r.Context())
for _, reg := range a.Challenges {
if data.HasValidChallenge(reg.Id()) {
data.State.ChallengeChecked(r, reg, r.URL.String(), logger)
if a.Continue {
return true, nil
}

View File

@@ -97,6 +97,10 @@ type StateInterface interface {
ChallengeFailed(r *http.Request, reg *Registration, err error, redirect string, logger *slog.Logger)
ChallengePassed(r *http.Request, reg *Registration, redirect string, logger *slog.Logger)
ChallengeIssued(r *http.Request, reg *Registration, redirect string, logger *slog.Logger)
ChallengeChecked(r *http.Request, reg *Registration, redirect string, logger *slog.Logger)
RuleHit(r *http.Request, name string, logger *slog.Logger)
RuleMiss(r *http.Request, name string, logger *slog.Logger)
Logger(r *http.Request) *slog.Logger

View File

@@ -43,7 +43,7 @@ func (state *State) ChallengeFailed(r *http.Request, reg *challenge.Registration
}
logger.Warn("challenge failed", "challenge", reg.Name, "err", err, "redirect", redirect)
//TODO: metrics
state.metrics.Challenge(reg.Name, "fail")
}
func (state *State) ChallengePassed(r *http.Request, reg *challenge.Registration, redirect string, logger *slog.Logger) {
@@ -52,7 +52,7 @@ func (state *State) ChallengePassed(r *http.Request, reg *challenge.Registration
}
logger.Warn("challenge passed", "challenge", reg.Name, "redirect", redirect)
//TODO: metrics
state.metrics.Challenge(reg.Name, "pass")
}
func (state *State) ChallengeIssued(r *http.Request, reg *challenge.Registration, redirect string, logger *slog.Logger) {
@@ -61,7 +61,19 @@ func (state *State) ChallengeIssued(r *http.Request, reg *challenge.Registration
}
logger.Info("challenge issued", "challenge", reg.Name, "redirect", redirect)
//TODO: metrics
state.metrics.Challenge(reg.Name, "issue")
}
func (state *State) ChallengeChecked(r *http.Request, reg *challenge.Registration, redirect string, logger *slog.Logger) {
state.metrics.Challenge(reg.Name, "check")
}
func (state *State) RuleHit(r *http.Request, name string, logger *slog.Logger) {
state.metrics.Rule(name, "hit")
}
func (state *State) RuleMiss(r *http.Request, name string, logger *slog.Logger) {
state.metrics.Rule(name, "miss")
}
func (state *State) Logger(r *http.Request) *slog.Logger {

32
lib/metrics.go Normal file
View File

@@ -0,0 +1,32 @@
package lib
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
type stateMetrics struct {
rules *prometheus.CounterVec
challenges *prometheus.CounterVec
}
func newMetrics() *stateMetrics {
return &stateMetrics{
rules: promauto.NewCounterVec(prometheus.CounterOpts{
Name: "go-away_rule_results",
Help: "The number of rule hits or misses",
}, []string{"rule", "result"}),
challenges: promauto.NewCounterVec(prometheus.CounterOpts{
Name: "go-away_challenge_actions",
Help: "The number of challenges issued, passed or explicitly failed",
}, []string{"challenge", "action"}),
}
}
func (metrics *stateMetrics) Rule(name, result string) {
metrics.rules.With(prometheus.Labels{"rule": name, "result": result}).Inc()
}
func (metrics *stateMetrics) Challenge(name, action string) {
metrics.challenges.With(prometheus.Labels{"challenge": name, "action": action}).Inc()
}

View File

@@ -107,6 +107,8 @@ func (rule RuleState) Evaluate(logger *slog.Logger, w http.ResponseWriter, r *ht
return false, fmt.Errorf("error: evaluating administrative rule %s/%s: %w", data.Id.String(), rule.Hash, err)
} else if out != nil && out.Type() == types.BoolType {
if out.Equal(types.True) == types.True {
data.State.RuleHit(r, rule.Name, logger)
next, err = rule.Handler.Handle(lg, w, r, func() http.Handler {
r.Header.Set("X-Away-Rule", rule.Name)
r.Header.Set("X-Away-Hash", rule.Hash)
@@ -134,7 +136,13 @@ func (rule RuleState) Evaluate(logger *slog.Logger, w http.ResponseWriter, r *ht
return next, nil
}
}
} else {
data.State.RuleMiss(r, rule.Name, logger)
}
} else if out != nil {
err := fmt.Errorf("return type not Bool, got %s", out.Type().TypeName())
lg.Error(err.Error())
return false, fmt.Errorf("error: evaluating administrative rule %s/%s: %w", data.Id.String(), rule.Hash, err)
}
return true, nil

View File

@@ -20,9 +20,16 @@ type Backend struct {
// TLSSkipVerify Disable TLS certificate verification, if any
TLSSkipVerify bool `yaml:"tls-skip-verify"`
// IpHeader HTTP header to set containing the IP header. Set - to forcefully ignore global defaults.
IpHeader string `yaml:"ip-header"`
}
func (b Backend) Create() (*httputil.ReverseProxy, error) {
if b.IpHeader == "-" {
b.IpHeader = ""
}
proxy, err := utils.MakeReverseProxy(b.URL)
if err != nil {
return nil, err
@@ -40,9 +47,19 @@ func (b Backend) Create() (*httputil.ReverseProxy, error) {
if b.Host != "" {
transport.TLSClientConfig.ServerName = b.Host
}
if b.IpHeader != "" || b.Host != "" {
director := proxy.Director
proxy.Director = func(req *http.Request) {
req.Host = b.Host
if b.IpHeader != "" {
if ip := utils.GetRemoteAddress(req.Context()); ip != nil {
req.Header.Set(b.IpHeader, ip.Addr().Unmap().String())
}
}
if b.Host != "" {
req.Host = b.Host
}
director(req)
}
}

View File

@@ -44,6 +44,8 @@ type State struct {
close chan struct{}
Mux *http.ServeMux
metrics *stateMetrics
}
func NewState(p policy.Policy, opt settings.Settings, settings policy.StateSettings) (handler http.Handler, err error) {
@@ -51,6 +53,7 @@ func NewState(p policy.Policy, opt settings.Settings, settings policy.StateSetti
state.close = make(chan struct{})
state.settings = settings
state.opt = opt
state.metrics = newMetrics()
state.client = &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
@@ -214,7 +217,6 @@ func NewState(p policy.Policy, opt settings.Settings, settings policy.StateSetti
}
for _, r := range p.Rules {
rule, err := NewRuleState(state, r, conditionReplacer, nil)
if err != nil {
return nil, fmt.Errorf("rule %s: %w", r.Name, err)