From ce111f6ae9e499f0ddf58ffa964785dcb4980475 Mon Sep 17 00:00:00 2001 From: WeebDataHoarder <57538841+WeebDataHoarder@users.noreply.github.com> Date: Tue, 8 Apr 2025 22:11:58 +0200 Subject: [PATCH] Add DNSBL querying in conditions --- Dockerfile | 2 ++ README.md | 3 ++ cmd/go-away/main.go | 12 +++++-- examples/forgejo.yml | 21 ++++++++----- lib/conditions.go | 49 +++++++++++++++++++++++++++++ lib/state.go | 30 ++++++++++++++++-- utils/decaymap.go | 73 ++++++++++++++++++++++++++++++++++++++++++ utils/dnsbl.go | 75 ++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 254 insertions(+), 11 deletions(-) create mode 100644 utils/decaymap.go create mode 100644 utils/dnsbl.go diff --git a/Dockerfile b/Dockerfile index 53e3094..c8ee672 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,7 @@ ENV GOAWAY_SLOG_LEVEL="WARN" ENV GOAWAY_CLIENT_IP_HEADER="" ENV GOAWAY_JWT_PRIVATE_KEY_SEED="" ENV GOAWAY_BACKEND="" +ENV GOAWAY_DNSBL="dnsbl.dronebl.org" EXPOSE 8080/tcp EXPOSE 8080/udp @@ -51,6 +52,7 @@ ENV JWT_PRIVATE_KEY_SEED="${GOAWAY_JWT_PRIVATE_KEY_SEED}" ENTRYPOINT /bin/go-away --bind ${GOAWAY_BIND} --bind-network ${GOAWAY_BIND_NETWORK} --socket-mode ${GOAWAY_SOCKET_MODE} \ --policy ${GOAWAY_POLICY} --client-ip-header ${GOAWAY_CLIENT_IP_HEADER} \ + --dnsbl ${GOAWAY_DNSBL} \ --challenge-template ${GOAWAY_CHALLENGE_TEMPLATE} --challenge-template-theme ${GOAWAY_CHALLENGE_TEMPLATE_THEME} \ --slog-level ${GOAWAY_SLOG_LEVEL} \ --backend ${GOAWAY_BACKEND} \ No newline at end of file diff --git a/README.md b/README.md index 8af99b6..e605e55 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,9 @@ services: GOAWAY_CHALLENGE_TEMPLATE: forgejo GOAWAY_CHALLENGE_TEMPLATE_THEME: forgejo-dark + # specify a DNSBL for usage in conditions. Defaults to DroneBL + # GOAWAY_DNSBL: "dnsbl.dronebl.org" + GOAWAY_BACKEND: "git.example.com=http://forgejo:3000" # additional backends can be specified via more command arguments diff --git a/cmd/go-away/main.go b/cmd/go-away/main.go index d6d7cbc..07d270c 100644 --- a/cmd/go-away/main.go +++ b/cmd/go-away/main.go @@ -104,6 +104,8 @@ func main() { clientIpHeader := flag.String("client-ip-header", "", "Client HTTP header to fetch their IP address from (X-Real-Ip, X-Client-Ip, X-Forwarded-For, Cf-Connecting-Ip, etc.)") + dnsbl := flag.String("dnsbl", "dnsbl.dronebl.org", "blocklist for DNSBL (default DroneBL)") + policyFile := flag.String("policy", "", "path to policy YAML file") challengeTemplate := flag.String("challenge-template", "anubis", "name or path of the challenge template to use (anubis, forgejo)") challengeTemplateTheme := flag.String("challenge-template-theme", "", "name of the challenge template theme to use (forgejo => [forgejo-dark, forgejo-light, gitea...])") @@ -247,7 +249,7 @@ func main() { }() } - state, err := lib.NewState(p, lib.StateSettings{ + settings := lib.StateSettings{ Backends: createdBackends, Debug: *debugMode, PackageName: *packageName, @@ -255,7 +257,13 @@ func main() { ChallengeTemplateTheme: *challengeTemplateTheme, PrivateKeySeed: seed, ClientIpHeader: *clientIpHeader, - }) + } + + if *dnsbl != "" { + settings.DNSBL = utils.NewDNSBL(*dnsbl, net.DefaultResolver) + } + + state, err := lib.NewState(p, settings) if err != nil { log.Fatal(fmt.Errorf("failed to create state: %w", err)) diff --git a/examples/forgejo.yml b/examples/forgejo.yml index a642044..2a3adb5 100644 --- a/examples/forgejo.yml +++ b/examples/forgejo.yml @@ -410,12 +410,6 @@ rules: - 'path.matches("(?i)^/(WeebDataHoarder|P2Pool|mirror|git|S\\.O\\.N\\.G|FM10K|Sillycom|pwgen2155|kaitou|metonym)/[^/]+$")' action: pass - - name: suspicious-fetchers - action: challenge - challenges: [js-pow-sha256, http-cookie-check] - conditions: - - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")' - # check a sequence of challenges - name: heavy-operations/0 action: check @@ -436,6 +430,19 @@ rules: - 'path.matches("^/[^/]+/[^/]+/media/") && ($is-generic-browser)' action: pass + # check DNSBL and serve harder challenges + - name: undesired-dnsbl + conditions: + - 'inDNSBL(remoteAddress)' + action: check + challenges: [js-pow-sha256, http-cookie-check] + + - name: suspicious-fetchers + action: check + challenges: [js-pow-sha256] + conditions: + - 'userAgent.contains("facebookexternalhit/") || userAgent.contains("facebookcatalog/")' + # Allow PUT/DELETE/PATCH/POST requests in general - name: non-get-request action: pass @@ -443,13 +450,13 @@ rules: - '!(method == "HEAD" || method == "GET")' - - name: standard-tools action: challenge challenges: [self-meta-refresh] conditions: - '($is-generic-robot-ua)' - '($is-tool-ua)' + - '!($is-generic-browser)' - name: standard-browser action: challenge diff --git a/lib/conditions.go b/lib/conditions.go index 322ad5b..e41058d 100644 --- a/lib/conditions.go +++ b/lib/conditions.go @@ -1,11 +1,14 @@ package lib import ( + "context" "fmt" "github.com/google/cel-go/cel" "github.com/google/cel-go/common/types" "github.com/google/cel-go/common/types/ref" + "log/slog" "net" + "time" ) func (state *State) initConditions() (err error) { @@ -20,6 +23,52 @@ func (state *State) initConditions() (err error) { // http.Header cel.Variable("headers", cel.MapType(cel.StringType, cel.StringType)), //TODO: dynamic type? + cel.Function("inDNSBL", + cel.Overload("inDNSBL_ip", + []*cel.Type{cel.AnyType}, + cel.BoolType, + cel.UnaryBinding(func(val ref.Val) ref.Val { + if state.Settings.DNSBL == nil { + return types.Bool(false) + } + + var ip net.IP + switch v := val.Value().(type) { + case []byte: + ip = v + case net.IP: + ip = v + case string: + ip = net.ParseIP(v) + } + + if ip == nil { + panic(fmt.Errorf("invalid ip %v", val.Value())) + } + + var key [net.IPv6len]byte + copy(key[:], ip.To16()) + + result, ok := state.DecayMap.Get(key) + if ok { + return types.Bool(result.Bad()) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + result, err := state.Settings.DNSBL.Lookup(ctx, ip) + if err != nil { + slog.Debug("dnsbl lookup failed", "address", ip.String(), "result", result, "err", err) + } else { + slog.Debug("dnsbl lookup", "address", ip.String(), "result", result) + } + //TODO: configure decay + state.DecayMap.Set(key, result, time.Hour) + + return types.Bool(result.Bad()) + }), + ), + ), cel.Function("inNetwork", cel.Overload("inNetwork_string_ip", []*cel.Type{cel.StringType, cel.AnyType}, diff --git a/lib/state.go b/lib/state.go index f9b001c..8bd0763 100644 --- a/lib/state.go +++ b/lib/state.go @@ -26,6 +26,7 @@ import ( "io" "io/fs" "log/slog" + "net" "net/http" "net/http/httputil" "net/url" @@ -60,6 +61,10 @@ type State struct { Poison map[string][]byte ChallengeSolve sync.Map + + DecayMap *utils.DecayMap[[net.IPv6len]byte, utils.DNSBLResponse] + + close chan struct{} } func (state *State) AwaitChallenge(key []byte, ctx context.Context) challenge.VerifyResult { @@ -107,10 +112,12 @@ type StateSettings struct { ChallengeTemplate string ChallengeTemplateTheme string ClientIpHeader string + DNSBL *utils.DNSBL } -func NewState(p policy.Policy, settings StateSettings) (state *State, err error) { - state = new(State) +func NewState(p policy.Policy, settings StateSettings) (handler http.Handler, err error) { + state := new(State) + state.close = make(chan struct{}) state.Settings = settings state.Client = &http.Client{ CheckRedirect: func(req *http.Request, via []*http.Request) error { @@ -119,6 +126,10 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error) } state.UrlPath = "/.well-known/." + state.Settings.PackageName + if state.Settings.DNSBL != nil { + state.DecayMap = utils.NewDecayMap[[net.IPv6len]byte, utils.DNSBLResponse]() + } + // set a reasonable configuration for default http proxy if there is none for _, backend := range state.Settings.Backends { if proxy, ok := backend.(*httputil.ReverseProxy); ok { @@ -220,6 +231,7 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error) idCounter := challenge.Id(1) + //TODO: move this to self-contained challenge files for challengeName, p := range p.Challenges { // allow nesting @@ -769,6 +781,20 @@ func NewState(p policy.Policy, settings StateSettings) (state *State, err error) return nil, err } + if state.DecayMap != nil { + go func() { + ticker := time.NewTicker(17 * time.Minute) + for { + select { + case <-ticker.C: + state.DecayMap.Decay() + case <-state.close: + return + } + } + }() + } + return state, nil } diff --git a/utils/decaymap.go b/utils/decaymap.go new file mode 100644 index 0000000..f712bdb --- /dev/null +++ b/utils/decaymap.go @@ -0,0 +1,73 @@ +package utils + +import ( + "sync" + "time" +) + +func zilch[T any]() T { + var zero T + return zero +} + +type DecayMap[K, V comparable] struct { + data map[K]DecayMapEntry[V] + lock sync.RWMutex +} + +type DecayMapEntry[V comparable] struct { + Value V + expiry time.Time +} + +func NewDecayMap[K, V comparable]() *DecayMap[K, V] { + return &DecayMap[K, V]{ + data: make(map[K]DecayMapEntry[V]), + } +} + +func (m *DecayMap[K, V]) Get(key K) (V, bool) { + m.lock.RLock() + value, ok := m.data[key] + m.lock.RUnlock() + + if !ok { + return zilch[V](), false + } + + if time.Now().After(value.expiry) { + m.lock.Lock() + // Since previously reading m.data[key], the value may have been updated. + // Delete the entry only if the expiry time is still the same. + if m.data[key].expiry == value.expiry { + delete(m.data, key) + } + m.lock.Unlock() + + return zilch[V](), false + } + + return value.Value, true +} + +func (m *DecayMap[K, V]) Set(key K, value V, ttl time.Duration) { + m.lock.Lock() + defer m.lock.Unlock() + + m.data[key] = DecayMapEntry[V]{ + Value: value, + expiry: time.Now().Add(ttl), + } +} + +func (m *DecayMap[K, V]) Decay() { + m.lock.Lock() + defer m.lock.Unlock() + + now := time.Now() + for key, entry := range m.data { + if now.After(entry.expiry) { + delete(m.data, key) + } + } +} diff --git a/utils/dnsbl.go b/utils/dnsbl.go new file mode 100644 index 0000000..3c3341b --- /dev/null +++ b/utils/dnsbl.go @@ -0,0 +1,75 @@ +package utils + +import ( + "context" + "net" + "strconv" +) + +type DNSBL struct { + target string + resolver *net.Resolver +} + +func NewDNSBL(target string, resolver *net.Resolver) *DNSBL { + if resolver == nil { + resolver = net.DefaultResolver + } + return &DNSBL{ + target: target, + resolver: resolver, + } +} + +var nibbleTable = [16]byte{ + '0', '1', '2', '3', + '4', '5', '6', '7', + '8', '9', 'a', 'b', + 'c', 'd', 'e', 'f', +} + +type DNSBLResponse uint8 + +func (r DNSBLResponse) Bad() bool { + return r != ResponseGood && r != ResponseUnknown +} + +const ( + ResponseGood = DNSBLResponse(0) + ResponseUnknown = DNSBLResponse(255) +) + +func (bl DNSBL) Lookup(ctx context.Context, ip net.IP) (DNSBLResponse, error) { + var target []byte + if ip4 := ip.To4(); ip4 != nil { + // max length preallocate + target = make([]byte, 0, len(bl.target)+1+len(ip4)*4) + + for i := len(ip4) - 1; i >= 0; i-- { + target = strconv.AppendUint(target, uint64(ip4[i]), 10) + target = append(target, '.') + } + } else { + // IPv6 + // max length preallocate + target = make([]byte, 0, len(bl.target)+1+len(ip)*4) + + for i := len(ip) - 1; i >= 0; i-- { + target = append(target, nibbleTable[ip[i]&0xf], '.', ip[i]>>4, '.') + } + } + + target = append(target, bl.target...) + + ips, err := bl.resolver.LookupIP(ctx, "ip4", string(target)) + if err != nil { + return ResponseUnknown, err + } + + for _, ip := range ips { + ip4 := ip.To4() + return DNSBLResponse(ip4[len(ip4)-1]), nil + } + + return ResponseUnknown, nil +}