diff --git a/internal/app/app.go b/internal/app/app.go index 3c8e2ca..945dc97 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -281,7 +281,7 @@ func Start(cfg config.Config, apiToken string) (*App, error) { } a.WebAddr = pubLn.Addr().String() a.lns = append(a.lns, pubLn) - a.pub = newHTTPServer(web.New(st, rt)) + a.pub = newHTTPServer(web.New(st, rt, cfg.WebTrustedProxies())) } a.lmtpOpen = &connTracker{Listener: serveLn, conns: map[*trackedConn]struct{}{}} diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 92f4410..01802b1 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -344,6 +344,38 @@ func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) { } } +func TestWebBelievesForwardedForOnlyFromTrustedProxies(t *testing.T) { + relayAddr, _ := startSink(t) + cfg := testConfig(t, relayAddr) + cfg.Web.TrustedProxies = []string{"127.0.0.1"} // the test client is the proxy + a, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + // The form's rate limit counts every POST, before the list is looked up: an unknown list will do. + post := func(xff string) int { + req, _ := http.NewRequest("POST", "http://"+a.WebAddr+"/subscribe/ghost@example.org", strings.NewReader("email=a%40x.com")) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("X-Forwarded-For", xff) + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + return resp.StatusCode + } + for i := 0; i < 10; i++ { + post("203.0.113.1") + } + if c := post("203.0.113.1"); c != 429 { + t.Fatalf("one client over its limit: %d, want 429", c) + } + if c := post("203.0.113.2"); c != 404 { + t.Fatalf("another client behind the same proxy: %d, want 404 (not limited)", c) + } +} + func TestWebDisabledWhenListenIsEmpty(t *testing.T) { relayAddr, _ := startSink(t) cfg := testConfig(t, relayAddr) diff --git a/internal/config/config.go b/internal/config/config.go index 7f8f8e8..29dee9a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -32,6 +32,10 @@ type Config struct { } `yaml:"api"` Web struct { Listen string `yaml:"listen"` // public archive and unsubscribe pages; empty disables them + // TrustedProxies are the reverse proxies (CIDR ranges or single addresses) whose + // X-Forwarded-For names the client of the subscribe form's rate limit. Empty, the default: + // the TCP peer is the client and the header is ignored. + TrustedProxies []string `yaml:"trusted_proxies"` } `yaml:"web"` // DKIM lists the signing keys, one per domain. The private keys stay in files. DKIM []DKIMKey `yaml:"dkim"` @@ -76,9 +80,15 @@ type DKIMKey struct { } // LMTPAllowed returns the parsed lmtp.allow list (empty: every client is accepted). -func (c Config) LMTPAllowed() []*net.IPNet { - nets := make([]*net.IPNet, 0, len(c.LMTP.Allow)) - for _, a := range c.LMTP.Allow { +func (c Config) LMTPAllowed() []*net.IPNet { return parseNets(c.LMTP.Allow) } + +// WebTrustedProxies returns the parsed web.trusted_proxies list (empty: X-Forwarded-For is ignored). +func (c Config) WebTrustedProxies() []*net.IPNet { return parseNets(c.Web.TrustedProxies) } + +// parseNets parses a list that validate has already checked. +func parseNets(list []string) []*net.IPNet { + nets := make([]*net.IPNet, 0, len(list)) + for _, a := range list { if n, err := parseNet(a); err == nil { nets = append(nets, n) } @@ -201,6 +211,11 @@ func (c Config) validate() error { return fmt.Errorf("lmtp.allow[%d]: %w", i, err) } } + for i, a := range c.Web.TrustedProxies { + if _, err := parseNet(a); err != nil { + return fmt.Errorf("web.trusted_proxies[%d]: %w", i, err) + } + } seen := map[string]bool{} for i, k := range c.DKIM { switch { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5ff9076..c2fd56a 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -308,3 +308,60 @@ func TestBaseURLIsChecked(t *testing.T) { } } } + +func TestLoadWebTrustedProxies(t *testing.T) { + c, err := Load(write(t, "web:\n trusted_proxies: [10.0.0.0/8, 192.168.1.1, \"::1\", \"fd00::/8\"]\n")) + if err != nil { + t.Fatal(err) + } + nets := c.WebTrustedProxies() + if len(nets) != 4 { + t.Fatalf("nets: %v", nets) + } + for ip, want := range map[string]bool{ + "10.9.8.7": true, "192.168.1.1": true, "192.168.1.2": false, "::1": true, "fd00::5": true, "2001:db8::1": false, + } { + got := false + for _, n := range nets { + if n.Contains(net.ParseIP(ip)) { + got = true + } + } + if got != want { + t.Errorf("%s trusted = %v, want %v", ip, got, want) + } + } + for _, empty := range []string{"{}\n", "web:\n trusted_proxies: []\n"} { + if c, err := Load(write(t, empty)); err != nil || len(c.WebTrustedProxies()) != 0 { + t.Fatalf("%q: no trusted proxy by default: %v %v", empty, c.WebTrustedProxies(), err) + } + } +} + +func TestLoadRejectsABadTrustedProxy(t *testing.T) { + for _, bad := range []string{"not-an-ip", "10.0.0.0/33", "fd00::/129", "10.0.0/8", "", "10.0.0.1/", "proxy.example.org"} { + _, err := Load(write(t, "web:\n trusted_proxies: [\"127.0.0.1\", \""+bad+"\"]\n")) + if err == nil || !strings.Contains(err.Error(), "web.trusted_proxies[1]") { + t.Errorf("%q: want an error naming the entry, got %v", bad, err) + } + } +} + +func TestExampleConfigLoadsAndDocumentsTheSettings(t *testing.T) { + b, err := os.ReadFile("../../parsimony.example.yaml") + if err != nil { + t.Fatal(err) + } + if _, err := Load(write(t, string(b))); err != nil { + t.Fatal(err) + } + // Uncommented, the documented lines are valid. + uncommented := strings.ReplaceAll(string(b), " # trusted_proxies:", " trusted_proxies:") + if uncommented == string(b) { + t.Fatal("web.trusted_proxies is not documented") + } + c, err := Load(write(t, uncommented)) + if err != nil || len(c.WebTrustedProxies()) == 0 { + t.Fatalf("documented trusted_proxies: %v %v", c.WebTrustedProxies(), err) + } +} diff --git a/internal/web/limit.go b/internal/web/limit.go index d6b8e18..d76624f 100644 --- a/internal/web/limit.go +++ b/internal/web/limit.go @@ -3,17 +3,21 @@ package web import ( "net" "net/http" + "strings" "sync" "time" ) // A person subscribes once; a script posting the form for other people's addresses is what this -// slows down: at most subscribeMax POSTs per subscribeWindow from one address. The window starts -// at the first request of an address and everything about it is forgotten when it ends. +// slows down: at most subscribeMax POSTs per subscribeWindow from one client. A client is an IPv4 +// address or an IPv6 /64 (one machine usually holds a whole /64). The window starts at the first +// request of a client and everything about it is forgotten when it ends. At most maxClients are +// remembered: past that, new clients are refused until a window ends, rather than the map growing. const ( subscribeMax = 10 subscribeWindow = time.Hour sweepEvery = time.Minute + maxClients = 100000 ) type visits struct { @@ -26,20 +30,27 @@ type subscribeLimiter struct { now func() time.Time byIP map[string]*visits lastSweep time.Time + max int } func newSubscribeLimiter(now func() time.Time) *subscribeLimiter { - return &subscribeLimiter{now: now, byIP: map[string]*visits{}, lastSweep: now()} + return &subscribeLimiter{now: now, byIP: map[string]*visits{}, lastSweep: now(), max: maxClients} } -// allow counts a request from ip. When the address is over its limit it returns false and how long -// until it may try again; a refused request is not counted. +// allow counts a request from ip. When the address is over its limit, or it is new and the map is +// full, it returns false and how long until it may try again; a refused request is not counted. func (l *subscribeLimiter) allow(ip string) (retryAfter time.Duration, ok bool) { l.mu.Lock() defer l.mu.Unlock() now := l.now() l.sweep(now) v := l.byIP[ip] + if v == nil && len(l.byIP) >= l.max { + l.lastSweep = now.Add(-sweepEvery) // force it + if wait := l.sweep(now); len(l.byIP) >= l.max { + return wait, false + } + } if v == nil || now.Sub(v.first) >= subscribeWindow { v = &visits{first: now} l.byIP[ip] = v @@ -51,17 +62,22 @@ func (l *subscribeLimiter) allow(ip string) (retryAfter time.Duration, ok bool) return 0, true } -// sweep forgets addresses whose window is over, so the map cannot grow forever. -func (l *subscribeLimiter) sweep(now time.Time) { +// sweep forgets addresses whose window is over, at most once every sweepEvery. It returns how long +// until the next window of a kept address ends (zero when it did not run). +func (l *subscribeLimiter) sweep(now time.Time) (next time.Duration) { if now.Sub(l.lastSweep) < sweepEvery { - return + return 0 } l.lastSweep = now + next = subscribeWindow for ip, v := range l.byIP { - if now.Sub(v.first) >= subscribeWindow { + if left := v.first.Add(subscribeWindow).Sub(now); left <= 0 { delete(l.byIP, ip) + } else { + next = min(next, left) } } + return next } func (l *subscribeLimiter) size() int { @@ -70,12 +86,51 @@ func (l *subscribeLimiter) size() int { return len(l.byIP) } -// clientIP is the address of the TCP peer. X-Forwarded-For is deliberately ignored: anyone can -// send it, so trusting it would let a script pick a fresh identity for every request. -func clientIP(r *http.Request) string { +// clientKey names the client of r for the limiter: an IPv4 address, or the /64 of an IPv6 one. +// +// The client is the TCP peer. X-Forwarded-For is read only when the peer is one of the trusted +// proxies: anyone can send the header, so trusting it from anybody would let a script pick a fresh +// identity for every request. Even then only the right-most entry that is not itself a trusted +// proxy is taken, since the entries on its left were written by the client. An empty or malformed +// header, or one listing only trusted proxies, leaves the peer. +func clientKey(r *http.Request, trusted []*net.IPNet) string { host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - return r.RemoteAddr + host = r.RemoteAddr } - return host + ip := net.ParseIP(host) + if ip == nil { + return host + } + if inAny(ip, trusted) { + ip = forwardedClient(r, trusted, ip) + } + if ip4 := ip.To4(); ip4 != nil { + return ip4.String() + } + return ip.Mask(net.CIDRMask(64, 128)).String() + "/64" +} + +// forwardedClient walks X-Forwarded-For (all its lines, in order) from the right. +func forwardedClient(r *http.Request, trusted []*net.IPNet, peer net.IP) net.IP { + entries := strings.Split(strings.Join(r.Header.Values("X-Forwarded-For"), ","), ",") + for i := len(entries) - 1; i >= 0; i-- { + ip := net.ParseIP(strings.TrimSpace(entries[i])) + if ip == nil { + return peer + } + if !inAny(ip, trusted) { + return ip + } + } + return peer +} + +func inAny(ip net.IP, nets []*net.IPNet) bool { + for _, n := range nets { + if n.Contains(ip) { + return true + } + } + return false } diff --git a/internal/web/web.go b/internal/web/web.go index b90cdf9..7eaf456 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -7,6 +7,7 @@ import ( "errors" "html/template" "log" + "net" "net/http" "net/mail" "net/url" @@ -31,14 +32,19 @@ type server struct { router *router.Router tmpl *template.Template limit *subscribeLimiter + // trusted are the reverse proxies whose X-Forwarded-For is believed. + trusted []*net.IPNet } -// New returns the public web handler. -func New(s *store.Store, rt *router.Router) http.Handler { return newHandler(s, rt, time.Now) } +// New returns the public web handler. trustedProxies are the reverse proxies allowed to name the +// client in X-Forwarded-For (none: the TCP peer is the client). +func New(s *store.Store, rt *router.Router, trustedProxies []*net.IPNet) http.Handler { + return newHandler(s, rt, time.Now, trustedProxies) +} // newHandler is New with an injectable clock, for the tests. -func newHandler(s *store.Store, rt *router.Router, now func() time.Time) http.Handler { - srv := &server{store: s, router: rt, limit: newSubscribeLimiter(now), tmpl: template.Must(template.New("").Funcs(template.FuncMap{ +func newHandler(s *store.Store, rt *router.Router, now func() time.Time, trusted []*net.IPNet) http.Handler { + srv := &server{store: s, router: rt, limit: newSubscribeLimiter(now), trusted: trusted, tmpl: template.Must(template.New("").Funcs(template.FuncMap{ "hide": archive.Obfuscate, // The logo is a trusted static asset: inline, so the strict CSP needs no image source for it. "mark": func() template.HTML { return template.HTML(assets.Mark) }, @@ -262,7 +268,7 @@ func (s *server) subscribeForm(w http.ResponseWriter, r *http.Request) { const maxSubscribeBody = 4 << 10 func (s *server) subscribe(w http.ResponseWriter, r *http.Request) { - if wait, ok := s.limit.allow(clientIP(r)); !ok { + if wait, ok := s.limit.allow(clientKey(r, s.trusted)); !ok { w.Header().Set("Retry-After", strconv.Itoa(int((wait+time.Second-1)/time.Second))) s.render(w, http.StatusTooManyRequests, "toomany", nil) return diff --git a/internal/web/web_test.go b/internal/web/web_test.go index 62fae73..554d69f 100644 --- a/internal/web/web_test.go +++ b/internal/web/web_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "fmt" "html" + "net" "net/http" "net/http/httptest" "net/url" @@ -53,10 +54,24 @@ func newFixture(t *testing.T) *fixture { rt := router.New(s) f := &fixture{s: s, rt: rt, pub: pub, alice: ms[0].ID, bobTok: rt.UnsubscribeToken(ms[1].ID), out: &outbound.Fake{}, clock: time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)} - f.h = newHandler(s, rt, func() time.Time { return f.clock }) + f.h = newHandler(s, rt, func() time.Time { return f.clock }, nil) return f } +// trust rebuilds the handler so that it believes X-Forwarded-For from the given proxies. +func (f *fixture) trust(t *testing.T, cidrs ...string) { + t.Helper() + var nets []*net.IPNet + for _, c := range cidrs { + _, n, err := net.ParseCIDR(c) + if err != nil { + t.Fatal(err) + } + nets = append(nets, n) + } + f.h = newHandler(f.s, f.rt, func() time.Time { return f.clock }, nets) +} + // flush sends what is queued, as the background worker would, and returns everything sent so far. func (f *fixture) flush(t *testing.T) []outbound.Sent { t.Helper() @@ -138,7 +153,7 @@ func TestIndexEscapesTitles(t *testing.T) { func TestIndexWithoutPublicLists(t *testing.T) { s, _ := store.Open(":memory:") defer s.Close() - rec := get(t, New(s, router.New(s)), "/") + rec := get(t, New(s, router.New(s), nil), "/") wantStatus(t, rec, 200) wantBody(t, rec, "No public archives") } @@ -367,7 +382,7 @@ func subscribeRaw(h http.Handler, list, ip, body string, hdr map[string]string) for k, v := range hdr { req.Header.Set(k, v) } - req.RemoteAddr = ip + ":4321" + req.RemoteAddr = net.JoinHostPort(ip, "4321") rec := httptest.NewRecorder() h.ServeHTTP(rec, req) return rec @@ -714,3 +729,127 @@ func TestSubscribeLimiterSweepsAnAddressWhoseHourIsOver(t *testing.T) { t.Fatalf("size = %d, want 1 (the first address is forgotten)", l.size()) } } + +func TestSubscribeBehindATrustedProxyLimitsEachClient(t *testing.T) { + f := newFixture(t) + f.trust(t, "10.0.0.0/8") + post := func(xff string) int { + return subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), map[string]string{"X-Forwarded-For": xff}).Code + } + for i := 0; i < 10; i++ { + if c := post("203.0.113.1"); c != 200 { + t.Fatalf("request %d: %d", i+1, c) + } + } + if c := post("203.0.113.1"); c != 429 { + t.Fatalf("the 11th request of one client: %d, want 429", c) + } + // Every visitor comes through the same proxy, yet another one has a budget of their own. + if c := post("203.0.113.2"); c != 200 { + t.Fatalf("another client behind the proxy: %d, want 200", c) + } +} + +func TestSubscribeForwardedForFromAnUntrustedPeerIsIgnored(t *testing.T) { + f := newFixture(t) + f.trust(t, "10.0.0.0/8") + for i := 0; i < 10; i++ { + hdr := map[string]string{"X-Forwarded-For": fmt.Sprintf("203.0.113.%d", i)} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.7", email("a@x.com").Encode(), hdr), 200) + } + hdr := map[string]string{"X-Forwarded-For": "203.0.113.200"} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.7", email("a@x.com").Encode(), hdr), 429) +} + +func TestSubscribeTakesTheRightMostUntrustedForwardedAddress(t *testing.T) { + f := newFixture(t) + f.trust(t, "10.0.0.0/8", "192.168.0.0/16") + // The client may write anything on the left; the proxies append on the right. + for i := 0; i < 10; i++ { + hdr := map[string]string{"X-Forwarded-For": fmt.Sprintf("198.51.100.%d, 203.0.113.5, 192.168.1.1", i)} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 200) + } + hdr := map[string]string{"X-Forwarded-For": "1.1.1.1,203.0.113.5"} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 429) + // The header may arrive as several lines: they are one list, in order. + req := httptest.NewRequest("POST", "/subscribe/dev@example.org", strings.NewReader(email("a@x.com").Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Add("X-Forwarded-For", "198.51.100.77") + req.Header.Add("X-Forwarded-For", "203.0.113.5") + req.RemoteAddr = "10.0.0.2:4321" + rec := httptest.NewRecorder() + f.h.ServeHTTP(rec, req) + wantStatus(t, rec, 429) + // The proxy's own address does not count against the client. + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), map[string]string{"X-Forwarded-For": "203.0.113.6"}), 200) +} + +func TestSubscribeBadForwardedForFallsBackToThePeer(t *testing.T) { + f := newFixture(t) + f.trust(t, "10.0.0.0/8") + for i, xff := range []string{"", "garbage", "203.0.113.1, nonsense", "203.0.113.1:5555", "10.0.0.7", " , ", "", "x", "", ""} { + hdr := map[string]string{} + if xff != "" { + hdr["X-Forwarded-For"] = xff + } + if c := subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr).Code; c != 200 { + t.Fatalf("request %d (%q): %d", i+1, xff, c) + } + } + // All ten were counted against the proxy itself. + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), nil), 429) + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), map[string]string{"X-Forwarded-For": "203.0.113.1"}), 200) +} + +func TestSubscribeIPv6ClientsShareTheirSlash64(t *testing.T) { + f := newFixture(t) + for i := 0; i < 10; i++ { + wantStatus(t, subscribe(t, f, "dev@example.org", fmt.Sprintf("2001:db8:1:2::%x", i+1), email("a@x.com")), 200) + } + wantStatus(t, subscribe(t, f, "dev@example.org", "2001:db8:1:2:ffff:ffff:ffff:ffff", email("a@x.com")), 429) + wantStatus(t, subscribe(t, f, "dev@example.org", "2001:db8:1:3::1", email("a@x.com")), 200) + // IPv4 addresses stay separate, one each. + for i := 0; i < 10; i++ { + wantStatus(t, subscribe(t, f, "dev@example.org", "192.0.2.1", email("a@x.com")), 200) + } + wantStatus(t, subscribe(t, f, "dev@example.org", "192.0.2.2", email("a@x.com")), 200) + // The same goes for an IPv6 client named by a trusted proxy. + f.trust(t, "10.0.0.0/8") + for i := 0; i < 10; i++ { + hdr := map[string]string{"X-Forwarded-For": fmt.Sprintf("2001:db8:9:9::%x", i+1)} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 200) + } + hdr := map[string]string{"X-Forwarded-For": "2001:db8:9:9:1::1"} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 429) +} + +func TestSubscribeLimiterIsCapped(t *testing.T) { + t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + now := t0 + l := newSubscribeLimiter(func() time.Time { return now }) + if l.max != 100000 { + t.Fatalf("max = %d, want 100000", l.max) + } + l.max = 2 + l.allow("192.0.2.1") + now = t0.Add(59*time.Minute + 50*time.Second) + l.allow("192.0.2.2") // the periodic sweep runs here and finds nothing to forget + if wait, ok := l.allow("192.0.2.3"); ok || wait != 10*time.Second { + t.Fatalf("a new address when the map is full: wait %v ok %v, want refused for 10s (until the oldest window ends)", wait, ok) + } + if _, ok := l.allow("192.0.2.2"); !ok { + t.Fatal("an address already known keeps its budget") + } + if l.size() != 2 { + t.Fatalf("size = %d, want 2", l.size()) + } + // 20 s later the periodic sweep is not due yet, but the first address has expired: a full map + // sweeps before it refuses. + now = t0.Add(time.Hour + 10*time.Second) + if _, ok := l.allow("192.0.2.3"); !ok { + t.Fatal("room was made by forgetting an expired address") + } + if l.size() != 2 { + t.Fatalf("size = %d, want 2", l.size()) + } +} diff --git a/parsimony.example.yaml b/parsimony.example.yaml index 216e426..95f87c6 100644 --- a/parsimony.example.yaml +++ b/parsimony.example.yaml @@ -15,6 +15,10 @@ relay: # rate_per_second: 14 # at most this many sends started per second, for relays that throttle (Amazon SES); 0 = unlimited web: listen: 0.0.0.0:8081 # public archive and unsubscribe pages; "" disables them + # Reverse proxies (addresses or CIDR ranges) allowed to name the visitor in X-Forwarded-For, for + # the subscribe form's limit of 10 posts per hour per IPv4 address or IPv6 /64. Empty (the + # default): the TCP peer is the visitor and the header is ignored. + # trusted_proxies: [172.16.0.0/12] api: listen: 0.0.0.0:8080 # The API token is read from the PARSIMONY_API_TOKEN environment variable.