fix(web): key the subscribe limit on the real client

Behind a reverse proxy every visitor shared the proxy's address, so the
whole site got 10 signups an hour and one person could block everyone.
Over IPv6 each address of a /64 was a fresh key, so the limit did not
bind and the map grew by one entry per address.

- web.trusted_proxies (addresses or CIDR ranges, default empty) is
  validated at load and passed to the web handler. X-Forwarded-For is
  read only when the TCP peer is a trusted proxy; the client is then the
  right-most entry that is not itself a trusted proxy. An empty or
  malformed header falls back to the peer. With no trusted proxy the
  behaviour is unchanged: the header is ignored.
- IPv6 clients are keyed by their /64, IPv4 clients by address.
- The limiter remembers at most 100000 clients. When full it sweeps
  expired entries first and otherwise answers 429 to new clients until
  the oldest window ends, instead of growing.
This commit is contained in:
tgiachi 2026-11-10 15:11:26 +01:00
commit 44b3f9d98a
7 changed files with 334 additions and 26 deletions

View file

@ -281,7 +281,7 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
}
a.WebAddr = pubLn.Addr().String()
a.lns = append(a.lns, pubLn)
a.pub = newHTTPServer(web.New(st, rt))
a.pub = newHTTPServer(web.New(st, rt, cfg.WebTrustedProxies()))
}
a.lmtpOpen = &connTracker{Listener: serveLn, conns: map[*trackedConn]struct{}{}}

View file

@ -344,6 +344,38 @@ func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) {
}
}
func TestWebBelievesForwardedForOnlyFromTrustedProxies(t *testing.T) {
relayAddr, _ := startSink(t)
cfg := testConfig(t, relayAddr)
cfg.Web.TrustedProxies = []string{"127.0.0.1"} // the test client is the proxy
a, err := Start(cfg, "tok")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { a.Close() })
// The form's rate limit counts every POST, before the list is looked up: an unknown list will do.
post := func(xff string) int {
req, _ := http.NewRequest("POST", "http://"+a.WebAddr+"/subscribe/ghost@example.org", strings.NewReader("email=a%40x.com"))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("X-Forwarded-For", xff)
resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
return resp.StatusCode
}
for i := 0; i < 10; i++ {
post("203.0.113.1")
}
if c := post("203.0.113.1"); c != 429 {
t.Fatalf("one client over its limit: %d, want 429", c)
}
if c := post("203.0.113.2"); c != 404 {
t.Fatalf("another client behind the same proxy: %d, want 404 (not limited)", c)
}
}
func TestWebDisabledWhenListenIsEmpty(t *testing.T) {
relayAddr, _ := startSink(t)
cfg := testConfig(t, relayAddr)

View file

@ -32,6 +32,10 @@ type Config struct {
} `yaml:"api"`
Web struct {
Listen string `yaml:"listen"` // public archive and unsubscribe pages; empty disables them
// TrustedProxies are the reverse proxies (CIDR ranges or single addresses) whose
// X-Forwarded-For names the client of the subscribe form's rate limit. Empty, the default:
// the TCP peer is the client and the header is ignored.
TrustedProxies []string `yaml:"trusted_proxies"`
} `yaml:"web"`
// DKIM lists the signing keys, one per domain. The private keys stay in files.
DKIM []DKIMKey `yaml:"dkim"`
@ -76,9 +80,15 @@ type DKIMKey struct {
}
// LMTPAllowed returns the parsed lmtp.allow list (empty: every client is accepted).
func (c Config) LMTPAllowed() []*net.IPNet {
nets := make([]*net.IPNet, 0, len(c.LMTP.Allow))
for _, a := range c.LMTP.Allow {
func (c Config) LMTPAllowed() []*net.IPNet { return parseNets(c.LMTP.Allow) }
// WebTrustedProxies returns the parsed web.trusted_proxies list (empty: X-Forwarded-For is ignored).
func (c Config) WebTrustedProxies() []*net.IPNet { return parseNets(c.Web.TrustedProxies) }
// parseNets parses a list that validate has already checked.
func parseNets(list []string) []*net.IPNet {
nets := make([]*net.IPNet, 0, len(list))
for _, a := range list {
if n, err := parseNet(a); err == nil {
nets = append(nets, n)
}
@ -201,6 +211,11 @@ func (c Config) validate() error {
return fmt.Errorf("lmtp.allow[%d]: %w", i, err)
}
}
for i, a := range c.Web.TrustedProxies {
if _, err := parseNet(a); err != nil {
return fmt.Errorf("web.trusted_proxies[%d]: %w", i, err)
}
}
seen := map[string]bool{}
for i, k := range c.DKIM {
switch {

View file

@ -308,3 +308,60 @@ func TestBaseURLIsChecked(t *testing.T) {
}
}
}
func TestLoadWebTrustedProxies(t *testing.T) {
c, err := Load(write(t, "web:\n trusted_proxies: [10.0.0.0/8, 192.168.1.1, \"::1\", \"fd00::/8\"]\n"))
if err != nil {
t.Fatal(err)
}
nets := c.WebTrustedProxies()
if len(nets) != 4 {
t.Fatalf("nets: %v", nets)
}
for ip, want := range map[string]bool{
"10.9.8.7": true, "192.168.1.1": true, "192.168.1.2": false, "::1": true, "fd00::5": true, "2001:db8::1": false,
} {
got := false
for _, n := range nets {
if n.Contains(net.ParseIP(ip)) {
got = true
}
}
if got != want {
t.Errorf("%s trusted = %v, want %v", ip, got, want)
}
}
for _, empty := range []string{"{}\n", "web:\n trusted_proxies: []\n"} {
if c, err := Load(write(t, empty)); err != nil || len(c.WebTrustedProxies()) != 0 {
t.Fatalf("%q: no trusted proxy by default: %v %v", empty, c.WebTrustedProxies(), err)
}
}
}
func TestLoadRejectsABadTrustedProxy(t *testing.T) {
for _, bad := range []string{"not-an-ip", "10.0.0.0/33", "fd00::/129", "10.0.0/8", "", "10.0.0.1/", "proxy.example.org"} {
_, err := Load(write(t, "web:\n trusted_proxies: [\"127.0.0.1\", \""+bad+"\"]\n"))
if err == nil || !strings.Contains(err.Error(), "web.trusted_proxies[1]") {
t.Errorf("%q: want an error naming the entry, got %v", bad, err)
}
}
}
func TestExampleConfigLoadsAndDocumentsTheSettings(t *testing.T) {
b, err := os.ReadFile("../../parsimony.example.yaml")
if err != nil {
t.Fatal(err)
}
if _, err := Load(write(t, string(b))); err != nil {
t.Fatal(err)
}
// Uncommented, the documented lines are valid.
uncommented := strings.ReplaceAll(string(b), " # trusted_proxies:", " trusted_proxies:")
if uncommented == string(b) {
t.Fatal("web.trusted_proxies is not documented")
}
c, err := Load(write(t, uncommented))
if err != nil || len(c.WebTrustedProxies()) == 0 {
t.Fatalf("documented trusted_proxies: %v %v", c.WebTrustedProxies(), err)
}
}

View file

@ -3,17 +3,21 @@ package web
import (
"net"
"net/http"
"strings"
"sync"
"time"
)
// A person subscribes once; a script posting the form for other people's addresses is what this
// slows down: at most subscribeMax POSTs per subscribeWindow from one address. The window starts
// at the first request of an address and everything about it is forgotten when it ends.
// slows down: at most subscribeMax POSTs per subscribeWindow from one client. A client is an IPv4
// address or an IPv6 /64 (one machine usually holds a whole /64). The window starts at the first
// request of a client and everything about it is forgotten when it ends. At most maxClients are
// remembered: past that, new clients are refused until a window ends, rather than the map growing.
const (
subscribeMax = 10
subscribeWindow = time.Hour
sweepEvery = time.Minute
maxClients = 100000
)
type visits struct {
@ -26,20 +30,27 @@ type subscribeLimiter struct {
now func() time.Time
byIP map[string]*visits
lastSweep time.Time
max int
}
func newSubscribeLimiter(now func() time.Time) *subscribeLimiter {
return &subscribeLimiter{now: now, byIP: map[string]*visits{}, lastSweep: now()}
return &subscribeLimiter{now: now, byIP: map[string]*visits{}, lastSweep: now(), max: maxClients}
}
// allow counts a request from ip. When the address is over its limit it returns false and how long
// until it may try again; a refused request is not counted.
// allow counts a request from ip. When the address is over its limit, or it is new and the map is
// full, it returns false and how long until it may try again; a refused request is not counted.
func (l *subscribeLimiter) allow(ip string) (retryAfter time.Duration, ok bool) {
l.mu.Lock()
defer l.mu.Unlock()
now := l.now()
l.sweep(now)
v := l.byIP[ip]
if v == nil && len(l.byIP) >= l.max {
l.lastSweep = now.Add(-sweepEvery) // force it
if wait := l.sweep(now); len(l.byIP) >= l.max {
return wait, false
}
}
if v == nil || now.Sub(v.first) >= subscribeWindow {
v = &visits{first: now}
l.byIP[ip] = v
@ -51,17 +62,22 @@ func (l *subscribeLimiter) allow(ip string) (retryAfter time.Duration, ok bool)
return 0, true
}
// sweep forgets addresses whose window is over, so the map cannot grow forever.
func (l *subscribeLimiter) sweep(now time.Time) {
// sweep forgets addresses whose window is over, at most once every sweepEvery. It returns how long
// until the next window of a kept address ends (zero when it did not run).
func (l *subscribeLimiter) sweep(now time.Time) (next time.Duration) {
if now.Sub(l.lastSweep) < sweepEvery {
return
return 0
}
l.lastSweep = now
next = subscribeWindow
for ip, v := range l.byIP {
if now.Sub(v.first) >= subscribeWindow {
if left := v.first.Add(subscribeWindow).Sub(now); left <= 0 {
delete(l.byIP, ip)
} else {
next = min(next, left)
}
}
return next
}
func (l *subscribeLimiter) size() int {
@ -70,12 +86,51 @@ func (l *subscribeLimiter) size() int {
return len(l.byIP)
}
// clientIP is the address of the TCP peer. X-Forwarded-For is deliberately ignored: anyone can
// send it, so trusting it would let a script pick a fresh identity for every request.
func clientIP(r *http.Request) string {
// clientKey names the client of r for the limiter: an IPv4 address, or the /64 of an IPv6 one.
//
// The client is the TCP peer. X-Forwarded-For is read only when the peer is one of the trusted
// proxies: anyone can send the header, so trusting it from anybody would let a script pick a fresh
// identity for every request. Even then only the right-most entry that is not itself a trusted
// proxy is taken, since the entries on its left were written by the client. An empty or malformed
// header, or one listing only trusted proxies, leaves the peer.
func clientKey(r *http.Request, trusted []*net.IPNet) string {
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
return r.RemoteAddr
host = r.RemoteAddr
}
return host
ip := net.ParseIP(host)
if ip == nil {
return host
}
if inAny(ip, trusted) {
ip = forwardedClient(r, trusted, ip)
}
if ip4 := ip.To4(); ip4 != nil {
return ip4.String()
}
return ip.Mask(net.CIDRMask(64, 128)).String() + "/64"
}
// forwardedClient walks X-Forwarded-For (all its lines, in order) from the right.
func forwardedClient(r *http.Request, trusted []*net.IPNet, peer net.IP) net.IP {
entries := strings.Split(strings.Join(r.Header.Values("X-Forwarded-For"), ","), ",")
for i := len(entries) - 1; i >= 0; i-- {
ip := net.ParseIP(strings.TrimSpace(entries[i]))
if ip == nil {
return peer
}
if !inAny(ip, trusted) {
return ip
}
}
return peer
}
func inAny(ip net.IP, nets []*net.IPNet) bool {
for _, n := range nets {
if n.Contains(ip) {
return true
}
}
return false
}

View file

@ -7,6 +7,7 @@ import (
"errors"
"html/template"
"log"
"net"
"net/http"
"net/mail"
"net/url"
@ -31,14 +32,19 @@ type server struct {
router *router.Router
tmpl *template.Template
limit *subscribeLimiter
// trusted are the reverse proxies whose X-Forwarded-For is believed.
trusted []*net.IPNet
}
// New returns the public web handler.
func New(s *store.Store, rt *router.Router) http.Handler { return newHandler(s, rt, time.Now) }
// New returns the public web handler. trustedProxies are the reverse proxies allowed to name the
// client in X-Forwarded-For (none: the TCP peer is the client).
func New(s *store.Store, rt *router.Router, trustedProxies []*net.IPNet) http.Handler {
return newHandler(s, rt, time.Now, trustedProxies)
}
// newHandler is New with an injectable clock, for the tests.
func newHandler(s *store.Store, rt *router.Router, now func() time.Time) http.Handler {
srv := &server{store: s, router: rt, limit: newSubscribeLimiter(now), tmpl: template.Must(template.New("").Funcs(template.FuncMap{
func newHandler(s *store.Store, rt *router.Router, now func() time.Time, trusted []*net.IPNet) http.Handler {
srv := &server{store: s, router: rt, limit: newSubscribeLimiter(now), trusted: trusted, tmpl: template.Must(template.New("").Funcs(template.FuncMap{
"hide": archive.Obfuscate,
// The logo is a trusted static asset: inline, so the strict CSP needs no image source for it.
"mark": func() template.HTML { return template.HTML(assets.Mark) },
@ -262,7 +268,7 @@ func (s *server) subscribeForm(w http.ResponseWriter, r *http.Request) {
const maxSubscribeBody = 4 << 10
func (s *server) subscribe(w http.ResponseWriter, r *http.Request) {
if wait, ok := s.limit.allow(clientIP(r)); !ok {
if wait, ok := s.limit.allow(clientKey(r, s.trusted)); !ok {
w.Header().Set("Retry-After", strconv.Itoa(int((wait+time.Second-1)/time.Second)))
s.render(w, http.StatusTooManyRequests, "toomany", nil)
return

View file

@ -5,6 +5,7 @@ import (
"encoding/base64"
"fmt"
"html"
"net"
"net/http"
"net/http/httptest"
"net/url"
@ -53,10 +54,24 @@ func newFixture(t *testing.T) *fixture {
rt := router.New(s)
f := &fixture{s: s, rt: rt, pub: pub, alice: ms[0].ID, bobTok: rt.UnsubscribeToken(ms[1].ID),
out: &outbound.Fake{}, clock: time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)}
f.h = newHandler(s, rt, func() time.Time { return f.clock })
f.h = newHandler(s, rt, func() time.Time { return f.clock }, nil)
return f
}
// trust rebuilds the handler so that it believes X-Forwarded-For from the given proxies.
func (f *fixture) trust(t *testing.T, cidrs ...string) {
t.Helper()
var nets []*net.IPNet
for _, c := range cidrs {
_, n, err := net.ParseCIDR(c)
if err != nil {
t.Fatal(err)
}
nets = append(nets, n)
}
f.h = newHandler(f.s, f.rt, func() time.Time { return f.clock }, nets)
}
// flush sends what is queued, as the background worker would, and returns everything sent so far.
func (f *fixture) flush(t *testing.T) []outbound.Sent {
t.Helper()
@ -138,7 +153,7 @@ func TestIndexEscapesTitles(t *testing.T) {
func TestIndexWithoutPublicLists(t *testing.T) {
s, _ := store.Open(":memory:")
defer s.Close()
rec := get(t, New(s, router.New(s)), "/")
rec := get(t, New(s, router.New(s), nil), "/")
wantStatus(t, rec, 200)
wantBody(t, rec, "No public archives")
}
@ -367,7 +382,7 @@ func subscribeRaw(h http.Handler, list, ip, body string, hdr map[string]string)
for k, v := range hdr {
req.Header.Set(k, v)
}
req.RemoteAddr = ip + ":4321"
req.RemoteAddr = net.JoinHostPort(ip, "4321")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
return rec
@ -714,3 +729,127 @@ func TestSubscribeLimiterSweepsAnAddressWhoseHourIsOver(t *testing.T) {
t.Fatalf("size = %d, want 1 (the first address is forgotten)", l.size())
}
}
func TestSubscribeBehindATrustedProxyLimitsEachClient(t *testing.T) {
f := newFixture(t)
f.trust(t, "10.0.0.0/8")
post := func(xff string) int {
return subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), map[string]string{"X-Forwarded-For": xff}).Code
}
for i := 0; i < 10; i++ {
if c := post("203.0.113.1"); c != 200 {
t.Fatalf("request %d: %d", i+1, c)
}
}
if c := post("203.0.113.1"); c != 429 {
t.Fatalf("the 11th request of one client: %d, want 429", c)
}
// Every visitor comes through the same proxy, yet another one has a budget of their own.
if c := post("203.0.113.2"); c != 200 {
t.Fatalf("another client behind the proxy: %d, want 200", c)
}
}
func TestSubscribeForwardedForFromAnUntrustedPeerIsIgnored(t *testing.T) {
f := newFixture(t)
f.trust(t, "10.0.0.0/8")
for i := 0; i < 10; i++ {
hdr := map[string]string{"X-Forwarded-For": fmt.Sprintf("203.0.113.%d", i)}
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.7", email("a@x.com").Encode(), hdr), 200)
}
hdr := map[string]string{"X-Forwarded-For": "203.0.113.200"}
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.7", email("a@x.com").Encode(), hdr), 429)
}
func TestSubscribeTakesTheRightMostUntrustedForwardedAddress(t *testing.T) {
f := newFixture(t)
f.trust(t, "10.0.0.0/8", "192.168.0.0/16")
// The client may write anything on the left; the proxies append on the right.
for i := 0; i < 10; i++ {
hdr := map[string]string{"X-Forwarded-For": fmt.Sprintf("198.51.100.%d, 203.0.113.5, 192.168.1.1", i)}
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 200)
}
hdr := map[string]string{"X-Forwarded-For": "1.1.1.1,203.0.113.5"}
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 429)
// The header may arrive as several lines: they are one list, in order.
req := httptest.NewRequest("POST", "/subscribe/dev@example.org", strings.NewReader(email("a@x.com").Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Add("X-Forwarded-For", "198.51.100.77")
req.Header.Add("X-Forwarded-For", "203.0.113.5")
req.RemoteAddr = "10.0.0.2:4321"
rec := httptest.NewRecorder()
f.h.ServeHTTP(rec, req)
wantStatus(t, rec, 429)
// The proxy's own address does not count against the client.
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), map[string]string{"X-Forwarded-For": "203.0.113.6"}), 200)
}
func TestSubscribeBadForwardedForFallsBackToThePeer(t *testing.T) {
f := newFixture(t)
f.trust(t, "10.0.0.0/8")
for i, xff := range []string{"", "garbage", "203.0.113.1, nonsense", "203.0.113.1:5555", "10.0.0.7", " , ", "", "x", "", ""} {
hdr := map[string]string{}
if xff != "" {
hdr["X-Forwarded-For"] = xff
}
if c := subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr).Code; c != 200 {
t.Fatalf("request %d (%q): %d", i+1, xff, c)
}
}
// All ten were counted against the proxy itself.
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), nil), 429)
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), map[string]string{"X-Forwarded-For": "203.0.113.1"}), 200)
}
func TestSubscribeIPv6ClientsShareTheirSlash64(t *testing.T) {
f := newFixture(t)
for i := 0; i < 10; i++ {
wantStatus(t, subscribe(t, f, "dev@example.org", fmt.Sprintf("2001:db8:1:2::%x", i+1), email("a@x.com")), 200)
}
wantStatus(t, subscribe(t, f, "dev@example.org", "2001:db8:1:2:ffff:ffff:ffff:ffff", email("a@x.com")), 429)
wantStatus(t, subscribe(t, f, "dev@example.org", "2001:db8:1:3::1", email("a@x.com")), 200)
// IPv4 addresses stay separate, one each.
for i := 0; i < 10; i++ {
wantStatus(t, subscribe(t, f, "dev@example.org", "192.0.2.1", email("a@x.com")), 200)
}
wantStatus(t, subscribe(t, f, "dev@example.org", "192.0.2.2", email("a@x.com")), 200)
// The same goes for an IPv6 client named by a trusted proxy.
f.trust(t, "10.0.0.0/8")
for i := 0; i < 10; i++ {
hdr := map[string]string{"X-Forwarded-For": fmt.Sprintf("2001:db8:9:9::%x", i+1)}
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 200)
}
hdr := map[string]string{"X-Forwarded-For": "2001:db8:9:9:1::1"}
wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 429)
}
func TestSubscribeLimiterIsCapped(t *testing.T) {
t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
now := t0
l := newSubscribeLimiter(func() time.Time { return now })
if l.max != 100000 {
t.Fatalf("max = %d, want 100000", l.max)
}
l.max = 2
l.allow("192.0.2.1")
now = t0.Add(59*time.Minute + 50*time.Second)
l.allow("192.0.2.2") // the periodic sweep runs here and finds nothing to forget
if wait, ok := l.allow("192.0.2.3"); ok || wait != 10*time.Second {
t.Fatalf("a new address when the map is full: wait %v ok %v, want refused for 10s (until the oldest window ends)", wait, ok)
}
if _, ok := l.allow("192.0.2.2"); !ok {
t.Fatal("an address already known keeps its budget")
}
if l.size() != 2 {
t.Fatalf("size = %d, want 2", l.size())
}
// 20 s later the periodic sweep is not due yet, but the first address has expired: a full map
// sweeps before it refuses.
now = t0.Add(time.Hour + 10*time.Second)
if _, ok := l.allow("192.0.2.3"); !ok {
t.Fatal("room was made by forgetting an expired address")
}
if l.size() != 2 {
t.Fatalf("size = %d, want 2", l.size())
}
}