diff --git a/README.md b/README.md index cffe065..fd61cf3 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,9 @@ A publisher's post passes the same identity check as any post: the `From` header **Rate limiting.** Relays such as Amazon SES throttle sending. Set `relay.rate_per_second` to the most sends the daemon may start in one second (default `0`, unlimited). One pacer sits in front of every send attempt, retries included, and spaces them evenly. Set it a little under your relay's limit (12 for a limit of 14). Each recipient is one message, so a post to N members takes about N divided by the rate in seconds. -**Subscribe form.** A list whose archive is public and whose join policy is open gets a signup page at `/subscribe/{address}` on the public web listener, linked from its archive page. It sends the usual double opt-in message. The page that follows is always the same, whether or not the address is a member or the list qualifies for mail, so the form cannot be used to find out who is subscribed. A hidden field catches bots; each IP (the TCP peer, `X-Forwarded-For` is ignored) may post 10 times per hour, then gets `429`; behind a reverse proxy every visitor shares its address, so limit at the proxy too. The body is capped at 4 KiB. +**Subscribe form.** A list whose archive is public and whose join policy is open gets a signup page at `/subscribe/{address}` on the public web listener, linked from its archive page. It sends the usual double opt-in message. The page that follows is always the same, whether or not the address is a member or the list qualifies for mail, so the form cannot be used to find out who is subscribed. A hidden field catches bots; each client (an IPv4 address or an IPv6 /64) may post 10 times per hour, then gets `429`. Behind a reverse proxy, list the proxy in `web.trusted_proxies` (addresses or CIDR ranges, default empty): `X-Forwarded-For` is then read only from those proxies, and the client is its right-most address that is not a trusted proxy. With the list empty the header is ignored and the TCP peer is the client, so every visitor behind a proxy shares one budget. Make sure the proxy appends to `X-Forwarded-For` and cannot be bypassed. At most 100000 clients are tracked; past that, new clients get `429` until a window ends. The form refuses addresses at `localhost`, at bare IPs or `[literals]`, and at names ending in `.local`, `.localhost`, `.internal` or `.invalid`. The body is capped at 4 KiB. + +**Confirmation cap.** Each subscribe or unsubscribe request sends a confirmation mail. At most `limits.pending_per_hour` of them (default 300) are accepted per hour across all lists. Past that, requests get no mail, the visitor still sees the usual page, and the daemon logs a warning at most once a minute. Raise it before a newsletter launch. Not included: scheduled sends and open or click tracking. @@ -242,6 +244,8 @@ relay: ## Delivery and the queue +Each delivery has a time limit per step rather than one overall. Connecting, the greeting, TLS, authentication and the envelope get 60 seconds together. Sending the message gets 60 seconds plus one second per 32 KiB, at most 10 minutes. The relay then has 10 minutes to accept it, as RFC 5321 recommends, so a relay that runs content filters is not cut off and does not receive the message twice. If the relay never answers after the whole message was sent, the delivery is retried and the error in the queue says the outcome is unknown: the relay may already have it. + LMTP answers `250` when the post is in the queue, so a down relay never bounces mail back to Postfix. It answers `451` only when the database itself fails. diff --git a/internal/app/app.go b/internal/app/app.go index 3c8e2ca..0ac8bf6 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -218,6 +218,9 @@ func Start(cfg config.Config, apiToken string) (*App, error) { return nil, fmt.Errorf("store: %w", err) } st.Now = nowFunc + if cfg.Limits.PendingPerHour > 0 { // zero: a Config not made by config.Load, keep the default + st.PendingPerHour = int(cfg.Limits.PendingPerHour) + } a := &App{store: st} rt := router.New(st) @@ -281,7 +284,7 @@ func Start(cfg config.Config, apiToken string) (*App, error) { } a.WebAddr = pubLn.Addr().String() a.lns = append(a.lns, pubLn) - a.pub = newHTTPServer(web.New(st, rt)) + a.pub = newHTTPServer(web.New(st, rt, cfg.WebTrustedProxies())) } a.lmtpOpen = &connTracker{Listener: serveLn, conns: map[*trackedConn]struct{}{}} diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 92f4410..4744fd6 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -344,6 +344,60 @@ func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) { } } +func TestWebBelievesForwardedForOnlyFromTrustedProxies(t *testing.T) { + relayAddr, _ := startSink(t) + cfg := testConfig(t, relayAddr) + cfg.Web.TrustedProxies = []string{"127.0.0.1"} // the test client is the proxy + a, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + // The form's rate limit counts every POST, before the list is looked up: an unknown list will do. + post := func(xff string) int { + req, _ := http.NewRequest("POST", "http://"+a.WebAddr+"/subscribe/ghost@example.org", strings.NewReader("email=a%40x.com")) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("X-Forwarded-For", xff) + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + return resp.StatusCode + } + for i := 0; i < 10; i++ { + post("203.0.113.1") + } + if c := post("203.0.113.1"); c != 429 { + t.Fatalf("one client over its limit: %d, want 429", c) + } + if c := post("203.0.113.2"); c != 404 { + t.Fatalf("another client behind the same proxy: %d, want 404 (not limited)", c) + } +} + +func TestPendingPerHourReachesTheStore(t *testing.T) { + relayAddr, relay := startSink(t) + cfg := testConfig(t, relayAddr) + cfg.Limits.PendingPerHour = 1 + a, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok") + for _, from := range []string{"carol@x.com", "dave@x.com"} { + if err := lmtpSend(t, a.LMTPAddr, from, "dev+join@example.org", ""); err != nil { + t.Fatal(err) + } + } + relay.waitCount(t, 1) + time.Sleep(200 * time.Millisecond) + if n := len(relay.all()); n != 1 { + t.Fatalf("limits.pending_per_hour = 1: %d confirmation mails, want 1", n) + } +} + func TestWebDisabledWhenListenIsEmpty(t *testing.T) { relayAddr, _ := startSink(t) cfg := testConfig(t, relayAddr) diff --git a/internal/config/config.go b/internal/config/config.go index 7f8f8e8..bfb8433 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -32,7 +32,16 @@ type Config struct { } `yaml:"api"` Web struct { Listen string `yaml:"listen"` // public archive and unsubscribe pages; empty disables them + // TrustedProxies are the reverse proxies (CIDR ranges or single addresses) whose + // X-Forwarded-For names the client of the subscribe form's rate limit. Empty, the default: + // the TCP peer is the client and the header is ignored. + TrustedProxies []string `yaml:"trusted_proxies"` } `yaml:"web"` + Limits struct { + // PendingPerHour is the most subscribe and unsubscribe requests (each one sends a + // confirmation mail) accepted in an hour, for all lists together. + PendingPerHour Count `yaml:"pending_per_hour"` + } `yaml:"limits"` // DKIM lists the signing keys, one per domain. The private keys stay in files. DKIM []DKIMKey `yaml:"dkim"` Relay struct { @@ -68,6 +77,20 @@ func (r *Rate) UnmarshalYAML(n *yaml.Node) error { return nil } +// Count is limits.pending_per_hour. It exists so that a value that is not a whole number is reported +// with the name of the key. +type Count int + +// UnmarshalYAML reads a YAML integer. +func (c *Count) UnmarshalYAML(n *yaml.Node) error { + var i int + if n.Tag != "!!int" || n.Decode(&i) != nil { + return fmt.Errorf("line %d: limits.pending_per_hour must be a whole number, not %q", n.Line, n.Value) + } + *c = Count(i) + return nil +} + // DKIMKey is the signing key of one domain. type DKIMKey struct { Domain string `yaml:"domain"` @@ -76,9 +99,15 @@ type DKIMKey struct { } // LMTPAllowed returns the parsed lmtp.allow list (empty: every client is accepted). -func (c Config) LMTPAllowed() []*net.IPNet { - nets := make([]*net.IPNet, 0, len(c.LMTP.Allow)) - for _, a := range c.LMTP.Allow { +func (c Config) LMTPAllowed() []*net.IPNet { return parseNets(c.LMTP.Allow) } + +// WebTrustedProxies returns the parsed web.trusted_proxies list (empty: X-Forwarded-For is ignored). +func (c Config) WebTrustedProxies() []*net.IPNet { return parseNets(c.Web.TrustedProxies) } + +// parseNets parses a list that validate has already checked. +func parseNets(list []string) []*net.IPNet { + nets := make([]*net.IPNet, 0, len(list)) + for _, a := range list { if n, err := parseNet(a); err == nil { nets = append(nets, n) } @@ -201,6 +230,14 @@ func (c Config) validate() error { return fmt.Errorf("lmtp.allow[%d]: %w", i, err) } } + if c.Limits.PendingPerHour < 1 { + return fmt.Errorf("limits.pending_per_hour must be 1 or more, not %d", c.Limits.PendingPerHour) + } + for i, a := range c.Web.TrustedProxies { + if _, err := parseNet(a); err != nil { + return fmt.Errorf("web.trusted_proxies[%d]: %w", i, err) + } + } seen := map[string]bool{} for i, k := range c.DKIM { switch { @@ -230,6 +267,7 @@ func Load(path string) (Config, error) { c.Relay.Concurrency = 4 c.API.Listen = "127.0.0.1:8080" c.Web.Listen = "127.0.0.1:8081" + c.Limits.PendingPerHour = 300 b, err := os.ReadFile(path) if err != nil { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5ff9076..a7b35bc 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -308,3 +308,80 @@ func TestBaseURLIsChecked(t *testing.T) { } } } + +func TestLoadWebTrustedProxies(t *testing.T) { + c, err := Load(write(t, "web:\n trusted_proxies: [10.0.0.0/8, 192.168.1.1, \"::1\", \"fd00::/8\"]\n")) + if err != nil { + t.Fatal(err) + } + nets := c.WebTrustedProxies() + if len(nets) != 4 { + t.Fatalf("nets: %v", nets) + } + for ip, want := range map[string]bool{ + "10.9.8.7": true, "192.168.1.1": true, "192.168.1.2": false, "::1": true, "fd00::5": true, "2001:db8::1": false, + } { + got := false + for _, n := range nets { + if n.Contains(net.ParseIP(ip)) { + got = true + } + } + if got != want { + t.Errorf("%s trusted = %v, want %v", ip, got, want) + } + } + for _, empty := range []string{"{}\n", "web:\n trusted_proxies: []\n"} { + if c, err := Load(write(t, empty)); err != nil || len(c.WebTrustedProxies()) != 0 { + t.Fatalf("%q: no trusted proxy by default: %v %v", empty, c.WebTrustedProxies(), err) + } + } +} + +func TestLoadRejectsABadTrustedProxy(t *testing.T) { + for _, bad := range []string{"not-an-ip", "10.0.0.0/33", "fd00::/129", "10.0.0/8", "", "10.0.0.1/", "proxy.example.org"} { + _, err := Load(write(t, "web:\n trusted_proxies: [\"127.0.0.1\", \""+bad+"\"]\n")) + if err == nil || !strings.Contains(err.Error(), "web.trusted_proxies[1]") { + t.Errorf("%q: want an error naming the entry, got %v", bad, err) + } + } +} + +func TestExampleConfigLoadsAndDocumentsTheSettings(t *testing.T) { + b, err := os.ReadFile("../../parsimony.example.yaml") + if err != nil { + t.Fatal(err) + } + if _, err := Load(write(t, string(b))); err != nil { + t.Fatal(err) + } + // Uncommented, the documented lines are valid. + uncommented := string(b) + for _, line := range []string{" # trusted_proxies:", "# limits:", "# pending_per_hour:"} { + if !strings.Contains(uncommented, line) { + t.Fatalf("%q is not documented", line) + } + uncommented = strings.ReplaceAll(uncommented, line, strings.Replace(line, "# ", "", 1)) + } + c, err := Load(write(t, uncommented)) + if err != nil || len(c.WebTrustedProxies()) == 0 || c.Limits.PendingPerHour != 300 { + t.Fatalf("documented settings: %v %d %v", c.WebTrustedProxies(), c.Limits.PendingPerHour, err) + } +} + +func TestLimitsPendingPerHour(t *testing.T) { + c, err := Load(write(t, "{}\n")) + if err != nil || c.Limits.PendingPerHour != 300 { + t.Fatalf("default: %d %v", c.Limits.PendingPerHour, err) + } + c, err = Load(write(t, "limits:\n pending_per_hour: 2000\n")) + if err != nil || c.Limits.PendingPerHour != 2000 { + t.Fatalf("value: %d %v", c.Limits.PendingPerHour, err) + } + for _, bad := range []string{"0", "-1", "lots", "1.5"} { + _, err := Load(write(t, "limits:\n pending_per_hour: "+bad+"\n")) + if err == nil || !strings.Contains(err.Error(), "pending_per_hour") { + t.Errorf("%s: want an error naming limits.pending_per_hour, got %v", bad, err) + } + } +} diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go index 1cfc525..ff84522 100644 --- a/internal/store/m2_test.go +++ b/internal/store/m2_test.go @@ -718,6 +718,7 @@ func TestConfirmPendingIsSingleUseUnderConcurrency(t *testing.T) { } } +<<<<<<< HEAD func TestMigrationAddsTheOutboxPriorityAloneAfterAHalfDoneUpgrade(t *testing.T) { // A crash between adding post_id and priority leaves outbox with the first and not the second. path := t.TempDir() + "/old.db" @@ -737,5 +738,69 @@ func TestMigrationAddsTheOutboxPriorityAloneAfterAHalfDoneUpgrade(t *testing.T) defer s2.Close() if ok, err := hasColumn(s2.db, "outbox", "priority"); err != nil || !ok { t.Fatalf("priority is added back: %v %v", ok, err) +======= +func TestConfirmationMailGlobalCapIsConfigurable(t *testing.T) { + s := newTestStore(t) + if s.PendingPerHour != 300 { + t.Fatalf("default PendingPerHour = %d, want 300", s.PendingPerHour) + } + s.PendingPerHour = 2 + s.Logf = func(string, ...any) {} + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + for i := 0; i < 2; i++ { + if _, created, err := s.CreatePending(l.ID, fmt.Sprintf("u%d@x.com", i), "join"); err != nil || !created { + t.Fatalf("request %d: %v %v", i+1, created, err) + } + } + if _, created, err := s.CreatePending(l.ID, "u2@x.com", "join"); err != nil || created { + t.Fatalf("the third request in the hour: created %v err %v, want refused", created, err) + } +} + +func TestGlobalCapRefusalsAreLoggedOncePerMinute(t *testing.T) { + s := newTestStore(t) + s.PendingPerHour = 1 + var logged []string + s.Logf = func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) } + t0 := time.Unix(1_000_000, 0) + now := t0 + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + s.CreatePending(l.ID, "first@x.com", "join") + s.CreatePending(l.ID, "first@x.com", "join") // a repeat is not the global cap + if len(logged) != 0 { + t.Fatalf("nothing refused by the global cap yet: %q", logged) + } + for i := 0; i < 5; i++ { + s.CreatePending(l.ID, fmt.Sprintf("u%d@x.com", i), "join") + } + if len(logged) != 1 || !strings.Contains(logged[0], "limits.pending_per_hour") || !strings.Contains(logged[0], "1") { + t.Fatalf("one warning naming the setting: %q", logged) + } + now = t0.Add(time.Minute - time.Second) + s.CreatePending(l.ID, "late@x.com", "join") + if len(logged) != 1 { + t.Fatalf("at most one line a minute: %q", logged) + } + now = t0.Add(time.Minute) + s.CreatePending(l.ID, "later@x.com", "join") + if len(logged) != 2 { + t.Fatalf("a minute later the next refusal is logged: %q", logged) + } +} + +func TestPerAddressCapRefusalsAreNotLogged(t *testing.T) { + s := newTestStore(t) + var logged []string + s.Logf = func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) } + for i := 0; i <= maxPendingPerAddress; i++ { + l, _ := s.CreateList(fmt.Sprintf("l%d", i), "example.org", "L") + s.CreatePending(l.ID, "victim@x.com", "join") + } + if len(logged) != 0 { + t.Fatalf("the per-address cap is not the global one: %q", logged) +>>>>>>> fix/sr-web-limits } } diff --git a/internal/store/store.go b/internal/store/store.go index 6f74622..80181ec 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -8,9 +8,11 @@ import ( "encoding/hex" "errors" "fmt" + "log" "mime" "net/mail" "strings" + "sync" "time" "unicode" @@ -70,7 +72,7 @@ const ( // Caps on confirmation mail, per hour. Anyone can name any address in a join request, so the // daemon must not mail one victim many times (maxPendingPerAddress, across all lists) or send - // an unbounded amount in total (maxPendingPerHour). + // an unbounded amount in total (Store.PendingPerHour, maxPendingPerHour by default). // A moderated list could be flooded with mail that nobody reads: the queue per list is bounded, // and a message nobody moderated for a month is dropped. maxHeldPerList = 500 @@ -78,6 +80,9 @@ const ( maxPendingPerAddress = 3 maxPendingPerHour = 300 + + // pendingCapLogEvery spaces the warnings about requests refused by the global cap. + pendingCapLogEvery = time.Minute ) // List is a mailing list. @@ -133,6 +138,14 @@ type Store struct { db *sql.DB // Now returns the current time; tests replace it. Now func() time.Time + // PendingPerHour is the most join and leave requests recorded in an hour, for all lists and + // addresses together (limits.pending_per_hour). + PendingPerHour int + // Logf receives the warning written when PendingPerHour refuses a request; tests replace it. + Logf func(format string, args ...any) + + capLogMu sync.Mutex + capLoggedAt time.Time } const schema = ` @@ -261,7 +274,7 @@ func Open(path string) (*Store, error) { db.Close() return nil, fmt.Errorf("migrate: %w", err) } - return &Store{db: db, Now: time.Now}, nil + return &Store{db: db, Now: time.Now, PendingPerHour: maxPendingPerHour, Logf: log.Printf}, nil } func hasColumn(db *sql.DB, table, column string) (bool, error) { @@ -777,9 +790,13 @@ func (s *Store) CreatePending(listID int64, addr, action string) (token string, if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE address=? AND created_at>?`, addr, since).Scan(&n); err != nil || n >= maxPendingPerAddress { return "", false, err } - if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE created_at>?`, since).Scan(&n); err != nil || n >= maxPendingPerHour { + if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE created_at>?`, since).Scan(&n); err != nil { return "", false, err } + if n >= s.PendingPerHour { + s.warnPendingCap(now, n) + return "", false, nil + } // 80 bits: unguessable within the 72 h life of a token, and short enough that // name(<=30)+"+confirm-"+token fits the 64 characters of a local part. token, err = randomHex(10) @@ -794,6 +811,18 @@ func (s *Store) CreatePending(listID int64, addr, action string) (token string, return token, true, nil } +// warnPendingCap tells the operator, at most once every pendingCapLogEvery, that requests are being +// dropped: the visitor still sees the neutral page and gets no mail. +func (s *Store) warnPendingCap(now time.Time, n int) { + s.capLogMu.Lock() + defer s.capLogMu.Unlock() + if now.Sub(s.capLoggedAt) < pendingCapLogEvery { + return + } + s.capLoggedAt = now + s.Logf("warning: %d join and leave requests in the last hour reached limits.pending_per_hour (%d): new requests get no confirmation mail until older ones are an hour old", n, s.PendingPerHour) +} + // ConfirmPending applies the request behind a token and consumes the token, in one transaction: a // token is only used up when its action happened, and two confirmations of the same token cannot // both apply (the first one to claim the row wins, the other gets ErrNotFound). diff --git a/internal/web/limit.go b/internal/web/limit.go index d6b8e18..d76624f 100644 --- a/internal/web/limit.go +++ b/internal/web/limit.go @@ -3,17 +3,21 @@ package web import ( "net" "net/http" + "strings" "sync" "time" ) // A person subscribes once; a script posting the form for other people's addresses is what this -// slows down: at most subscribeMax POSTs per subscribeWindow from one address. The window starts -// at the first request of an address and everything about it is forgotten when it ends. +// slows down: at most subscribeMax POSTs per subscribeWindow from one client. A client is an IPv4 +// address or an IPv6 /64 (one machine usually holds a whole /64). The window starts at the first +// request of a client and everything about it is forgotten when it ends. At most maxClients are +// remembered: past that, new clients are refused until a window ends, rather than the map growing. const ( subscribeMax = 10 subscribeWindow = time.Hour sweepEvery = time.Minute + maxClients = 100000 ) type visits struct { @@ -26,20 +30,27 @@ type subscribeLimiter struct { now func() time.Time byIP map[string]*visits lastSweep time.Time + max int } func newSubscribeLimiter(now func() time.Time) *subscribeLimiter { - return &subscribeLimiter{now: now, byIP: map[string]*visits{}, lastSweep: now()} + return &subscribeLimiter{now: now, byIP: map[string]*visits{}, lastSweep: now(), max: maxClients} } -// allow counts a request from ip. When the address is over its limit it returns false and how long -// until it may try again; a refused request is not counted. +// allow counts a request from ip. When the address is over its limit, or it is new and the map is +// full, it returns false and how long until it may try again; a refused request is not counted. func (l *subscribeLimiter) allow(ip string) (retryAfter time.Duration, ok bool) { l.mu.Lock() defer l.mu.Unlock() now := l.now() l.sweep(now) v := l.byIP[ip] + if v == nil && len(l.byIP) >= l.max { + l.lastSweep = now.Add(-sweepEvery) // force it + if wait := l.sweep(now); len(l.byIP) >= l.max { + return wait, false + } + } if v == nil || now.Sub(v.first) >= subscribeWindow { v = &visits{first: now} l.byIP[ip] = v @@ -51,17 +62,22 @@ func (l *subscribeLimiter) allow(ip string) (retryAfter time.Duration, ok bool) return 0, true } -// sweep forgets addresses whose window is over, so the map cannot grow forever. -func (l *subscribeLimiter) sweep(now time.Time) { +// sweep forgets addresses whose window is over, at most once every sweepEvery. It returns how long +// until the next window of a kept address ends (zero when it did not run). +func (l *subscribeLimiter) sweep(now time.Time) (next time.Duration) { if now.Sub(l.lastSweep) < sweepEvery { - return + return 0 } l.lastSweep = now + next = subscribeWindow for ip, v := range l.byIP { - if now.Sub(v.first) >= subscribeWindow { + if left := v.first.Add(subscribeWindow).Sub(now); left <= 0 { delete(l.byIP, ip) + } else { + next = min(next, left) } } + return next } func (l *subscribeLimiter) size() int { @@ -70,12 +86,51 @@ func (l *subscribeLimiter) size() int { return len(l.byIP) } -// clientIP is the address of the TCP peer. X-Forwarded-For is deliberately ignored: anyone can -// send it, so trusting it would let a script pick a fresh identity for every request. -func clientIP(r *http.Request) string { +// clientKey names the client of r for the limiter: an IPv4 address, or the /64 of an IPv6 one. +// +// The client is the TCP peer. X-Forwarded-For is read only when the peer is one of the trusted +// proxies: anyone can send the header, so trusting it from anybody would let a script pick a fresh +// identity for every request. Even then only the right-most entry that is not itself a trusted +// proxy is taken, since the entries on its left were written by the client. An empty or malformed +// header, or one listing only trusted proxies, leaves the peer. +func clientKey(r *http.Request, trusted []*net.IPNet) string { host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - return r.RemoteAddr + host = r.RemoteAddr } - return host + ip := net.ParseIP(host) + if ip == nil { + return host + } + if inAny(ip, trusted) { + ip = forwardedClient(r, trusted, ip) + } + if ip4 := ip.To4(); ip4 != nil { + return ip4.String() + } + return ip.Mask(net.CIDRMask(64, 128)).String() + "/64" +} + +// forwardedClient walks X-Forwarded-For (all its lines, in order) from the right. +func forwardedClient(r *http.Request, trusted []*net.IPNet, peer net.IP) net.IP { + entries := strings.Split(strings.Join(r.Header.Values("X-Forwarded-For"), ","), ",") + for i := len(entries) - 1; i >= 0; i-- { + ip := net.ParseIP(strings.TrimSpace(entries[i])) + if ip == nil { + return peer + } + if !inAny(ip, trusted) { + return ip + } + } + return peer +} + +func inAny(ip net.IP, nets []*net.IPNet) bool { + for _, n := range nets { + if n.Contains(ip) { + return true + } + } + return false } diff --git a/internal/web/web.go b/internal/web/web.go index b90cdf9..ae06a79 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -7,6 +7,7 @@ import ( "errors" "html/template" "log" + "net" "net/http" "net/mail" "net/url" @@ -31,14 +32,19 @@ type server struct { router *router.Router tmpl *template.Template limit *subscribeLimiter + // trusted are the reverse proxies whose X-Forwarded-For is believed. + trusted []*net.IPNet } -// New returns the public web handler. -func New(s *store.Store, rt *router.Router) http.Handler { return newHandler(s, rt, time.Now) } +// New returns the public web handler. trustedProxies are the reverse proxies allowed to name the +// client in X-Forwarded-For (none: the TCP peer is the client). +func New(s *store.Store, rt *router.Router, trustedProxies []*net.IPNet) http.Handler { + return newHandler(s, rt, time.Now, trustedProxies) +} // newHandler is New with an injectable clock, for the tests. -func newHandler(s *store.Store, rt *router.Router, now func() time.Time) http.Handler { - srv := &server{store: s, router: rt, limit: newSubscribeLimiter(now), tmpl: template.Must(template.New("").Funcs(template.FuncMap{ +func newHandler(s *store.Store, rt *router.Router, now func() time.Time, trusted []*net.IPNet) http.Handler { + srv := &server{store: s, router: rt, limit: newSubscribeLimiter(now), trusted: trusted, tmpl: template.Must(template.New("").Funcs(template.FuncMap{ "hide": archive.Obfuscate, // The logo is a trusted static asset: inline, so the strict CSP needs no image source for it. "mark": func() template.HTML { return template.HTML(assets.Mark) }, @@ -262,7 +268,7 @@ func (s *server) subscribeForm(w http.ResponseWriter, r *http.Request) { const maxSubscribeBody = 4 << 10 func (s *server) subscribe(w http.ResponseWriter, r *http.Request) { - if wait, ok := s.limit.allow(clientIP(r)); !ok { + if wait, ok := s.limit.allow(clientKey(r, s.trusted)); !ok { w.Header().Set("Retry-After", strconv.Itoa(int((wait+time.Second-1)/time.Second))) s.render(w, http.StatusTooManyRequests, "toomany", nil) return @@ -313,7 +319,7 @@ func printable(s string) string { } // plainAddress accepts one bare ASCII address of at most 254 characters: no display name, no angle -// brackets, no list, no quoting, no control characters. +// brackets, no list, no quoting, no control characters, and a public domain (see publicDomain). func plainAddress(s string) (string, bool) { s = strings.TrimSpace(s) if s == "" || len(s) > 254 { @@ -325,12 +331,40 @@ func plainAddress(s string) (string, bool) { } } a, err := mail.ParseAddress(s) - if err != nil || a.Name != "" || a.Address != s { + if err != nil || a.Name != "" || a.Address != s || !publicDomain(s[strings.LastIndexByte(s, '@')+1:]) { return "", false } return s, true } +// publicDomain accepts a domain name that can be on the Internet: dotted labels of letters, digits +// and hyphens, not an IP address, and not one of the names reserved for local use. A domain literal +// ([127.0.0.1]), a bare host (localhost) or a .local name would make the daemon mail itself or its +// network. +func publicDomain(d string) bool { + d = strings.ToLower(d) + if !strings.Contains(d, ".") { + return false + } + for _, label := range strings.Split(d, ".") { + if label == "" || strings.ContainsFunc(label, func(r rune) bool { + return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '-') + }) { + return false + } + } + // No top-level domain is all digits: such a name is an IP address (a@127.0.0.1). + if strings.Trim(d[strings.LastIndexByte(d, '.')+1:], "0123456789") == "" { + return false + } + for _, local := range []string{".localhost", ".local", ".internal", ".invalid"} { + if strings.HasSuffix(d, local) { + return false + } + } + return true +} + func (s *server) unsubscribePage(w http.ResponseWriter, r *http.Request) { l, err := s.router.CheckUnsubscribe(r.PathValue("token")) switch { diff --git a/internal/web/web_test.go b/internal/web/web_test.go index 62fae73..f48340e 100644 --- a/internal/web/web_test.go +++ b/internal/web/web_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "fmt" "html" + "net" "net/http" "net/http/httptest" "net/url" @@ -53,10 +54,24 @@ func newFixture(t *testing.T) *fixture { rt := router.New(s) f := &fixture{s: s, rt: rt, pub: pub, alice: ms[0].ID, bobTok: rt.UnsubscribeToken(ms[1].ID), out: &outbound.Fake{}, clock: time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)} - f.h = newHandler(s, rt, func() time.Time { return f.clock }) + f.h = newHandler(s, rt, func() time.Time { return f.clock }, nil) return f } +// trust rebuilds the handler so that it believes X-Forwarded-For from the given proxies. +func (f *fixture) trust(t *testing.T, cidrs ...string) { + t.Helper() + var nets []*net.IPNet + for _, c := range cidrs { + _, n, err := net.ParseCIDR(c) + if err != nil { + t.Fatal(err) + } + nets = append(nets, n) + } + f.h = newHandler(f.s, f.rt, func() time.Time { return f.clock }, nets) +} + // flush sends what is queued, as the background worker would, and returns everything sent so far. func (f *fixture) flush(t *testing.T) []outbound.Sent { t.Helper() @@ -138,7 +153,7 @@ func TestIndexEscapesTitles(t *testing.T) { func TestIndexWithoutPublicLists(t *testing.T) { s, _ := store.Open(":memory:") defer s.Close() - rec := get(t, New(s, router.New(s)), "/") + rec := get(t, New(s, router.New(s), nil), "/") wantStatus(t, rec, 200) wantBody(t, rec, "No public archives") } @@ -367,7 +382,7 @@ func subscribeRaw(h http.Handler, list, ip, body string, hdr map[string]string) for k, v := range hdr { req.Header.Set(k, v) } - req.RemoteAddr = ip + ":4321" + req.RemoteAddr = net.JoinHostPort(ip, "4321") rec := httptest.NewRecorder() h.ServeHTTP(rec, req) return rec @@ -517,6 +532,20 @@ func TestSubscribeRejectsBadAddressesAndEscapesWhatWasTyped(t *testing.T) { {"line break", "a@x.com\r\nBcc: v@x.com"}, {"non-ASCII", "cärol@x.com"}, {"too long", long}, + {"no dot in the domain", "a@b"}, + {"localhost", "root@localhost"}, + {"localhost in capitals", "root@LOCALHOST"}, + {"under localhost", "a@app.localhost"}, + {"mDNS", "a@printer.local"}, + {"internal", "a@svc.internal"}, + {"invalid", "a@x.invalid"}, + {"domain literal", "a@[127.0.0.1]"}, + {"IPv6 domain literal", "a@[IPv6:::1]"}, + {"trailing dot", "a@x.com."}, + {"underscore in the domain", "a@x_y.com"}, + {"symbol in the domain", "a@x!y.com"}, + {"plus in the domain", "a@x+y.com"}, + {"bare IP address", "a@127.0.0.1"}, } { rec := subscribe(t, f, "dev@example.org", fmt.Sprintf("192.0.2.%d", i+1), email(c.in)) // one IP each: the limit is per IP if rec.Code != 400 { @@ -540,6 +569,15 @@ func TestSubscribeRejectsBadAddressesAndEscapesWhatWasTyped(t *testing.T) { wantBody(t, subscribe(t, f, "dev@example.org", "192.0.2.101", email("a"+ok)), "valid email address") } +func TestSubscribeAcceptsOrdinaryDomains(t *testing.T) { + f := newFixture(t) + for i, addr := range []string{"a@members.test", "a@example.com", "a@mail.example", "a@my-host.Example.ORG", "a@x1.co", "a@localhost.example.com"} { + rec := subscribe(t, f, "dev@example.org", fmt.Sprintf("192.0.2.%d", i+1), email(addr)) + wantStatus(t, rec, 200) + wantBody(t, rec, neutralText) + } +} + func TestSubscribeBodyIsLimitedTo4KiB(t *testing.T) { f := newFixture(t) body := func(n int) string { @@ -714,3 +752,141 @@ func TestSubscribeLimiterSweepsAnAddressWhoseHourIsOver(t *testing.T) { t.Fatalf("size = %d, want 1 (the first address is forgotten)", l.size()) } } + +func TestSubscribeBehindATrustedProxyLimitsEachClient(t *testing.T) { + f := newFixture(t) + f.trust(t, "10.0.0.0/8") + post := func(xff string) int { + return subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), map[string]string{"X-Forwarded-For": xff}).Code + } + for i := 0; i < 10; i++ { + if c := post("203.0.113.1"); c != 200 { + t.Fatalf("request %d: %d", i+1, c) + } + } + if c := post("203.0.113.1"); c != 429 { + t.Fatalf("the 11th request of one client: %d, want 429", c) + } + // Every visitor comes through the same proxy, yet another one has a budget of their own. + if c := post("203.0.113.2"); c != 200 { + t.Fatalf("another client behind the proxy: %d, want 200", c) + } +} + +func TestSubscribeForwardedForFromAnUntrustedPeerIsIgnored(t *testing.T) { + f := newFixture(t) + f.trust(t, "10.0.0.0/8") + for i := 0; i < 10; i++ { + hdr := map[string]string{"X-Forwarded-For": fmt.Sprintf("203.0.113.%d", i)} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.7", email("a@x.com").Encode(), hdr), 200) + } + hdr := map[string]string{"X-Forwarded-For": "203.0.113.200"} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.7", email("a@x.com").Encode(), hdr), 429) +} + +func TestSubscribeTakesTheRightMostUntrustedForwardedAddress(t *testing.T) { + f := newFixture(t) + f.trust(t, "10.0.0.0/8", "192.168.0.0/16") + // The client may write anything on the left; the proxies append on the right. + for i := 0; i < 10; i++ { + hdr := map[string]string{"X-Forwarded-For": fmt.Sprintf("198.51.100.%d, 203.0.113.5, 192.168.1.1", i)} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 200) + } + hdr := map[string]string{"X-Forwarded-For": "1.1.1.1,203.0.113.5"} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 429) + // The header may arrive as several lines: they are one list, in order. + req := httptest.NewRequest("POST", "/subscribe/dev@example.org", strings.NewReader(email("a@x.com").Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Add("X-Forwarded-For", "198.51.100.77") + req.Header.Add("X-Forwarded-For", "203.0.113.5") + req.RemoteAddr = "10.0.0.2:4321" + rec := httptest.NewRecorder() + f.h.ServeHTTP(rec, req) + wantStatus(t, rec, 429) + // The proxy's own address does not count against the client. + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), map[string]string{"X-Forwarded-For": "203.0.113.6"}), 200) +} + +func TestSubscribeBadForwardedForFallsBackToThePeer(t *testing.T) { + f := newFixture(t) + f.trust(t, "10.0.0.0/8") + for i, xff := range []string{"", "garbage", "203.0.113.1, nonsense", "203.0.113.1:5555", "10.0.0.7", " , ", "", "x", "", ""} { + hdr := map[string]string{} + if xff != "" { + hdr["X-Forwarded-For"] = xff + } + if c := subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr).Code; c != 200 { + t.Fatalf("request %d (%q): %d", i+1, xff, c) + } + } + // All ten were counted against the proxy itself. + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), nil), 429) + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), map[string]string{"X-Forwarded-For": "203.0.113.1"}), 200) +} + +func TestSubscribeIPv6ClientsShareTheirSlash64(t *testing.T) { + f := newFixture(t) + for i := 0; i < 10; i++ { + wantStatus(t, subscribe(t, f, "dev@example.org", fmt.Sprintf("2001:db8:1:2::%x", i+1), email("a@x.com")), 200) + } + wantStatus(t, subscribe(t, f, "dev@example.org", "2001:db8:1:2:ffff:ffff:ffff:ffff", email("a@x.com")), 429) + wantStatus(t, subscribe(t, f, "dev@example.org", "2001:db8:1:3::1", email("a@x.com")), 200) + // IPv4 addresses stay separate, one each. + for i := 0; i < 10; i++ { + wantStatus(t, subscribe(t, f, "dev@example.org", "192.0.2.1", email("a@x.com")), 200) + } + wantStatus(t, subscribe(t, f, "dev@example.org", "192.0.2.2", email("a@x.com")), 200) + // The same goes for an IPv6 client named by a trusted proxy. + f.trust(t, "10.0.0.0/8") + for i := 0; i < 10; i++ { + hdr := map[string]string{"X-Forwarded-For": fmt.Sprintf("2001:db8:9:9::%x", i+1)} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 200) + } + hdr := map[string]string{"X-Forwarded-For": "2001:db8:9:9:1::1"} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "10.0.0.2", email("a@x.com").Encode(), hdr), 429) +} + +func TestSubscribeLimiterIsCapped(t *testing.T) { + t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + now := t0 + l := newSubscribeLimiter(func() time.Time { return now }) + if l.max != 100000 { + t.Fatalf("max = %d, want 100000", l.max) + } + l.max = 2 + l.allow("192.0.2.1") + now = t0.Add(59*time.Minute + 50*time.Second) + l.allow("192.0.2.2") // the periodic sweep runs here and finds nothing to forget + if wait, ok := l.allow("192.0.2.3"); ok || wait != 10*time.Second { + t.Fatalf("a new address when the map is full: wait %v ok %v, want refused for 10s (until the oldest window ends)", wait, ok) + } + if _, ok := l.allow("192.0.2.2"); !ok { + t.Fatal("an address already known keeps its budget") + } + if l.size() != 2 { + t.Fatalf("size = %d, want 2", l.size()) + } + // 20 s later the periodic sweep is not due yet, but the first address has expired: a full map + // sweeps before it refuses. + now = t0.Add(time.Hour + 10*time.Second) + if _, ok := l.allow("192.0.2.3"); !ok { + t.Fatal("room was made by forgetting an expired address") + } + if l.size() != 2 { + t.Fatalf("size = %d, want 2", l.size()) + } +} + +// mail.ParseAddress already refuses most of these; publicDomain must not depend on it. +func TestPublicDomain(t *testing.T) { + for d, want := range map[string]bool{ + "example.com": true, "members.test": true, "a-b.example": true, "X1.Example.ORG": true, + "x..com": false, ".x.com": false, "x.com.": false, "x": false, "localhost": false, "[127.0.0.1]": false, + "a.localhost": false, "a.local": false, "a.internal": false, "a.invalid": false, "x_y.com": false, + "127.0.0.1": false, "10.1.2.3": false, "x.123": false, "123.example": true, "x.c0m": true, + } { + if got := publicDomain(d); got != want { + t.Errorf("publicDomain(%q) = %v, want %v", d, got, want) + } + } +} diff --git a/parsimony.example.yaml b/parsimony.example.yaml index 216e426..17a95be 100644 --- a/parsimony.example.yaml +++ b/parsimony.example.yaml @@ -15,8 +15,17 @@ relay: # rate_per_second: 14 # at most this many sends started per second, for relays that throttle (Amazon SES); 0 = unlimited web: listen: 0.0.0.0:8081 # public archive and unsubscribe pages; "" disables them + # Reverse proxies (addresses or CIDR ranges) allowed to name the visitor in X-Forwarded-For, for + # the subscribe form's limit of 10 posts per hour per IPv4 address or IPv6 /64. Empty (the + # default): the TCP peer is the visitor and the header is ignored. + # trusted_proxies: [172.16.0.0/12] api: listen: 0.0.0.0:8080 +# Each subscribe or unsubscribe request sends a confirmation mail. At most this many are accepted +# per hour, for all lists together (default 300); past it, requests get no mail and a warning is +# logged once a minute. Raise it before a newsletter launch. +# limits: +# pending_per_hour: 300 # The API token is read from the PARSIMONY_API_TOKEN environment variable. # Sign outgoing mail with DKIM, one key per domain. The private key stays in a file # (create it with: parsimony dkim keygen lists.example.org --selector mail).