feat: configurable cap on confirmation requests, with a warning
When the global cap of unconfirmed requests (300 an hour) was reached, a subscribe request silently got no mail while the visitor saw "Check your email"; a newsletter launch can pass 300 signups an hour. - limits.pending_per_hour (default 300, must be a whole number >= 1) is validated at load and set on the store by the app. - When the global cap refuses a request the store logs a warning naming the setting, at most once a minute. The visitor still gets the neutral page, so nothing about members is revealed. The per-address cap is unchanged and is not logged. - parsimony.example.yaml documents the setting.
This commit is contained in:
parent
96a1243c2f
commit
8d28e48360
7 changed files with 175 additions and 8 deletions
|
|
@ -218,6 +218,9 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
||||||
return nil, fmt.Errorf("store: %w", err)
|
return nil, fmt.Errorf("store: %w", err)
|
||||||
}
|
}
|
||||||
st.Now = nowFunc
|
st.Now = nowFunc
|
||||||
|
if cfg.Limits.PendingPerHour > 0 { // zero: a Config not made by config.Load, keep the default
|
||||||
|
st.PendingPerHour = int(cfg.Limits.PendingPerHour)
|
||||||
|
}
|
||||||
a := &App{store: st}
|
a := &App{store: st}
|
||||||
|
|
||||||
rt := router.New(st)
|
rt := router.New(st)
|
||||||
|
|
|
||||||
|
|
@ -376,6 +376,28 @@ func TestWebBelievesForwardedForOnlyFromTrustedProxies(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPendingPerHourReachesTheStore(t *testing.T) {
|
||||||
|
relayAddr, relay := startSink(t)
|
||||||
|
cfg := testConfig(t, relayAddr)
|
||||||
|
cfg.Limits.PendingPerHour = 1
|
||||||
|
a, err := Start(cfg, "tok")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { a.Close() })
|
||||||
|
call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok")
|
||||||
|
for _, from := range []string{"carol@x.com", "dave@x.com"} {
|
||||||
|
if err := lmtpSend(t, a.LMTPAddr, from, "dev+join@example.org", ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
relay.waitCount(t, 1)
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
if n := len(relay.all()); n != 1 {
|
||||||
|
t.Fatalf("limits.pending_per_hour = 1: %d confirmation mails, want 1", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestWebDisabledWhenListenIsEmpty(t *testing.T) {
|
func TestWebDisabledWhenListenIsEmpty(t *testing.T) {
|
||||||
relayAddr, _ := startSink(t)
|
relayAddr, _ := startSink(t)
|
||||||
cfg := testConfig(t, relayAddr)
|
cfg := testConfig(t, relayAddr)
|
||||||
|
|
|
||||||
|
|
@ -37,6 +37,11 @@ type Config struct {
|
||||||
// the TCP peer is the client and the header is ignored.
|
// the TCP peer is the client and the header is ignored.
|
||||||
TrustedProxies []string `yaml:"trusted_proxies"`
|
TrustedProxies []string `yaml:"trusted_proxies"`
|
||||||
} `yaml:"web"`
|
} `yaml:"web"`
|
||||||
|
Limits struct {
|
||||||
|
// PendingPerHour is the most subscribe and unsubscribe requests (each one sends a
|
||||||
|
// confirmation mail) accepted in an hour, for all lists together.
|
||||||
|
PendingPerHour Count `yaml:"pending_per_hour"`
|
||||||
|
} `yaml:"limits"`
|
||||||
// DKIM lists the signing keys, one per domain. The private keys stay in files.
|
// DKIM lists the signing keys, one per domain. The private keys stay in files.
|
||||||
DKIM []DKIMKey `yaml:"dkim"`
|
DKIM []DKIMKey `yaml:"dkim"`
|
||||||
Relay struct {
|
Relay struct {
|
||||||
|
|
@ -72,6 +77,20 @@ func (r *Rate) UnmarshalYAML(n *yaml.Node) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Count is limits.pending_per_hour. It exists so that a value that is not a whole number is reported
|
||||||
|
// with the name of the key.
|
||||||
|
type Count int
|
||||||
|
|
||||||
|
// UnmarshalYAML reads a YAML integer.
|
||||||
|
func (c *Count) UnmarshalYAML(n *yaml.Node) error {
|
||||||
|
var i int
|
||||||
|
if n.Tag != "!!int" || n.Decode(&i) != nil {
|
||||||
|
return fmt.Errorf("line %d: limits.pending_per_hour must be a whole number, not %q", n.Line, n.Value)
|
||||||
|
}
|
||||||
|
*c = Count(i)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// DKIMKey is the signing key of one domain.
|
// DKIMKey is the signing key of one domain.
|
||||||
type DKIMKey struct {
|
type DKIMKey struct {
|
||||||
Domain string `yaml:"domain"`
|
Domain string `yaml:"domain"`
|
||||||
|
|
@ -211,6 +230,9 @@ func (c Config) validate() error {
|
||||||
return fmt.Errorf("lmtp.allow[%d]: %w", i, err)
|
return fmt.Errorf("lmtp.allow[%d]: %w", i, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if c.Limits.PendingPerHour < 1 {
|
||||||
|
return fmt.Errorf("limits.pending_per_hour must be 1 or more, not %d", c.Limits.PendingPerHour)
|
||||||
|
}
|
||||||
for i, a := range c.Web.TrustedProxies {
|
for i, a := range c.Web.TrustedProxies {
|
||||||
if _, err := parseNet(a); err != nil {
|
if _, err := parseNet(a); err != nil {
|
||||||
return fmt.Errorf("web.trusted_proxies[%d]: %w", i, err)
|
return fmt.Errorf("web.trusted_proxies[%d]: %w", i, err)
|
||||||
|
|
@ -245,6 +267,7 @@ func Load(path string) (Config, error) {
|
||||||
c.Relay.Concurrency = 4
|
c.Relay.Concurrency = 4
|
||||||
c.API.Listen = "127.0.0.1:8080"
|
c.API.Listen = "127.0.0.1:8080"
|
||||||
c.Web.Listen = "127.0.0.1:8081"
|
c.Web.Listen = "127.0.0.1:8081"
|
||||||
|
c.Limits.PendingPerHour = 300
|
||||||
|
|
||||||
b, err := os.ReadFile(path)
|
b, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
||||||
|
|
@ -356,12 +356,32 @@ func TestExampleConfigLoadsAndDocumentsTheSettings(t *testing.T) {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
// Uncommented, the documented lines are valid.
|
// Uncommented, the documented lines are valid.
|
||||||
uncommented := strings.ReplaceAll(string(b), " # trusted_proxies:", " trusted_proxies:")
|
uncommented := string(b)
|
||||||
if uncommented == string(b) {
|
for _, line := range []string{" # trusted_proxies:", "# limits:", "# pending_per_hour:"} {
|
||||||
t.Fatal("web.trusted_proxies is not documented")
|
if !strings.Contains(uncommented, line) {
|
||||||
|
t.Fatalf("%q is not documented", line)
|
||||||
|
}
|
||||||
|
uncommented = strings.ReplaceAll(uncommented, line, strings.Replace(line, "# ", "", 1))
|
||||||
}
|
}
|
||||||
c, err := Load(write(t, uncommented))
|
c, err := Load(write(t, uncommented))
|
||||||
if err != nil || len(c.WebTrustedProxies()) == 0 {
|
if err != nil || len(c.WebTrustedProxies()) == 0 || c.Limits.PendingPerHour != 300 {
|
||||||
t.Fatalf("documented trusted_proxies: %v %v", c.WebTrustedProxies(), err)
|
t.Fatalf("documented settings: %v %d %v", c.WebTrustedProxies(), c.Limits.PendingPerHour, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLimitsPendingPerHour(t *testing.T) {
|
||||||
|
c, err := Load(write(t, "{}\n"))
|
||||||
|
if err != nil || c.Limits.PendingPerHour != 300 {
|
||||||
|
t.Fatalf("default: %d %v", c.Limits.PendingPerHour, err)
|
||||||
|
}
|
||||||
|
c, err = Load(write(t, "limits:\n pending_per_hour: 2000\n"))
|
||||||
|
if err != nil || c.Limits.PendingPerHour != 2000 {
|
||||||
|
t.Fatalf("value: %d %v", c.Limits.PendingPerHour, err)
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"0", "-1", "lots", "1.5"} {
|
||||||
|
_, err := Load(write(t, "limits:\n pending_per_hour: "+bad+"\n"))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "pending_per_hour") {
|
||||||
|
t.Errorf("%s: want an error naming limits.pending_per_hour, got %v", bad, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -717,3 +717,68 @@ func TestConfirmPendingIsSingleUseUnderConcurrency(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfirmationMailGlobalCapIsConfigurable(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
if s.PendingPerHour != 300 {
|
||||||
|
t.Fatalf("default PendingPerHour = %d, want 300", s.PendingPerHour)
|
||||||
|
}
|
||||||
|
s.PendingPerHour = 2
|
||||||
|
s.Logf = func(string, ...any) {}
|
||||||
|
now := time.Unix(1_000_000, 0)
|
||||||
|
s.Now = func() time.Time { return now }
|
||||||
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
if _, created, err := s.CreatePending(l.ID, fmt.Sprintf("u%d@x.com", i), "join"); err != nil || !created {
|
||||||
|
t.Fatalf("request %d: %v %v", i+1, created, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, created, err := s.CreatePending(l.ID, "u2@x.com", "join"); err != nil || created {
|
||||||
|
t.Fatalf("the third request in the hour: created %v err %v, want refused", created, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGlobalCapRefusalsAreLoggedOncePerMinute(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
s.PendingPerHour = 1
|
||||||
|
var logged []string
|
||||||
|
s.Logf = func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }
|
||||||
|
t0 := time.Unix(1_000_000, 0)
|
||||||
|
now := t0
|
||||||
|
s.Now = func() time.Time { return now }
|
||||||
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||||
|
s.CreatePending(l.ID, "first@x.com", "join")
|
||||||
|
s.CreatePending(l.ID, "first@x.com", "join") // a repeat is not the global cap
|
||||||
|
if len(logged) != 0 {
|
||||||
|
t.Fatalf("nothing refused by the global cap yet: %q", logged)
|
||||||
|
}
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
s.CreatePending(l.ID, fmt.Sprintf("u%d@x.com", i), "join")
|
||||||
|
}
|
||||||
|
if len(logged) != 1 || !strings.Contains(logged[0], "limits.pending_per_hour") || !strings.Contains(logged[0], "1") {
|
||||||
|
t.Fatalf("one warning naming the setting: %q", logged)
|
||||||
|
}
|
||||||
|
now = t0.Add(time.Minute - time.Second)
|
||||||
|
s.CreatePending(l.ID, "late@x.com", "join")
|
||||||
|
if len(logged) != 1 {
|
||||||
|
t.Fatalf("at most one line a minute: %q", logged)
|
||||||
|
}
|
||||||
|
now = t0.Add(time.Minute)
|
||||||
|
s.CreatePending(l.ID, "later@x.com", "join")
|
||||||
|
if len(logged) != 2 {
|
||||||
|
t.Fatalf("a minute later the next refusal is logged: %q", logged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPerAddressCapRefusalsAreNotLogged(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
var logged []string
|
||||||
|
s.Logf = func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }
|
||||||
|
for i := 0; i <= maxPendingPerAddress; i++ {
|
||||||
|
l, _ := s.CreateList(fmt.Sprintf("l%d", i), "example.org", "L")
|
||||||
|
s.CreatePending(l.ID, "victim@x.com", "join")
|
||||||
|
}
|
||||||
|
if len(logged) != 0 {
|
||||||
|
t.Fatalf("the per-address cap is not the global one: %q", logged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,9 +8,11 @@ import (
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"mime"
|
"mime"
|
||||||
"net/mail"
|
"net/mail"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"unicode"
|
"unicode"
|
||||||
|
|
||||||
|
|
@ -70,7 +72,7 @@ const (
|
||||||
|
|
||||||
// Caps on confirmation mail, per hour. Anyone can name any address in a join request, so the
|
// Caps on confirmation mail, per hour. Anyone can name any address in a join request, so the
|
||||||
// daemon must not mail one victim many times (maxPendingPerAddress, across all lists) or send
|
// daemon must not mail one victim many times (maxPendingPerAddress, across all lists) or send
|
||||||
// an unbounded amount in total (maxPendingPerHour).
|
// an unbounded amount in total (Store.PendingPerHour, maxPendingPerHour by default).
|
||||||
// A moderated list could be flooded with mail that nobody reads: the queue per list is bounded,
|
// A moderated list could be flooded with mail that nobody reads: the queue per list is bounded,
|
||||||
// and a message nobody moderated for a month is dropped.
|
// and a message nobody moderated for a month is dropped.
|
||||||
maxHeldPerList = 500
|
maxHeldPerList = 500
|
||||||
|
|
@ -78,6 +80,9 @@ const (
|
||||||
|
|
||||||
maxPendingPerAddress = 3
|
maxPendingPerAddress = 3
|
||||||
maxPendingPerHour = 300
|
maxPendingPerHour = 300
|
||||||
|
|
||||||
|
// pendingCapLogEvery spaces the warnings about requests refused by the global cap.
|
||||||
|
pendingCapLogEvery = time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
// List is a mailing list.
|
// List is a mailing list.
|
||||||
|
|
@ -133,6 +138,14 @@ type Store struct {
|
||||||
db *sql.DB
|
db *sql.DB
|
||||||
// Now returns the current time; tests replace it.
|
// Now returns the current time; tests replace it.
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
|
// PendingPerHour is the most join and leave requests recorded in an hour, for all lists and
|
||||||
|
// addresses together (limits.pending_per_hour).
|
||||||
|
PendingPerHour int
|
||||||
|
// Logf receives the warning written when PendingPerHour refuses a request; tests replace it.
|
||||||
|
Logf func(format string, args ...any)
|
||||||
|
|
||||||
|
capLogMu sync.Mutex
|
||||||
|
capLoggedAt time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
const schema = `
|
const schema = `
|
||||||
|
|
@ -261,7 +274,7 @@ func Open(path string) (*Store, error) {
|
||||||
db.Close()
|
db.Close()
|
||||||
return nil, fmt.Errorf("migrate: %w", err)
|
return nil, fmt.Errorf("migrate: %w", err)
|
||||||
}
|
}
|
||||||
return &Store{db: db, Now: time.Now}, nil
|
return &Store{db: db, Now: time.Now, PendingPerHour: maxPendingPerHour, Logf: log.Printf}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func hasColumn(db *sql.DB, table, column string) (bool, error) {
|
func hasColumn(db *sql.DB, table, column string) (bool, error) {
|
||||||
|
|
@ -773,9 +786,13 @@ func (s *Store) CreatePending(listID int64, addr, action string) (token string,
|
||||||
if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE address=? AND created_at>?`, addr, since).Scan(&n); err != nil || n >= maxPendingPerAddress {
|
if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE address=? AND created_at>?`, addr, since).Scan(&n); err != nil || n >= maxPendingPerAddress {
|
||||||
return "", false, err
|
return "", false, err
|
||||||
}
|
}
|
||||||
if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE created_at>?`, since).Scan(&n); err != nil || n >= maxPendingPerHour {
|
if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE created_at>?`, since).Scan(&n); err != nil {
|
||||||
return "", false, err
|
return "", false, err
|
||||||
}
|
}
|
||||||
|
if n >= s.PendingPerHour {
|
||||||
|
s.warnPendingCap(now, n)
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
// 80 bits: unguessable within the 72 h life of a token, and short enough that
|
// 80 bits: unguessable within the 72 h life of a token, and short enough that
|
||||||
// name(<=30)+"+confirm-"+token fits the 64 characters of a local part.
|
// name(<=30)+"+confirm-"+token fits the 64 characters of a local part.
|
||||||
token, err = randomHex(10)
|
token, err = randomHex(10)
|
||||||
|
|
@ -790,6 +807,18 @@ func (s *Store) CreatePending(listID int64, addr, action string) (token string,
|
||||||
return token, true, nil
|
return token, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// warnPendingCap tells the operator, at most once every pendingCapLogEvery, that requests are being
|
||||||
|
// dropped: the visitor still sees the neutral page and gets no mail.
|
||||||
|
func (s *Store) warnPendingCap(now time.Time, n int) {
|
||||||
|
s.capLogMu.Lock()
|
||||||
|
defer s.capLogMu.Unlock()
|
||||||
|
if !s.capLoggedAt.IsZero() && now.Sub(s.capLoggedAt) < pendingCapLogEvery {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.capLoggedAt = now
|
||||||
|
s.Logf("warning: %d join and leave requests in the last hour reached limits.pending_per_hour (%d): new requests get no confirmation mail until older ones are an hour old", n, s.PendingPerHour)
|
||||||
|
}
|
||||||
|
|
||||||
// ConfirmPending applies the request behind a token and consumes the token, in one transaction: a
|
// ConfirmPending applies the request behind a token and consumes the token, in one transaction: a
|
||||||
// token is only used up when its action happened, and two confirmations of the same token cannot
|
// token is only used up when its action happened, and two confirmations of the same token cannot
|
||||||
// both apply (the first one to claim the row wins, the other gets ErrNotFound).
|
// both apply (the first one to claim the row wins, the other gets ErrNotFound).
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,11 @@ web:
|
||||||
# trusted_proxies: [172.16.0.0/12]
|
# trusted_proxies: [172.16.0.0/12]
|
||||||
api:
|
api:
|
||||||
listen: 0.0.0.0:8080
|
listen: 0.0.0.0:8080
|
||||||
|
# Each subscribe or unsubscribe request sends a confirmation mail. At most this many are accepted
|
||||||
|
# per hour, for all lists together (default 300); past it, requests get no mail and a warning is
|
||||||
|
# logged once a minute. Raise it before a newsletter launch.
|
||||||
|
# limits:
|
||||||
|
# pending_per_hour: 300
|
||||||
# The API token is read from the PARSIMONY_API_TOKEN environment variable.
|
# The API token is read from the PARSIMONY_API_TOKEN environment variable.
|
||||||
# Sign outgoing mail with DKIM, one key per domain. The private key stays in a file
|
# Sign outgoing mail with DKIM, one key per domain. The private key stays in a file
|
||||||
# (create it with: parsimony dkim keygen lists.example.org --selector mail).
|
# (create it with: parsimony dkim keygen lists.example.org --selector mail).
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue