diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index ff494f8..d205820 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -153,7 +153,7 @@ func TestListLifecycle(t *testing.T) { e.fails(t, 1, "already exists", "list", "add", "dev@example.org") e.fails(t, 2, "name@domain", "list", "add", "nodomain") e.fails(t, 2, "name@domain", "list", "add", "@example.org") - e.fails(t, 1, "name and domain", "list", "add", "a+b@example.org") + e.fails(t, 1, "name must be", "list", "add", "a+b@example.org") for _, ref := range []string{"dev@example.org", "DEV@Example.org", "1"} { out = e.ok(t, "list", "show", ref) diff --git a/internal/router/commands.go b/internal/router/commands.go index 499c303..d9628e7 100644 --- a/internal/router/commands.go +++ b/internal/router/commands.go @@ -85,8 +85,9 @@ func (r *Router) join(l store.List, sender string) error { if l.JoinPolicy != store.JoinOpen { return ErrJoinClosed } - if ok, err := r.store.IsMember(l.ID, sender); err != nil || ok { - return err // members get no mail: it would leak membership + // A member disabled by bounces is treated like a non-member: confirming re-enables it. + if ok, err := r.store.IsActiveMember(l.ID, sender); err != nil || ok { + return err // active members get no mail: it would leak membership } return r.askConfirmation(l, sender, store.ActionJoin) } @@ -126,25 +127,18 @@ func (r *Router) askConfirmation(l store.List, addr, action string) error { } func (r *Router) confirm(l store.List, token string) error { - p, err := r.store.ConfirmPending(token) - if errors.Is(err, store.ErrNotFound) || (err == nil && p.ListID != l.ID) { - return ErrBadToken - } - if err != nil { - return err - } - switch p.Action { - case store.ActionJoin: - if l.JoinPolicy != store.JoinOpen { + // The token is used up only if the action happens: a closed list or a database error leaves it + // valid, and two replies to the same mail cannot both apply. + _, err := r.store.ConfirmPending(token, l.ID, func(p store.Pending) error { + if p.Action == store.ActionJoin && l.JoinPolicy != store.JoinOpen { return ErrJoinClosed } - return r.store.AddMember(l.ID, p.Address, "") - case store.ActionLeave: - if err := r.store.RemoveMember(l.ID, p.Address); err != nil && !errors.Is(err, store.ErrNotFound) { - return err - } + return nil + }) + if errors.Is(err, store.ErrNotFound) { + return ErrBadToken } - return nil + return err } func (r *Router) bounce(l store.List, tag string, data []byte) error { diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go index 1a1d1c6..669a6e6 100644 --- a/internal/router/m2_test.go +++ b/internal/router/m2_test.go @@ -236,6 +236,81 @@ func TestConfirmAfterListClosed(t *testing.T) { if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); ok { t.Fatal("must not join a closed list") } + // The refusal did not use the token up: once the list is open again the same mail works. + e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinOpen, store.ArchivePrivate) + if _, err := e.command("carol@x.com", "dev+confirm-"+tok+"@example.org", nil); err != nil { + t.Fatalf("the token must survive a refusal: %v", err) + } + if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); !ok { + t.Fatal("carol must join once the list is open") + } +} + +func TestConfirmWithTokenOfAnotherListKeepsItUsable(t *testing.T) { + e := newEnv(t) + e.s.CreateList("ops", "example.org", "Ops") + e.command("carol@x.com", "dev+join@example.org", nil) + tok := token(t, lastMail(t, e.out)) + if _, err := e.command("carol@x.com", "ops+confirm-"+tok+"@example.org", nil); !errors.Is(err, ErrBadToken) { + t.Fatalf("want ErrBadToken, got %v", err) + } + if _, err := e.command("carol@x.com", "dev+confirm-"+tok+"@example.org", nil); err != nil { + t.Fatalf("the right list still accepts it: %v", err) + } +} + +// A member disabled by bounces must be able to rejoin: +join sends the confirmation, confirming +// re-enables the subscription and clears the bounce state. +func TestDisabledMemberCanRejoin(t *testing.T) { + e := newEnv(t) + for i := 0; i < store.BounceThreshold; i++ { + e.s.RecordHardBounce(e.alice, store.BounceThreshold) + } + if m, _ := e.s.MemberByID(e.alice); !m.Disabled { + t.Fatal("setup: alice must be disabled") + } + n := len(e.out.Sent) + if _, err := e.command("alice@example.com", "dev+join@example.org", nil); err != nil { + t.Fatal(err) + } + if len(e.out.Sent) != n+1 { + t.Fatalf("a disabled member must get the confirmation, sent %d new mails", len(e.out.Sent)-n) + } + m := lastMail(t, e.out) + if len(m.To) != 1 || m.To[0] != "alice@example.com" { + t.Fatalf("to = %v", m.To) + } + if got, _ := e.s.MemberByID(e.alice); !got.Disabled { + t.Fatal("not re-enabled before confirming") + } + if _, err := e.command("alice@example.com", "dev+confirm-"+token(t, m)+"@example.org", nil); err != nil { + t.Fatal(err) + } + got, err := e.s.MemberByID(e.alice) + if err != nil || got.Disabled || got.BounceCount != 0 || got.ID != e.alice { + t.Fatalf("after confirming: %+v %v", got, err) + } + // She receives list mail again. + res, err := e.r.Route("bob@example.com", "dev@example.org", postFrom("Bob ")) + if err != nil { + t.Fatal(err) + } + seen := false + for _, r := range res.Recipients { + seen = seen || r.Address == "alice@example.com" + } + if !seen { + t.Fatalf("alice must receive posts again: %+v", res.Recipients) + } +} + +// An active member still gets nothing: no mail that would reveal membership. +func TestJoinStaysSilentForActiveMembers(t *testing.T) { + e := newEnv(t) + e.command("alice@example.com", "dev+join@example.org", nil) + if len(e.out.Sent) != 0 { + t.Fatal("active members get no mail") + } } func TestLeaveFlow(t *testing.T) { diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go index 5af346c..0c71628 100644 --- a/internal/store/m2_test.go +++ b/internal/store/m2_test.go @@ -6,6 +6,8 @@ import ( "fmt" "path/filepath" "strings" + "sync" + "sync/atomic" "testing" "time" ) @@ -82,20 +84,20 @@ func TestPendingTokens(t *testing.T) { if _, created, _ := s.CreatePending(l.ID, "a@x.com", "leave"); !created { t.Fatal("leave is a separate request") } - p, err := s.ConfirmPending(tok) + p, err := s.ConfirmPending(tok, l.ID, nil) if err != nil || p.Address != "a@x.com" || p.Action != "join" || p.ListID != l.ID { t.Fatalf("confirm: %+v %v", p, err) } - if _, err := s.ConfirmPending(tok); !errors.Is(err, ErrNotFound) { + if _, err := s.ConfirmPending(tok, l.ID, nil); !errors.Is(err, ErrNotFound) { t.Fatal("token is single use") } - if _, err := s.ConfirmPending("nope"); !errors.Is(err, ErrNotFound) { + if _, err := s.ConfirmPending("nope", l.ID, nil); !errors.Is(err, ErrNotFound) { t.Fatal("unknown token") } // Expiry. tok2, _, _ := s.CreatePending(l.ID, "b@x.com", "join") now = now.Add(73 * time.Hour) - if _, err := s.ConfirmPending(tok2); !errors.Is(err, ErrNotFound) { + if _, err := s.ConfirmPending(tok2, l.ID, nil); !errors.Is(err, ErrNotFound) { t.Fatal("token must expire after 72h") } } @@ -511,3 +513,204 @@ INSERT INTO members(list_id, address, display_name) VALUES(1,'a@x.com','A'),(99, t.Fatal("members_v1 was left behind") } } + +func pendingRows(t *testing.T, s *Store) int { + t.Helper() + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM pending`).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func TestCreatePendingPurgesExpiredRows(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + s.CreatePending(l.ID, "old@x.com", "join") + now = now.Add(70 * time.Hour) + s.CreatePending(l.ID, "recent@x.com", "join") + if n := pendingRows(t, s); n != 2 { + t.Fatalf("rows within the TTL must stay: %d", n) + } + // 72h after the first request (and a little more): it goes, the other stays. + now = now.Add(3 * time.Hour) + s.CreatePending(l.ID, "third@x.com", "join") + if n := pendingRows(t, s); n != 2 { + t.Fatalf("the expired row must be purged: %d rows", n) + } + var old int + s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE address='old@x.com'`).Scan(&old) + if old != 0 { + t.Fatal("the expired request is still there") + } + // Exactly at the TTL a token is still valid, so it is not purged either. + now = now.Add(pendingTTL) + s.CreatePending(l.ID, "fourth@x.com", "join") + var recent int + s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE address='third@x.com'`).Scan(&recent) + if recent != 1 { + t.Fatal("a row exactly at the TTL must not be purged yet") + } +} + +func TestConfirmPendingAppliesTheActionAndConsumesTheToken(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + tok, _, _ := s.CreatePending(l.ID, "carol@x.com", ActionJoin) + p, err := s.ConfirmPending(tok, l.ID, nil) + if err != nil || p.Address != "carol@x.com" || p.Action != ActionJoin { + t.Fatalf("%+v %v", p, err) + } + if ok, _ := s.IsMember(l.ID, "carol@x.com"); !ok { + t.Fatal("join must subscribe") + } + if pendingRows(t, s) != 0 { + t.Fatal("the token must be consumed") + } + + tok, _, _ = s.CreatePending(l.ID, "carol@x.com", ActionLeave) + if _, err := s.ConfirmPending(tok, l.ID, nil); err != nil { + t.Fatal(err) + } + if ok, _ := s.IsMember(l.ID, "carol@x.com"); ok { + t.Fatal("leave must unsubscribe") + } + // Leaving when already gone still succeeds and still uses the token. + tok, _, _ = s.CreatePending(l.ID, "ghost@x.com", ActionLeave) + if _, err := s.ConfirmPending(tok, l.ID, nil); err != nil { + t.Fatalf("leave of a non-member: %v", err) + } + if pendingRows(t, s) != 0 { + t.Fatal("the token must be consumed") + } +} + +func TestConfirmPendingJoinReenablesADisabledMember(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.AddMember(l.ID, "bob@x.com", "Bob") + ms, _ := s.Members(l.ID) + for i := 0; i < BounceThreshold; i++ { + s.RecordHardBounce(ms[0].ID, BounceThreshold) + } + tok, _, _ := s.CreatePending(l.ID, "bob@x.com", ActionJoin) + if _, err := s.ConfirmPending(tok, l.ID, nil); err != nil { + t.Fatal(err) + } + m, _ := s.MemberByID(ms[0].ID) + if m.Disabled || m.BounceCount != 0 { + t.Fatalf("%+v", m) + } +} + +// A refusal or a failure leaves the token usable: the confirmation is not lost. +func TestConfirmPendingKeepsTheTokenWhenTheActionDoesNotHappen(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + tok, _, _ := s.CreatePending(l.ID, "carol@x.com", ActionJoin) + + refused := errors.New("closed") + if _, err := s.ConfirmPending(tok, l.ID, func(Pending) error { return refused }); !errors.Is(err, refused) { + t.Fatalf("want the check's error, got %v", err) + } + if pendingRows(t, s) != 1 { + t.Fatal("a refused confirmation must keep its token") + } + + // A database error while subscribing: force one with a trigger. + if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON members BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil { + t.Fatal(err) + } + if _, err := s.ConfirmPending(tok, l.ID, nil); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("want the database error, got %v", err) + } + if pendingRows(t, s) != 1 { + t.Fatal("a failed AddMember must keep the token") + } + s.db.Exec(`DROP TRIGGER boom`) + if _, err := s.ConfirmPending(tok, l.ID, nil); err != nil { + t.Fatalf("the token must still work: %v", err) + } + if ok, _ := s.IsMember(l.ID, "carol@x.com"); !ok { + t.Fatal("carol must be subscribed") + } +} + +func TestConfirmPendingForAnotherListIsNotFoundAndKeepsTheToken(t *testing.T) { + s := newTestStore(t) + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + tok, _, _ := s.CreatePending(dev.ID, "carol@x.com", ActionJoin) + if _, err := s.ConfirmPending(tok, ops.ID, nil); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if ok, _ := s.IsMember(ops.ID, "carol@x.com"); ok { + t.Fatal("must not join the wrong list") + } + if _, err := s.ConfirmPending(tok, dev.ID, nil); err != nil { + t.Fatalf("the right list can still use it: %v", err) + } +} + +func TestConfirmPendingExpiredIsNotFound(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + tok, _, _ := s.CreatePending(l.ID, "carol@x.com", ActionJoin) + now = now.Add(pendingTTL) // exactly at the limit: still valid + if _, err := s.ConfirmPending(tok, l.ID, nil); err != nil { + t.Fatalf("at the TTL: %v", err) + } + tok, _, _ = s.CreatePending(l.ID, "dave@x.com", ActionJoin) + now = now.Add(pendingTTL + time.Second) + checked := false + if _, err := s.ConfirmPending(tok, l.ID, func(Pending) error { checked = true; return nil }); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if checked { + t.Fatal("an expired token must not reach the check") + } + if ok, _ := s.IsMember(l.ID, "dave@x.com"); ok { + t.Fatal("an expired token must not subscribe") + } +} + +// Two confirmations of one token at the same time: exactly one applies. +func TestConfirmPendingIsSingleUseUnderConcurrency(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "p.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + for round := 0; round < 20; round++ { + addr := fmt.Sprintf("u%d@x.com", round) + tok, _, _ := s.CreatePending(l.ID, addr, ActionJoin) + var applied, notFound, other atomic.Int32 + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, err := s.ConfirmPending(tok, l.ID, func(Pending) error { applied.Add(1); return nil }) + switch { + case err == nil: + case errors.Is(err, ErrNotFound): + notFound.Add(1) + default: + other.Add(1) + } + }() + } + close(start) + wg.Wait() + if applied.Load() != 1 || notFound.Load() != 7 || other.Load() != 0 { + t.Fatalf("round %d: applied %d, not found %d, other errors %d", round, applied.Load(), notFound.Load(), other.Load()) + } + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 38581ec..832aa38 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -503,7 +503,11 @@ func scanMember(row interface{ Scan(...any) error }) (Member, error) { // AddMember subscribes an address to a list. Re-adding keeps the member id and // clears the bounce state. func (s *Store) AddMember(listID int64, addr, displayName string) error { - _, err := s.db.Exec(`INSERT INTO members(list_id, address, display_name) VALUES(?,?,?) + return addMember(s.db, listID, addr, displayName) +} + +func addMember(q querier, listID int64, addr, displayName string) error { + _, err := q.Exec(`INSERT INTO members(list_id, address, display_name) VALUES(?,?,?) ON CONFLICT(list_id, address) DO UPDATE SET display_name=excluded.display_name, disabled=0, bounce_count=0`, listID, norm(addr), displayName) return err @@ -516,6 +520,13 @@ func (s *Store) IsMember(listID int64, addr string) (bool, error) { return n > 0, err } +// IsActiveMember reports whether addr is subscribed and not disabled by bounces. +func (s *Store) IsActiveMember(listID int64, addr string) (bool, error) { + var n int + err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=? AND disabled=0`, listID, norm(addr)).Scan(&n) + return n > 0, err +} + func (s *Store) queryMembers(where string, args ...any) ([]Member, error) { rows, err := s.db.Query(`SELECT `+memberCols+` FROM members WHERE `+where+` ORDER BY address`, args...) if err != nil { @@ -550,7 +561,11 @@ func (s *Store) MemberByID(id int64) (Member, error) { // RemoveMember unsubscribes an address from a list. func (s *Store) RemoveMember(listID int64, addr string) error { - return affected(s.db.Exec(`DELETE FROM members WHERE list_id=? AND address=?`, listID, norm(addr))) + return removeMember(s.db, listID, addr) +} + +func removeMember(q querier, listID int64, addr string) error { + return affected(q.Exec(`DELETE FROM members WHERE list_id=? AND address=?`, listID, norm(addr))) } // DeleteMember unsubscribes a member by id. @@ -666,6 +681,11 @@ func randomHex(n int) (string, error) { func (s *Store) CreatePending(listID int64, addr, action string) (token string, created bool, err error) { addr = norm(addr) now := s.Now() + // Expired requests are dead weight (a token older than the TTL is refused): drop them here, so + // the table stays small without a background job. + if _, err = s.db.Exec(`DELETE FROM pending WHERE created_at?`, listID, addr, action, now.Add(-pendingRateLimit).Unix()).Scan(&n) @@ -691,23 +711,56 @@ func (s *Store) CreatePending(listID int64, addr, action string) (token string, return token, true, nil } -// ConfirmPending consumes a token. Unknown, used and expired tokens give ErrNotFound. -func (s *Store) ConfirmPending(token string) (Pending, error) { +// ConfirmPending applies the request behind a token and consumes the token, in one transaction: a +// token is only used up when its action happened, and two confirmations of the same token cannot +// both apply (the first one to claim the row wins, the other gets ErrNotFound). +// +// Unknown, used, expired tokens and tokens of another list give ErrNotFound (the last two are left +// alone). check, when not nil, runs before anything changes; its error is returned and the token +// stays valid. A "join" subscribes the address (re-enabling a disabled member), a "leave" +// unsubscribes it (already gone is fine). +func (s *Store) ConfirmPending(token string, listID int64, check func(Pending) error) (Pending, error) { + tx, err := s.db.Begin() + if err != nil { + return Pending{}, err + } + defer tx.Rollback() var p Pending var ts int64 - err := s.db.QueryRow(`SELECT token, list_id, address, action, created_at FROM pending WHERE token=?`, token). - Scan(&p.Token, &p.ListID, &p.Address, &p.Action, &ts) + // DELETE ... RETURNING takes the write lock first, so concurrent confirmations queue up here. + err = tx.QueryRow(`DELETE FROM pending WHERE token=? AND list_id=? RETURNING token, list_id, address, action, created_at`, + token, listID).Scan(&p.Token, &p.ListID, &p.Address, &p.Action, &ts) if errors.Is(err, sql.ErrNoRows) { return Pending{}, ErrNotFound } if err != nil { return Pending{}, err } - if _, err := s.db.Exec(`DELETE FROM pending WHERE token=?`, token); err != nil { + if s.Now().Sub(time.Unix(ts, 0)) > pendingTTL { + // Expired: the row is gone for good. + if err := tx.Commit(); err != nil { + return Pending{}, err + } + return Pending{}, ErrNotFound + } + if check != nil { + if err := check(p); err != nil { + return Pending{}, err + } + } + switch p.Action { + case ActionJoin: + err = addMember(tx, p.ListID, p.Address, "") + case ActionLeave: + if err = removeMember(tx, p.ListID, p.Address); errors.Is(err, ErrNotFound) { + err = nil + } + } + if err != nil { return Pending{}, err } - if s.Now().Sub(time.Unix(ts, 0)) > pendingTTL { - return Pending{}, ErrNotFound + if err := tx.Commit(); err != nil { + return Pending{}, err } return p, nil }