diff --git a/internal/app/app.go b/internal/app/app.go index ea90216..cf34233 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -19,11 +19,15 @@ import ( "codeberg.org/squidd/parsimony/internal/config" "codeberg.org/squidd/parsimony/internal/ingest" "codeberg.org/squidd/parsimony/internal/outbound" + "codeberg.org/squidd/parsimony/internal/queue" "codeberg.org/squidd/parsimony/internal/router" "codeberg.org/squidd/parsimony/internal/store" "codeberg.org/squidd/parsimony/internal/web" ) +// nowFunc is the clock of the store (queue schedules, token expiry). Tests move it. +var nowFunc = time.Now + // App is a running daemon. type App struct { // LMTPAddr and APIAddr are the addresses actually bound. @@ -38,16 +42,14 @@ type App struct { web *http.Server pub *http.Server wg sync.WaitGroup + stop context.CancelFunc // stops the queue worker once sync.Once } // releaser lets the API approve held messages through the router. -type releaser struct { - rt *router.Router - out outbound.Sender -} +type releaser struct{ rt *router.Router } -func (r releaser) Release(ctx context.Context, id int64) error { return r.rt.Release(ctx, r.out, id) } +func (r releaser) Release(_ context.Context, id int64) error { return r.rt.Release(id) } // Start opens the database and begins serving. The REST API runs only when // apiToken is not empty. @@ -56,12 +58,18 @@ func Start(cfg config.Config, apiToken string) (*App, error) { if err != nil { return nil, fmt.Errorf("store: %w", err) } + st.Now = nowFunc a := &App{store: st} rt := router.New(st) rt.BaseURL = cfg.BaseURL - relay := &outbound.SMTPRelay{Addr: cfg.Relay.Addr} - a.lmtp = ingest.NewServer(rt, relay) + worker := &queue.Worker{ + Store: st, + Sender: &outbound.SMTPRelay{Addr: cfg.Relay.Addr}, + Concurrency: cfg.Relay.Concurrency, + } + rt.OnEnqueue = worker.Wake + a.lmtp = ingest.NewServer(rt) if cfg.LMTP.Network == "unix" { os.Remove(cfg.LMTP.Addr) @@ -86,7 +94,7 @@ func Start(cfg config.Config, apiToken string) (*App, error) { a.lns = append(a.lns, apiLn) // The admin UI and the API share a listener, so the UI needs no CORS. mux := http.NewServeMux() - mux.Handle("/api/", api.New(st, apiToken, releaser{rt, relay})) + mux.Handle("/api/", api.New(st, apiToken, releaser{rt})) mux.Handle("/", adminui.Embedded()) a.web = &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second} } @@ -106,6 +114,13 @@ func Start(cfg config.Config, apiToken string) (*App, error) { a.pub = &http.Server{Handler: web.New(st, rt), ReadHeaderTimeout: 10 * time.Second} } + var workerCtx context.Context + workerCtx, a.stop = context.WithCancel(context.Background()) + a.wg.Add(1) + go func() { + defer a.wg.Done() + worker.Run(workerCtx) + }() a.wg.Add(1) go func() { defer a.wg.Done() @@ -147,6 +162,7 @@ func (a *App) Close() { if a.pub != nil { a.pub.Close() } + a.stop() // in-flight sends are interrupted and stay queued a.wg.Wait() a.store.Close() }) diff --git a/internal/app/app_test.go b/internal/app/app_test.go index a931e22..3b47ae8 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -6,12 +6,14 @@ import ( "net/http" "strings" "sync" + "sync/atomic" "testing" "time" "github.com/emersion/go-smtp" "codeberg.org/squidd/parsimony/internal/config" + "codeberg.org/squidd/parsimony/internal/store" ) // sink is a fake SMTP relay that records what it receives. @@ -58,6 +60,18 @@ func (s *sink) count() int { return len(s.msgs) } +// waitCount waits until the relay holds n messages. +func (s *sink) waitCount(t *testing.T, n int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for s.count() < n { + if time.Now().After(deadline) { + t.Fatalf("relay has %d messages, want %d", s.count(), n) + } + time.Sleep(5 * time.Millisecond) + } +} + func (s *sink) all() []sunk { s.mu.Lock() defer s.mu.Unlock() @@ -168,16 +182,15 @@ func TestEndToEnd(t *testing.T) { if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { t.Fatalf("lmtp: %v", err) } - if relay.count() != 2 { - t.Fatalf("relay got %d messages, want 2", relay.count()) - } - got := relay.msgs[0] + relay.waitCount(t, 2) + got := relay.all()[0] if !strings.Contains(got.data, "List-Id: Dev ") || !strings.HasPrefix(got.from, "dev+bounce-") { t.Fatalf("unexpected relayed mail: %+v", got) } if err := lmtpSend(t, a.LMTPAddr, "mallory@evil.com", "dev@example.org", msg); err == nil { t.Fatal("non-member must be rejected") } + time.Sleep(100 * time.Millisecond) // give a wrongly queued message time to show up if relay.count() != 2 { t.Fatal("rejected mail must not be relayed") } @@ -275,6 +288,7 @@ func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) { } // The relayed copy carries a working one-click link for that recipient. + relay.waitCount(t, 2) var link string for _, m := range relay.all() { if len(m.to) == 1 && m.to[0] == "bob@example.com" { @@ -300,6 +314,8 @@ func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) { "From: Alice \r\nSubject: Second\r\nMessage-ID: \r\n\r\nhi\r\n"); err != nil { t.Fatal(err) } + relay.waitCount(t, before+1) + time.Sleep(100 * time.Millisecond) if got := relay.count() - before; got != 1 { t.Fatalf("after bob left only alice should get a copy, got %d", got) } @@ -386,3 +402,122 @@ func TestAdminUIAndAPIShareTheListener(t *testing.T) { } } } + +// deadAddr returns an address nothing listens on. +func deadAddr(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := ln.Addr().String() + ln.Close() + return addr +} + +func TestQueuedMailSurvivesARestartAndIsRetriedWhenTheRelayReturns(t *testing.T) { + var skew atomic.Int64 + nowFunc = func() time.Time { return time.Now().Add(time.Duration(skew.Load())) } + t.Cleanup(func() { nowFunc = time.Now }) + + // 1. The relay is down: the post is still accepted and stays queued. + cfg := testConfig(t, deadAddr(t)) + a, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok") + for _, m := range []string{"alice@example.com", "bob@example.com"} { + call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"`+m+`"}`, "tok") + } + msg := "From: Alice \r\nSubject: queued\r\nMessage-ID: \r\n\r\nbody\r\n" + if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { + t.Fatalf("a post must be accepted while the relay is down: %v", err) + } + peek, err := store.Open(cfg.DB) + if err != nil { + t.Fatal(err) + } + defer peek.Close() + deadline := time.Now().Add(5 * time.Second) + for { + st, _ := peek.OutboxStats() + if st.Pending == 2 && st.Retrying == 2 { + break + } + if time.Now().After(deadline) { + t.Fatalf("queue after the first attempts: %+v", st) + } + time.Sleep(10 * time.Millisecond) + } + + // 2. Restart with a working relay, after the backoff has passed. + a.Close() + relayAddr, relay := startSink(t) + cfg.Relay.Addr = relayAddr + skew.Store(int64(10 * time.Minute)) + b, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { b.Close() }) + relay.waitCount(t, 2) + got := relay.all() + if !strings.Contains(got[0].data, "List-Id: Dev ") || !strings.Contains(got[0].data, "Subject: queued") { + t.Fatalf("delivered mail: %q", got[0].data) + } + deadline = time.Now().Add(5 * time.Second) + for { + st, _ := peek.OutboxStats() + if st.Pending == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("delivered mail must leave the queue: %+v", st) + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestCloseInterruptsASlowSendWithoutLosingIt(t *testing.T) { + // A relay that accepts the connection and never answers. + ln, _ := net.Listen("tcp", "127.0.0.1:0") + t.Cleanup(func() { ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + defer c.Close() + } + }() + cfg := testConfig(t, ln.Addr().String()) + a, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok") + call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok") + msg := "From: Alice \r\nSubject: slow\r\nMessage-ID: \r\n\r\nbody\r\n" + if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { + t.Fatal(err) + } + time.Sleep(200 * time.Millisecond) // the worker is now stuck talking to the silent relay + + closed := make(chan struct{}) + go func() { a.Close(); close(closed) }() + select { + case <-closed: + case <-time.After(5 * time.Second): + t.Fatal("Close must not wait for a hung relay") + } + peek, err := store.Open(cfg.DB) + if err != nil { + t.Fatal(err) + } + defer peek.Close() + if st, _ := peek.OutboxStats(); st.Pending != 1 { + t.Fatalf("the interrupted delivery must stay queued: %+v", st) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index dd33a8b..92b60c4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,7 +24,8 @@ type Config struct { Listen string `yaml:"listen"` // public archive and unsubscribe pages; empty disables them } `yaml:"web"` Relay struct { - Addr string `yaml:"addr"` // host:port of the outgoing SMTP relay + Addr string `yaml:"addr"` // host:port of the outgoing SMTP relay + Concurrency int `yaml:"concurrency"` // simultaneous deliveries } `yaml:"relay"` } @@ -35,6 +36,7 @@ func Load(path string) (Config, error) { c.LMTP.Network = "tcp" c.LMTP.Addr = "127.0.0.1:2424" c.Relay.Addr = "127.0.0.1:25" + c.Relay.Concurrency = 4 c.API.Listen = "127.0.0.1:8080" c.Web.Listen = "127.0.0.1:8081" diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0e39f69..df78e2d 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -20,17 +20,17 @@ func TestLoadDefaults(t *testing.T) { if err != nil { t.Fatal(err) } - if c.DB != "parsimony.db" || c.LMTP.Network != "tcp" || c.LMTP.Addr != "127.0.0.1:2424" || c.Relay.Addr != "127.0.0.1:25" || c.API.Listen != "127.0.0.1:8080" || c.Web.Listen != "127.0.0.1:8081" { + if c.DB != "parsimony.db" || c.LMTP.Network != "tcp" || c.LMTP.Addr != "127.0.0.1:2424" || c.Relay.Addr != "127.0.0.1:25" || c.Relay.Concurrency != 4 || c.API.Listen != "127.0.0.1:8080" || c.Web.Listen != "127.0.0.1:8081" { t.Fatalf("defaults: %+v", c) } } func TestLoadValues(t *testing.T) { - c, err := Load(write(t, "db: /var/lib/p.db\nbase_url: https://l.example.org\nlmtp:\n network: unix\n addr: /run/p.sock\nrelay:\n addr: mx:587\napi:\n listen: 0.0.0.0:9000\nweb:\n listen: 0.0.0.0:9001\n")) + c, err := Load(write(t, "db: /var/lib/p.db\nbase_url: https://l.example.org\nlmtp:\n network: unix\n addr: /run/p.sock\nrelay:\n addr: mx:587\n concurrency: 9\napi:\n listen: 0.0.0.0:9000\nweb:\n listen: 0.0.0.0:9001\n")) if err != nil { t.Fatal(err) } - if c.DB != "/var/lib/p.db" || c.BaseURL != "https://l.example.org" || c.LMTP.Network != "unix" || c.LMTP.Addr != "/run/p.sock" || c.Relay.Addr != "mx:587" || c.API.Listen != "0.0.0.0:9000" || c.Web.Listen != "0.0.0.0:9001" { + if c.DB != "/var/lib/p.db" || c.BaseURL != "https://l.example.org" || c.LMTP.Network != "unix" || c.LMTP.Addr != "/run/p.sock" || c.Relay.Addr != "mx:587" || c.Relay.Concurrency != 9 || c.API.Listen != "0.0.0.0:9000" || c.Web.Listen != "0.0.0.0:9001" { t.Fatalf("values: %+v", c) } } diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 3fa60c0..1d4d8b2 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -2,7 +2,6 @@ package ingest import ( - "context" "errors" "io" "log" @@ -10,16 +9,16 @@ import ( "github.com/emersion/go-smtp" - "codeberg.org/squidd/parsimony/internal/outbound" "codeberg.org/squidd/parsimony/internal/router" ) const maxMessageBytes = 25 << 20 -// NewServer builds an LMTP server that routes incoming mail and relays it. +// NewServer builds an LMTP server that routes incoming mail into the outbound queue. +// A message is accepted only once it is durably queued. // Set Network/Addr on the result (or call Serve) to start it. -func NewServer(r *router.Router, out outbound.Sender) *smtp.Server { - srv := smtp.NewServer(&backend{router: r, out: out}) +func NewServer(r *router.Router) *smtp.Server { + srv := smtp.NewServer(&backend{router: r}) srv.LMTP = true srv.Domain = "localhost" srv.MaxMessageBytes = maxMessageBytes @@ -30,7 +29,6 @@ func NewServer(r *router.Router, out outbound.Sender) *smtp.Server { type backend struct { router *router.Router - out outbound.Sender } func (b *backend) NewSession(*smtp.Conn) (smtp.Session, error) { @@ -67,25 +65,22 @@ func (s *session) LMTPData(r io.Reader, status smtp.StatusCollector) error { if err != nil { return err } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - for _, rcpt := range s.rcpts { - status.SetStatus(rcpt, s.deliver(ctx, rcpt, data)) + status.SetStatus(rcpt, s.deliver(rcpt, data)) } return nil } -func (s *session) deliver(ctx context.Context, rcpt string, data []byte) error { +func (s *session) deliver(rcpt string, data []byte) error { rt := s.b.router - if handled, err := rt.Command(ctx, s.b.out, s.from, rcpt, data); handled { + if handled, err := rt.Command(s.from, rcpt, data); handled { return s.status(rcpt, err) } res, err := rt.Route(s.from, rcpt, data) if err != nil { return s.status(rcpt, err) } - return s.status(rcpt, rt.Send(ctx, s.b.out, res)) + return s.status(rcpt, rt.Deliver(res)) } // status turns a router result into an LMTP reply. diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 4535268..27d17f9 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -6,17 +6,26 @@ import ( "net" "strings" "testing" + "time" "github.com/emersion/go-smtp" "codeberg.org/squidd/parsimony/internal/outbound" + "codeberg.org/squidd/parsimony/internal/queue" "codeberg.org/squidd/parsimony/internal/router" "codeberg.org/squidd/parsimony/internal/store" ) const msg = "From: Alice \r\nSubject: hi\r\n\r\nbody\r\n" -func setup(t *testing.T) (addr string, relay *outbound.Fake) { +// stack is the LMTP server with a real queue worker in front of a fake relay. +type stack struct { + addr string + relay *outbound.Fake + store *store.Store +} + +func newStack(t *testing.T, sender outbound.Sender) *stack { t.Helper() s, err := store.Open(":memory:") if err != nil { @@ -27,15 +36,52 @@ func setup(t *testing.T) (addr string, relay *outbound.Fake) { s.AddMember(l.ID, "alice@example.com", "Alice") s.AddMember(l.ID, "bob@example.com", "Bob") - relay = &outbound.Fake{} - srv := NewServer(router.New(s), relay) + relay := &outbound.Fake{} + if sender == nil { + sender = relay + } + rt := router.New(s) + w := &queue.Worker{Store: s, Sender: sender, Interval: 10 * time.Millisecond} + rt.OnEnqueue = w.Wake + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { w.Run(ctx); close(done) }() + t.Cleanup(func() { cancel(); <-done }) + + srv := NewServer(rt) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } go srv.Serve(ln) t.Cleanup(func() { srv.Close() }) - return ln.Addr().String(), relay + return &stack{addr: ln.Addr().String(), relay: relay, store: s} +} + +// sent waits until the relay has received n messages and returns them. +func (st *stack) sent(t *testing.T, n int) []outbound.Sent { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for { + got := st.relay.Messages() + if len(got) >= n { + return got + } + if time.Now().After(deadline) { + t.Fatalf("relay got %d messages, want %d", len(got), n) + } + time.Sleep(5 * time.Millisecond) + } +} + +// queued is how many deliveries still wait in the outbox. +func (st *stack) queued(t *testing.T) int { + t.Helper() + stats, err := st.store.OutboxStats() + if err != nil { + t.Fatal(err) + } + return stats.Pending } // deliver sends msg over LMTP and returns the per-recipient errors. @@ -80,41 +126,45 @@ func deliver(t *testing.T, addr, from string, rcpts []string) map[string]error { } func TestDeliverFansOut(t *testing.T) { - addr, relay := setup(t) - res := deliver(t, addr, "alice@example.com", []string{"dev@example.org"}) + st := newStack(t, nil) + res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"}) if res["dev@example.org"] != nil { t.Fatalf("unexpected error: %v", res) } - if len(relay.Sent) != 2 { - t.Fatalf("want one send per member, got %d", len(relay.Sent)) + got := st.sent(t, 2) + if len(got) != 2 { + t.Fatalf("want one send per member, got %d", len(got)) } - s := relay.Sent[0] - if len(s.To) != 1 { - t.Errorf("recipients = %v", s.To) + m := got[0] + if len(m.To) != 1 { + t.Errorf("recipients = %v", m.To) } - if !strings.HasPrefix(s.From, "dev+bounce-") { - t.Errorf("envelope from = %q", s.From) + if !strings.HasPrefix(m.From, "dev+bounce-") { + t.Errorf("envelope from = %q", m.From) } - if !strings.Contains(string(s.Data), "List-Id: Dev ") { - t.Errorf("missing List-Id: %q", s.Data) + if !strings.Contains(string(m.Data), "List-Id: Dev ") { + t.Errorf("missing List-Id: %q", m.Data) + } + if !strings.Contains(string(m.Data), "List-Unsubscribe: ") { + t.Errorf("missing per-recipient unsubscribe header: %q", m.Data) } } func TestNonMemberRejectedPermanently(t *testing.T) { - addr, relay := setup(t) - res := deliver(t, addr, "mallory@evil.com", []string{"dev@example.org"}) + st := newStack(t, nil) + res := deliver(t, st.addr, "mallory@evil.com", []string{"dev@example.org"}) var se *smtp.SMTPError if !errors.As(res["dev@example.org"], &se) || se.Code != 550 { t.Fatalf("want 550, got %v", res["dev@example.org"]) } - if len(relay.Sent) != 0 { - t.Fatal("nothing must be relayed") + if st.queued(t) != 0 { + t.Fatal("nothing may be queued") } } func TestUnknownListRejected(t *testing.T) { - addr, _ := setup(t) - res := deliver(t, addr, "alice@example.com", []string{"nope@example.org"}) + st := newStack(t, nil) + res := deliver(t, st.addr, "alice@example.com", []string{"nope@example.org"}) var se *smtp.SMTPError if !errors.As(res["nope@example.org"], &se) || se.Code != 550 { t.Fatalf("want 550, got %v", res["nope@example.org"]) @@ -122,79 +172,83 @@ func TestUnknownListRejected(t *testing.T) { } func TestMixedRecipients(t *testing.T) { - addr, relay := setup(t) - res := deliver(t, addr, "alice@example.com", []string{"dev@example.org", "nope@example.org"}) + st := newStack(t, nil) + res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org", "nope@example.org"}) if res["dev@example.org"] != nil || res["nope@example.org"] == nil { t.Fatalf("results: %v", res) } - if len(relay.Sent) != 2 { - t.Fatalf("sent = %d", len(relay.Sent)) + st.sent(t, 2) +} + +// down is a relay that refuses everything. +type down struct{} + +func (down) Send(context.Context, string, []string, []byte) error { return errors.New("relay down") } + +func TestRelayDownStillAcceptsAndKeepsTheMailQueued(t *testing.T) { + st := newStack(t, down{}) + res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"}) + if res["dev@example.org"] != nil { + t.Fatalf("a queued post is accepted even when the relay is down: %v", res) + } + // The worker tries and schedules retries; nothing is lost. + deadline := time.Now().Add(3 * time.Second) + for { + stats, _ := st.store.OutboxStats() + if stats.Pending == 2 && stats.Retrying == 2 { + break + } + if time.Now().After(deadline) { + t.Fatalf("queue: %+v", stats) + } + time.Sleep(5 * time.Millisecond) } } -type failing struct{} - -func (failing) Send(_ context.Context, _ string, _ []string, _ []byte) error { - return errors.New("relay down") -} - -func TestRelayFailureIsTemporary(t *testing.T) { - s, _ := store.Open(":memory:") - t.Cleanup(func() { s.Close() }) - l, _ := s.CreateList("dev", "example.org", "Dev") - s.AddMember(l.ID, "alice@example.com", "Alice") - srv := NewServer(router.New(s), failing{}) - ln, _ := net.Listen("tcp", "127.0.0.1:0") - go srv.Serve(ln) - t.Cleanup(func() { srv.Close() }) - - res := deliver(t, ln.Addr().String(), "alice@example.com", []string{"dev@example.org"}) +func TestQueueFailureIsTemporary(t *testing.T) { + st := newStack(t, nil) + st.store.Close() // the database goes away under the server + res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"}) var se *smtp.SMTPError if !errors.As(res["dev@example.org"], &se) || se.Code != 451 { - t.Fatalf("want 451, got %v", res["dev@example.org"]) + t.Fatalf("want 451 so the MTA retries, got %v", res["dev@example.org"]) } } func TestJoinCommandOverLMTP(t *testing.T) { - addr, relay := setup(t) - res := deliver(t, addr, "carol@x.com", []string{"dev+join@example.org"}) + st := newStack(t, nil) + res := deliver(t, st.addr, "carol@x.com", []string{"dev+join@example.org"}) if res["dev+join@example.org"] != nil { t.Fatalf("join: %v", res) } - if len(relay.Sent) != 1 || relay.Sent[0].To[0] != "carol@x.com" { - t.Fatalf("confirmation mail: %+v", relay.Sent) + got := st.sent(t, 1) + if len(got[0].To) != 1 || got[0].To[0] != "carol@x.com" { + t.Fatalf("confirmation mail: %+v", got) } } func TestBadTokenRejected(t *testing.T) { - addr, _ := setup(t) - res := deliver(t, addr, "carol@x.com", []string{"dev+confirm-deadbeef@example.org"}) + st := newStack(t, nil) + res := deliver(t, st.addr, "carol@x.com", []string{"dev+confirm-deadbeef@example.org"}) var se *smtp.SMTPError if !errors.As(res["dev+confirm-deadbeef@example.org"], &se) || se.Code != 550 { t.Fatalf("want 550, got %v", res) } } -func TestModeratedPostIsAcceptedButNotRelayed(t *testing.T) { - s, _ := store.Open(":memory:") - t.Cleanup(func() { s.Close() }) - l, _ := s.CreateList("dev", "example.org", "Dev") - s.AddMember(l.ID, "alice@example.com", "Alice") - s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate) - relay := &outbound.Fake{} - srv := NewServer(router.New(s), relay) - ln, _ := net.Listen("tcp", "127.0.0.1:0") - go srv.Serve(ln) - t.Cleanup(func() { srv.Close() }) +func TestModeratedPostIsAcceptedButNotQueued(t *testing.T) { + st := newStack(t, nil) + l, _ := st.store.FindList("dev@example.org") + st.store.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate) - res := deliver(t, ln.Addr().String(), "stranger@x.com", []string{"dev@example.org"}) + res := deliver(t, st.addr, "stranger@x.com", []string{"dev@example.org"}) if res["dev@example.org"] != nil { t.Fatalf("held mail must be accepted: %v", res) } - if len(relay.Sent) != 0 { - t.Fatal("nothing may be relayed") + if st.queued(t) != 0 { + t.Fatal("nothing may be queued for delivery") } - if h, _ := s.ListHeld(l.ID); len(h) != 1 { + if h, _ := st.store.ListHeld(l.ID); len(h) != 1 { t.Fatalf("held = %d", len(h)) } } diff --git a/internal/outbound/outbound.go b/internal/outbound/outbound.go index cd374ab..b53088d 100644 --- a/internal/outbound/outbound.go +++ b/internal/outbound/outbound.go @@ -43,12 +43,32 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b return nil } -func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) error { +func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) (err error) { d := net.Dialer{Timeout: 10 * time.Second} conn, err := d.DialContext(ctx, "tcp", r.Addr) if err != nil { return err } + // A relay that stops answering must not hold us past the context: apply its + // deadline to the connection and close the connection if it is cancelled. + if dl, ok := ctx.Deadline(); ok { + conn.SetDeadline(dl) + } + stop := make(chan struct{}) + defer close(stop) + go func() { + select { + case <-ctx.Done(): + conn.Close() + case <-stop: + } + }() + defer func() { + if err != nil && ctx.Err() != nil { + err = ctx.Err() // report the cause, not "use of closed connection" + } + }() + c := smtp.NewClient(conn) defer c.Close() if err := c.Hello("localhost"); err != nil { @@ -73,6 +93,13 @@ type Fake struct { Sent []Sent } +// Messages returns a copy of what was recorded; safe while sends are running. +func (f *Fake) Messages() []Sent { + f.mu.Lock() + defer f.mu.Unlock() + return append([]Sent(nil), f.Sent...) +} + // Send records the message. func (f *Fake) Send(_ context.Context, from string, to []string, data []byte) error { f.mu.Lock() diff --git a/internal/outbound/outbound_test.go b/internal/outbound/outbound_test.go index 0459e55..47b0566 100644 --- a/internal/outbound/outbound_test.go +++ b/internal/outbound/outbound_test.go @@ -8,6 +8,7 @@ import ( "net" "sync" "testing" + "time" "github.com/emersion/go-smtp" ) @@ -196,3 +197,68 @@ func TestRelayRejectionsAreClassifiedFromTheWire(t *testing.T) { } } } + +// silent accepts connections and never says a word. +func silent(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + var conns []net.Conn + var mu sync.Mutex + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + conns = append(conns, c) + mu.Unlock() + } + }() + t.Cleanup(func() { + ln.Close() + mu.Lock() + for _, c := range conns { + c.Close() + } + mu.Unlock() + }) + return ln.Addr().String() +} + +func TestSendStopsWhenTheContextIsCancelled(t *testing.T) { + r := &SMTPRelay{Addr: silent(t)} + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { errc <- r.Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("x")) }() + time.Sleep(100 * time.Millisecond) + cancel() + select { + case err := <-errc: + if err == nil { + t.Fatal("want an error") + } + case <-time.After(2 * time.Second): + t.Fatal("Send ignored the cancelled context while talking to a silent relay") + } +} + +func TestSendHonoursTheContextDeadline(t *testing.T) { + r := &SMTPRelay{Addr: silent(t)} + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + start := time.Now() + err := r.Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("x")) + if err == nil { + t.Fatal("want an error") + } + if took := time.Since(start); took > 2*time.Second { + t.Fatalf("took %v", took) + } + if IsPermanent(err) { + t.Fatalf("a timeout is temporary: %v", err) + } +} diff --git a/internal/router/commands.go b/internal/router/commands.go index 7b2e0f7..499c303 100644 --- a/internal/router/commands.go +++ b/internal/router/commands.go @@ -1,7 +1,6 @@ package router import ( - "context" "crypto/hmac" "crypto/sha256" "encoding/hex" @@ -14,7 +13,6 @@ import ( "strings" "time" - "codeberg.org/squidd/parsimony/internal/outbound" "codeberg.org/squidd/parsimony/internal/store" ) @@ -48,7 +46,7 @@ func (r *Router) bounceAddress(l store.List, memberID int64) string { // Command handles list command addresses such as dev+join@example.org. // handled is false when rcpt is a plain list address (use Route for those). -func (r *Router) Command(ctx context.Context, out outbound.Sender, sender, rcpt string, data []byte) (handled bool, err error) { +func (r *Router) Command(sender, rcpt string, data []byte) (handled bool, err error) { local, domain, ok := strings.Cut(strings.ToLower(strings.TrimSpace(rcpt)), "@") if !ok { return false, nil @@ -67,9 +65,9 @@ func (r *Router) Command(ctx context.Context, out outbound.Sender, sender, rcpt switch { case tag == "join": - return true, r.join(ctx, out, l, sender) + return true, r.join(l, sender) case tag == "leave": - return true, r.leave(ctx, out, l, sender) + return true, r.leave(l, sender) case strings.HasPrefix(tag, "confirm-"): return true, r.confirm(l, strings.TrimPrefix(tag, "confirm-")) case tag == "bounce": @@ -80,7 +78,7 @@ func (r *Router) Command(ctx context.Context, out outbound.Sender, sender, rcpt return true, ErrUnknownList } -func (r *Router) join(ctx context.Context, out outbound.Sender, l store.List, sender string) error { +func (r *Router) join(l store.List, sender string) error { if sender == "" { return nil } @@ -90,20 +88,20 @@ func (r *Router) join(ctx context.Context, out outbound.Sender, l store.List, se if ok, err := r.store.IsMember(l.ID, sender); err != nil || ok { return err // members get no mail: it would leak membership } - return r.askConfirmation(ctx, out, l, sender, store.ActionJoin) + return r.askConfirmation(l, sender, store.ActionJoin) } -func (r *Router) leave(ctx context.Context, out outbound.Sender, l store.List, sender string) error { +func (r *Router) leave(l store.List, sender string) error { if sender == "" { return nil } if ok, err := r.store.IsMember(l.ID, sender); err != nil || !ok { return err } - return r.askConfirmation(ctx, out, l, sender, store.ActionLeave) + return r.askConfirmation(l, sender, store.ActionLeave) } -func (r *Router) askConfirmation(ctx context.Context, out outbound.Sender, l store.List, addr, action string) error { +func (r *Router) askConfirmation(l store.List, addr, action string) error { token, created, err := r.store.CreatePending(l.ID, addr, action) if err != nil || !created { return err @@ -124,7 +122,7 @@ func (r *Router) askConfirmation(ctx context.Context, out outbound.Sender, l sto time.Now().Format(time.RFC1123Z), token[:8], strconv.FormatInt(time.Now().UnixNano(), 36), l.Domain, verb, l.Address()) - return out.Send(ctx, l.Name+"+bounce@"+l.Domain, []string{addr}, []byte(msg)) + return r.enqueue([]byte(msg), []store.NewOutbound{{From: l.Name + "+bounce@" + l.Domain, Rcpt: addr}}) } func (r *Router) confirm(l store.List, token string) error { diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go index c1b0e20..5d9f4d5 100644 --- a/internal/router/m2_test.go +++ b/internal/router/m2_test.go @@ -8,6 +8,7 @@ import ( "testing" "codeberg.org/squidd/parsimony/internal/outbound" + "codeberg.org/squidd/parsimony/internal/queue" "codeberg.org/squidd/parsimony/internal/store" ) @@ -34,6 +35,37 @@ func newEnv(t *testing.T) *env { return &env{r: New(s), s: s, out: &outbound.Fake{}, list: l, ctx: context.Background(), alice: ms[0].ID} } +// flush sends everything queued, as the background worker would. +func (e *env) flush() { + w := &queue.Worker{Store: e.s, Sender: e.out} + for { + st, err := w.RunOnce(e.ctx) + if err != nil || st == (queue.Stats{}) { + return + } + } +} + +func (e *env) send(res Result) error { + if err := e.r.Deliver(res); err != nil { + return err + } + e.flush() + return nil +} + +func (e *env) command(sender, rcpt string, data []byte) (bool, error) { + handled, err := e.r.Command(sender, rcpt, data) + e.flush() + return handled, err +} + +func (e *env) release(id int64) error { + err := e.r.Release(id) + e.flush() + return err +} + func lastMail(t *testing.T, out *outbound.Fake) outbound.Sent { t.Helper() if len(out.Sent) == 0 { @@ -48,7 +80,7 @@ func TestSendUsesVERPPerRecipient(t *testing.T) { if err != nil { t.Fatal(err) } - if err := e.r.Send(e.ctx, e.out, res); err != nil { + if err := e.send(res); err != nil { t.Fatal(err) } if len(e.out.Sent) != 2 { @@ -107,7 +139,7 @@ func TestReleaseHeld(t *testing.T) { e.r.Route("stranger@x.com", "dev@example.org", []byte(sample)) held, _ := e.s.ListHeld(e.list.ID) - if err := e.r.Release(e.ctx, e.out, held[0].ID); err != nil { + if err := e.release(held[0].ID); err != nil { t.Fatal(err) } if len(e.out.Sent) != 2 { @@ -119,7 +151,7 @@ func TestReleaseHeld(t *testing.T) { if rest, _ := e.s.ListHeld(e.list.ID); len(rest) != 0 { t.Fatal("released message must leave the queue") } - if err := e.r.Release(e.ctx, e.out, held[0].ID); !errors.Is(err, store.ErrNotFound) { + if err := e.release(held[0].ID); !errors.Is(err, store.ErrNotFound) { t.Fatalf("want ErrNotFound, got %v", err) } } @@ -137,7 +169,7 @@ func token(t *testing.T, m outbound.Sent) string { func TestJoinFlow(t *testing.T) { e := newEnv(t) - handled, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) + handled, err := e.command("carol@x.com", "dev+join@example.org", nil) if !handled || err != nil { t.Fatalf("join: %v %v", handled, err) } @@ -150,14 +182,14 @@ func TestJoinFlow(t *testing.T) { } tok := token(t, m) - handled, err = e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil) + handled, err = e.command("carol@x.com", "dev+confirm-"+tok+"@example.org", nil) if !handled || err != nil { t.Fatalf("confirm: %v %v", handled, err) } if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); !ok { t.Fatal("carol must be a member after confirming") } - _, err = e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil) + _, err = e.command("carol@x.com", "dev+confirm-"+tok+"@example.org", nil) if !errors.Is(err, ErrBadToken) { t.Fatalf("reused token: want ErrBadToken, got %v", err) } @@ -165,16 +197,16 @@ func TestJoinFlow(t *testing.T) { func TestJoinRateLimitedAndSilentForMembers(t *testing.T) { e := newEnv(t) - e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) - e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) + e.command("carol@x.com", "dev+join@example.org", nil) + e.command("carol@x.com", "dev+join@example.org", nil) if len(e.out.Sent) != 1 { t.Fatalf("second request must not send mail, sent %d", len(e.out.Sent)) } - e.r.Command(e.ctx, e.out, "alice@example.com", "dev+join@example.org", nil) + e.command("alice@example.com", "dev+join@example.org", nil) if len(e.out.Sent) != 1 { t.Fatal("existing members get no mail") } - e.r.Command(e.ctx, e.out, "", "dev+join@example.org", nil) + e.command("", "dev+join@example.org", nil) if len(e.out.Sent) != 1 { t.Fatal("empty sender is ignored") } @@ -183,7 +215,7 @@ func TestJoinRateLimitedAndSilentForMembers(t *testing.T) { func TestJoinClosed(t *testing.T) { e := newEnv(t) e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate) - _, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) + _, err := e.command("carol@x.com", "dev+join@example.org", nil) if !errors.Is(err, ErrJoinClosed) { t.Fatalf("want ErrJoinClosed, got %v", err) } @@ -194,10 +226,10 @@ func TestJoinClosed(t *testing.T) { func TestConfirmAfterListClosed(t *testing.T) { e := newEnv(t) - e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) + e.command("carol@x.com", "dev+join@example.org", nil) tok := token(t, lastMail(t, e.out)) e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate) - _, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil) + _, err := e.command("carol@x.com", "dev+confirm-"+tok+"@example.org", nil) if !errors.Is(err, ErrJoinClosed) { t.Fatalf("want ErrJoinClosed, got %v", err) } @@ -208,7 +240,7 @@ func TestConfirmAfterListClosed(t *testing.T) { func TestLeaveFlow(t *testing.T) { e := newEnv(t) - e.r.Command(e.ctx, e.out, "alice@example.com", "dev+leave@example.org", nil) + e.command("alice@example.com", "dev+leave@example.org", nil) m := lastMail(t, e.out) if m.To[0] != "alice@example.com" { t.Fatalf("to = %v", m.To) @@ -216,13 +248,13 @@ func TestLeaveFlow(t *testing.T) { if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); !ok { t.Fatal("still a member before confirming") } - e.r.Command(e.ctx, e.out, "alice@example.com", "dev+confirm-"+token(t, m)+"@example.org", nil) + e.command("alice@example.com", "dev+confirm-"+token(t, m)+"@example.org", nil) if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); ok { t.Fatal("alice must be gone") } // Leaving when not a member sends nothing. n := len(e.out.Sent) - e.r.Command(e.ctx, e.out, "zed@x.com", "dev+leave@example.org", nil) + e.command("zed@x.com", "dev+leave@example.org", nil) if len(e.out.Sent) != n { t.Fatal("non-members get no mail") } @@ -231,9 +263,9 @@ func TestLeaveFlow(t *testing.T) { func TestConfirmTokenFromOtherList(t *testing.T) { e := newEnv(t) e.s.CreateList("ops", "example.org", "Ops") - e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil) + e.command("carol@x.com", "dev+join@example.org", nil) tok := token(t, lastMail(t, e.out)) - _, err := e.r.Command(e.ctx, e.out, "carol@x.com", "ops+confirm-"+tok+"@example.org", nil) + _, err := e.command("carol@x.com", "ops+confirm-"+tok+"@example.org", nil) if !errors.Is(err, ErrBadToken) { t.Fatalf("want ErrBadToken, got %v", err) } @@ -241,7 +273,7 @@ func TestConfirmTokenFromOtherList(t *testing.T) { func TestCommandNotHandledForPlainAddress(t *testing.T) { e := newEnv(t) - handled, err := e.r.Command(e.ctx, e.out, "a@x.com", "dev@example.org", nil) + handled, err := e.command("a@x.com", "dev@example.org", nil) if handled || err != nil { t.Fatalf("plain address: %v %v", handled, err) } @@ -250,7 +282,7 @@ func TestCommandNotHandledForPlainAddress(t *testing.T) { func TestUnknownCommandAndList(t *testing.T) { e := newEnv(t) for _, rcpt := range []string{"dev+bogus@example.org", "nope+join@example.org"} { - handled, err := e.r.Command(e.ctx, e.out, "a@x.com", rcpt, nil) + handled, err := e.command("a@x.com", rcpt, nil) if !handled || !errors.Is(err, ErrUnknownList) { t.Errorf("%s: %v %v", rcpt, handled, err) } @@ -265,7 +297,7 @@ func TestHardBouncesDisableMember(t *testing.T) { e := newEnv(t) addr := e.r.bounceAddress(e.list, e.alice) for i := 0; i < 3; i++ { - handled, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1")) + handled, err := e.command("", addr, bounceMsg("5.1.1")) if !handled || err != nil { t.Fatalf("bounce: %v %v", handled, err) } @@ -278,7 +310,7 @@ func TestHardBouncesDisableMember(t *testing.T) { func TestSoftBounceIgnored(t *testing.T) { e := newEnv(t) - e.r.Command(e.ctx, e.out, "", e.r.bounceAddress(e.list, e.alice), bounceMsg("4.2.2")) + e.command("", e.r.bounceAddress(e.list, e.alice), bounceMsg("4.2.2")) if m, _ := e.s.MemberByID(e.alice); m.BounceCount != 0 { t.Fatalf("soft bounce counted: %+v", m) } @@ -287,7 +319,7 @@ func TestSoftBounceIgnored(t *testing.T) { func TestForgedBounceRejected(t *testing.T) { e := newEnv(t) addr := fmt.Sprintf("dev+bounce-%d-000000000000@example.org", e.alice) - _, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1")) + _, err := e.command("", addr, bounceMsg("5.1.1")) if !errors.Is(err, ErrBadToken) { t.Fatalf("want ErrBadToken, got %v", err) } @@ -301,7 +333,7 @@ func TestBounceFromOtherListRejected(t *testing.T) { e.s.CreateList("ops", "example.org", "Ops") ops, _ := e.s.FindList("ops@example.org") addr := e.r.bounceAddress(ops, e.alice) // valid MAC but alice is not in ops - _, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1")) + _, err := e.command("", addr, bounceMsg("5.1.1")) if !errors.Is(err, ErrBadToken) { t.Fatalf("want ErrBadToken, got %v", err) } @@ -309,47 +341,81 @@ func TestBounceFromOtherListRejected(t *testing.T) { func TestSystemBounceAddressDiscarded(t *testing.T) { e := newEnv(t) - handled, err := e.r.Command(e.ctx, e.out, "", "dev+bounce@example.org", bounceMsg("5.1.1")) + handled, err := e.command("", "dev+bounce@example.org", bounceMsg("5.1.1")) if !handled || err != nil { t.Fatalf("%v %v", handled, err) } } -// flaky fails for the listed addresses. -type flaky struct { - bad map[string]bool - sent []string -} - -func (f *flaky) Send(_ context.Context, _ string, to []string, _ []byte) error { - if f.bad[to[0]] { - return errors.New("rejected") - } - f.sent = append(f.sent, to[0]) - return nil -} - -func TestSendSkipsFailedRecipients(t *testing.T) { +func TestDeliverQueuesOneEntryPerRecipientAtomically(t *testing.T) { e := newEnv(t) res, _ := e.r.Route("alice@example.com", "dev@example.org", []byte(sample)) - f := &flaky{bad: map[string]bool{"alice@example.com": true}} - if err := e.r.Send(e.ctx, f, res); err != nil { - t.Fatalf("partial failure must not fail the delivery: %v", err) + if err := e.r.Deliver(res); err != nil { + t.Fatal(err) } - if len(f.sent) != 1 || f.sent[0] != "bob@example.com" { - t.Fatalf("sent: %v", f.sent) + if len(e.out.Sent) != 0 { + t.Fatal("Deliver only queues; the worker sends") + } + due, _ := e.s.DueOutbox(10) + if len(due) != 2 { + t.Fatalf("queued = %d", len(due)) + } + if due[0].MsgID != due[1].MsgID { + t.Fatal("recipients must share one stored body") + } + for _, d := range due { + if d.MemberID == 0 || !strings.HasPrefix(d.From, "dev+bounce-") || !strings.Contains(string(d.Prefix), "List-Unsubscribe:") { + t.Fatalf("item: %+v", d) + } + } + body, _ := e.s.OutboxData(due[0].MsgID) + if !strings.Contains(string(body), "List-Id:") || strings.Contains(string(body), "List-Unsubscribe:") { + t.Fatalf("shared body has the common headers only: %q", body) } } -func TestSendFailsWhenNobodyIsReached(t *testing.T) { +func TestDeliverFailsCleanlyWhenTheQueueIsUnavailable(t *testing.T) { e := newEnv(t) res, _ := e.r.Route("alice@example.com", "dev@example.org", []byte(sample)) - f := &flaky{bad: map[string]bool{"alice@example.com": true, "bob@example.com": true}} - if err := e.r.Send(e.ctx, f, res); err == nil { + e.s.Close() + if err := e.r.Deliver(res); err == nil { t.Fatal("want an error so the MTA retries") } } +func TestDeliverWithoutRecipientsQueuesNothing(t *testing.T) { + e := newEnv(t) + woke := 0 + e.r.OnEnqueue = func() { woke++ } + if err := e.r.Deliver(Result{List: e.list, Data: []byte("x")}); err != nil { + t.Fatal(err) + } + if st, _ := e.s.OutboxStats(); st.Pending != 0 || woke != 0 { + t.Fatalf("pending=%d woke=%d", st.Pending, woke) + } +} + +func TestQueuedMailWakesTheSender(t *testing.T) { + e := newEnv(t) + woke := 0 + e.r.OnEnqueue = func() { woke++ } + res, _ := e.r.Route("alice@example.com", "dev@example.org", []byte(sample)) + e.r.Deliver(res) + e.r.Command("carol@x.com", "dev+join@example.org", nil) + if woke != 2 { + t.Fatalf("OnEnqueue called %d times, want once per queued batch (2)", woke) + } +} + +func TestConfirmationMailIsQueuedAsSystemMail(t *testing.T) { + e := newEnv(t) + e.r.Command("carol@x.com", "dev+join@example.org", nil) + due, _ := e.s.DueOutbox(10) + if len(due) != 1 || due[0].Rcpt != "carol@x.com" || due[0].MemberID != 0 || due[0].From != "dev+bounce@example.org" { + t.Fatalf("queue: %+v", due) + } +} + func TestIncomingListHeadersAreStripped(t *testing.T) { r := setup(t) // no BaseURL: One-Click must not survive from the outside msg := "From: Alice \r\nList-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n" + diff --git a/internal/router/m3_test.go b/internal/router/m3_test.go index 6769095..ca43821 100644 --- a/internal/router/m3_test.go +++ b/internal/router/m3_test.go @@ -58,7 +58,7 @@ func TestRejectedAndHeldPostsAreNotArchived(t *testing.T) { t.Fatalf("archived = %d", total) } held, _ := e.s.ListHeld(e.list.ID) - if err := e.r.Release(e.ctx, e.out, held[0].ID); err != nil { + if err := e.release(held[0].ID); err != nil { t.Fatal(err) } if _, total, _ := e.s.ListMessages(e.list.ID, 10, 0); total != 1 { @@ -85,7 +85,7 @@ func sendPost(t *testing.T, e *env) { if err != nil { t.Fatal(err) } - if err := e.r.Send(e.ctx, e.out, res); err != nil { + if err := e.send(res); err != nil { t.Fatal(err) } } @@ -134,7 +134,7 @@ func TestSpoofedUnsubscribeHeadersNeverReachRecipients(t *testing.T) { e := newEnv(t) msg := "From: Alice \r\nList-Unsubscribe: \r\nList-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n\r\nx\r\n" res, _ := e.r.Route("alice@example.com", "dev@example.org", []byte(msg)) - e.r.Send(e.ctx, e.out, res) + e.send(res) for _, m := range e.out.Sent { h := header(t, m.Data) if strings.Contains(h.Get("List-Unsubscribe"), "evil") || h.Get("List-Unsubscribe-Post") != "" { diff --git a/internal/router/router.go b/internal/router/router.go index 43d5989..86fc1fc 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -2,16 +2,13 @@ package router import ( - "context" "errors" - "fmt" "log" "strings" "sync" "time" "codeberg.org/squidd/parsimony/internal/archive" - "codeberg.org/squidd/parsimony/internal/outbound" "codeberg.org/squidd/parsimony/internal/store" ) @@ -48,6 +45,8 @@ type Router struct { store *store.Store // BaseURL is the public web URL; enables one-click unsubscribe when set. BaseURL string + // OnEnqueue, when set, is called after mail is queued (to wake the sender). + OnEnqueue func() secretOnce sync.Once secretVal []byte @@ -101,24 +100,30 @@ func (r *Router) prepare(l store.List, data []byte) (Result, error) { return Result{List: l, Recipients: rcpts, Data: out}, nil } -// Send delivers the message to every recipient, each with its own signed -// bounce address (VERP). Individual failures are logged and skipped; an error is -// returned only when nobody could be reached, so a retry cannot duplicate mail. -func (r *Router) Send(ctx context.Context, out outbound.Sender, res Result) error { - failed := 0 +// Deliver queues the message for every recipient, each with its own signed +// bounce address (VERP) and unsubscribe headers. The queue write is atomic: on +// error nothing was queued and the caller may safely retry. +func (r *Router) Deliver(res Result) error { + items := make([]store.NewOutbound, 0, len(res.Recipients)) for _, rc := range res.Recipients { from := r.bounceAddress(res.List, rc.ID) if from == "" { return errors.New("cannot create bounce address") } - data := append(r.unsubscribeHeaders(res.List, rc.ID), res.Data...) - if err := out.Send(ctx, from, []string{rc.Address}, data); err != nil { - failed++ - log.Printf("deliver %s to %s: %v", res.List.Address(), rc.Address, err) - } + items = append(items, store.NewOutbound{ + From: from, Rcpt: rc.Address, MemberID: rc.ID, Prefix: r.unsubscribeHeaders(res.List, rc.ID), + }) } - if failed > 0 && failed == len(res.Recipients) { - return fmt.Errorf("delivery failed for all %d recipients", failed) + return r.enqueue(res.Data, items) +} + +// enqueue stores deliveries and wakes the sender. +func (r *Router) enqueue(data []byte, items []store.NewOutbound) error { + if err := r.store.Enqueue(data, items); err != nil { + return err + } + if r.OnEnqueue != nil && len(items) > 0 { + r.OnEnqueue() } return nil } @@ -156,8 +161,9 @@ func (r *Router) archive(l store.List, data []byte) { } } -// Release approves a held message and delivers it. -func (r *Router) Release(ctx context.Context, out outbound.Sender, heldID int64) error { +// Release approves a held message: it is queued for delivery and leaves the +// moderation queue. +func (r *Router) Release(heldID int64) error { h, err := r.store.GetHeld(heldID) if err != nil { return err @@ -170,7 +176,7 @@ func (r *Router) Release(ctx context.Context, out outbound.Sender, heldID int64) if err != nil { return err } - if err := r.Send(ctx, out, res); err != nil { + if err := r.Deliver(res); err != nil { return err } return r.store.DeleteHeld(heldID)