feat: deliver through the persistent queue
- Router.Deliver queues one entry per recipient atomically; posts are accepted once queued - LMTP answers 451 only when the database fails, never because the relay is down - Confirmation mail uses the same queue - Wire the worker into the daemon: relay.concurrency config, wake on enqueue, interrupt on Close - Make SMTPRelay honour the context deadline and cancellation on a silent relay - Test restart survival, relay outage with retry, and Close with a hung relay
This commit is contained in:
parent
b207312cef
commit
6c95184adc
12 changed files with 534 additions and 169 deletions
|
|
@ -19,11 +19,15 @@ import (
|
|||
"codeberg.org/squidd/parsimony/internal/config"
|
||||
"codeberg.org/squidd/parsimony/internal/ingest"
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/queue"
|
||||
"codeberg.org/squidd/parsimony/internal/router"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
"codeberg.org/squidd/parsimony/internal/web"
|
||||
)
|
||||
|
||||
// nowFunc is the clock of the store (queue schedules, token expiry). Tests move it.
|
||||
var nowFunc = time.Now
|
||||
|
||||
// App is a running daemon.
|
||||
type App struct {
|
||||
// LMTPAddr and APIAddr are the addresses actually bound.
|
||||
|
|
@ -38,16 +42,14 @@ type App struct {
|
|||
web *http.Server
|
||||
pub *http.Server
|
||||
wg sync.WaitGroup
|
||||
stop context.CancelFunc // stops the queue worker
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
// releaser lets the API approve held messages through the router.
|
||||
type releaser struct {
|
||||
rt *router.Router
|
||||
out outbound.Sender
|
||||
}
|
||||
type releaser struct{ rt *router.Router }
|
||||
|
||||
func (r releaser) Release(ctx context.Context, id int64) error { return r.rt.Release(ctx, r.out, id) }
|
||||
func (r releaser) Release(_ context.Context, id int64) error { return r.rt.Release(id) }
|
||||
|
||||
// Start opens the database and begins serving. The REST API runs only when
|
||||
// apiToken is not empty.
|
||||
|
|
@ -56,12 +58,18 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("store: %w", err)
|
||||
}
|
||||
st.Now = nowFunc
|
||||
a := &App{store: st}
|
||||
|
||||
rt := router.New(st)
|
||||
rt.BaseURL = cfg.BaseURL
|
||||
relay := &outbound.SMTPRelay{Addr: cfg.Relay.Addr}
|
||||
a.lmtp = ingest.NewServer(rt, relay)
|
||||
worker := &queue.Worker{
|
||||
Store: st,
|
||||
Sender: &outbound.SMTPRelay{Addr: cfg.Relay.Addr},
|
||||
Concurrency: cfg.Relay.Concurrency,
|
||||
}
|
||||
rt.OnEnqueue = worker.Wake
|
||||
a.lmtp = ingest.NewServer(rt)
|
||||
|
||||
if cfg.LMTP.Network == "unix" {
|
||||
os.Remove(cfg.LMTP.Addr)
|
||||
|
|
@ -86,7 +94,7 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
a.lns = append(a.lns, apiLn)
|
||||
// The admin UI and the API share a listener, so the UI needs no CORS.
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/api/", api.New(st, apiToken, releaser{rt, relay}))
|
||||
mux.Handle("/api/", api.New(st, apiToken, releaser{rt}))
|
||||
mux.Handle("/", adminui.Embedded())
|
||||
a.web = &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second}
|
||||
}
|
||||
|
|
@ -106,6 +114,13 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
a.pub = &http.Server{Handler: web.New(st, rt), ReadHeaderTimeout: 10 * time.Second}
|
||||
}
|
||||
|
||||
var workerCtx context.Context
|
||||
workerCtx, a.stop = context.WithCancel(context.Background())
|
||||
a.wg.Add(1)
|
||||
go func() {
|
||||
defer a.wg.Done()
|
||||
worker.Run(workerCtx)
|
||||
}()
|
||||
a.wg.Add(1)
|
||||
go func() {
|
||||
defer a.wg.Done()
|
||||
|
|
@ -147,6 +162,7 @@ func (a *App) Close() {
|
|||
if a.pub != nil {
|
||||
a.pub.Close()
|
||||
}
|
||||
a.stop() // in-flight sends are interrupted and stay queued
|
||||
a.wg.Wait()
|
||||
a.store.Close()
|
||||
})
|
||||
|
|
|
|||
|
|
@ -6,12 +6,14 @@ import (
|
|||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/emersion/go-smtp"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/config"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
// sink is a fake SMTP relay that records what it receives.
|
||||
|
|
@ -58,6 +60,18 @@ func (s *sink) count() int {
|
|||
return len(s.msgs)
|
||||
}
|
||||
|
||||
// waitCount waits until the relay holds n messages.
|
||||
func (s *sink) waitCount(t *testing.T, n int) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for s.count() < n {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("relay has %d messages, want %d", s.count(), n)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sink) all() []sunk {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
|
@ -168,16 +182,15 @@ func TestEndToEnd(t *testing.T) {
|
|||
if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil {
|
||||
t.Fatalf("lmtp: %v", err)
|
||||
}
|
||||
if relay.count() != 2 {
|
||||
t.Fatalf("relay got %d messages, want 2", relay.count())
|
||||
}
|
||||
got := relay.msgs[0]
|
||||
relay.waitCount(t, 2)
|
||||
got := relay.all()[0]
|
||||
if !strings.Contains(got.data, "List-Id: Dev <dev.example.org>") || !strings.HasPrefix(got.from, "dev+bounce-") {
|
||||
t.Fatalf("unexpected relayed mail: %+v", got)
|
||||
}
|
||||
if err := lmtpSend(t, a.LMTPAddr, "mallory@evil.com", "dev@example.org", msg); err == nil {
|
||||
t.Fatal("non-member must be rejected")
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond) // give a wrongly queued message time to show up
|
||||
if relay.count() != 2 {
|
||||
t.Fatal("rejected mail must not be relayed")
|
||||
}
|
||||
|
|
@ -275,6 +288,7 @@ func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) {
|
|||
}
|
||||
|
||||
// The relayed copy carries a working one-click link for that recipient.
|
||||
relay.waitCount(t, 2)
|
||||
var link string
|
||||
for _, m := range relay.all() {
|
||||
if len(m.to) == 1 && m.to[0] == "bob@example.com" {
|
||||
|
|
@ -300,6 +314,8 @@ func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) {
|
|||
"From: Alice <alice@example.com>\r\nSubject: Second\r\nMessage-ID: <q2@example.com>\r\n\r\nhi\r\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
relay.waitCount(t, before+1)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if got := relay.count() - before; got != 1 {
|
||||
t.Fatalf("after bob left only alice should get a copy, got %d", got)
|
||||
}
|
||||
|
|
@ -386,3 +402,122 @@ func TestAdminUIAndAPIShareTheListener(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// deadAddr returns an address nothing listens on.
|
||||
func deadAddr(t *testing.T) string {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
ln.Close()
|
||||
return addr
|
||||
}
|
||||
|
||||
func TestQueuedMailSurvivesARestartAndIsRetriedWhenTheRelayReturns(t *testing.T) {
|
||||
var skew atomic.Int64
|
||||
nowFunc = func() time.Time { return time.Now().Add(time.Duration(skew.Load())) }
|
||||
t.Cleanup(func() { nowFunc = time.Now })
|
||||
|
||||
// 1. The relay is down: the post is still accepted and stays queued.
|
||||
cfg := testConfig(t, deadAddr(t))
|
||||
a, err := Start(cfg, "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok")
|
||||
for _, m := range []string{"alice@example.com", "bob@example.com"} {
|
||||
call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"`+m+`"}`, "tok")
|
||||
}
|
||||
msg := "From: Alice <alice@example.com>\r\nSubject: queued\r\nMessage-ID: <q@x>\r\n\r\nbody\r\n"
|
||||
if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil {
|
||||
t.Fatalf("a post must be accepted while the relay is down: %v", err)
|
||||
}
|
||||
peek, err := store.Open(cfg.DB)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer peek.Close()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
st, _ := peek.OutboxStats()
|
||||
if st.Pending == 2 && st.Retrying == 2 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("queue after the first attempts: %+v", st)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
// 2. Restart with a working relay, after the backoff has passed.
|
||||
a.Close()
|
||||
relayAddr, relay := startSink(t)
|
||||
cfg.Relay.Addr = relayAddr
|
||||
skew.Store(int64(10 * time.Minute))
|
||||
b, err := Start(cfg, "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { b.Close() })
|
||||
relay.waitCount(t, 2)
|
||||
got := relay.all()
|
||||
if !strings.Contains(got[0].data, "List-Id: Dev <dev.example.org>") || !strings.Contains(got[0].data, "Subject: queued") {
|
||||
t.Fatalf("delivered mail: %q", got[0].data)
|
||||
}
|
||||
deadline = time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
st, _ := peek.OutboxStats()
|
||||
if st.Pending == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("delivered mail must leave the queue: %+v", st)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseInterruptsASlowSendWithoutLosingIt(t *testing.T) {
|
||||
// A relay that accepts the connection and never answers.
|
||||
ln, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
}
|
||||
}()
|
||||
cfg := testConfig(t, ln.Addr().String())
|
||||
a, err := Start(cfg, "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok")
|
||||
call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok")
|
||||
msg := "From: Alice <alice@example.com>\r\nSubject: slow\r\nMessage-ID: <s@x>\r\n\r\nbody\r\n"
|
||||
if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond) // the worker is now stuck talking to the silent relay
|
||||
|
||||
closed := make(chan struct{})
|
||||
go func() { a.Close(); close(closed) }()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Close must not wait for a hung relay")
|
||||
}
|
||||
peek, err := store.Open(cfg.DB)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer peek.Close()
|
||||
if st, _ := peek.OutboxStats(); st.Pending != 1 {
|
||||
t.Fatalf("the interrupted delivery must stay queued: %+v", st)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@ type Config struct {
|
|||
Listen string `yaml:"listen"` // public archive and unsubscribe pages; empty disables them
|
||||
} `yaml:"web"`
|
||||
Relay struct {
|
||||
Addr string `yaml:"addr"` // host:port of the outgoing SMTP relay
|
||||
Addr string `yaml:"addr"` // host:port of the outgoing SMTP relay
|
||||
Concurrency int `yaml:"concurrency"` // simultaneous deliveries
|
||||
} `yaml:"relay"`
|
||||
}
|
||||
|
||||
|
|
@ -35,6 +36,7 @@ func Load(path string) (Config, error) {
|
|||
c.LMTP.Network = "tcp"
|
||||
c.LMTP.Addr = "127.0.0.1:2424"
|
||||
c.Relay.Addr = "127.0.0.1:25"
|
||||
c.Relay.Concurrency = 4
|
||||
c.API.Listen = "127.0.0.1:8080"
|
||||
c.Web.Listen = "127.0.0.1:8081"
|
||||
|
||||
|
|
|
|||
|
|
@ -20,17 +20,17 @@ func TestLoadDefaults(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.DB != "parsimony.db" || c.LMTP.Network != "tcp" || c.LMTP.Addr != "127.0.0.1:2424" || c.Relay.Addr != "127.0.0.1:25" || c.API.Listen != "127.0.0.1:8080" || c.Web.Listen != "127.0.0.1:8081" {
|
||||
if c.DB != "parsimony.db" || c.LMTP.Network != "tcp" || c.LMTP.Addr != "127.0.0.1:2424" || c.Relay.Addr != "127.0.0.1:25" || c.Relay.Concurrency != 4 || c.API.Listen != "127.0.0.1:8080" || c.Web.Listen != "127.0.0.1:8081" {
|
||||
t.Fatalf("defaults: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadValues(t *testing.T) {
|
||||
c, err := Load(write(t, "db: /var/lib/p.db\nbase_url: https://l.example.org\nlmtp:\n network: unix\n addr: /run/p.sock\nrelay:\n addr: mx:587\napi:\n listen: 0.0.0.0:9000\nweb:\n listen: 0.0.0.0:9001\n"))
|
||||
c, err := Load(write(t, "db: /var/lib/p.db\nbase_url: https://l.example.org\nlmtp:\n network: unix\n addr: /run/p.sock\nrelay:\n addr: mx:587\n concurrency: 9\napi:\n listen: 0.0.0.0:9000\nweb:\n listen: 0.0.0.0:9001\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.DB != "/var/lib/p.db" || c.BaseURL != "https://l.example.org" || c.LMTP.Network != "unix" || c.LMTP.Addr != "/run/p.sock" || c.Relay.Addr != "mx:587" || c.API.Listen != "0.0.0.0:9000" || c.Web.Listen != "0.0.0.0:9001" {
|
||||
if c.DB != "/var/lib/p.db" || c.BaseURL != "https://l.example.org" || c.LMTP.Network != "unix" || c.LMTP.Addr != "/run/p.sock" || c.Relay.Addr != "mx:587" || c.Relay.Concurrency != 9 || c.API.Listen != "0.0.0.0:9000" || c.Web.Listen != "0.0.0.0:9001" {
|
||||
t.Fatalf("values: %+v", c)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
package ingest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
|
|
@ -10,16 +9,16 @@ import (
|
|||
|
||||
"github.com/emersion/go-smtp"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/router"
|
||||
)
|
||||
|
||||
const maxMessageBytes = 25 << 20
|
||||
|
||||
// NewServer builds an LMTP server that routes incoming mail and relays it.
|
||||
// NewServer builds an LMTP server that routes incoming mail into the outbound queue.
|
||||
// A message is accepted only once it is durably queued.
|
||||
// Set Network/Addr on the result (or call Serve) to start it.
|
||||
func NewServer(r *router.Router, out outbound.Sender) *smtp.Server {
|
||||
srv := smtp.NewServer(&backend{router: r, out: out})
|
||||
func NewServer(r *router.Router) *smtp.Server {
|
||||
srv := smtp.NewServer(&backend{router: r})
|
||||
srv.LMTP = true
|
||||
srv.Domain = "localhost"
|
||||
srv.MaxMessageBytes = maxMessageBytes
|
||||
|
|
@ -30,7 +29,6 @@ func NewServer(r *router.Router, out outbound.Sender) *smtp.Server {
|
|||
|
||||
type backend struct {
|
||||
router *router.Router
|
||||
out outbound.Sender
|
||||
}
|
||||
|
||||
func (b *backend) NewSession(*smtp.Conn) (smtp.Session, error) {
|
||||
|
|
@ -67,25 +65,22 @@ func (s *session) LMTPData(r io.Reader, status smtp.StatusCollector) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
for _, rcpt := range s.rcpts {
|
||||
status.SetStatus(rcpt, s.deliver(ctx, rcpt, data))
|
||||
status.SetStatus(rcpt, s.deliver(rcpt, data))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *session) deliver(ctx context.Context, rcpt string, data []byte) error {
|
||||
func (s *session) deliver(rcpt string, data []byte) error {
|
||||
rt := s.b.router
|
||||
if handled, err := rt.Command(ctx, s.b.out, s.from, rcpt, data); handled {
|
||||
if handled, err := rt.Command(s.from, rcpt, data); handled {
|
||||
return s.status(rcpt, err)
|
||||
}
|
||||
res, err := rt.Route(s.from, rcpt, data)
|
||||
if err != nil {
|
||||
return s.status(rcpt, err)
|
||||
}
|
||||
return s.status(rcpt, rt.Send(ctx, s.b.out, res))
|
||||
return s.status(rcpt, rt.Deliver(res))
|
||||
}
|
||||
|
||||
// status turns a router result into an LMTP reply.
|
||||
|
|
|
|||
|
|
@ -6,17 +6,26 @@ import (
|
|||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/emersion/go-smtp"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/queue"
|
||||
"codeberg.org/squidd/parsimony/internal/router"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
const msg = "From: Alice <alice@example.com>\r\nSubject: hi\r\n\r\nbody\r\n"
|
||||
|
||||
func setup(t *testing.T) (addr string, relay *outbound.Fake) {
|
||||
// stack is the LMTP server with a real queue worker in front of a fake relay.
|
||||
type stack struct {
|
||||
addr string
|
||||
relay *outbound.Fake
|
||||
store *store.Store
|
||||
}
|
||||
|
||||
func newStack(t *testing.T, sender outbound.Sender) *stack {
|
||||
t.Helper()
|
||||
s, err := store.Open(":memory:")
|
||||
if err != nil {
|
||||
|
|
@ -27,15 +36,52 @@ func setup(t *testing.T) (addr string, relay *outbound.Fake) {
|
|||
s.AddMember(l.ID, "alice@example.com", "Alice")
|
||||
s.AddMember(l.ID, "bob@example.com", "Bob")
|
||||
|
||||
relay = &outbound.Fake{}
|
||||
srv := NewServer(router.New(s), relay)
|
||||
relay := &outbound.Fake{}
|
||||
if sender == nil {
|
||||
sender = relay
|
||||
}
|
||||
rt := router.New(s)
|
||||
w := &queue.Worker{Store: s, Sender: sender, Interval: 10 * time.Millisecond}
|
||||
rt.OnEnqueue = w.Wake
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { w.Run(ctx); close(done) }()
|
||||
t.Cleanup(func() { cancel(); <-done })
|
||||
|
||||
srv := NewServer(rt)
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Serve(ln)
|
||||
t.Cleanup(func() { srv.Close() })
|
||||
return ln.Addr().String(), relay
|
||||
return &stack{addr: ln.Addr().String(), relay: relay, store: s}
|
||||
}
|
||||
|
||||
// sent waits until the relay has received n messages and returns them.
|
||||
func (st *stack) sent(t *testing.T, n int) []outbound.Sent {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
got := st.relay.Messages()
|
||||
if len(got) >= n {
|
||||
return got
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("relay got %d messages, want %d", len(got), n)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// queued is how many deliveries still wait in the outbox.
|
||||
func (st *stack) queued(t *testing.T) int {
|
||||
t.Helper()
|
||||
stats, err := st.store.OutboxStats()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return stats.Pending
|
||||
}
|
||||
|
||||
// deliver sends msg over LMTP and returns the per-recipient errors.
|
||||
|
|
@ -80,41 +126,45 @@ func deliver(t *testing.T, addr, from string, rcpts []string) map[string]error {
|
|||
}
|
||||
|
||||
func TestDeliverFansOut(t *testing.T) {
|
||||
addr, relay := setup(t)
|
||||
res := deliver(t, addr, "alice@example.com", []string{"dev@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
|
||||
if res["dev@example.org"] != nil {
|
||||
t.Fatalf("unexpected error: %v", res)
|
||||
}
|
||||
if len(relay.Sent) != 2 {
|
||||
t.Fatalf("want one send per member, got %d", len(relay.Sent))
|
||||
got := st.sent(t, 2)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want one send per member, got %d", len(got))
|
||||
}
|
||||
s := relay.Sent[0]
|
||||
if len(s.To) != 1 {
|
||||
t.Errorf("recipients = %v", s.To)
|
||||
m := got[0]
|
||||
if len(m.To) != 1 {
|
||||
t.Errorf("recipients = %v", m.To)
|
||||
}
|
||||
if !strings.HasPrefix(s.From, "dev+bounce-") {
|
||||
t.Errorf("envelope from = %q", s.From)
|
||||
if !strings.HasPrefix(m.From, "dev+bounce-") {
|
||||
t.Errorf("envelope from = %q", m.From)
|
||||
}
|
||||
if !strings.Contains(string(s.Data), "List-Id: Dev <dev.example.org>") {
|
||||
t.Errorf("missing List-Id: %q", s.Data)
|
||||
if !strings.Contains(string(m.Data), "List-Id: Dev <dev.example.org>") {
|
||||
t.Errorf("missing List-Id: %q", m.Data)
|
||||
}
|
||||
if !strings.Contains(string(m.Data), "List-Unsubscribe: <mailto:dev+leave@example.org>") {
|
||||
t.Errorf("missing per-recipient unsubscribe header: %q", m.Data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonMemberRejectedPermanently(t *testing.T) {
|
||||
addr, relay := setup(t)
|
||||
res := deliver(t, addr, "mallory@evil.com", []string{"dev@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "mallory@evil.com", []string{"dev@example.org"})
|
||||
var se *smtp.SMTPError
|
||||
if !errors.As(res["dev@example.org"], &se) || se.Code != 550 {
|
||||
t.Fatalf("want 550, got %v", res["dev@example.org"])
|
||||
}
|
||||
if len(relay.Sent) != 0 {
|
||||
t.Fatal("nothing must be relayed")
|
||||
if st.queued(t) != 0 {
|
||||
t.Fatal("nothing may be queued")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownListRejected(t *testing.T) {
|
||||
addr, _ := setup(t)
|
||||
res := deliver(t, addr, "alice@example.com", []string{"nope@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "alice@example.com", []string{"nope@example.org"})
|
||||
var se *smtp.SMTPError
|
||||
if !errors.As(res["nope@example.org"], &se) || se.Code != 550 {
|
||||
t.Fatalf("want 550, got %v", res["nope@example.org"])
|
||||
|
|
@ -122,79 +172,83 @@ func TestUnknownListRejected(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestMixedRecipients(t *testing.T) {
|
||||
addr, relay := setup(t)
|
||||
res := deliver(t, addr, "alice@example.com", []string{"dev@example.org", "nope@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org", "nope@example.org"})
|
||||
if res["dev@example.org"] != nil || res["nope@example.org"] == nil {
|
||||
t.Fatalf("results: %v", res)
|
||||
}
|
||||
if len(relay.Sent) != 2 {
|
||||
t.Fatalf("sent = %d", len(relay.Sent))
|
||||
st.sent(t, 2)
|
||||
}
|
||||
|
||||
// down is a relay that refuses everything.
|
||||
type down struct{}
|
||||
|
||||
func (down) Send(context.Context, string, []string, []byte) error { return errors.New("relay down") }
|
||||
|
||||
func TestRelayDownStillAcceptsAndKeepsTheMailQueued(t *testing.T) {
|
||||
st := newStack(t, down{})
|
||||
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
|
||||
if res["dev@example.org"] != nil {
|
||||
t.Fatalf("a queued post is accepted even when the relay is down: %v", res)
|
||||
}
|
||||
// The worker tries and schedules retries; nothing is lost.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
stats, _ := st.store.OutboxStats()
|
||||
if stats.Pending == 2 && stats.Retrying == 2 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("queue: %+v", stats)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
type failing struct{}
|
||||
|
||||
func (failing) Send(_ context.Context, _ string, _ []string, _ []byte) error {
|
||||
return errors.New("relay down")
|
||||
}
|
||||
|
||||
func TestRelayFailureIsTemporary(t *testing.T) {
|
||||
s, _ := store.Open(":memory:")
|
||||
t.Cleanup(func() { s.Close() })
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.AddMember(l.ID, "alice@example.com", "Alice")
|
||||
srv := NewServer(router.New(s), failing{})
|
||||
ln, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
go srv.Serve(ln)
|
||||
t.Cleanup(func() { srv.Close() })
|
||||
|
||||
res := deliver(t, ln.Addr().String(), "alice@example.com", []string{"dev@example.org"})
|
||||
func TestQueueFailureIsTemporary(t *testing.T) {
|
||||
st := newStack(t, nil)
|
||||
st.store.Close() // the database goes away under the server
|
||||
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
|
||||
var se *smtp.SMTPError
|
||||
if !errors.As(res["dev@example.org"], &se) || se.Code != 451 {
|
||||
t.Fatalf("want 451, got %v", res["dev@example.org"])
|
||||
t.Fatalf("want 451 so the MTA retries, got %v", res["dev@example.org"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestJoinCommandOverLMTP(t *testing.T) {
|
||||
addr, relay := setup(t)
|
||||
res := deliver(t, addr, "carol@x.com", []string{"dev+join@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "carol@x.com", []string{"dev+join@example.org"})
|
||||
if res["dev+join@example.org"] != nil {
|
||||
t.Fatalf("join: %v", res)
|
||||
}
|
||||
if len(relay.Sent) != 1 || relay.Sent[0].To[0] != "carol@x.com" {
|
||||
t.Fatalf("confirmation mail: %+v", relay.Sent)
|
||||
got := st.sent(t, 1)
|
||||
if len(got[0].To) != 1 || got[0].To[0] != "carol@x.com" {
|
||||
t.Fatalf("confirmation mail: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBadTokenRejected(t *testing.T) {
|
||||
addr, _ := setup(t)
|
||||
res := deliver(t, addr, "carol@x.com", []string{"dev+confirm-deadbeef@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "carol@x.com", []string{"dev+confirm-deadbeef@example.org"})
|
||||
var se *smtp.SMTPError
|
||||
if !errors.As(res["dev+confirm-deadbeef@example.org"], &se) || se.Code != 550 {
|
||||
t.Fatalf("want 550, got %v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModeratedPostIsAcceptedButNotRelayed(t *testing.T) {
|
||||
s, _ := store.Open(":memory:")
|
||||
t.Cleanup(func() { s.Close() })
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.AddMember(l.ID, "alice@example.com", "Alice")
|
||||
s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
|
||||
relay := &outbound.Fake{}
|
||||
srv := NewServer(router.New(s), relay)
|
||||
ln, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
go srv.Serve(ln)
|
||||
t.Cleanup(func() { srv.Close() })
|
||||
func TestModeratedPostIsAcceptedButNotQueued(t *testing.T) {
|
||||
st := newStack(t, nil)
|
||||
l, _ := st.store.FindList("dev@example.org")
|
||||
st.store.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
|
||||
|
||||
res := deliver(t, ln.Addr().String(), "stranger@x.com", []string{"dev@example.org"})
|
||||
res := deliver(t, st.addr, "stranger@x.com", []string{"dev@example.org"})
|
||||
if res["dev@example.org"] != nil {
|
||||
t.Fatalf("held mail must be accepted: %v", res)
|
||||
}
|
||||
if len(relay.Sent) != 0 {
|
||||
t.Fatal("nothing may be relayed")
|
||||
if st.queued(t) != 0 {
|
||||
t.Fatal("nothing may be queued for delivery")
|
||||
}
|
||||
if h, _ := s.ListHeld(l.ID); len(h) != 1 {
|
||||
if h, _ := st.store.ListHeld(l.ID); len(h) != 1 {
|
||||
t.Fatalf("held = %d", len(h))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,12 +43,32 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b
|
|||
return nil
|
||||
}
|
||||
|
||||
func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) error {
|
||||
func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) (err error) {
|
||||
d := net.Dialer{Timeout: 10 * time.Second}
|
||||
conn, err := d.DialContext(ctx, "tcp", r.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A relay that stops answering must not hold us past the context: apply its
|
||||
// deadline to the connection and close the connection if it is cancelled.
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
conn.SetDeadline(dl)
|
||||
}
|
||||
stop := make(chan struct{})
|
||||
defer close(stop)
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
conn.Close()
|
||||
case <-stop:
|
||||
}
|
||||
}()
|
||||
defer func() {
|
||||
if err != nil && ctx.Err() != nil {
|
||||
err = ctx.Err() // report the cause, not "use of closed connection"
|
||||
}
|
||||
}()
|
||||
|
||||
c := smtp.NewClient(conn)
|
||||
defer c.Close()
|
||||
if err := c.Hello("localhost"); err != nil {
|
||||
|
|
@ -73,6 +93,13 @@ type Fake struct {
|
|||
Sent []Sent
|
||||
}
|
||||
|
||||
// Messages returns a copy of what was recorded; safe while sends are running.
|
||||
func (f *Fake) Messages() []Sent {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]Sent(nil), f.Sent...)
|
||||
}
|
||||
|
||||
// Send records the message.
|
||||
func (f *Fake) Send(_ context.Context, from string, to []string, data []byte) error {
|
||||
f.mu.Lock()
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/emersion/go-smtp"
|
||||
)
|
||||
|
|
@ -196,3 +197,68 @@ func TestRelayRejectionsAreClassifiedFromTheWire(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// silent accepts connections and never says a word.
|
||||
func silent(t *testing.T) string {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var conns []net.Conn
|
||||
var mu sync.Mutex
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
conns = append(conns, c)
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
ln.Close()
|
||||
mu.Lock()
|
||||
for _, c := range conns {
|
||||
c.Close()
|
||||
}
|
||||
mu.Unlock()
|
||||
})
|
||||
return ln.Addr().String()
|
||||
}
|
||||
|
||||
func TestSendStopsWhenTheContextIsCancelled(t *testing.T) {
|
||||
r := &SMTPRelay{Addr: silent(t)}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
errc := make(chan error, 1)
|
||||
go func() { errc <- r.Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("x")) }()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-errc:
|
||||
if err == nil {
|
||||
t.Fatal("want an error")
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Send ignored the cancelled context while talking to a silent relay")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendHonoursTheContextDeadline(t *testing.T) {
|
||||
r := &SMTPRelay{Addr: silent(t)}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
err := r.Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("x"))
|
||||
if err == nil {
|
||||
t.Fatal("want an error")
|
||||
}
|
||||
if took := time.Since(start); took > 2*time.Second {
|
||||
t.Fatalf("took %v", took)
|
||||
}
|
||||
if IsPermanent(err) {
|
||||
t.Fatalf("a timeout is temporary: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
|
|
@ -14,7 +13,6 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
|
|
@ -48,7 +46,7 @@ func (r *Router) bounceAddress(l store.List, memberID int64) string {
|
|||
|
||||
// Command handles list command addresses such as dev+join@example.org.
|
||||
// handled is false when rcpt is a plain list address (use Route for those).
|
||||
func (r *Router) Command(ctx context.Context, out outbound.Sender, sender, rcpt string, data []byte) (handled bool, err error) {
|
||||
func (r *Router) Command(sender, rcpt string, data []byte) (handled bool, err error) {
|
||||
local, domain, ok := strings.Cut(strings.ToLower(strings.TrimSpace(rcpt)), "@")
|
||||
if !ok {
|
||||
return false, nil
|
||||
|
|
@ -67,9 +65,9 @@ func (r *Router) Command(ctx context.Context, out outbound.Sender, sender, rcpt
|
|||
|
||||
switch {
|
||||
case tag == "join":
|
||||
return true, r.join(ctx, out, l, sender)
|
||||
return true, r.join(l, sender)
|
||||
case tag == "leave":
|
||||
return true, r.leave(ctx, out, l, sender)
|
||||
return true, r.leave(l, sender)
|
||||
case strings.HasPrefix(tag, "confirm-"):
|
||||
return true, r.confirm(l, strings.TrimPrefix(tag, "confirm-"))
|
||||
case tag == "bounce":
|
||||
|
|
@ -80,7 +78,7 @@ func (r *Router) Command(ctx context.Context, out outbound.Sender, sender, rcpt
|
|||
return true, ErrUnknownList
|
||||
}
|
||||
|
||||
func (r *Router) join(ctx context.Context, out outbound.Sender, l store.List, sender string) error {
|
||||
func (r *Router) join(l store.List, sender string) error {
|
||||
if sender == "" {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -90,20 +88,20 @@ func (r *Router) join(ctx context.Context, out outbound.Sender, l store.List, se
|
|||
if ok, err := r.store.IsMember(l.ID, sender); err != nil || ok {
|
||||
return err // members get no mail: it would leak membership
|
||||
}
|
||||
return r.askConfirmation(ctx, out, l, sender, store.ActionJoin)
|
||||
return r.askConfirmation(l, sender, store.ActionJoin)
|
||||
}
|
||||
|
||||
func (r *Router) leave(ctx context.Context, out outbound.Sender, l store.List, sender string) error {
|
||||
func (r *Router) leave(l store.List, sender string) error {
|
||||
if sender == "" {
|
||||
return nil
|
||||
}
|
||||
if ok, err := r.store.IsMember(l.ID, sender); err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
return r.askConfirmation(ctx, out, l, sender, store.ActionLeave)
|
||||
return r.askConfirmation(l, sender, store.ActionLeave)
|
||||
}
|
||||
|
||||
func (r *Router) askConfirmation(ctx context.Context, out outbound.Sender, l store.List, addr, action string) error {
|
||||
func (r *Router) askConfirmation(l store.List, addr, action string) error {
|
||||
token, created, err := r.store.CreatePending(l.ID, addr, action)
|
||||
if err != nil || !created {
|
||||
return err
|
||||
|
|
@ -124,7 +122,7 @@ func (r *Router) askConfirmation(ctx context.Context, out outbound.Sender, l sto
|
|||
time.Now().Format(time.RFC1123Z),
|
||||
token[:8], strconv.FormatInt(time.Now().UnixNano(), 36), l.Domain,
|
||||
verb, l.Address())
|
||||
return out.Send(ctx, l.Name+"+bounce@"+l.Domain, []string{addr}, []byte(msg))
|
||||
return r.enqueue([]byte(msg), []store.NewOutbound{{From: l.Name + "+bounce@" + l.Domain, Rcpt: addr}})
|
||||
}
|
||||
|
||||
func (r *Router) confirm(l store.List, token string) error {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"testing"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/queue"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
|
|
@ -34,6 +35,37 @@ func newEnv(t *testing.T) *env {
|
|||
return &env{r: New(s), s: s, out: &outbound.Fake{}, list: l, ctx: context.Background(), alice: ms[0].ID}
|
||||
}
|
||||
|
||||
// flush sends everything queued, as the background worker would.
|
||||
func (e *env) flush() {
|
||||
w := &queue.Worker{Store: e.s, Sender: e.out}
|
||||
for {
|
||||
st, err := w.RunOnce(e.ctx)
|
||||
if err != nil || st == (queue.Stats{}) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *env) send(res Result) error {
|
||||
if err := e.r.Deliver(res); err != nil {
|
||||
return err
|
||||
}
|
||||
e.flush()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *env) command(sender, rcpt string, data []byte) (bool, error) {
|
||||
handled, err := e.r.Command(sender, rcpt, data)
|
||||
e.flush()
|
||||
return handled, err
|
||||
}
|
||||
|
||||
func (e *env) release(id int64) error {
|
||||
err := e.r.Release(id)
|
||||
e.flush()
|
||||
return err
|
||||
}
|
||||
|
||||
func lastMail(t *testing.T, out *outbound.Fake) outbound.Sent {
|
||||
t.Helper()
|
||||
if len(out.Sent) == 0 {
|
||||
|
|
@ -48,7 +80,7 @@ func TestSendUsesVERPPerRecipient(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.r.Send(e.ctx, e.out, res); err != nil {
|
||||
if err := e.send(res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(e.out.Sent) != 2 {
|
||||
|
|
@ -107,7 +139,7 @@ func TestReleaseHeld(t *testing.T) {
|
|||
e.r.Route("stranger@x.com", "dev@example.org", []byte(sample))
|
||||
held, _ := e.s.ListHeld(e.list.ID)
|
||||
|
||||
if err := e.r.Release(e.ctx, e.out, held[0].ID); err != nil {
|
||||
if err := e.release(held[0].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(e.out.Sent) != 2 {
|
||||
|
|
@ -119,7 +151,7 @@ func TestReleaseHeld(t *testing.T) {
|
|||
if rest, _ := e.s.ListHeld(e.list.ID); len(rest) != 0 {
|
||||
t.Fatal("released message must leave the queue")
|
||||
}
|
||||
if err := e.r.Release(e.ctx, e.out, held[0].ID); !errors.Is(err, store.ErrNotFound) {
|
||||
if err := e.release(held[0].ID); !errors.Is(err, store.ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -137,7 +169,7 @@ func token(t *testing.T, m outbound.Sent) string {
|
|||
|
||||
func TestJoinFlow(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
handled, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
|
||||
handled, err := e.command("carol@x.com", "dev+join@example.org", nil)
|
||||
if !handled || err != nil {
|
||||
t.Fatalf("join: %v %v", handled, err)
|
||||
}
|
||||
|
|
@ -150,14 +182,14 @@ func TestJoinFlow(t *testing.T) {
|
|||
}
|
||||
tok := token(t, m)
|
||||
|
||||
handled, err = e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil)
|
||||
handled, err = e.command("carol@x.com", "dev+confirm-"+tok+"@example.org", nil)
|
||||
if !handled || err != nil {
|
||||
t.Fatalf("confirm: %v %v", handled, err)
|
||||
}
|
||||
if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); !ok {
|
||||
t.Fatal("carol must be a member after confirming")
|
||||
}
|
||||
_, err = e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil)
|
||||
_, err = e.command("carol@x.com", "dev+confirm-"+tok+"@example.org", nil)
|
||||
if !errors.Is(err, ErrBadToken) {
|
||||
t.Fatalf("reused token: want ErrBadToken, got %v", err)
|
||||
}
|
||||
|
|
@ -165,16 +197,16 @@ func TestJoinFlow(t *testing.T) {
|
|||
|
||||
func TestJoinRateLimitedAndSilentForMembers(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
|
||||
e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
|
||||
e.command("carol@x.com", "dev+join@example.org", nil)
|
||||
e.command("carol@x.com", "dev+join@example.org", nil)
|
||||
if len(e.out.Sent) != 1 {
|
||||
t.Fatalf("second request must not send mail, sent %d", len(e.out.Sent))
|
||||
}
|
||||
e.r.Command(e.ctx, e.out, "alice@example.com", "dev+join@example.org", nil)
|
||||
e.command("alice@example.com", "dev+join@example.org", nil)
|
||||
if len(e.out.Sent) != 1 {
|
||||
t.Fatal("existing members get no mail")
|
||||
}
|
||||
e.r.Command(e.ctx, e.out, "", "dev+join@example.org", nil)
|
||||
e.command("", "dev+join@example.org", nil)
|
||||
if len(e.out.Sent) != 1 {
|
||||
t.Fatal("empty sender is ignored")
|
||||
}
|
||||
|
|
@ -183,7 +215,7 @@ func TestJoinRateLimitedAndSilentForMembers(t *testing.T) {
|
|||
func TestJoinClosed(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate)
|
||||
_, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
|
||||
_, err := e.command("carol@x.com", "dev+join@example.org", nil)
|
||||
if !errors.Is(err, ErrJoinClosed) {
|
||||
t.Fatalf("want ErrJoinClosed, got %v", err)
|
||||
}
|
||||
|
|
@ -194,10 +226,10 @@ func TestJoinClosed(t *testing.T) {
|
|||
|
||||
func TestConfirmAfterListClosed(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
|
||||
e.command("carol@x.com", "dev+join@example.org", nil)
|
||||
tok := token(t, lastMail(t, e.out))
|
||||
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate)
|
||||
_, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil)
|
||||
_, err := e.command("carol@x.com", "dev+confirm-"+tok+"@example.org", nil)
|
||||
if !errors.Is(err, ErrJoinClosed) {
|
||||
t.Fatalf("want ErrJoinClosed, got %v", err)
|
||||
}
|
||||
|
|
@ -208,7 +240,7 @@ func TestConfirmAfterListClosed(t *testing.T) {
|
|||
|
||||
func TestLeaveFlow(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.r.Command(e.ctx, e.out, "alice@example.com", "dev+leave@example.org", nil)
|
||||
e.command("alice@example.com", "dev+leave@example.org", nil)
|
||||
m := lastMail(t, e.out)
|
||||
if m.To[0] != "alice@example.com" {
|
||||
t.Fatalf("to = %v", m.To)
|
||||
|
|
@ -216,13 +248,13 @@ func TestLeaveFlow(t *testing.T) {
|
|||
if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); !ok {
|
||||
t.Fatal("still a member before confirming")
|
||||
}
|
||||
e.r.Command(e.ctx, e.out, "alice@example.com", "dev+confirm-"+token(t, m)+"@example.org", nil)
|
||||
e.command("alice@example.com", "dev+confirm-"+token(t, m)+"@example.org", nil)
|
||||
if ok, _ := e.s.IsMember(e.list.ID, "alice@example.com"); ok {
|
||||
t.Fatal("alice must be gone")
|
||||
}
|
||||
// Leaving when not a member sends nothing.
|
||||
n := len(e.out.Sent)
|
||||
e.r.Command(e.ctx, e.out, "zed@x.com", "dev+leave@example.org", nil)
|
||||
e.command("zed@x.com", "dev+leave@example.org", nil)
|
||||
if len(e.out.Sent) != n {
|
||||
t.Fatal("non-members get no mail")
|
||||
}
|
||||
|
|
@ -231,9 +263,9 @@ func TestLeaveFlow(t *testing.T) {
|
|||
func TestConfirmTokenFromOtherList(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.s.CreateList("ops", "example.org", "Ops")
|
||||
e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
|
||||
e.command("carol@x.com", "dev+join@example.org", nil)
|
||||
tok := token(t, lastMail(t, e.out))
|
||||
_, err := e.r.Command(e.ctx, e.out, "carol@x.com", "ops+confirm-"+tok+"@example.org", nil)
|
||||
_, err := e.command("carol@x.com", "ops+confirm-"+tok+"@example.org", nil)
|
||||
if !errors.Is(err, ErrBadToken) {
|
||||
t.Fatalf("want ErrBadToken, got %v", err)
|
||||
}
|
||||
|
|
@ -241,7 +273,7 @@ func TestConfirmTokenFromOtherList(t *testing.T) {
|
|||
|
||||
func TestCommandNotHandledForPlainAddress(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
handled, err := e.r.Command(e.ctx, e.out, "a@x.com", "dev@example.org", nil)
|
||||
handled, err := e.command("a@x.com", "dev@example.org", nil)
|
||||
if handled || err != nil {
|
||||
t.Fatalf("plain address: %v %v", handled, err)
|
||||
}
|
||||
|
|
@ -250,7 +282,7 @@ func TestCommandNotHandledForPlainAddress(t *testing.T) {
|
|||
func TestUnknownCommandAndList(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
for _, rcpt := range []string{"dev+bogus@example.org", "nope+join@example.org"} {
|
||||
handled, err := e.r.Command(e.ctx, e.out, "a@x.com", rcpt, nil)
|
||||
handled, err := e.command("a@x.com", rcpt, nil)
|
||||
if !handled || !errors.Is(err, ErrUnknownList) {
|
||||
t.Errorf("%s: %v %v", rcpt, handled, err)
|
||||
}
|
||||
|
|
@ -265,7 +297,7 @@ func TestHardBouncesDisableMember(t *testing.T) {
|
|||
e := newEnv(t)
|
||||
addr := e.r.bounceAddress(e.list, e.alice)
|
||||
for i := 0; i < 3; i++ {
|
||||
handled, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1"))
|
||||
handled, err := e.command("", addr, bounceMsg("5.1.1"))
|
||||
if !handled || err != nil {
|
||||
t.Fatalf("bounce: %v %v", handled, err)
|
||||
}
|
||||
|
|
@ -278,7 +310,7 @@ func TestHardBouncesDisableMember(t *testing.T) {
|
|||
|
||||
func TestSoftBounceIgnored(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.r.Command(e.ctx, e.out, "", e.r.bounceAddress(e.list, e.alice), bounceMsg("4.2.2"))
|
||||
e.command("", e.r.bounceAddress(e.list, e.alice), bounceMsg("4.2.2"))
|
||||
if m, _ := e.s.MemberByID(e.alice); m.BounceCount != 0 {
|
||||
t.Fatalf("soft bounce counted: %+v", m)
|
||||
}
|
||||
|
|
@ -287,7 +319,7 @@ func TestSoftBounceIgnored(t *testing.T) {
|
|||
func TestForgedBounceRejected(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
addr := fmt.Sprintf("dev+bounce-%d-000000000000@example.org", e.alice)
|
||||
_, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1"))
|
||||
_, err := e.command("", addr, bounceMsg("5.1.1"))
|
||||
if !errors.Is(err, ErrBadToken) {
|
||||
t.Fatalf("want ErrBadToken, got %v", err)
|
||||
}
|
||||
|
|
@ -301,7 +333,7 @@ func TestBounceFromOtherListRejected(t *testing.T) {
|
|||
e.s.CreateList("ops", "example.org", "Ops")
|
||||
ops, _ := e.s.FindList("ops@example.org")
|
||||
addr := e.r.bounceAddress(ops, e.alice) // valid MAC but alice is not in ops
|
||||
_, err := e.r.Command(e.ctx, e.out, "", addr, bounceMsg("5.1.1"))
|
||||
_, err := e.command("", addr, bounceMsg("5.1.1"))
|
||||
if !errors.Is(err, ErrBadToken) {
|
||||
t.Fatalf("want ErrBadToken, got %v", err)
|
||||
}
|
||||
|
|
@ -309,47 +341,81 @@ func TestBounceFromOtherListRejected(t *testing.T) {
|
|||
|
||||
func TestSystemBounceAddressDiscarded(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
handled, err := e.r.Command(e.ctx, e.out, "", "dev+bounce@example.org", bounceMsg("5.1.1"))
|
||||
handled, err := e.command("", "dev+bounce@example.org", bounceMsg("5.1.1"))
|
||||
if !handled || err != nil {
|
||||
t.Fatalf("%v %v", handled, err)
|
||||
}
|
||||
}
|
||||
|
||||
// flaky fails for the listed addresses.
|
||||
type flaky struct {
|
||||
bad map[string]bool
|
||||
sent []string
|
||||
}
|
||||
|
||||
func (f *flaky) Send(_ context.Context, _ string, to []string, _ []byte) error {
|
||||
if f.bad[to[0]] {
|
||||
return errors.New("rejected")
|
||||
}
|
||||
f.sent = append(f.sent, to[0])
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestSendSkipsFailedRecipients(t *testing.T) {
|
||||
func TestDeliverQueuesOneEntryPerRecipientAtomically(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
res, _ := e.r.Route("alice@example.com", "dev@example.org", []byte(sample))
|
||||
f := &flaky{bad: map[string]bool{"alice@example.com": true}}
|
||||
if err := e.r.Send(e.ctx, f, res); err != nil {
|
||||
t.Fatalf("partial failure must not fail the delivery: %v", err)
|
||||
if err := e.r.Deliver(res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.sent) != 1 || f.sent[0] != "bob@example.com" {
|
||||
t.Fatalf("sent: %v", f.sent)
|
||||
if len(e.out.Sent) != 0 {
|
||||
t.Fatal("Deliver only queues; the worker sends")
|
||||
}
|
||||
due, _ := e.s.DueOutbox(10)
|
||||
if len(due) != 2 {
|
||||
t.Fatalf("queued = %d", len(due))
|
||||
}
|
||||
if due[0].MsgID != due[1].MsgID {
|
||||
t.Fatal("recipients must share one stored body")
|
||||
}
|
||||
for _, d := range due {
|
||||
if d.MemberID == 0 || !strings.HasPrefix(d.From, "dev+bounce-") || !strings.Contains(string(d.Prefix), "List-Unsubscribe:") {
|
||||
t.Fatalf("item: %+v", d)
|
||||
}
|
||||
}
|
||||
body, _ := e.s.OutboxData(due[0].MsgID)
|
||||
if !strings.Contains(string(body), "List-Id:") || strings.Contains(string(body), "List-Unsubscribe:") {
|
||||
t.Fatalf("shared body has the common headers only: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendFailsWhenNobodyIsReached(t *testing.T) {
|
||||
func TestDeliverFailsCleanlyWhenTheQueueIsUnavailable(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
res, _ := e.r.Route("alice@example.com", "dev@example.org", []byte(sample))
|
||||
f := &flaky{bad: map[string]bool{"alice@example.com": true, "bob@example.com": true}}
|
||||
if err := e.r.Send(e.ctx, f, res); err == nil {
|
||||
e.s.Close()
|
||||
if err := e.r.Deliver(res); err == nil {
|
||||
t.Fatal("want an error so the MTA retries")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliverWithoutRecipientsQueuesNothing(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
woke := 0
|
||||
e.r.OnEnqueue = func() { woke++ }
|
||||
if err := e.r.Deliver(Result{List: e.list, Data: []byte("x")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st, _ := e.s.OutboxStats(); st.Pending != 0 || woke != 0 {
|
||||
t.Fatalf("pending=%d woke=%d", st.Pending, woke)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueuedMailWakesTheSender(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
woke := 0
|
||||
e.r.OnEnqueue = func() { woke++ }
|
||||
res, _ := e.r.Route("alice@example.com", "dev@example.org", []byte(sample))
|
||||
e.r.Deliver(res)
|
||||
e.r.Command("carol@x.com", "dev+join@example.org", nil)
|
||||
if woke != 2 {
|
||||
t.Fatalf("OnEnqueue called %d times, want once per queued batch (2)", woke)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfirmationMailIsQueuedAsSystemMail(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.r.Command("carol@x.com", "dev+join@example.org", nil)
|
||||
due, _ := e.s.DueOutbox(10)
|
||||
if len(due) != 1 || due[0].Rcpt != "carol@x.com" || due[0].MemberID != 0 || due[0].From != "dev+bounce@example.org" {
|
||||
t.Fatalf("queue: %+v", due)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncomingListHeadersAreStripped(t *testing.T) {
|
||||
r := setup(t) // no BaseURL: One-Click must not survive from the outside
|
||||
msg := "From: Alice <alice@example.com>\r\nList-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n" +
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ func TestRejectedAndHeldPostsAreNotArchived(t *testing.T) {
|
|||
t.Fatalf("archived = %d", total)
|
||||
}
|
||||
held, _ := e.s.ListHeld(e.list.ID)
|
||||
if err := e.r.Release(e.ctx, e.out, held[0].ID); err != nil {
|
||||
if err := e.release(held[0].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, total, _ := e.s.ListMessages(e.list.ID, 10, 0); total != 1 {
|
||||
|
|
@ -85,7 +85,7 @@ func sendPost(t *testing.T, e *env) {
|
|||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.r.Send(e.ctx, e.out, res); err != nil {
|
||||
if err := e.send(res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
|
@ -134,7 +134,7 @@ func TestSpoofedUnsubscribeHeadersNeverReachRecipients(t *testing.T) {
|
|||
e := newEnv(t)
|
||||
msg := "From: Alice <alice@example.com>\r\nList-Unsubscribe: <https://evil.example/u>\r\nList-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n\r\nx\r\n"
|
||||
res, _ := e.r.Route("alice@example.com", "dev@example.org", []byte(msg))
|
||||
e.r.Send(e.ctx, e.out, res)
|
||||
e.send(res)
|
||||
for _, m := range e.out.Sent {
|
||||
h := header(t, m.Data)
|
||||
if strings.Contains(h.Get("List-Unsubscribe"), "evil") || h.Get("List-Unsubscribe-Post") != "" {
|
||||
|
|
|
|||
|
|
@ -2,16 +2,13 @@
|
|||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/archive"
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
|
|
@ -48,6 +45,8 @@ type Router struct {
|
|||
store *store.Store
|
||||
// BaseURL is the public web URL; enables one-click unsubscribe when set.
|
||||
BaseURL string
|
||||
// OnEnqueue, when set, is called after mail is queued (to wake the sender).
|
||||
OnEnqueue func()
|
||||
|
||||
secretOnce sync.Once
|
||||
secretVal []byte
|
||||
|
|
@ -101,24 +100,30 @@ func (r *Router) prepare(l store.List, data []byte) (Result, error) {
|
|||
return Result{List: l, Recipients: rcpts, Data: out}, nil
|
||||
}
|
||||
|
||||
// Send delivers the message to every recipient, each with its own signed
|
||||
// bounce address (VERP). Individual failures are logged and skipped; an error is
|
||||
// returned only when nobody could be reached, so a retry cannot duplicate mail.
|
||||
func (r *Router) Send(ctx context.Context, out outbound.Sender, res Result) error {
|
||||
failed := 0
|
||||
// Deliver queues the message for every recipient, each with its own signed
|
||||
// bounce address (VERP) and unsubscribe headers. The queue write is atomic: on
|
||||
// error nothing was queued and the caller may safely retry.
|
||||
func (r *Router) Deliver(res Result) error {
|
||||
items := make([]store.NewOutbound, 0, len(res.Recipients))
|
||||
for _, rc := range res.Recipients {
|
||||
from := r.bounceAddress(res.List, rc.ID)
|
||||
if from == "" {
|
||||
return errors.New("cannot create bounce address")
|
||||
}
|
||||
data := append(r.unsubscribeHeaders(res.List, rc.ID), res.Data...)
|
||||
if err := out.Send(ctx, from, []string{rc.Address}, data); err != nil {
|
||||
failed++
|
||||
log.Printf("deliver %s to %s: %v", res.List.Address(), rc.Address, err)
|
||||
}
|
||||
items = append(items, store.NewOutbound{
|
||||
From: from, Rcpt: rc.Address, MemberID: rc.ID, Prefix: r.unsubscribeHeaders(res.List, rc.ID),
|
||||
})
|
||||
}
|
||||
if failed > 0 && failed == len(res.Recipients) {
|
||||
return fmt.Errorf("delivery failed for all %d recipients", failed)
|
||||
return r.enqueue(res.Data, items)
|
||||
}
|
||||
|
||||
// enqueue stores deliveries and wakes the sender.
|
||||
func (r *Router) enqueue(data []byte, items []store.NewOutbound) error {
|
||||
if err := r.store.Enqueue(data, items); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.OnEnqueue != nil && len(items) > 0 {
|
||||
r.OnEnqueue()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -156,8 +161,9 @@ func (r *Router) archive(l store.List, data []byte) {
|
|||
}
|
||||
}
|
||||
|
||||
// Release approves a held message and delivers it.
|
||||
func (r *Router) Release(ctx context.Context, out outbound.Sender, heldID int64) error {
|
||||
// Release approves a held message: it is queued for delivery and leaves the
|
||||
// moderation queue.
|
||||
func (r *Router) Release(heldID int64) error {
|
||||
h, err := r.store.GetHeld(heldID)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -170,7 +176,7 @@ func (r *Router) Release(ctx context.Context, out outbound.Sender, heldID int64)
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.Send(ctx, out, res); err != nil {
|
||||
if err := r.Deliver(res); err != nil {
|
||||
return err
|
||||
}
|
||||
return r.store.DeleteHeld(heldID)
|
||||
|
|
|
|||
Loading…
Reference in a new issue