diff --git a/README.md b/README.md index ba46316..e7db3b7 100644 --- a/README.md +++ b/README.md @@ -190,8 +190,15 @@ It answers `451` only when the database itself fails. - Hard bounces count for 30 days: when the last one is older, the next one starts again from 1. A disabled member stays disabled until it is added again. - A relay that accepted the message is never asked again because `QUIT` went wrong. -- Delivery is at-least-once: a crash between the relay accepting a message and the queue entry being - removed can send one message to one recipient twice. +- The same post is queued once. If the `250` for a post never reaches Postfix, Postfix delivers the post again; a list + remembers each post it queued for an hour (by `Message-ID`, or by a hash of the whole message when it has none), + answers `250` to a copy and does not queue it again (the log says so). An approved held post is always delivered and is + remembered too, so a copy that arrives later is not. +- On stop, the daemon refuses new connections and lets the LMTP sessions and API or web requests in flight finish, so a + post being received gets its `250`. After 30 seconds it cuts whatever is still open. +- What remains is at-least-once delivery across the relay: a crash between the relay accepting a message and the queue + entry being removed can send one message to one recipient twice. A copy of a post that arrives more than an hour later + is also delivered again. - `GET /api/v1/queue` and the Queue page show what is pending or failing, and a stuck delivery can be discarded. ## Admin UI diff --git a/internal/app/app.go b/internal/app/app.go index 270b888..5115942 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -26,6 +26,9 @@ import ( "codeberg.org/squidd/parsimony/internal/web" ) +// shutdownTimeout bounds how long Close waits for the sessions in flight before it cuts them. +var shutdownTimeout = 30 * time.Second + // nowFunc is the clock of the store (queue schedules, token expiry). Tests move it. var nowFunc = time.Now @@ -109,6 +112,51 @@ func lmtpIsOpen(network, addr string) bool { return ip == nil || !ip.IsLoopback() } +// connTracker remembers the open connections of a listener, so Close can cut the ones still open +// when the grace period ends: once Shutdown has begun, smtp.Server.Close does nothing. +type connTracker struct { + net.Listener + mu sync.Mutex + conns map[*trackedConn]struct{} +} + +func (t *connTracker) Accept() (net.Conn, error) { + c, err := t.Listener.Accept() + if err != nil { + return nil, err + } + tc := &trackedConn{Conn: c, t: t} + t.mu.Lock() + t.conns[tc] = struct{}{} + t.mu.Unlock() + return tc, nil +} + +// closeAll cuts every connection still open. +func (t *connTracker) closeAll() { + t.mu.Lock() + open := make([]*trackedConn, 0, len(t.conns)) + for c := range t.conns { + open = append(open, c) + } + t.mu.Unlock() + for _, c := range open { + c.Close() + } +} + +type trackedConn struct { + net.Conn + t *connTracker +} + +func (c *trackedConn) Close() error { + c.t.mu.Lock() + delete(c.t.conns, c) + c.t.mu.Unlock() + return c.Conn.Close() +} + // App is a running daemon. type App struct { // LMTPAddr and APIAddr are the addresses actually bound. @@ -117,14 +165,16 @@ type App struct { APIAddr string WebAddr string // empty when the public web is disabled - store *store.Store - lmtp *smtp.Server - lns []net.Listener // closed by Close so Serve returns even if Close wins the race with it - web *http.Server - pub *http.Server - wg sync.WaitGroup - stop context.CancelFunc // stops the queue worker - once sync.Once + store *store.Store + lmtp *smtp.Server + lmtpOpen *connTracker // the LMTP connections, cut by Close when the grace period ends + lmtpDone chan struct{} // closed when the LMTP Serve returns + lns []net.Listener // closed by Close so Serve returns even if Close wins the race with it + web *http.Server + pub *http.Server + wg sync.WaitGroup + stop context.CancelFunc // stops the queue worker + once sync.Once } // releaser lets the API approve held messages through the router. @@ -233,6 +283,9 @@ func Start(cfg config.Config, apiToken string) (*App, error) { a.pub = newHTTPServer(web.New(st, rt)) } + a.lmtpOpen = &connTracker{Listener: serveLn, conns: map[*trackedConn]struct{}{}} + a.lmtpDone = make(chan struct{}) + var workerCtx context.Context workerCtx, a.stop = context.WithCancel(context.Background()) a.wg.Add(1) @@ -243,7 +296,8 @@ func Start(cfg config.Config, apiToken string) (*App, error) { a.wg.Add(1) go func() { defer a.wg.Done() - if err := a.lmtp.Serve(serveLn); err != nil && !errors.Is(err, smtp.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { + defer close(a.lmtpDone) + if err := a.lmtp.Serve(a.lmtpOpen); err != nil && !errors.Is(err, smtp.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { log.Printf("lmtp: %v", err) } }() @@ -268,18 +322,26 @@ func Start(cfg config.Config, apiToken string) (*App, error) { return a, nil } -// Close stops the servers and closes the database. It is safe to call twice. +// Close stops the daemon and closes the database. It is safe to call twice. It stops accepting +// connections, then lets the LMTP sessions and HTTP requests in flight finish, so a post being +// received is queued and gets its 250 (a cut reply makes the MTA send the post again). After +// shutdownTimeout it cuts whatever is still open. func (a *App) Close() { a.once.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() for _, ln := range a.lns { ln.Close() } - a.lmtp.Close() - if a.web != nil { - a.web.Close() + <-a.lmtpDone // Serve has returned: no connection is added while Shutdown waits + if err := a.lmtp.Shutdown(ctx); errors.Is(err, context.DeadlineExceeded) { + log.Printf("lmtp: sessions still open after %v, closing them", shutdownTimeout) + a.lmtpOpen.closeAll() } - if a.pub != nil { - a.pub.Close() + for _, srv := range []*http.Server{a.web, a.pub} { + if srv != nil && srv.Shutdown(ctx) != nil { + srv.Close() + } } a.stop() // in-flight sends are interrupted and stay queued a.wg.Wait() diff --git a/internal/app/app_test.go b/internal/app/app_test.go index a1f6cf3..e96ea79 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -1,6 +1,7 @@ package app import ( + "bufio" "bytes" "crypto/ecdsa" "crypto/elliptic" @@ -12,6 +13,7 @@ import ( "encoding/base64" "encoding/pem" "errors" + "fmt" "io" "log" "math/big" @@ -120,13 +122,17 @@ func testConfig(t *testing.T, relay string) config.Config { return c } +// client opens one connection per request: a pooled connection the transport dialed and never +// used would hold an HTTP server's graceful shutdown for 5 s. +var client = &http.Client{Transport: &http.Transport{DisableKeepAlives: true}} + func call(t *testing.T, a *App, method, path, body, token string) int { t.Helper() req, _ := http.NewRequest(method, "http://"+a.APIAddr+path, strings.NewReader(body)) if token != "" { req.Header.Set("Authorization", "Bearer "+token) } - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { t.Fatal(err) } @@ -137,7 +143,7 @@ func call(t *testing.T, a *App, method, path, body, token string) int { func fetch(t *testing.T, method, url string) (int, string) { t.Helper() req, _ := http.NewRequest(method, url, nil) - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { t.Fatal(err) } @@ -353,6 +359,7 @@ func TestWebDisabledWhenListenIsEmpty(t *testing.T) { } func TestCloseDropsOpenConnections(t *testing.T) { + setShutdownTimeout(t, 200*time.Millisecond) relayAddr, _ := startSink(t) a, err := Start(testConfig(t, relayAddr), "tok") if err != nil { @@ -432,6 +439,193 @@ func deadAddr(t *testing.T) string { return addr } +func setShutdownTimeout(t *testing.T, d time.Duration) { + old := shutdownTimeout + shutdownTimeout = d + t.Cleanup(func() { shutdownTimeout = old }) +} + +// waitRefused waits until the LMTP listener no longer accepts connections. +func waitRefused(t *testing.T, addr string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + c, err := net.Dial("tcp", addr) + if err != nil { + return + } + c.Close() + if time.Now().After(deadline) { + t.Fatal("the LMTP listener still accepts connections") + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestCloseLetsAnLMTPDeliveryInFlightFinish(t *testing.T) { + cfg := testConfig(t, deadAddr(t)) // the post stays queued, where it can be counted + a, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok") + call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok") + + conn, err := net.Dial("tcp", a.LMTPAddr) + if err != nil { + t.Fatal(err) + } + c := smtp.NewClientLMTP(conn) + defer c.Close() + if err := c.Hello("localhost"); err != nil { + t.Fatal(err) + } + if err := c.Mail("alice@example.com", nil); err != nil { + t.Fatal(err) + } + if err := c.Rcpt("dev@example.org", nil); err != nil { + t.Fatal(err) + } + w, err := c.Data() + if err != nil { + t.Fatal(err) + } + w.Write([]byte("From: alice@example.com\r\nSubject: in flight\r\nMessage-ID: \r\n\r\nbody\r\n")) + + // The daemon stops while the post is being sent. + closed := make(chan struct{}) + go func() { a.Close(); close(closed) }() + waitRefused(t, a.LMTPAddr) + if _, err := w.CloseWithLMTPResponse(); err != nil { + t.Fatalf("the post in flight must get its 250: %v", err) + } + if err := c.Quit(); err != nil { + t.Fatalf("quit: %v", err) + } + select { + case <-closed: + case <-time.After(5 * time.Second): + t.Fatal("Close must return once the session is over") + } + peek, err := store.Open(cfg.DB) + if err != nil { + t.Fatal(err) + } + defer peek.Close() + if st, _ := peek.OutboxStats(); st.Pending != 1 { + t.Fatalf("the post must be queued: %+v", st) + } +} + +func TestCloseCutsAHungLMTPClientAfterTheGracePeriod(t *testing.T) { + setShutdownTimeout(t, 200*time.Millisecond) + relayAddr, _ := startSink(t) + a, err := Start(testConfig(t, relayAddr), "tok") + if err != nil { + t.Fatal(err) + } + conn, err := net.Dial("tcp", a.LMTPAddr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + c := smtp.NewClientLMTP(conn) + if err := c.Hello("localhost"); err != nil { + t.Fatal(err) + } + if err := c.Mail("alice@example.com", nil); err != nil { + t.Fatal(err) + } + if err := c.Rcpt("dev@example.org", nil); err != nil { + t.Fatal(err) + } + if _, err := c.Data(); err != nil { + t.Fatal(err) + } + // The client never ends the message. + start := time.Now() + closed := make(chan struct{}) + go func() { a.Close(); close(closed) }() + select { + case <-closed: + case <-time.After(5 * time.Second): + t.Fatal("Close must not wait for a hung client beyond the grace period") + } + if d := time.Since(start); d < 150*time.Millisecond { + t.Fatalf("Close returned after %v: it must give the session its grace period", d) + } + conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + buf := make([]byte, 512) + for { + if _, err := conn.Read(buf); err != nil { + if ne, ok := err.(net.Error); ok && ne.Timeout() { + t.Fatal("the hung connection is still open after Close") + } + break + } + } +} + +func TestCloseLetsAnAPIRequestInFlightFinish(t *testing.T) { + relayAddr, _ := startSink(t) + a, err := Start(testConfig(t, relayAddr), "tok") + if err != nil { + t.Fatal(err) + } + conn, err := net.Dial("tcp", a.APIAddr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + body := `{"name":"dev","domain":"example.org"}` + fmt.Fprintf(conn, "POST /api/v1/lists HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer tok\r\nContent-Type: application/json\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s", len(body), body[:10]) + time.Sleep(100 * time.Millisecond) // the handler is now reading the body + + closed := make(chan struct{}) + go func() { a.Close(); close(closed) }() + waitRefused(t, a.APIAddr) + io.WriteString(conn, body[10:]) + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + resp, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + t.Fatalf("the request in flight must get its answer: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d", resp.StatusCode) + } + select { + case <-closed: + case <-time.After(5 * time.Second): + t.Fatal("Close must return once the request is over") + } +} + +func TestFinishedLMTPSessionsAreForgotten(t *testing.T) { + relayAddr, _ := startSink(t) + a, err := Start(testConfig(t, relayAddr), "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + for i := 0; i < 3; i++ { + lmtpSend(t, a.LMTPAddr, "alice@example.com", "nope@example.org", "Subject: x\r\n\r\nx\r\n") + } + deadline := time.Now().Add(5 * time.Second) + for { + a.lmtpOpen.mu.Lock() + n := len(a.lmtpOpen.conns) + a.lmtpOpen.mu.Unlock() + if n == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("%d closed sessions are still tracked", n) + } + time.Sleep(5 * time.Millisecond) + } +} + func TestQueuedMailSurvivesARestartAndIsRetriedWhenTheRelayReturns(t *testing.T) { var skew atomic.Int64 nowFunc = func() time.Time { return time.Now().Add(time.Duration(skew.Load())) } diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 02f2beb..42a0e20 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -86,7 +86,7 @@ func (s *session) deliver(rcpt string, data []byte) error { // status turns a router result into an LMTP reply. func (s *session) status(rcpt string, err error) error { switch { - case err == nil, errors.Is(err, router.ErrHeld), errors.Is(err, router.ErrDiscarded): + case err == nil, errors.Is(err, router.ErrHeld), errors.Is(err, router.ErrDiscarded), errors.Is(err, router.ErrDuplicate): return nil case errors.Is(err, router.ErrUnknownList): return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such list"} diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index cd281eb..fa8be14 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -339,3 +339,15 @@ func TestAMessageTooBigToHoldAnswers552(t *testing.T) { t.Fatalf("a permanent refusal: the sender is told to send it elsewhere: %v", err) } } + +func TestTheSamePostDeliveredAgainIsAcceptedButNotQueuedTwice(t *testing.T) { + st := newStack(t, down{}) // the deliveries stay in the queue, where they can be counted + for i := 0; i < 2; i++ { + if res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"}); res["dev@example.org"] != nil { + t.Fatalf("delivery %d: a duplicate is acknowledged with 250, got %v", i+1, res) + } + } + if n := st.queued(t); n != 2 { + t.Fatalf("queued = %d, want one delivery per member", n) + } +} diff --git a/internal/router/dup_test.go b/internal/router/dup_test.go new file mode 100644 index 0000000..1463aef --- /dev/null +++ b/internal/router/dup_test.go @@ -0,0 +1,126 @@ +package router + +import ( + "errors" + "testing" +) + +func pending(t *testing.T, e *env) int { + t.Helper() + st, err := e.s.OutboxStats() + if err != nil { + t.Fatal(err) + } + return st.Pending +} + +func (e *env) post(t *testing.T, msg string) error { + t.Helper() + res, err := e.r.Route("alice@example.com", "dev@example.org", []byte(msg)) + if err != nil { + t.Fatalf("route: %v", err) + } + return e.r.Deliver(res) +} + +func TestTheSamePostIsDeliveredOnce(t *testing.T) { + e := newEnv(t) + woke := 0 + e.r.OnEnqueue = func() { woke++ } + msg := "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: \r\n\r\nbody\r\n" + if err := e.post(t, msg); err != nil { + t.Fatal(err) + } + if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) { + t.Fatalf("want ErrDuplicate, got %v", err) + } + if n := pending(t, e); n != 2 { + t.Fatalf("pending = %d, want one delivery per member", n) + } + if woke != 1 { + t.Fatalf("OnEnqueue called %d times for a duplicate", woke) + } +} + +func TestAnotherMessageIDIsAnotherPost(t *testing.T) { + e := newEnv(t) + if err := e.post(t, "From: alice@example.com\r\nMessage-ID: \r\n\r\nbody\r\n"); err != nil { + t.Fatal(err) + } + if err := e.post(t, "From: alice@example.com\r\nMessage-ID: \r\n\r\nbody\r\n"); err != nil { + t.Fatal(err) + } + if n := pending(t, e); n != 4 { + t.Fatalf("pending = %d", n) + } +} + +func TestAPostWithoutMessageIDIsKnownByItsContent(t *testing.T) { + e := newEnv(t) + msg := "From: alice@example.com\r\nSubject: hi\r\n\r\nbody\r\n" + if err := e.post(t, msg); err != nil { + t.Fatal(err) + } + if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) { + t.Fatalf("the same bytes again: want ErrDuplicate, got %v", err) + } + if err := e.post(t, msg+"more\r\n"); err != nil { + t.Fatalf("another body is another post: %v", err) + } + if n := pending(t, e); n != 4 { + t.Fatalf("pending = %d", n) + } +} + +func TestPostIDIgnoresTheRestWhenThereIsAMessageID(t *testing.T) { + a := postID([]byte("Message-ID: \r\nSubject: a\r\n\r\nx")) + b := postID([]byte("Subject: b\r\nMessage-ID: \r\n\r\ny")) + if a != b || a != "mid:" { + t.Fatalf("%q %q", a, b) + } + if c := postID([]byte("Subject: b\r\n\r\ny")); c == postID([]byte("Subject: b\r\n\r\nz")) || len(c) < 10 { + t.Fatalf("content hash: %q", c) + } + if postID([]byte("Message-ID: \r\n\r\n")) == postID([]byte("Message-ID: \r\n\r\n")) { + t.Fatal("Message-IDs are case sensitive") + } +} + +func TestAnApprovedPostIsDeliveredEvenIfItWasSeen(t *testing.T) { + e := newEnv(t) + msg := "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: \r\n\r\nbody\r\n" + if err := e.post(t, msg); err != nil { + t.Fatal(err) + } + id, err := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(msg), "r") + if err != nil { + t.Fatal(err) + } + if err := e.r.Release(id); err != nil { + t.Fatalf("release: %v", err) + } + if n := pending(t, e); n != 4 { + t.Fatalf("pending = %d, the approval must deliver", n) + } + if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) { + t.Fatalf("a copy after the approval: want ErrDuplicate, got %v", err) + } +} + +func TestAnApprovedPostIsRemembered(t *testing.T) { + e := newEnv(t) + msg := "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: \r\n\r\nbody\r\n" + id, err := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(msg), "r") + if err != nil { + t.Fatal(err) + } + if err := e.r.Release(id); err != nil { + t.Fatalf("release: %v", err) + } + if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) { + t.Fatalf("the MTA's copy of an approved post: want ErrDuplicate, got %v", err) + } + if n := pending(t, e); n != 2 { + t.Fatalf("pending = %d", n) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index e735ced..7e95813 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -3,6 +3,8 @@ package router import ( "bytes" + "crypto/sha256" + "encoding/hex" "errors" "log" "net/mail" @@ -34,6 +36,9 @@ var ( ErrGone = errors.New("already unsubscribed") // ErrJoinClosed means the list does not accept self-service subscriptions. ErrJoinClosed = errors.New("list is closed") + // ErrDuplicate means the list already queued the same post less than an hour ago: the MTA sent + // it again because our reply was lost. It is acknowledged and not queued again. + ErrDuplicate = store.ErrDuplicate ) // Recipient is a subscriber that receives a message. @@ -126,13 +131,36 @@ func (r *Router) prepare(l store.List, data []byte) (Result, error) { // Deliver queues the message for every recipient, each with its own signed // bounce address (VERP) and unsubscribe headers. The queue write is atomic: on -// error nothing was queued and the caller may safely retry. +// error nothing was queued and the caller may safely retry. A post the list +// already queued in the last hour is not queued again: Deliver returns ErrDuplicate. func (r *Router) Deliver(res Result) error { items, err := r.outbound(res) if err != nil { return err } - return r.enqueue(res.Data, items) + id := postID(res.Data) + if err := r.store.EnqueuePost(res.List.ID, id, res.Data, items); err != nil { + if errors.Is(err, ErrDuplicate) { + log.Printf("duplicate post to %s not delivered again: %s", res.List.Address(), id) + } + return err + } + if r.OnEnqueue != nil && len(items) > 0 { + r.OnEnqueue() + } + return nil +} + +// postID identifies a post for the duplicate check: its Message-ID, or a hash of the whole +// message when it has none. +func postID(data []byte) string { + if m, err := mail.ReadMessage(bytes.NewReader(data)); err == nil { + if id := m.Header.Get("Message-ID"); id != "" { + return "mid:" + id + } + } + sum := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(sum[:]) } // outbound builds one queue entry per recipient. @@ -213,7 +241,7 @@ func (r *Router) Release(heldID int64) error { if err != nil { return err } - if err := r.store.EnqueueFromHeld(heldID, res.Data, items); err != nil { + if err := r.store.EnqueueFromHeld(heldID, postID(res.Data), res.Data, items); err != nil { return err } if r.OnEnqueue != nil && len(items) > 0 { diff --git a/internal/store/dup_test.go b/internal/store/dup_test.go new file mode 100644 index 0000000..f8bc235 --- /dev/null +++ b/internal/store/dup_test.go @@ -0,0 +1,149 @@ +package store + +import ( + "errors" + "testing" + "time" +) + +func TestEnqueuePostRefusesTheSamePostTwiceWithinAnHour(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(2)); err != nil { + t.Fatal(err) + } + now = now.Add(59 * time.Minute) + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(2)); !errors.Is(err, ErrDuplicate) { + t.Fatalf("want ErrDuplicate, got %v", err) + } + if st, _ := s.OutboxStats(); st.Pending != 2 { + t.Fatalf("a duplicate must queue nothing, pending = %d", st.Pending) + } +} + +func TestEnqueuePostTellsPostsAndListsApart(t *testing.T) { + s := newTestStore(t) + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + if err := s.EnqueuePost(dev.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatal(err) + } + if err := s.EnqueuePost(dev.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatalf("another Message-ID: %v", err) + } + if err := s.EnqueuePost(ops.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatalf("the same post to another list: %v", err) + } + if st, _ := s.OutboxStats(); st.Pending != 3 { + t.Fatalf("pending = %d", st.Pending) + } +} + +func TestEnqueuePostForgetsAPostAfterAnHour(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatal(err) + } + now = now.Add(time.Hour) + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(1)); !errors.Is(err, ErrDuplicate) { + t.Fatalf("exactly one hour later it is still a duplicate, got %v", err) + } + now = now.Add(time.Second) + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatalf("after an hour the post is accepted again: %v", err) + } + if st, _ := s.OutboxStats(); st.Pending != 2 { + t.Fatalf("pending = %d", st.Pending) + } +} + +func TestEnqueuePurgesExpiredRecordsOfEveryList(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + s.EnqueuePost(dev.ID, "mid:", []byte("m"), items(1)) + now = now.Add(2 * time.Hour) + s.EnqueuePost(ops.ID, "mid:", []byte("m"), items(1)) + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM delivered`).Scan(&n); err != nil || n != 1 { + t.Fatalf("records = %d (%v), want only the fresh one", n, err) + } +} + +func TestEnqueuePostRecordsAndQueuesTogether(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON outbox BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil { + t.Fatal(err) + } + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(2)); err == nil { + t.Fatal("want the queue error") + } + s.db.Exec(`DROP TRIGGER boom`) + // The failed attempt left no record behind: the MTA's retry is queued. + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(2)); err != nil { + t.Fatalf("retry after a failed queue write: %v", err) + } + if st, _ := s.OutboxStats(); st.Pending != 2 { + t.Fatalf("pending = %d", st.Pending) + } +} + +func TestEnqueuePostRecordsAPostWithNoRecipients(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), nil); err != nil { + t.Fatal(err) + } + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), nil); !errors.Is(err, ErrDuplicate) { + t.Fatalf("want ErrDuplicate, got %v", err) + } + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM outbox_msgs`).Scan(&n); err != nil || n != 0 { + t.Fatalf("stored messages = %d (%v), want none without recipients", n, err) + } +} + +func TestReleasingAHeldPostDeliversItEvenIfSeenAndRecordsIt(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatal(err) + } + hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + now = now.Add(50 * time.Minute) + if err := s.EnqueueFromHeld(hid, "mid:", []byte("final"), items(2)); err != nil { + t.Fatalf("an approval always delivers: %v", err) + } + if st, _ := s.OutboxStats(); st.Pending != 3 { + t.Fatalf("pending = %d", st.Pending) + } + // The approval refreshed the record: a retry of the same post 20 minutes later is a duplicate. + now = now.Add(20 * time.Minute) + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(1)); !errors.Is(err, ErrDuplicate) { + t.Fatalf("want ErrDuplicate after a release, got %v", err) + } +} + +func TestDeletingAListForgetsItsPosts(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte("m"), nil) + if err := s.DeleteList(l.ID); err != nil { + t.Fatal(err) + } + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM delivered`).Scan(&n); err != nil || n != 0 { + t.Fatalf("records = %d (%v)", n, err) + } +} diff --git a/internal/store/outbox.go b/internal/store/outbox.go index aaa3e20..c72abea 100644 --- a/internal/store/outbox.go +++ b/internal/store/outbox.go @@ -58,19 +58,61 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error { return tx.Commit() } -// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues -// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the -// others get ErrNotFound, so a message is never delivered twice. With no recipients the message -// is still released. -func (s *Store) EnqueueFromHeld(heldID int64, msg []byte, items []NewOutbound) error { +// postMemory is how long a post queued for a list is remembered, to refuse it if the MTA delivers it again. +const postMemory = time.Hour + +// EnqueuePost queues a post to a list, like Enqueue, and records postID (the identity of the post) +// in the same transaction, so a crash never leaves one without the other. If the list already +// queued the same postID within postMemory it queues nothing and returns ErrDuplicate: the MTA +// delivers the post again when our reply to the first delivery was lost. +func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []NewOutbound) error { + now := s.Now() tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() + if _, err := tx.Exec(`DELETE FROM delivered WHERE created_at 0 { + if err := enqueueTx(tx, now.Unix(), msg, items); err != nil { + return err + } + } + return tx.Commit() +} + +// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues +// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the +// others get ErrNotFound, so a message is never delivered twice. With no recipients the message +// is still released. An approval always delivers, even a post already seen, and records postID +// as EnqueuePost does, so a later copy of the same post is refused. +func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items []NewOutbound) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var listID int64 + if err := tx.QueryRow(`SELECT list_id FROM held WHERE id=?`, heldID).Scan(&listID); errors.Is(err, sql.ErrNoRows) { + return ErrNotFound + } else if err != nil { + return err + } if err := affected(tx.Exec(`DELETE FROM held WHERE id=?`, heldID)); err != nil { return err } + if _, err := tx.Exec(`INSERT INTO delivered(list_id, post_id, created_at) VALUES(?,?,?) + ON CONFLICT(list_id, post_id) DO UPDATE SET created_at=excluded.created_at`, listID, postID, s.Now().Unix()); err != nil { + return err + } if len(items) > 0 { if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil { return err diff --git a/internal/store/outbox_test.go b/internal/store/outbox_test.go index 70d95a9..3458932 100644 --- a/internal/store/outbox_test.go +++ b/internal/store/outbox_test.go @@ -194,7 +194,7 @@ func TestEnqueueFromHeldMovesTheMessageAtomically(t *testing.T) { l, _ := s.CreateList("dev", "example.org", "Dev") hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("Subject: s\r\n\r\nb"), "r") - if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); err != nil { + if err := s.EnqueueFromHeld(hid, "mid:", []byte("final"), items(2)); err != nil { t.Fatal(err) } if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) { @@ -204,7 +204,7 @@ func TestEnqueueFromHeldMovesTheMessageAtomically(t *testing.T) { t.Fatalf("pending = %d", st.Pending) } // Approving again must not queue anything. - if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); !errors.Is(err, ErrNotFound) { + if err := s.EnqueueFromHeld(hid, "mid:", []byte("final"), items(2)); !errors.Is(err, ErrNotFound) { t.Fatalf("want ErrNotFound, got %v", err) } if st, _ := s.OutboxStats(); st.Pending != 2 { @@ -216,7 +216,7 @@ func TestEnqueueFromHeldWithNoRecipientsStillReleasesTheMessage(t *testing.T) { s := newTestStore(t) l, _ := s.CreateList("dev", "example.org", "Dev") hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") - if err := s.EnqueueFromHeld(hid, []byte("final"), nil); err != nil { + if err := s.EnqueueFromHeld(hid, "mid:", []byte("final"), nil); err != nil { t.Fatal(err) } if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) { @@ -237,7 +237,7 @@ func TestOnlyOneOfSeveralSimultaneousApprovalsWins(t *testing.T) { for i := 0; i < n; i++ { go func() { <-start - results <- s.EnqueueFromHeld(hid, []byte("final"), items(3)) + results <- s.EnqueueFromHeld(hid, "mid:", []byte("final"), items(3)) }() } close(start) diff --git a/internal/store/store.go b/internal/store/store.go index 9286c1d..f54c25a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -26,6 +26,8 @@ var ( ErrHeldTooLarge = errors.New("message too large to hold for moderation") // ErrExists is returned when creating something that already exists. ErrExists = errors.New("already exists") + // ErrDuplicate is returned when the same post was already queued for the list within postMemory. + ErrDuplicate = errors.New("post already delivered") ) // Policy values. @@ -212,6 +214,12 @@ CREATE TABLE IF NOT EXISTS outbox ( failed INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id); +CREATE TABLE IF NOT EXISTS delivered ( + list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, + post_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (list_id, post_id) +); CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL