Merge branch 'fix/dup-and-shutdown' into fix/fable-mediums
This commit is contained in:
commit
5af20f5891
11 changed files with 657 additions and 29 deletions
11
README.md
11
README.md
|
|
@ -190,8 +190,15 @@ It answers `451` only when the database itself fails.
|
|||
- Hard bounces count for 30 days: when the last one is older, the next one starts again from 1. A disabled member
|
||||
stays disabled until it is added again.
|
||||
- A relay that accepted the message is never asked again because `QUIT` went wrong.
|
||||
- Delivery is at-least-once: a crash between the relay accepting a message and the queue entry being
|
||||
removed can send one message to one recipient twice.
|
||||
- The same post is queued once. If the `250` for a post never reaches Postfix, Postfix delivers the post again; a list
|
||||
remembers each post it queued for an hour (by `Message-ID`, or by a hash of the whole message when it has none),
|
||||
answers `250` to a copy and does not queue it again (the log says so). An approved held post is always delivered and is
|
||||
remembered too, so a copy that arrives later is not.
|
||||
- On stop, the daemon refuses new connections and lets the LMTP sessions and API or web requests in flight finish, so a
|
||||
post being received gets its `250`. After 30 seconds it cuts whatever is still open.
|
||||
- What remains is at-least-once delivery across the relay: a crash between the relay accepting a message and the queue
|
||||
entry being removed can send one message to one recipient twice. A copy of a post that arrives more than an hour later
|
||||
is also delivered again.
|
||||
- `GET /api/v1/queue` and the Queue page show what is pending or failing, and a stuck delivery can be discarded.
|
||||
|
||||
## Admin UI
|
||||
|
|
|
|||
|
|
@ -26,6 +26,9 @@ import (
|
|||
"codeberg.org/squidd/parsimony/internal/web"
|
||||
)
|
||||
|
||||
// shutdownTimeout bounds how long Close waits for the sessions in flight before it cuts them.
|
||||
var shutdownTimeout = 30 * time.Second
|
||||
|
||||
// nowFunc is the clock of the store (queue schedules, token expiry). Tests move it.
|
||||
var nowFunc = time.Now
|
||||
|
||||
|
|
@ -109,6 +112,51 @@ func lmtpIsOpen(network, addr string) bool {
|
|||
return ip == nil || !ip.IsLoopback()
|
||||
}
|
||||
|
||||
// connTracker remembers the open connections of a listener, so Close can cut the ones still open
|
||||
// when the grace period ends: once Shutdown has begun, smtp.Server.Close does nothing.
|
||||
type connTracker struct {
|
||||
net.Listener
|
||||
mu sync.Mutex
|
||||
conns map[*trackedConn]struct{}
|
||||
}
|
||||
|
||||
func (t *connTracker) Accept() (net.Conn, error) {
|
||||
c, err := t.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tc := &trackedConn{Conn: c, t: t}
|
||||
t.mu.Lock()
|
||||
t.conns[tc] = struct{}{}
|
||||
t.mu.Unlock()
|
||||
return tc, nil
|
||||
}
|
||||
|
||||
// closeAll cuts every connection still open.
|
||||
func (t *connTracker) closeAll() {
|
||||
t.mu.Lock()
|
||||
open := make([]*trackedConn, 0, len(t.conns))
|
||||
for c := range t.conns {
|
||||
open = append(open, c)
|
||||
}
|
||||
t.mu.Unlock()
|
||||
for _, c := range open {
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
|
||||
type trackedConn struct {
|
||||
net.Conn
|
||||
t *connTracker
|
||||
}
|
||||
|
||||
func (c *trackedConn) Close() error {
|
||||
c.t.mu.Lock()
|
||||
delete(c.t.conns, c)
|
||||
c.t.mu.Unlock()
|
||||
return c.Conn.Close()
|
||||
}
|
||||
|
||||
// App is a running daemon.
|
||||
type App struct {
|
||||
// LMTPAddr and APIAddr are the addresses actually bound.
|
||||
|
|
@ -117,14 +165,16 @@ type App struct {
|
|||
APIAddr string
|
||||
WebAddr string // empty when the public web is disabled
|
||||
|
||||
store *store.Store
|
||||
lmtp *smtp.Server
|
||||
lns []net.Listener // closed by Close so Serve returns even if Close wins the race with it
|
||||
web *http.Server
|
||||
pub *http.Server
|
||||
wg sync.WaitGroup
|
||||
stop context.CancelFunc // stops the queue worker
|
||||
once sync.Once
|
||||
store *store.Store
|
||||
lmtp *smtp.Server
|
||||
lmtpOpen *connTracker // the LMTP connections, cut by Close when the grace period ends
|
||||
lmtpDone chan struct{} // closed when the LMTP Serve returns
|
||||
lns []net.Listener // closed by Close so Serve returns even if Close wins the race with it
|
||||
web *http.Server
|
||||
pub *http.Server
|
||||
wg sync.WaitGroup
|
||||
stop context.CancelFunc // stops the queue worker
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
// releaser lets the API approve held messages through the router.
|
||||
|
|
@ -233,6 +283,9 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
a.pub = newHTTPServer(web.New(st, rt))
|
||||
}
|
||||
|
||||
a.lmtpOpen = &connTracker{Listener: serveLn, conns: map[*trackedConn]struct{}{}}
|
||||
a.lmtpDone = make(chan struct{})
|
||||
|
||||
var workerCtx context.Context
|
||||
workerCtx, a.stop = context.WithCancel(context.Background())
|
||||
a.wg.Add(1)
|
||||
|
|
@ -243,7 +296,8 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
a.wg.Add(1)
|
||||
go func() {
|
||||
defer a.wg.Done()
|
||||
if err := a.lmtp.Serve(serveLn); err != nil && !errors.Is(err, smtp.ErrServerClosed) && !errors.Is(err, net.ErrClosed) {
|
||||
defer close(a.lmtpDone)
|
||||
if err := a.lmtp.Serve(a.lmtpOpen); err != nil && !errors.Is(err, smtp.ErrServerClosed) && !errors.Is(err, net.ErrClosed) {
|
||||
log.Printf("lmtp: %v", err)
|
||||
}
|
||||
}()
|
||||
|
|
@ -268,18 +322,26 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
return a, nil
|
||||
}
|
||||
|
||||
// Close stops the servers and closes the database. It is safe to call twice.
|
||||
// Close stops the daemon and closes the database. It is safe to call twice. It stops accepting
|
||||
// connections, then lets the LMTP sessions and HTTP requests in flight finish, so a post being
|
||||
// received is queued and gets its 250 (a cut reply makes the MTA send the post again). After
|
||||
// shutdownTimeout it cuts whatever is still open.
|
||||
func (a *App) Close() {
|
||||
a.once.Do(func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
||||
defer cancel()
|
||||
for _, ln := range a.lns {
|
||||
ln.Close()
|
||||
}
|
||||
a.lmtp.Close()
|
||||
if a.web != nil {
|
||||
a.web.Close()
|
||||
<-a.lmtpDone // Serve has returned: no connection is added while Shutdown waits
|
||||
if err := a.lmtp.Shutdown(ctx); errors.Is(err, context.DeadlineExceeded) {
|
||||
log.Printf("lmtp: sessions still open after %v, closing them", shutdownTimeout)
|
||||
a.lmtpOpen.closeAll()
|
||||
}
|
||||
if a.pub != nil {
|
||||
a.pub.Close()
|
||||
for _, srv := range []*http.Server{a.web, a.pub} {
|
||||
if srv != nil && srv.Shutdown(ctx) != nil {
|
||||
srv.Close()
|
||||
}
|
||||
}
|
||||
a.stop() // in-flight sends are interrupted and stay queued
|
||||
a.wg.Wait()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package app
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
|
|
@ -12,6 +13,7 @@ import (
|
|||
"encoding/base64"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math/big"
|
||||
|
|
@ -120,13 +122,17 @@ func testConfig(t *testing.T, relay string) config.Config {
|
|||
return c
|
||||
}
|
||||
|
||||
// client opens one connection per request: a pooled connection the transport dialed and never
|
||||
// used would hold an HTTP server's graceful shutdown for 5 s.
|
||||
var client = &http.Client{Transport: &http.Transport{DisableKeepAlives: true}}
|
||||
|
||||
func call(t *testing.T, a *App, method, path, body, token string) int {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(method, "http://"+a.APIAddr+path, strings.NewReader(body))
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -137,7 +143,7 @@ func call(t *testing.T, a *App, method, path, body, token string) int {
|
|||
func fetch(t *testing.T, method, url string) (int, string) {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(method, url, nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -353,6 +359,7 @@ func TestWebDisabledWhenListenIsEmpty(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestCloseDropsOpenConnections(t *testing.T) {
|
||||
setShutdownTimeout(t, 200*time.Millisecond)
|
||||
relayAddr, _ := startSink(t)
|
||||
a, err := Start(testConfig(t, relayAddr), "tok")
|
||||
if err != nil {
|
||||
|
|
@ -432,6 +439,193 @@ func deadAddr(t *testing.T) string {
|
|||
return addr
|
||||
}
|
||||
|
||||
func setShutdownTimeout(t *testing.T, d time.Duration) {
|
||||
old := shutdownTimeout
|
||||
shutdownTimeout = d
|
||||
t.Cleanup(func() { shutdownTimeout = old })
|
||||
}
|
||||
|
||||
// waitRefused waits until the LMTP listener no longer accepts connections.
|
||||
func waitRefused(t *testing.T, addr string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
c, err := net.Dial("tcp", addr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.Close()
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the LMTP listener still accepts connections")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseLetsAnLMTPDeliveryInFlightFinish(t *testing.T) {
|
||||
cfg := testConfig(t, deadAddr(t)) // the post stays queued, where it can be counted
|
||||
a, err := Start(cfg, "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok")
|
||||
call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok")
|
||||
|
||||
conn, err := net.Dial("tcp", a.LMTPAddr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := smtp.NewClientLMTP(conn)
|
||||
defer c.Close()
|
||||
if err := c.Hello("localhost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Mail("alice@example.com", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Rcpt("dev@example.org", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w, err := c.Data()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write([]byte("From: alice@example.com\r\nSubject: in flight\r\nMessage-ID: <f@x>\r\n\r\nbody\r\n"))
|
||||
|
||||
// The daemon stops while the post is being sent.
|
||||
closed := make(chan struct{})
|
||||
go func() { a.Close(); close(closed) }()
|
||||
waitRefused(t, a.LMTPAddr)
|
||||
if _, err := w.CloseWithLMTPResponse(); err != nil {
|
||||
t.Fatalf("the post in flight must get its 250: %v", err)
|
||||
}
|
||||
if err := c.Quit(); err != nil {
|
||||
t.Fatalf("quit: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Close must return once the session is over")
|
||||
}
|
||||
peek, err := store.Open(cfg.DB)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer peek.Close()
|
||||
if st, _ := peek.OutboxStats(); st.Pending != 1 {
|
||||
t.Fatalf("the post must be queued: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseCutsAHungLMTPClientAfterTheGracePeriod(t *testing.T) {
|
||||
setShutdownTimeout(t, 200*time.Millisecond)
|
||||
relayAddr, _ := startSink(t)
|
||||
a, err := Start(testConfig(t, relayAddr), "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn, err := net.Dial("tcp", a.LMTPAddr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
c := smtp.NewClientLMTP(conn)
|
||||
if err := c.Hello("localhost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Mail("alice@example.com", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Rcpt("dev@example.org", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := c.Data(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The client never ends the message.
|
||||
start := time.Now()
|
||||
closed := make(chan struct{})
|
||||
go func() { a.Close(); close(closed) }()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Close must not wait for a hung client beyond the grace period")
|
||||
}
|
||||
if d := time.Since(start); d < 150*time.Millisecond {
|
||||
t.Fatalf("Close returned after %v: it must give the session its grace period", d)
|
||||
}
|
||||
conn.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
buf := make([]byte, 512)
|
||||
for {
|
||||
if _, err := conn.Read(buf); err != nil {
|
||||
if ne, ok := err.(net.Error); ok && ne.Timeout() {
|
||||
t.Fatal("the hung connection is still open after Close")
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseLetsAnAPIRequestInFlightFinish(t *testing.T) {
|
||||
relayAddr, _ := startSink(t)
|
||||
a, err := Start(testConfig(t, relayAddr), "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn, err := net.Dial("tcp", a.APIAddr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
body := `{"name":"dev","domain":"example.org"}`
|
||||
fmt.Fprintf(conn, "POST /api/v1/lists HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer tok\r\nContent-Type: application/json\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s", len(body), body[:10])
|
||||
time.Sleep(100 * time.Millisecond) // the handler is now reading the body
|
||||
|
||||
closed := make(chan struct{})
|
||||
go func() { a.Close(); close(closed) }()
|
||||
waitRefused(t, a.APIAddr)
|
||||
io.WriteString(conn, body[10:])
|
||||
conn.SetReadDeadline(time.Now().Add(5 * time.Second))
|
||||
resp, err := http.ReadResponse(bufio.NewReader(conn), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the request in flight must get its answer: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("status = %d", resp.StatusCode)
|
||||
}
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Close must return once the request is over")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinishedLMTPSessionsAreForgotten(t *testing.T) {
|
||||
relayAddr, _ := startSink(t)
|
||||
a, err := Start(testConfig(t, relayAddr), "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { a.Close() })
|
||||
for i := 0; i < 3; i++ {
|
||||
lmtpSend(t, a.LMTPAddr, "alice@example.com", "nope@example.org", "Subject: x\r\n\r\nx\r\n")
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
a.lmtpOpen.mu.Lock()
|
||||
n := len(a.lmtpOpen.conns)
|
||||
a.lmtpOpen.mu.Unlock()
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%d closed sessions are still tracked", n)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueuedMailSurvivesARestartAndIsRetriedWhenTheRelayReturns(t *testing.T) {
|
||||
var skew atomic.Int64
|
||||
nowFunc = func() time.Time { return time.Now().Add(time.Duration(skew.Load())) }
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ func (s *session) deliver(rcpt string, data []byte) error {
|
|||
// status turns a router result into an LMTP reply.
|
||||
func (s *session) status(rcpt string, err error) error {
|
||||
switch {
|
||||
case err == nil, errors.Is(err, router.ErrHeld), errors.Is(err, router.ErrDiscarded):
|
||||
case err == nil, errors.Is(err, router.ErrHeld), errors.Is(err, router.ErrDiscarded), errors.Is(err, router.ErrDuplicate):
|
||||
return nil
|
||||
case errors.Is(err, router.ErrUnknownList):
|
||||
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such list"}
|
||||
|
|
|
|||
|
|
@ -339,3 +339,15 @@ func TestAMessageTooBigToHoldAnswers552(t *testing.T) {
|
|||
t.Fatalf("a permanent refusal: the sender is told to send it elsewhere: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSamePostDeliveredAgainIsAcceptedButNotQueuedTwice(t *testing.T) {
|
||||
st := newStack(t, down{}) // the deliveries stay in the queue, where they can be counted
|
||||
for i := 0; i < 2; i++ {
|
||||
if res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"}); res["dev@example.org"] != nil {
|
||||
t.Fatalf("delivery %d: a duplicate is acknowledged with 250, got %v", i+1, res)
|
||||
}
|
||||
}
|
||||
if n := st.queued(t); n != 2 {
|
||||
t.Fatalf("queued = %d, want one delivery per member", n)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
126
internal/router/dup_test.go
Normal file
126
internal/router/dup_test.go
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
package router
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func pending(t *testing.T, e *env) int {
|
||||
t.Helper()
|
||||
st, err := e.s.OutboxStats()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return st.Pending
|
||||
}
|
||||
|
||||
func (e *env) post(t *testing.T, msg string) error {
|
||||
t.Helper()
|
||||
res, err := e.r.Route("alice@example.com", "dev@example.org", []byte(msg))
|
||||
if err != nil {
|
||||
t.Fatalf("route: %v", err)
|
||||
}
|
||||
return e.r.Deliver(res)
|
||||
}
|
||||
|
||||
func TestTheSamePostIsDeliveredOnce(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
woke := 0
|
||||
e.r.OnEnqueue = func() { woke++ }
|
||||
msg := "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: <one@x>\r\n\r\nbody\r\n"
|
||||
if err := e.post(t, msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) {
|
||||
t.Fatalf("want ErrDuplicate, got %v", err)
|
||||
}
|
||||
if n := pending(t, e); n != 2 {
|
||||
t.Fatalf("pending = %d, want one delivery per member", n)
|
||||
}
|
||||
if woke != 1 {
|
||||
t.Fatalf("OnEnqueue called %d times for a duplicate", woke)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnotherMessageIDIsAnotherPost(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
if err := e.post(t, "From: alice@example.com\r\nMessage-ID: <one@x>\r\n\r\nbody\r\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.post(t, "From: alice@example.com\r\nMessage-ID: <two@x>\r\n\r\nbody\r\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := pending(t, e); n != 4 {
|
||||
t.Fatalf("pending = %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPostWithoutMessageIDIsKnownByItsContent(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
msg := "From: alice@example.com\r\nSubject: hi\r\n\r\nbody\r\n"
|
||||
if err := e.post(t, msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) {
|
||||
t.Fatalf("the same bytes again: want ErrDuplicate, got %v", err)
|
||||
}
|
||||
if err := e.post(t, msg+"more\r\n"); err != nil {
|
||||
t.Fatalf("another body is another post: %v", err)
|
||||
}
|
||||
if n := pending(t, e); n != 4 {
|
||||
t.Fatalf("pending = %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostIDIgnoresTheRestWhenThereIsAMessageID(t *testing.T) {
|
||||
a := postID([]byte("Message-ID: <one@x> \r\nSubject: a\r\n\r\nx"))
|
||||
b := postID([]byte("Subject: b\r\nMessage-ID: <one@x>\r\n\r\ny"))
|
||||
if a != b || a != "mid:<one@x>" {
|
||||
t.Fatalf("%q %q", a, b)
|
||||
}
|
||||
if c := postID([]byte("Subject: b\r\n\r\ny")); c == postID([]byte("Subject: b\r\n\r\nz")) || len(c) < 10 {
|
||||
t.Fatalf("content hash: %q", c)
|
||||
}
|
||||
if postID([]byte("Message-ID: <one@x>\r\n\r\n")) == postID([]byte("Message-ID: <One@x>\r\n\r\n")) {
|
||||
t.Fatal("Message-IDs are case sensitive")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnApprovedPostIsDeliveredEvenIfItWasSeen(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
msg := "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: <one@x>\r\n\r\nbody\r\n"
|
||||
if err := e.post(t, msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, err := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(msg), "r")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.r.Release(id); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
if n := pending(t, e); n != 4 {
|
||||
t.Fatalf("pending = %d, the approval must deliver", n)
|
||||
}
|
||||
if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) {
|
||||
t.Fatalf("a copy after the approval: want ErrDuplicate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnApprovedPostIsRemembered(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
msg := "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: <held@x>\r\n\r\nbody\r\n"
|
||||
id, err := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(msg), "r")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.r.Release(id); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) {
|
||||
t.Fatalf("the MTA's copy of an approved post: want ErrDuplicate, got %v", err)
|
||||
}
|
||||
if n := pending(t, e); n != 2 {
|
||||
t.Fatalf("pending = %d", n)
|
||||
}
|
||||
}
|
||||
|
|
@ -3,6 +3,8 @@ package router
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"log"
|
||||
"net/mail"
|
||||
|
|
@ -34,6 +36,9 @@ var (
|
|||
ErrGone = errors.New("already unsubscribed")
|
||||
// ErrJoinClosed means the list does not accept self-service subscriptions.
|
||||
ErrJoinClosed = errors.New("list is closed")
|
||||
// ErrDuplicate means the list already queued the same post less than an hour ago: the MTA sent
|
||||
// it again because our reply was lost. It is acknowledged and not queued again.
|
||||
ErrDuplicate = store.ErrDuplicate
|
||||
)
|
||||
|
||||
// Recipient is a subscriber that receives a message.
|
||||
|
|
@ -126,13 +131,36 @@ func (r *Router) prepare(l store.List, data []byte) (Result, error) {
|
|||
|
||||
// Deliver queues the message for every recipient, each with its own signed
|
||||
// bounce address (VERP) and unsubscribe headers. The queue write is atomic: on
|
||||
// error nothing was queued and the caller may safely retry.
|
||||
// error nothing was queued and the caller may safely retry. A post the list
|
||||
// already queued in the last hour is not queued again: Deliver returns ErrDuplicate.
|
||||
func (r *Router) Deliver(res Result) error {
|
||||
items, err := r.outbound(res)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return r.enqueue(res.Data, items)
|
||||
id := postID(res.Data)
|
||||
if err := r.store.EnqueuePost(res.List.ID, id, res.Data, items); err != nil {
|
||||
if errors.Is(err, ErrDuplicate) {
|
||||
log.Printf("duplicate post to %s not delivered again: %s", res.List.Address(), id)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if r.OnEnqueue != nil && len(items) > 0 {
|
||||
r.OnEnqueue()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// postID identifies a post for the duplicate check: its Message-ID, or a hash of the whole
|
||||
// message when it has none.
|
||||
func postID(data []byte) string {
|
||||
if m, err := mail.ReadMessage(bytes.NewReader(data)); err == nil {
|
||||
if id := m.Header.Get("Message-ID"); id != "" {
|
||||
return "mid:" + id
|
||||
}
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// outbound builds one queue entry per recipient.
|
||||
|
|
@ -213,7 +241,7 @@ func (r *Router) Release(heldID int64) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.store.EnqueueFromHeld(heldID, res.Data, items); err != nil {
|
||||
if err := r.store.EnqueueFromHeld(heldID, postID(res.Data), res.Data, items); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.OnEnqueue != nil && len(items) > 0 {
|
||||
|
|
|
|||
149
internal/store/dup_test.go
Normal file
149
internal/store/dup_test.go
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestEnqueuePostRefusesTheSamePostTwiceWithinAnHour(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(59 * time.Minute)
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(2)); !errors.Is(err, ErrDuplicate) {
|
||||
t.Fatalf("want ErrDuplicate, got %v", err)
|
||||
}
|
||||
if st, _ := s.OutboxStats(); st.Pending != 2 {
|
||||
t.Fatalf("a duplicate must queue nothing, pending = %d", st.Pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueuePostTellsPostsAndListsApart(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
dev, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
ops, _ := s.CreateList("ops", "example.org", "Ops")
|
||||
if err := s.EnqueuePost(dev.ID, "mid:<a@x>", []byte("m"), items(1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.EnqueuePost(dev.ID, "mid:<b@x>", []byte("m"), items(1)); err != nil {
|
||||
t.Fatalf("another Message-ID: %v", err)
|
||||
}
|
||||
if err := s.EnqueuePost(ops.ID, "mid:<a@x>", []byte("m"), items(1)); err != nil {
|
||||
t.Fatalf("the same post to another list: %v", err)
|
||||
}
|
||||
if st, _ := s.OutboxStats(); st.Pending != 3 {
|
||||
t.Fatalf("pending = %d", st.Pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueuePostForgetsAPostAfterAnHour(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(time.Hour)
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(1)); !errors.Is(err, ErrDuplicate) {
|
||||
t.Fatalf("exactly one hour later it is still a duplicate, got %v", err)
|
||||
}
|
||||
now = now.Add(time.Second)
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(1)); err != nil {
|
||||
t.Fatalf("after an hour the post is accepted again: %v", err)
|
||||
}
|
||||
if st, _ := s.OutboxStats(); st.Pending != 2 {
|
||||
t.Fatalf("pending = %d", st.Pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueuePurgesExpiredRecordsOfEveryList(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
dev, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
ops, _ := s.CreateList("ops", "example.org", "Ops")
|
||||
s.EnqueuePost(dev.ID, "mid:<a@x>", []byte("m"), items(1))
|
||||
now = now.Add(2 * time.Hour)
|
||||
s.EnqueuePost(ops.ID, "mid:<b@x>", []byte("m"), items(1))
|
||||
var n int
|
||||
if err := s.db.QueryRow(`SELECT COUNT(*) FROM delivered`).Scan(&n); err != nil || n != 1 {
|
||||
t.Fatalf("records = %d (%v), want only the fresh one", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueuePostRecordsAndQueuesTogether(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON outbox BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(2)); err == nil {
|
||||
t.Fatal("want the queue error")
|
||||
}
|
||||
s.db.Exec(`DROP TRIGGER boom`)
|
||||
// The failed attempt left no record behind: the MTA's retry is queued.
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(2)); err != nil {
|
||||
t.Fatalf("retry after a failed queue write: %v", err)
|
||||
}
|
||||
if st, _ := s.OutboxStats(); st.Pending != 2 {
|
||||
t.Fatalf("pending = %d", st.Pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueuePostRecordsAPostWithNoRecipients(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), nil); !errors.Is(err, ErrDuplicate) {
|
||||
t.Fatalf("want ErrDuplicate, got %v", err)
|
||||
}
|
||||
var n int
|
||||
if err := s.db.QueryRow(`SELECT COUNT(*) FROM outbox_msgs`).Scan(&n); err != nil || n != 0 {
|
||||
t.Fatalf("stored messages = %d (%v), want none without recipients", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleasingAHeldPostDeliversItEvenIfSeenAndRecordsIt(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
|
||||
now = now.Add(50 * time.Minute)
|
||||
if err := s.EnqueueFromHeld(hid, "mid:<a@x>", []byte("final"), items(2)); err != nil {
|
||||
t.Fatalf("an approval always delivers: %v", err)
|
||||
}
|
||||
if st, _ := s.OutboxStats(); st.Pending != 3 {
|
||||
t.Fatalf("pending = %d", st.Pending)
|
||||
}
|
||||
// The approval refreshed the record: a retry of the same post 20 minutes later is a duplicate.
|
||||
now = now.Add(20 * time.Minute)
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(1)); !errors.Is(err, ErrDuplicate) {
|
||||
t.Fatalf("want ErrDuplicate after a release, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeletingAListForgetsItsPosts(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), nil)
|
||||
if err := s.DeleteList(l.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var n int
|
||||
if err := s.db.QueryRow(`SELECT COUNT(*) FROM delivered`).Scan(&n); err != nil || n != 0 {
|
||||
t.Fatalf("records = %d (%v)", n, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -58,19 +58,61 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error {
|
|||
return tx.Commit()
|
||||
}
|
||||
|
||||
// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues
|
||||
// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the
|
||||
// others get ErrNotFound, so a message is never delivered twice. With no recipients the message
|
||||
// is still released.
|
||||
func (s *Store) EnqueueFromHeld(heldID int64, msg []byte, items []NewOutbound) error {
|
||||
// postMemory is how long a post queued for a list is remembered, to refuse it if the MTA delivers it again.
|
||||
const postMemory = time.Hour
|
||||
|
||||
// EnqueuePost queues a post to a list, like Enqueue, and records postID (the identity of the post)
|
||||
// in the same transaction, so a crash never leaves one without the other. If the list already
|
||||
// queued the same postID within postMemory it queues nothing and returns ErrDuplicate: the MTA
|
||||
// delivers the post again when our reply to the first delivery was lost.
|
||||
func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []NewOutbound) error {
|
||||
now := s.Now()
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.Exec(`DELETE FROM delivered WHERE created_at<?`, now.Add(-postMemory).Unix()); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO delivered(list_id, post_id, created_at) VALUES(?,?,?)`, listID, postID, now.Unix()); err != nil {
|
||||
if isUnique(err) {
|
||||
return ErrDuplicate
|
||||
}
|
||||
return err
|
||||
}
|
||||
if len(items) > 0 {
|
||||
if err := enqueueTx(tx, now.Unix(), msg, items); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues
|
||||
// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the
|
||||
// others get ErrNotFound, so a message is never delivered twice. With no recipients the message
|
||||
// is still released. An approval always delivers, even a post already seen, and records postID
|
||||
// as EnqueuePost does, so a later copy of the same post is refused.
|
||||
func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items []NewOutbound) error {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var listID int64
|
||||
if err := tx.QueryRow(`SELECT list_id FROM held WHERE id=?`, heldID).Scan(&listID); errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := affected(tx.Exec(`DELETE FROM held WHERE id=?`, heldID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO delivered(list_id, post_id, created_at) VALUES(?,?,?)
|
||||
ON CONFLICT(list_id, post_id) DO UPDATE SET created_at=excluded.created_at`, listID, postID, s.Now().Unix()); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(items) > 0 {
|
||||
if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -194,7 +194,7 @@ func TestEnqueueFromHeldMovesTheMessageAtomically(t *testing.T) {
|
|||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("Subject: s\r\n\r\nb"), "r")
|
||||
|
||||
if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); err != nil {
|
||||
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), items(2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) {
|
||||
|
|
@ -204,7 +204,7 @@ func TestEnqueueFromHeldMovesTheMessageAtomically(t *testing.T) {
|
|||
t.Fatalf("pending = %d", st.Pending)
|
||||
}
|
||||
// Approving again must not queue anything.
|
||||
if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); !errors.Is(err, ErrNotFound) {
|
||||
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), items(2)); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
if st, _ := s.OutboxStats(); st.Pending != 2 {
|
||||
|
|
@ -216,7 +216,7 @@ func TestEnqueueFromHeldWithNoRecipientsStillReleasesTheMessage(t *testing.T) {
|
|||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
|
||||
if err := s.EnqueueFromHeld(hid, []byte("final"), nil); err != nil {
|
||||
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) {
|
||||
|
|
@ -237,7 +237,7 @@ func TestOnlyOneOfSeveralSimultaneousApprovalsWins(t *testing.T) {
|
|||
for i := 0; i < n; i++ {
|
||||
go func() {
|
||||
<-start
|
||||
results <- s.EnqueueFromHeld(hid, []byte("final"), items(3))
|
||||
results <- s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), items(3))
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ var (
|
|||
ErrHeldTooLarge = errors.New("message too large to hold for moderation")
|
||||
// ErrExists is returned when creating something that already exists.
|
||||
ErrExists = errors.New("already exists")
|
||||
// ErrDuplicate is returned when the same post was already queued for the list within postMemory.
|
||||
ErrDuplicate = errors.New("post already delivered")
|
||||
)
|
||||
|
||||
// Policy values.
|
||||
|
|
@ -212,6 +214,12 @@ CREATE TABLE IF NOT EXISTS outbox (
|
|||
failed INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id);
|
||||
CREATE TABLE IF NOT EXISTS delivered (
|
||||
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
|
||||
post_id TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (list_id, post_id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
|
|
|
|||
Loading…
Reference in a new issue