From c66fd427b249ff66248d5e1aaa55b9cba590126c Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 15:07:11 +0100 Subject: [PATCH 1/3] fix(store): add the outbox post_id and priority columns separately The two columns were added in one non-transactional Exec guarded by post_id alone. A crash between the two statements left a database with post_id but no priority, and every later start failed at CREATE INDEX outbox_due_prio. Each column is now checked and added on its own, as the members columns are. --- internal/store/m2_test.go | 22 ++++++++++++++++++++++ internal/store/store.go | 8 ++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go index 79765bf..1cfc525 100644 --- a/internal/store/m2_test.go +++ b/internal/store/m2_test.go @@ -717,3 +717,25 @@ func TestConfirmPendingIsSingleUseUnderConcurrency(t *testing.T) { } } } + +func TestMigrationAddsTheOutboxPriorityAloneAfterAHalfDoneUpgrade(t *testing.T) { + // A crash between adding post_id and priority leaves outbox with the first and not the second. + path := t.TempDir() + "/old.db" + s, _ := Open(path) + if _, err := s.db.Exec(`DROP INDEX outbox_due_prio`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`ALTER TABLE outbox DROP COLUMN priority`); err != nil { + t.Fatal(err) + } + s.Close() + + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + if ok, err := hasColumn(s2.db, "outbox", "priority"); err != nil || !ok { + t.Fatalf("priority is added back: %v %v", ok, err) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index bfe150e..6f74622 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -325,8 +325,12 @@ func migrate(db *sql.DB) error { } } if has, _ := hasColumn(db, "outbox", "post_id"); !has { - if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN post_id INTEGER NOT NULL DEFAULT 0; - ALTER TABLE outbox ADD COLUMN priority INTEGER NOT NULL DEFAULT 0;`); err != nil { + if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN post_id INTEGER NOT NULL DEFAULT 0`); err != nil { + return err + } + } + if has, _ := hasColumn(db, "outbox", "priority"); !has { + if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN priority INTEGER NOT NULL DEFAULT 0`); err != nil { return err } } From f14faf52d538bc327afed0556591d706d16a9e6f Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 15:20:49 +0100 Subject: [PATCH 2/3] fix(outbound): give each SMTP phase its own deadline A relay can take minutes to answer the final dot (content filter, large message, slow link). One deadline over the whole transaction cut it after the message had been accepted, so the delivery was retried and sent again. The connection now moves its deadline as the transaction advances: setup (dial, greeting, TLS, AUTH, MAIL, RCPT, DATA) has SMTPRelay.Timeout, 60 s by default; the upload has 60 s plus the size at 32 KiB/s, at most 10 minutes; the reply to the final dot has 10 minutes (RFC 5321 4.5.3.2.6). go-smtp sets and clears its own per-command deadlines, so a wrapper clips them to the current phase. Context cancellation still closes the connection at once. A timeout in any phase is temporary; after the final dot the error says the outcome is unknown. --- internal/outbound/outbound.go | 101 ++++++++++--- internal/outbound/timeout_test.go | 231 ++++++++++++++++++++++++++++++ 2 files changed, 313 insertions(+), 19 deletions(-) create mode 100644 internal/outbound/timeout_test.go diff --git a/internal/outbound/outbound.go b/internal/outbound/outbound.go index 09ebcaa..c2de797 100644 --- a/internal/outbound/outbound.go +++ b/internal/outbound/outbound.go @@ -45,6 +45,59 @@ type SMTPRelay struct { Username string // SMTP AUTH PLAIN; only over TLS or to a loopback address Password string Helo string // EHLO name; "localhost" by default + // Timeout bounds the setup of each transaction: connection, greeting, TLS, AUTH, MAIL FROM, + // RCPT TO and the DATA command. DefaultTimeout if zero. The upload of the message and the + // wait for its acceptance have their own budgets (see uploadBudget and finalReplyTimeout). + Timeout time.Duration +} + +// DefaultTimeout is the default SMTPRelay.Timeout. +const DefaultTimeout = time.Minute + +// Budgets of the later phases of a transaction. Variables so that tests can shrink them. +var ( + dataFloor = time.Minute // least time to upload a message + dataRate = 32 << 10 // slowest upload we wait for, in bytes per second + dataCap = 10 * time.Minute // most time to upload a message + // finalReplyTimeout is how long the relay may take to answer the final dot: RFC 5321 4.5.3.2.6 + // asks for 10 minutes, because content filters run at that point. + finalReplyTimeout = 10 * time.Minute +) + +// uploadBudget is the time allowed to upload a message of size bytes: dataFloor plus the time +// it takes at dataRate, at most dataCap. +func uploadBudget(size int) time.Duration { + b := dataFloor + time.Duration(float64(size)/float64(dataRate)*float64(time.Second)) + return min(b, dataCap) +} + +func (r *SMTPRelay) timeout() time.Duration { + if r.Timeout > 0 { + return r.Timeout + } + return DefaultTimeout +} + +// phasedConn keeps the connection within the budget of the current phase. go-smtp sets its own +// deadline around each command and clears it afterwards; here no deadline it sets, cleared or not, +// may go past the limit of the phase. +type phasedConn struct { + net.Conn + limit time.Time +} + +func (p *phasedConn) SetDeadline(t time.Time) error { + if t.IsZero() || p.limit.Before(t) { + t = p.limit + } + return p.Conn.SetDeadline(t) +} + +// phase gives the connection d to finish the next phase. The context's own deadline and its +// cancellation close the connection instead (see dial), so that the error is the context's. +func (p *phasedConn) phase(d time.Duration) { + p.limit = time.Now().Add(d) + p.Conn.SetDeadline(p.limit) } // Send splits recipients in batches and sends one transaction per batch. @@ -65,20 +118,21 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b return nil } -// dial opens the connection and speaks EHLO, TLS and AUTH as configured. -// The returned release function must be called when the client is done: until then the connection +// dial opens the connection and speaks EHLO, TLS and AUTH as configured. The connection's deadline +// is the end of the setup budget, which starts now. The returned release function must be called when the client is done: until then the connection // is closed if ctx is cancelled, even in the middle of EHLO, STARTTLS or AUTH with a silent relay. -func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), err error) { +func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, conn *phasedConn, release func(), err error) { + setupEnd := time.Now().Add(r.timeout()) host, _, err := net.SplitHostPort(r.Addr) if err != nil { - return nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err) + return nil, nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err) } mode := r.TLS if mode == "" { mode = TLSNone } if r.Username != "" && mode == TLSNone && !isLoopback(host) { - return nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls") + return nil, nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls") } helo := r.Helo if helo == "" { @@ -89,21 +143,20 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e tlsConf.ServerName = r.ServerName } - d := &net.Dialer{Timeout: 10 * time.Second} - var conn net.Conn + d := &net.Dialer{Timeout: 10 * time.Second, Deadline: setupEnd} + var raw net.Conn if mode == TLSImplicit { - conn, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr) + raw, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr) } else { - conn, err = d.DialContext(ctx, "tcp", r.Addr) + raw, err = d.DialContext(ctx, "tcp", r.Addr) } if err != nil { - return nil, nil, err - } - // A relay that stops answering must not hold us past the context: apply its deadline to the - // connection and close the connection if the context is cancelled. - if dl, ok := ctx.Deadline(); ok { - conn.SetDeadline(dl) + return nil, nil, nil, err } + // A relay that stops answering must not hold us past the budget or the context: apply the + // earlier deadline to the connection, and close the connection if the context is cancelled. + conn = &phasedConn{Conn: raw} + conn.phase(time.Until(setupEnd)) stop := make(chan struct{}) go func() { select { @@ -113,10 +166,10 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e } }() release = func() { close(stop) } - fail := func(err error) (*smtp.Client, func(), error) { + fail := func(err error) (*smtp.Client, *phasedConn, func(), error) { release() conn.Close() - return nil, nil, err + return nil, nil, nil, err } if mode == TLSStartTLS { @@ -139,7 +192,7 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e return fail(authError(err)) } } - return c, release, nil + return c, conn, release, nil } func isLoopback(host string) bool { @@ -162,7 +215,7 @@ func authError(err error) error { } func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) (err error) { - c, release, err := r.dial(ctx) + c, conn, release, err := r.dial(ctx) if err != nil { if ctx.Err() != nil { err = ctx.Err() // report the cause, not "use of closed connection" @@ -194,13 +247,23 @@ func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, dat if err != nil { return authError(&PhaseError{Phase: "DATA", Err: err}) } + conn.phase(uploadBudget(len(data))) if _, err := io.Copy(w, bytes.NewReader(data)); err != nil { return &PhaseError{Phase: "DATA", Err: err} } + // Close sends the rest of the message and the final dot, then waits for the verdict. + conn.phase(finalReplyTimeout) if err := w.Close(); err != nil { + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + // Temporary like any timeout, but the relay may have taken the message: a retry can + // deliver it twice. Say so, so the admin can tell. + err = fmt.Errorf("no reply to the end of the message, outcome unknown: %w", err) + } return &PhaseError{Phase: "DATA", Err: err} } // The relay has accepted the message. If QUIT now fails, retrying would deliver it twice. + conn.phase(r.timeout()) c.Quit() return nil } diff --git a/internal/outbound/timeout_test.go b/internal/outbound/timeout_test.go new file mode 100644 index 0000000..b1103d9 --- /dev/null +++ b/internal/outbound/timeout_test.go @@ -0,0 +1,231 @@ +package outbound + +import ( + "bufio" + "context" + "errors" + "net" + "strings" + "sync" + "testing" + "time" +) + +// scaleTimeouts shrinks the phase budgets so the tests run in milliseconds. +func scaleTimeouts(t *testing.T, floor, finalReply time.Duration) { + t.Helper() + oldFloor, oldRate, oldCap, oldReply := dataFloor, dataRate, dataCap, finalReplyTimeout + dataFloor, finalReplyTimeout = floor, finalReply + t.Cleanup(func() { dataFloor, dataRate, dataCap, finalReplyTimeout = oldFloor, oldRate, oldCap, oldReply }) +} + +// slowRelay is a bare SMTP server that waits dotDelay before answering the final dot +// (forever when dotDelay is negative). +type slowRelay struct { + mu sync.Mutex + accepted int +} + +func startSlowRelay(t *testing.T, dotDelay time.Duration) (*slowRelay, string) { + t.Helper() + s := &slowRelay{} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + t.Cleanup(func() { close(done); ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + r := bufio.NewReader(c) + c.Write([]byte("220 slow ESMTP\r\n")) + inData := false + for { + l, err := r.ReadString('\n') + if err != nil { + return + } + l = strings.TrimRight(l, "\r\n") + switch { + case inData: + if l != "." { + continue + } + inData = false + if dotDelay < 0 { + <-done + return + } + time.Sleep(dotDelay) + s.mu.Lock() + s.accepted++ + s.mu.Unlock() + c.Write([]byte("250 2.0.0 queued\r\n")) + case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"): + c.Write([]byte("250 slow\r\n")) + case strings.HasPrefix(l, "MAIL"), strings.HasPrefix(l, "RCPT"): + c.Write([]byte("250 2.1.0 ok\r\n")) + case l == "DATA": + inData = true + c.Write([]byte("354 go ahead\r\n")) + case l == "QUIT": + c.Write([]byte("221 bye\r\n")) + return + } + } + }() + } + }() + return s, ln.Addr().String() +} + +func TestASlowReplyToTheFinalDotIsWaitedForAndDeliversOnce(t *testing.T) { + // The relay takes longer than the whole setup budget to accept the message (a content filter): + // that is not a failure, or the retry would deliver it a second time. + scaleTimeouts(t, 100*time.Millisecond, 3*time.Second) + s, addr := startSlowRelay(t, 400*time.Millisecond) + r := &SMTPRelay{Addr: addr, Timeout: 100 * time.Millisecond} + if err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")); err != nil { + t.Fatalf("the relay accepted the message: %v", err) + } + s.mu.Lock() + defer s.mu.Unlock() + if s.accepted != 1 { + t.Fatalf("accepted %d times", s.accepted) + } +} + +func TestNoReplyToTheFinalDotTimesOutAsATemporaryFailure(t *testing.T) { + scaleTimeouts(t, 100*time.Millisecond, 300*time.Millisecond) + _, addr := startSlowRelay(t, -1) + r := &SMTPRelay{Addr: addr, Timeout: 5 * time.Second} + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() { + t.Fatalf("want a timeout, got %v", err) + } + if IsPermanent(err) { + t.Fatalf("an unknown outcome is retried, not failed: %v", err) + } + if !strings.Contains(err.Error(), "outcome unknown") { + t.Fatalf("the error says the relay may have the message: %v", err) + } + if took := time.Since(start); took < 250*time.Millisecond || took > 3*time.Second { + t.Fatalf("the final reply has its own budget, took %v", took) + } +} + +func TestARelayThatStallsAtTheGreetingTimesOutInTheShortBudget(t *testing.T) { + r := &SMTPRelay{Addr: silent(t), Timeout: 150 * time.Millisecond} + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("x")) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() { + t.Fatalf("want a timeout, got %v", err) + } + if IsPermanent(err) { + t.Fatalf("a timeout is temporary: %v", err) + } + if took := time.Since(start); took > 2*time.Second { + t.Fatalf("took %v", took) + } +} + +func TestTheUploadBudgetGrowsWithTheMessageSize(t *testing.T) { + if got := uploadBudget(0); got != time.Minute { + t.Fatalf("floor: %v", got) + } + small, big := uploadBudget(10<<10), uploadBudget(10<<20) + if big <= small { + t.Fatalf("a big message gets more time: small %v, big %v", small, big) + } + if want := time.Minute + 320*time.Second; big != want { // 10 MiB at 32 KiB/s + t.Fatalf("10 MiB: got %v, want %v", big, want) + } + if got := uploadBudget(1 << 40); got != 10*time.Minute { + t.Fatalf("cap: %v", got) + } +} + +func TestCancellingTheContextAbortsTheWaitForTheFinalReply(t *testing.T) { + _, addr := startSlowRelay(t, -1) // the full 10 minutes apply + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { + errc <- (&SMTPRelay{Addr: addr}).Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")) + }() + time.Sleep(200 * time.Millisecond) + cancel() + select { + case err := <-errc: + if !errors.Is(err, context.Canceled) { + t.Fatalf("want context.Canceled, got %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Send ignored the cancelled context while waiting for the final reply") + } +} + +// stopsReadingAtData answers up to DATA, then never reads again. +func stopsReadingAtData(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + t.Cleanup(func() { close(done); ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + r := bufio.NewReader(c) + c.Write([]byte("220 stuck ESMTP\r\n")) + for { + l, err := r.ReadString('\n') + if err != nil { + return + } + switch l = strings.TrimRight(l, "\r\n"); { + case l == "DATA": + c.Write([]byte("354 go ahead\r\n")) + <-done + return + case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"): + c.Write([]byte("250 stuck\r\n")) + default: + c.Write([]byte("250 ok\r\n")) + } + } + }() + } + }() + return ln.Addr().String() +} + +func TestAStalledUploadTimesOutInItsOwnBudget(t *testing.T) { + scaleTimeouts(t, 200*time.Millisecond, time.Minute) + dataRate = 1 << 40 // the budget is the floor + r := &SMTPRelay{Addr: stopsReadingAtData(t), Timeout: time.Minute} + big := []byte(strings.Repeat(strings.Repeat("x", 998)+"\r\n", 32<<10)) // 32 MiB, more than the socket buffers + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, big) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() || IsPermanent(err) { + t.Fatalf("want a temporary timeout, got %v", err) + } + if took := time.Since(start); took > 5*time.Second { + t.Fatalf("the upload has its own budget, took %v", took) + } +} From af487b8c8475b6618245e7a6d3b80f5b0ea575c7 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 15:20:49 +0100 Subject: [PATCH 3/3] fix(queue): stop putting one deadline on the whole send Worker.Timeout defaulted to one minute over the entire SMTP transaction, cutting relays that are slow to accept a message and causing duplicate deliveries. The default is now no total deadline: SMTPRelay bounds each phase itself. A positive Timeout still applies. --- internal/queue/queue.go | 14 +++++++------- internal/queue/queue_test.go | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/internal/queue/queue.go b/internal/queue/queue.go index de5a634..aad7fb6 100644 --- a/internal/queue/queue.go +++ b/internal/queue/queue.go @@ -20,7 +20,6 @@ const MaxAge = 72 * time.Hour const ( defaultConcurrency = 4 defaultBatch = 50 - defaultTimeout = time.Minute defaultInterval = 5 * time.Second // purgeEvery is the least time between two scans that forget old failed deliveries. @@ -55,7 +54,7 @@ type Worker struct { Sender outbound.Sender Concurrency int // simultaneous sends Batch int // deliveries taken per run - Timeout time.Duration // per send + Timeout time.Duration // per send; zero leaves the deadlines to Sender (SMTPRelay bounds each SMTP phase) Interval time.Duration // idle polling, to pick up retries that have become due // Sign, when set, signs the final message (DKIM). On error the message is sent unsigned: // not delivering is worse than a missed signature. @@ -245,15 +244,16 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem } } - timeout := w.Timeout - if timeout <= 0 { - timeout = defaultTimeout - } // Wait for the slot before starting the timeout, so waiting does not eat into the send. if err := w.pace().Wait(ctx); err != nil { return Stats{} // shutting down: the delivery stays queued, unchanged } - sctx, cancel := context.WithTimeout(ctx, timeout) + // No total deadline by default: one that covers the whole transaction cut relays that take + // minutes to accept a message, which was then retried and delivered twice. + sctx, cancel := ctx, context.CancelFunc(func() {}) + if w.Timeout > 0 { + sctx, cancel = context.WithTimeout(ctx, w.Timeout) + } err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg) cancel() diff --git a/internal/queue/queue_test.go b/internal/queue/queue_test.go index 24ae823..abd1c29 100644 --- a/internal/queue/queue_test.go +++ b/internal/queue/queue_test.go @@ -405,6 +405,24 @@ func TestSlowRelayTimesOutAndIsRetried(t *testing.T) { } } +func TestTheQueueLeavesTheSendDeadlinesToTheSender(t *testing.T) { + // One total deadline cut a relay that took over a minute to answer the final dot: the message + // was accepted, then retried and delivered again. The sender bounds each SMTP phase itself. + e := setup(t) + e.enqueue(t, 1) + var hasDeadline bool + e.snd.fn = func(ctx context.Context, _ string, _ []string) error { + _, hasDeadline = ctx.Deadline() + return nil + } + if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 { + t.Fatalf("sent: %+v", st) + } + if hasDeadline { + t.Fatal("the queue must not put a deadline on the whole send") + } +} + func waitFor(t *testing.T, what string, cond func() bool) { t.Helper() deadline := time.Now().Add(3 * time.Second)