Merge branch 'fix/sr-send-timeout' into fix/second-review

This commit is contained in:
tgiachi 2026-11-10 15:21:24 +01:00
commit c17ff1fed1
6 changed files with 366 additions and 28 deletions

View file

@ -45,6 +45,59 @@ type SMTPRelay struct {
Username string // SMTP AUTH PLAIN; only over TLS or to a loopback address
Password string
Helo string // EHLO name; "localhost" by default
// Timeout bounds the setup of each transaction: connection, greeting, TLS, AUTH, MAIL FROM,
// RCPT TO and the DATA command. DefaultTimeout if zero. The upload of the message and the
// wait for its acceptance have their own budgets (see uploadBudget and finalReplyTimeout).
Timeout time.Duration
}
// DefaultTimeout is the default SMTPRelay.Timeout.
const DefaultTimeout = time.Minute
// Budgets of the later phases of a transaction. Variables so that tests can shrink them.
var (
dataFloor = time.Minute // least time to upload a message
dataRate = 32 << 10 // slowest upload we wait for, in bytes per second
dataCap = 10 * time.Minute // most time to upload a message
// finalReplyTimeout is how long the relay may take to answer the final dot: RFC 5321 4.5.3.2.6
// asks for 10 minutes, because content filters run at that point.
finalReplyTimeout = 10 * time.Minute
)
// uploadBudget is the time allowed to upload a message of size bytes: dataFloor plus the time
// it takes at dataRate, at most dataCap.
func uploadBudget(size int) time.Duration {
b := dataFloor + time.Duration(float64(size)/float64(dataRate)*float64(time.Second))
return min(b, dataCap)
}
func (r *SMTPRelay) timeout() time.Duration {
if r.Timeout > 0 {
return r.Timeout
}
return DefaultTimeout
}
// phasedConn keeps the connection within the budget of the current phase. go-smtp sets its own
// deadline around each command and clears it afterwards; here no deadline it sets, cleared or not,
// may go past the limit of the phase.
type phasedConn struct {
net.Conn
limit time.Time
}
func (p *phasedConn) SetDeadline(t time.Time) error {
if t.IsZero() || p.limit.Before(t) {
t = p.limit
}
return p.Conn.SetDeadline(t)
}
// phase gives the connection d to finish the next phase. The context's own deadline and its
// cancellation close the connection instead (see dial), so that the error is the context's.
func (p *phasedConn) phase(d time.Duration) {
p.limit = time.Now().Add(d)
p.Conn.SetDeadline(p.limit)
}
// Send splits recipients in batches and sends one transaction per batch.
@ -65,20 +118,21 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b
return nil
}
// dial opens the connection and speaks EHLO, TLS and AUTH as configured.
// The returned release function must be called when the client is done: until then the connection
// dial opens the connection and speaks EHLO, TLS and AUTH as configured. The connection's deadline
// is the end of the setup budget, which starts now. The returned release function must be called when the client is done: until then the connection
// is closed if ctx is cancelled, even in the middle of EHLO, STARTTLS or AUTH with a silent relay.
func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), err error) {
func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, conn *phasedConn, release func(), err error) {
setupEnd := time.Now().Add(r.timeout())
host, _, err := net.SplitHostPort(r.Addr)
if err != nil {
return nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err)
return nil, nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err)
}
mode := r.TLS
if mode == "" {
mode = TLSNone
}
if r.Username != "" && mode == TLSNone && !isLoopback(host) {
return nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls")
return nil, nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls")
}
helo := r.Helo
if helo == "" {
@ -89,21 +143,20 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e
tlsConf.ServerName = r.ServerName
}
d := &net.Dialer{Timeout: 10 * time.Second}
var conn net.Conn
d := &net.Dialer{Timeout: 10 * time.Second, Deadline: setupEnd}
var raw net.Conn
if mode == TLSImplicit {
conn, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr)
raw, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr)
} else {
conn, err = d.DialContext(ctx, "tcp", r.Addr)
raw, err = d.DialContext(ctx, "tcp", r.Addr)
}
if err != nil {
return nil, nil, err
}
// A relay that stops answering must not hold us past the context: apply its deadline to the
// connection and close the connection if the context is cancelled.
if dl, ok := ctx.Deadline(); ok {
conn.SetDeadline(dl)
return nil, nil, nil, err
}
// A relay that stops answering must not hold us past the budget or the context: apply the
// earlier deadline to the connection, and close the connection if the context is cancelled.
conn = &phasedConn{Conn: raw}
conn.phase(time.Until(setupEnd))
stop := make(chan struct{})
go func() {
select {
@ -113,10 +166,10 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e
}
}()
release = func() { close(stop) }
fail := func(err error) (*smtp.Client, func(), error) {
fail := func(err error) (*smtp.Client, *phasedConn, func(), error) {
release()
conn.Close()
return nil, nil, err
return nil, nil, nil, err
}
if mode == TLSStartTLS {
@ -139,7 +192,7 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e
return fail(authError(err))
}
}
return c, release, nil
return c, conn, release, nil
}
func isLoopback(host string) bool {
@ -162,7 +215,7 @@ func authError(err error) error {
}
func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) (err error) {
c, release, err := r.dial(ctx)
c, conn, release, err := r.dial(ctx)
if err != nil {
if ctx.Err() != nil {
err = ctx.Err() // report the cause, not "use of closed connection"
@ -194,13 +247,23 @@ func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, dat
if err != nil {
return authError(&PhaseError{Phase: "DATA", Err: err})
}
conn.phase(uploadBudget(len(data)))
if _, err := io.Copy(w, bytes.NewReader(data)); err != nil {
return &PhaseError{Phase: "DATA", Err: err}
}
// Close sends the rest of the message and the final dot, then waits for the verdict.
conn.phase(finalReplyTimeout)
if err := w.Close(); err != nil {
var ne net.Error
if errors.As(err, &ne) && ne.Timeout() {
// Temporary like any timeout, but the relay may have taken the message: a retry can
// deliver it twice. Say so, so the admin can tell.
err = fmt.Errorf("no reply to the end of the message, outcome unknown: %w", err)
}
return &PhaseError{Phase: "DATA", Err: err}
}
// The relay has accepted the message. If QUIT now fails, retrying would deliver it twice.
conn.phase(r.timeout())
c.Quit()
return nil
}

View file

@ -0,0 +1,231 @@
package outbound
import (
"bufio"
"context"
"errors"
"net"
"strings"
"sync"
"testing"
"time"
)
// scaleTimeouts shrinks the phase budgets so the tests run in milliseconds.
func scaleTimeouts(t *testing.T, floor, finalReply time.Duration) {
t.Helper()
oldFloor, oldRate, oldCap, oldReply := dataFloor, dataRate, dataCap, finalReplyTimeout
dataFloor, finalReplyTimeout = floor, finalReply
t.Cleanup(func() { dataFloor, dataRate, dataCap, finalReplyTimeout = oldFloor, oldRate, oldCap, oldReply })
}
// slowRelay is a bare SMTP server that waits dotDelay before answering the final dot
// (forever when dotDelay is negative).
type slowRelay struct {
mu sync.Mutex
accepted int
}
func startSlowRelay(t *testing.T, dotDelay time.Duration) (*slowRelay, string) {
t.Helper()
s := &slowRelay{}
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
done := make(chan struct{})
t.Cleanup(func() { close(done); ln.Close() })
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
go func() {
defer c.Close()
r := bufio.NewReader(c)
c.Write([]byte("220 slow ESMTP\r\n"))
inData := false
for {
l, err := r.ReadString('\n')
if err != nil {
return
}
l = strings.TrimRight(l, "\r\n")
switch {
case inData:
if l != "." {
continue
}
inData = false
if dotDelay < 0 {
<-done
return
}
time.Sleep(dotDelay)
s.mu.Lock()
s.accepted++
s.mu.Unlock()
c.Write([]byte("250 2.0.0 queued\r\n"))
case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"):
c.Write([]byte("250 slow\r\n"))
case strings.HasPrefix(l, "MAIL"), strings.HasPrefix(l, "RCPT"):
c.Write([]byte("250 2.1.0 ok\r\n"))
case l == "DATA":
inData = true
c.Write([]byte("354 go ahead\r\n"))
case l == "QUIT":
c.Write([]byte("221 bye\r\n"))
return
}
}
}()
}
}()
return s, ln.Addr().String()
}
func TestASlowReplyToTheFinalDotIsWaitedForAndDeliversOnce(t *testing.T) {
// The relay takes longer than the whole setup budget to accept the message (a content filter):
// that is not a failure, or the retry would deliver it a second time.
scaleTimeouts(t, 100*time.Millisecond, 3*time.Second)
s, addr := startSlowRelay(t, 400*time.Millisecond)
r := &SMTPRelay{Addr: addr, Timeout: 100 * time.Millisecond}
if err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")); err != nil {
t.Fatalf("the relay accepted the message: %v", err)
}
s.mu.Lock()
defer s.mu.Unlock()
if s.accepted != 1 {
t.Fatalf("accepted %d times", s.accepted)
}
}
func TestNoReplyToTheFinalDotTimesOutAsATemporaryFailure(t *testing.T) {
scaleTimeouts(t, 100*time.Millisecond, 300*time.Millisecond)
_, addr := startSlowRelay(t, -1)
r := &SMTPRelay{Addr: addr, Timeout: 5 * time.Second}
start := time.Now()
err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n"))
var ne net.Error
if !errors.As(err, &ne) || !ne.Timeout() {
t.Fatalf("want a timeout, got %v", err)
}
if IsPermanent(err) {
t.Fatalf("an unknown outcome is retried, not failed: %v", err)
}
if !strings.Contains(err.Error(), "outcome unknown") {
t.Fatalf("the error says the relay may have the message: %v", err)
}
if took := time.Since(start); took < 250*time.Millisecond || took > 3*time.Second {
t.Fatalf("the final reply has its own budget, took %v", took)
}
}
func TestARelayThatStallsAtTheGreetingTimesOutInTheShortBudget(t *testing.T) {
r := &SMTPRelay{Addr: silent(t), Timeout: 150 * time.Millisecond}
start := time.Now()
err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("x"))
var ne net.Error
if !errors.As(err, &ne) || !ne.Timeout() {
t.Fatalf("want a timeout, got %v", err)
}
if IsPermanent(err) {
t.Fatalf("a timeout is temporary: %v", err)
}
if took := time.Since(start); took > 2*time.Second {
t.Fatalf("took %v", took)
}
}
func TestTheUploadBudgetGrowsWithTheMessageSize(t *testing.T) {
if got := uploadBudget(0); got != time.Minute {
t.Fatalf("floor: %v", got)
}
small, big := uploadBudget(10<<10), uploadBudget(10<<20)
if big <= small {
t.Fatalf("a big message gets more time: small %v, big %v", small, big)
}
if want := time.Minute + 320*time.Second; big != want { // 10 MiB at 32 KiB/s
t.Fatalf("10 MiB: got %v, want %v", big, want)
}
if got := uploadBudget(1 << 40); got != 10*time.Minute {
t.Fatalf("cap: %v", got)
}
}
func TestCancellingTheContextAbortsTheWaitForTheFinalReply(t *testing.T) {
_, addr := startSlowRelay(t, -1) // the full 10 minutes apply
ctx, cancel := context.WithCancel(context.Background())
errc := make(chan error, 1)
go func() {
errc <- (&SMTPRelay{Addr: addr}).Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n"))
}()
time.Sleep(200 * time.Millisecond)
cancel()
select {
case err := <-errc:
if !errors.Is(err, context.Canceled) {
t.Fatalf("want context.Canceled, got %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("Send ignored the cancelled context while waiting for the final reply")
}
}
// stopsReadingAtData answers up to DATA, then never reads again.
func stopsReadingAtData(t *testing.T) string {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
done := make(chan struct{})
t.Cleanup(func() { close(done); ln.Close() })
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
go func() {
defer c.Close()
r := bufio.NewReader(c)
c.Write([]byte("220 stuck ESMTP\r\n"))
for {
l, err := r.ReadString('\n')
if err != nil {
return
}
switch l = strings.TrimRight(l, "\r\n"); {
case l == "DATA":
c.Write([]byte("354 go ahead\r\n"))
<-done
return
case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"):
c.Write([]byte("250 stuck\r\n"))
default:
c.Write([]byte("250 ok\r\n"))
}
}
}()
}
}()
return ln.Addr().String()
}
func TestAStalledUploadTimesOutInItsOwnBudget(t *testing.T) {
scaleTimeouts(t, 200*time.Millisecond, time.Minute)
dataRate = 1 << 40 // the budget is the floor
r := &SMTPRelay{Addr: stopsReadingAtData(t), Timeout: time.Minute}
big := []byte(strings.Repeat(strings.Repeat("x", 998)+"\r\n", 32<<10)) // 32 MiB, more than the socket buffers
start := time.Now()
err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, big)
var ne net.Error
if !errors.As(err, &ne) || !ne.Timeout() || IsPermanent(err) {
t.Fatalf("want a temporary timeout, got %v", err)
}
if took := time.Since(start); took > 5*time.Second {
t.Fatalf("the upload has its own budget, took %v", took)
}
}

View file

@ -20,7 +20,6 @@ const MaxAge = 72 * time.Hour
const (
defaultConcurrency = 4
defaultBatch = 50
defaultTimeout = time.Minute
defaultInterval = 5 * time.Second
// purgeEvery is the least time between two scans that forget old failed deliveries.
@ -55,7 +54,7 @@ type Worker struct {
Sender outbound.Sender
Concurrency int // simultaneous sends
Batch int // deliveries taken per run
Timeout time.Duration // per send
Timeout time.Duration // per send; zero leaves the deadlines to Sender (SMTPRelay bounds each SMTP phase)
Interval time.Duration // idle polling, to pick up retries that have become due
// Sign, when set, signs the final message (DKIM). On error the message is sent unsigned:
// not delivering is worse than a missed signature.
@ -245,15 +244,16 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem
}
}
timeout := w.Timeout
if timeout <= 0 {
timeout = defaultTimeout
}
// Wait for the slot before starting the timeout, so waiting does not eat into the send.
if err := w.pace().Wait(ctx); err != nil {
return Stats{} // shutting down: the delivery stays queued, unchanged
}
sctx, cancel := context.WithTimeout(ctx, timeout)
// No total deadline by default: one that covers the whole transaction cut relays that take
// minutes to accept a message, which was then retried and delivered twice.
sctx, cancel := ctx, context.CancelFunc(func() {})
if w.Timeout > 0 {
sctx, cancel = context.WithTimeout(ctx, w.Timeout)
}
err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg)
cancel()

View file

@ -405,6 +405,24 @@ func TestSlowRelayTimesOutAndIsRetried(t *testing.T) {
}
}
func TestTheQueueLeavesTheSendDeadlinesToTheSender(t *testing.T) {
// One total deadline cut a relay that took over a minute to answer the final dot: the message
// was accepted, then retried and delivered again. The sender bounds each SMTP phase itself.
e := setup(t)
e.enqueue(t, 1)
var hasDeadline bool
e.snd.fn = func(ctx context.Context, _ string, _ []string) error {
_, hasDeadline = ctx.Deadline()
return nil
}
if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 {
t.Fatalf("sent: %+v", st)
}
if hasDeadline {
t.Fatal("the queue must not put a deadline on the whole send")
}
}
func waitFor(t *testing.T, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(3 * time.Second)

View file

@ -717,3 +717,25 @@ func TestConfirmPendingIsSingleUseUnderConcurrency(t *testing.T) {
}
}
}
func TestMigrationAddsTheOutboxPriorityAloneAfterAHalfDoneUpgrade(t *testing.T) {
// A crash between adding post_id and priority leaves outbox with the first and not the second.
path := t.TempDir() + "/old.db"
s, _ := Open(path)
if _, err := s.db.Exec(`DROP INDEX outbox_due_prio`); err != nil {
t.Fatal(err)
}
if _, err := s.db.Exec(`ALTER TABLE outbox DROP COLUMN priority`); err != nil {
t.Fatal(err)
}
s.Close()
s2, err := Open(path)
if err != nil {
t.Fatalf("migrate: %v", err)
}
defer s2.Close()
if ok, err := hasColumn(s2.db, "outbox", "priority"); err != nil || !ok {
t.Fatalf("priority is added back: %v %v", ok, err)
}
}

View file

@ -325,8 +325,12 @@ func migrate(db *sql.DB) error {
}
}
if has, _ := hasColumn(db, "outbox", "post_id"); !has {
if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN post_id INTEGER NOT NULL DEFAULT 0;
ALTER TABLE outbox ADD COLUMN priority INTEGER NOT NULL DEFAULT 0;`); err != nil {
if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN post_id INTEGER NOT NULL DEFAULT 0`); err != nil {
return err
}
}
if has, _ := hasColumn(db, "outbox", "priority"); !has {
if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN priority INTEGER NOT NULL DEFAULT 0`); err != nil {
return err
}
}