From f14faf52d538bc327afed0556591d706d16a9e6f Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 15:20:49 +0100 Subject: [PATCH] fix(outbound): give each SMTP phase its own deadline A relay can take minutes to answer the final dot (content filter, large message, slow link). One deadline over the whole transaction cut it after the message had been accepted, so the delivery was retried and sent again. The connection now moves its deadline as the transaction advances: setup (dial, greeting, TLS, AUTH, MAIL, RCPT, DATA) has SMTPRelay.Timeout, 60 s by default; the upload has 60 s plus the size at 32 KiB/s, at most 10 minutes; the reply to the final dot has 10 minutes (RFC 5321 4.5.3.2.6). go-smtp sets and clears its own per-command deadlines, so a wrapper clips them to the current phase. Context cancellation still closes the connection at once. A timeout in any phase is temporary; after the final dot the error says the outcome is unknown. --- internal/outbound/outbound.go | 101 ++++++++++--- internal/outbound/timeout_test.go | 231 ++++++++++++++++++++++++++++++ 2 files changed, 313 insertions(+), 19 deletions(-) create mode 100644 internal/outbound/timeout_test.go diff --git a/internal/outbound/outbound.go b/internal/outbound/outbound.go index 09ebcaa..c2de797 100644 --- a/internal/outbound/outbound.go +++ b/internal/outbound/outbound.go @@ -45,6 +45,59 @@ type SMTPRelay struct { Username string // SMTP AUTH PLAIN; only over TLS or to a loopback address Password string Helo string // EHLO name; "localhost" by default + // Timeout bounds the setup of each transaction: connection, greeting, TLS, AUTH, MAIL FROM, + // RCPT TO and the DATA command. DefaultTimeout if zero. The upload of the message and the + // wait for its acceptance have their own budgets (see uploadBudget and finalReplyTimeout). + Timeout time.Duration +} + +// DefaultTimeout is the default SMTPRelay.Timeout. +const DefaultTimeout = time.Minute + +// Budgets of the later phases of a transaction. Variables so that tests can shrink them. +var ( + dataFloor = time.Minute // least time to upload a message + dataRate = 32 << 10 // slowest upload we wait for, in bytes per second + dataCap = 10 * time.Minute // most time to upload a message + // finalReplyTimeout is how long the relay may take to answer the final dot: RFC 5321 4.5.3.2.6 + // asks for 10 minutes, because content filters run at that point. + finalReplyTimeout = 10 * time.Minute +) + +// uploadBudget is the time allowed to upload a message of size bytes: dataFloor plus the time +// it takes at dataRate, at most dataCap. +func uploadBudget(size int) time.Duration { + b := dataFloor + time.Duration(float64(size)/float64(dataRate)*float64(time.Second)) + return min(b, dataCap) +} + +func (r *SMTPRelay) timeout() time.Duration { + if r.Timeout > 0 { + return r.Timeout + } + return DefaultTimeout +} + +// phasedConn keeps the connection within the budget of the current phase. go-smtp sets its own +// deadline around each command and clears it afterwards; here no deadline it sets, cleared or not, +// may go past the limit of the phase. +type phasedConn struct { + net.Conn + limit time.Time +} + +func (p *phasedConn) SetDeadline(t time.Time) error { + if t.IsZero() || p.limit.Before(t) { + t = p.limit + } + return p.Conn.SetDeadline(t) +} + +// phase gives the connection d to finish the next phase. The context's own deadline and its +// cancellation close the connection instead (see dial), so that the error is the context's. +func (p *phasedConn) phase(d time.Duration) { + p.limit = time.Now().Add(d) + p.Conn.SetDeadline(p.limit) } // Send splits recipients in batches and sends one transaction per batch. @@ -65,20 +118,21 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b return nil } -// dial opens the connection and speaks EHLO, TLS and AUTH as configured. -// The returned release function must be called when the client is done: until then the connection +// dial opens the connection and speaks EHLO, TLS and AUTH as configured. The connection's deadline +// is the end of the setup budget, which starts now. The returned release function must be called when the client is done: until then the connection // is closed if ctx is cancelled, even in the middle of EHLO, STARTTLS or AUTH with a silent relay. -func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), err error) { +func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, conn *phasedConn, release func(), err error) { + setupEnd := time.Now().Add(r.timeout()) host, _, err := net.SplitHostPort(r.Addr) if err != nil { - return nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err) + return nil, nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err) } mode := r.TLS if mode == "" { mode = TLSNone } if r.Username != "" && mode == TLSNone && !isLoopback(host) { - return nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls") + return nil, nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls") } helo := r.Helo if helo == "" { @@ -89,21 +143,20 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e tlsConf.ServerName = r.ServerName } - d := &net.Dialer{Timeout: 10 * time.Second} - var conn net.Conn + d := &net.Dialer{Timeout: 10 * time.Second, Deadline: setupEnd} + var raw net.Conn if mode == TLSImplicit { - conn, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr) + raw, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr) } else { - conn, err = d.DialContext(ctx, "tcp", r.Addr) + raw, err = d.DialContext(ctx, "tcp", r.Addr) } if err != nil { - return nil, nil, err - } - // A relay that stops answering must not hold us past the context: apply its deadline to the - // connection and close the connection if the context is cancelled. - if dl, ok := ctx.Deadline(); ok { - conn.SetDeadline(dl) + return nil, nil, nil, err } + // A relay that stops answering must not hold us past the budget or the context: apply the + // earlier deadline to the connection, and close the connection if the context is cancelled. + conn = &phasedConn{Conn: raw} + conn.phase(time.Until(setupEnd)) stop := make(chan struct{}) go func() { select { @@ -113,10 +166,10 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e } }() release = func() { close(stop) } - fail := func(err error) (*smtp.Client, func(), error) { + fail := func(err error) (*smtp.Client, *phasedConn, func(), error) { release() conn.Close() - return nil, nil, err + return nil, nil, nil, err } if mode == TLSStartTLS { @@ -139,7 +192,7 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e return fail(authError(err)) } } - return c, release, nil + return c, conn, release, nil } func isLoopback(host string) bool { @@ -162,7 +215,7 @@ func authError(err error) error { } func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) (err error) { - c, release, err := r.dial(ctx) + c, conn, release, err := r.dial(ctx) if err != nil { if ctx.Err() != nil { err = ctx.Err() // report the cause, not "use of closed connection" @@ -194,13 +247,23 @@ func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, dat if err != nil { return authError(&PhaseError{Phase: "DATA", Err: err}) } + conn.phase(uploadBudget(len(data))) if _, err := io.Copy(w, bytes.NewReader(data)); err != nil { return &PhaseError{Phase: "DATA", Err: err} } + // Close sends the rest of the message and the final dot, then waits for the verdict. + conn.phase(finalReplyTimeout) if err := w.Close(); err != nil { + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + // Temporary like any timeout, but the relay may have taken the message: a retry can + // deliver it twice. Say so, so the admin can tell. + err = fmt.Errorf("no reply to the end of the message, outcome unknown: %w", err) + } return &PhaseError{Phase: "DATA", Err: err} } // The relay has accepted the message. If QUIT now fails, retrying would deliver it twice. + conn.phase(r.timeout()) c.Quit() return nil } diff --git a/internal/outbound/timeout_test.go b/internal/outbound/timeout_test.go new file mode 100644 index 0000000..b1103d9 --- /dev/null +++ b/internal/outbound/timeout_test.go @@ -0,0 +1,231 @@ +package outbound + +import ( + "bufio" + "context" + "errors" + "net" + "strings" + "sync" + "testing" + "time" +) + +// scaleTimeouts shrinks the phase budgets so the tests run in milliseconds. +func scaleTimeouts(t *testing.T, floor, finalReply time.Duration) { + t.Helper() + oldFloor, oldRate, oldCap, oldReply := dataFloor, dataRate, dataCap, finalReplyTimeout + dataFloor, finalReplyTimeout = floor, finalReply + t.Cleanup(func() { dataFloor, dataRate, dataCap, finalReplyTimeout = oldFloor, oldRate, oldCap, oldReply }) +} + +// slowRelay is a bare SMTP server that waits dotDelay before answering the final dot +// (forever when dotDelay is negative). +type slowRelay struct { + mu sync.Mutex + accepted int +} + +func startSlowRelay(t *testing.T, dotDelay time.Duration) (*slowRelay, string) { + t.Helper() + s := &slowRelay{} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + t.Cleanup(func() { close(done); ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + r := bufio.NewReader(c) + c.Write([]byte("220 slow ESMTP\r\n")) + inData := false + for { + l, err := r.ReadString('\n') + if err != nil { + return + } + l = strings.TrimRight(l, "\r\n") + switch { + case inData: + if l != "." { + continue + } + inData = false + if dotDelay < 0 { + <-done + return + } + time.Sleep(dotDelay) + s.mu.Lock() + s.accepted++ + s.mu.Unlock() + c.Write([]byte("250 2.0.0 queued\r\n")) + case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"): + c.Write([]byte("250 slow\r\n")) + case strings.HasPrefix(l, "MAIL"), strings.HasPrefix(l, "RCPT"): + c.Write([]byte("250 2.1.0 ok\r\n")) + case l == "DATA": + inData = true + c.Write([]byte("354 go ahead\r\n")) + case l == "QUIT": + c.Write([]byte("221 bye\r\n")) + return + } + } + }() + } + }() + return s, ln.Addr().String() +} + +func TestASlowReplyToTheFinalDotIsWaitedForAndDeliversOnce(t *testing.T) { + // The relay takes longer than the whole setup budget to accept the message (a content filter): + // that is not a failure, or the retry would deliver it a second time. + scaleTimeouts(t, 100*time.Millisecond, 3*time.Second) + s, addr := startSlowRelay(t, 400*time.Millisecond) + r := &SMTPRelay{Addr: addr, Timeout: 100 * time.Millisecond} + if err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")); err != nil { + t.Fatalf("the relay accepted the message: %v", err) + } + s.mu.Lock() + defer s.mu.Unlock() + if s.accepted != 1 { + t.Fatalf("accepted %d times", s.accepted) + } +} + +func TestNoReplyToTheFinalDotTimesOutAsATemporaryFailure(t *testing.T) { + scaleTimeouts(t, 100*time.Millisecond, 300*time.Millisecond) + _, addr := startSlowRelay(t, -1) + r := &SMTPRelay{Addr: addr, Timeout: 5 * time.Second} + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() { + t.Fatalf("want a timeout, got %v", err) + } + if IsPermanent(err) { + t.Fatalf("an unknown outcome is retried, not failed: %v", err) + } + if !strings.Contains(err.Error(), "outcome unknown") { + t.Fatalf("the error says the relay may have the message: %v", err) + } + if took := time.Since(start); took < 250*time.Millisecond || took > 3*time.Second { + t.Fatalf("the final reply has its own budget, took %v", took) + } +} + +func TestARelayThatStallsAtTheGreetingTimesOutInTheShortBudget(t *testing.T) { + r := &SMTPRelay{Addr: silent(t), Timeout: 150 * time.Millisecond} + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("x")) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() { + t.Fatalf("want a timeout, got %v", err) + } + if IsPermanent(err) { + t.Fatalf("a timeout is temporary: %v", err) + } + if took := time.Since(start); took > 2*time.Second { + t.Fatalf("took %v", took) + } +} + +func TestTheUploadBudgetGrowsWithTheMessageSize(t *testing.T) { + if got := uploadBudget(0); got != time.Minute { + t.Fatalf("floor: %v", got) + } + small, big := uploadBudget(10<<10), uploadBudget(10<<20) + if big <= small { + t.Fatalf("a big message gets more time: small %v, big %v", small, big) + } + if want := time.Minute + 320*time.Second; big != want { // 10 MiB at 32 KiB/s + t.Fatalf("10 MiB: got %v, want %v", big, want) + } + if got := uploadBudget(1 << 40); got != 10*time.Minute { + t.Fatalf("cap: %v", got) + } +} + +func TestCancellingTheContextAbortsTheWaitForTheFinalReply(t *testing.T) { + _, addr := startSlowRelay(t, -1) // the full 10 minutes apply + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { + errc <- (&SMTPRelay{Addr: addr}).Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")) + }() + time.Sleep(200 * time.Millisecond) + cancel() + select { + case err := <-errc: + if !errors.Is(err, context.Canceled) { + t.Fatalf("want context.Canceled, got %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Send ignored the cancelled context while waiting for the final reply") + } +} + +// stopsReadingAtData answers up to DATA, then never reads again. +func stopsReadingAtData(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + t.Cleanup(func() { close(done); ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + r := bufio.NewReader(c) + c.Write([]byte("220 stuck ESMTP\r\n")) + for { + l, err := r.ReadString('\n') + if err != nil { + return + } + switch l = strings.TrimRight(l, "\r\n"); { + case l == "DATA": + c.Write([]byte("354 go ahead\r\n")) + <-done + return + case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"): + c.Write([]byte("250 stuck\r\n")) + default: + c.Write([]byte("250 ok\r\n")) + } + } + }() + } + }() + return ln.Addr().String() +} + +func TestAStalledUploadTimesOutInItsOwnBudget(t *testing.T) { + scaleTimeouts(t, 200*time.Millisecond, time.Minute) + dataRate = 1 << 40 // the budget is the floor + r := &SMTPRelay{Addr: stopsReadingAtData(t), Timeout: time.Minute} + big := []byte(strings.Repeat(strings.Repeat("x", 998)+"\r\n", 32<<10)) // 32 MiB, more than the socket buffers + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, big) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() || IsPermanent(err) { + t.Fatalf("want a temporary timeout, got %v", err) + } + if took := time.Since(start); took > 5*time.Second { + t.Fatalf("the upload has its own budget, took %v", took) + } +}