fix(outbound): give each SMTP phase its own deadline
A relay can take minutes to answer the final dot (content filter, large message, slow link). One deadline over the whole transaction cut it after the message had been accepted, so the delivery was retried and sent again. The connection now moves its deadline as the transaction advances: setup (dial, greeting, TLS, AUTH, MAIL, RCPT, DATA) has SMTPRelay.Timeout, 60 s by default; the upload has 60 s plus the size at 32 KiB/s, at most 10 minutes; the reply to the final dot has 10 minutes (RFC 5321 4.5.3.2.6). go-smtp sets and clears its own per-command deadlines, so a wrapper clips them to the current phase. Context cancellation still closes the connection at once. A timeout in any phase is temporary; after the final dot the error says the outcome is unknown.
This commit is contained in:
parent
c66fd427b2
commit
f14faf52d5
2 changed files with 313 additions and 19 deletions
|
|
@ -45,6 +45,59 @@ type SMTPRelay struct {
|
|||
Username string // SMTP AUTH PLAIN; only over TLS or to a loopback address
|
||||
Password string
|
||||
Helo string // EHLO name; "localhost" by default
|
||||
// Timeout bounds the setup of each transaction: connection, greeting, TLS, AUTH, MAIL FROM,
|
||||
// RCPT TO and the DATA command. DefaultTimeout if zero. The upload of the message and the
|
||||
// wait for its acceptance have their own budgets (see uploadBudget and finalReplyTimeout).
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// DefaultTimeout is the default SMTPRelay.Timeout.
|
||||
const DefaultTimeout = time.Minute
|
||||
|
||||
// Budgets of the later phases of a transaction. Variables so that tests can shrink them.
|
||||
var (
|
||||
dataFloor = time.Minute // least time to upload a message
|
||||
dataRate = 32 << 10 // slowest upload we wait for, in bytes per second
|
||||
dataCap = 10 * time.Minute // most time to upload a message
|
||||
// finalReplyTimeout is how long the relay may take to answer the final dot: RFC 5321 4.5.3.2.6
|
||||
// asks for 10 minutes, because content filters run at that point.
|
||||
finalReplyTimeout = 10 * time.Minute
|
||||
)
|
||||
|
||||
// uploadBudget is the time allowed to upload a message of size bytes: dataFloor plus the time
|
||||
// it takes at dataRate, at most dataCap.
|
||||
func uploadBudget(size int) time.Duration {
|
||||
b := dataFloor + time.Duration(float64(size)/float64(dataRate)*float64(time.Second))
|
||||
return min(b, dataCap)
|
||||
}
|
||||
|
||||
func (r *SMTPRelay) timeout() time.Duration {
|
||||
if r.Timeout > 0 {
|
||||
return r.Timeout
|
||||
}
|
||||
return DefaultTimeout
|
||||
}
|
||||
|
||||
// phasedConn keeps the connection within the budget of the current phase. go-smtp sets its own
|
||||
// deadline around each command and clears it afterwards; here no deadline it sets, cleared or not,
|
||||
// may go past the limit of the phase.
|
||||
type phasedConn struct {
|
||||
net.Conn
|
||||
limit time.Time
|
||||
}
|
||||
|
||||
func (p *phasedConn) SetDeadline(t time.Time) error {
|
||||
if t.IsZero() || p.limit.Before(t) {
|
||||
t = p.limit
|
||||
}
|
||||
return p.Conn.SetDeadline(t)
|
||||
}
|
||||
|
||||
// phase gives the connection d to finish the next phase. The context's own deadline and its
|
||||
// cancellation close the connection instead (see dial), so that the error is the context's.
|
||||
func (p *phasedConn) phase(d time.Duration) {
|
||||
p.limit = time.Now().Add(d)
|
||||
p.Conn.SetDeadline(p.limit)
|
||||
}
|
||||
|
||||
// Send splits recipients in batches and sends one transaction per batch.
|
||||
|
|
@ -65,20 +118,21 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b
|
|||
return nil
|
||||
}
|
||||
|
||||
// dial opens the connection and speaks EHLO, TLS and AUTH as configured.
|
||||
// The returned release function must be called when the client is done: until then the connection
|
||||
// dial opens the connection and speaks EHLO, TLS and AUTH as configured. The connection's deadline
|
||||
// is the end of the setup budget, which starts now. The returned release function must be called when the client is done: until then the connection
|
||||
// is closed if ctx is cancelled, even in the middle of EHLO, STARTTLS or AUTH with a silent relay.
|
||||
func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), err error) {
|
||||
func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, conn *phasedConn, release func(), err error) {
|
||||
setupEnd := time.Now().Add(r.timeout())
|
||||
host, _, err := net.SplitHostPort(r.Addr)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err)
|
||||
return nil, nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err)
|
||||
}
|
||||
mode := r.TLS
|
||||
if mode == "" {
|
||||
mode = TLSNone
|
||||
}
|
||||
if r.Username != "" && mode == TLSNone && !isLoopback(host) {
|
||||
return nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls")
|
||||
return nil, nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls")
|
||||
}
|
||||
helo := r.Helo
|
||||
if helo == "" {
|
||||
|
|
@ -89,21 +143,20 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e
|
|||
tlsConf.ServerName = r.ServerName
|
||||
}
|
||||
|
||||
d := &net.Dialer{Timeout: 10 * time.Second}
|
||||
var conn net.Conn
|
||||
d := &net.Dialer{Timeout: 10 * time.Second, Deadline: setupEnd}
|
||||
var raw net.Conn
|
||||
if mode == TLSImplicit {
|
||||
conn, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr)
|
||||
raw, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr)
|
||||
} else {
|
||||
conn, err = d.DialContext(ctx, "tcp", r.Addr)
|
||||
raw, err = d.DialContext(ctx, "tcp", r.Addr)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// A relay that stops answering must not hold us past the context: apply its deadline to the
|
||||
// connection and close the connection if the context is cancelled.
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
conn.SetDeadline(dl)
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
// A relay that stops answering must not hold us past the budget or the context: apply the
|
||||
// earlier deadline to the connection, and close the connection if the context is cancelled.
|
||||
conn = &phasedConn{Conn: raw}
|
||||
conn.phase(time.Until(setupEnd))
|
||||
stop := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
|
|
@ -113,10 +166,10 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e
|
|||
}
|
||||
}()
|
||||
release = func() { close(stop) }
|
||||
fail := func(err error) (*smtp.Client, func(), error) {
|
||||
fail := func(err error) (*smtp.Client, *phasedConn, func(), error) {
|
||||
release()
|
||||
conn.Close()
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
if mode == TLSStartTLS {
|
||||
|
|
@ -139,7 +192,7 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e
|
|||
return fail(authError(err))
|
||||
}
|
||||
}
|
||||
return c, release, nil
|
||||
return c, conn, release, nil
|
||||
}
|
||||
|
||||
func isLoopback(host string) bool {
|
||||
|
|
@ -162,7 +215,7 @@ func authError(err error) error {
|
|||
}
|
||||
|
||||
func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) (err error) {
|
||||
c, release, err := r.dial(ctx)
|
||||
c, conn, release, err := r.dial(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
err = ctx.Err() // report the cause, not "use of closed connection"
|
||||
|
|
@ -194,13 +247,23 @@ func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, dat
|
|||
if err != nil {
|
||||
return authError(&PhaseError{Phase: "DATA", Err: err})
|
||||
}
|
||||
conn.phase(uploadBudget(len(data)))
|
||||
if _, err := io.Copy(w, bytes.NewReader(data)); err != nil {
|
||||
return &PhaseError{Phase: "DATA", Err: err}
|
||||
}
|
||||
// Close sends the rest of the message and the final dot, then waits for the verdict.
|
||||
conn.phase(finalReplyTimeout)
|
||||
if err := w.Close(); err != nil {
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() {
|
||||
// Temporary like any timeout, but the relay may have taken the message: a retry can
|
||||
// deliver it twice. Say so, so the admin can tell.
|
||||
err = fmt.Errorf("no reply to the end of the message, outcome unknown: %w", err)
|
||||
}
|
||||
return &PhaseError{Phase: "DATA", Err: err}
|
||||
}
|
||||
// The relay has accepted the message. If QUIT now fails, retrying would deliver it twice.
|
||||
conn.phase(r.timeout())
|
||||
c.Quit()
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
231
internal/outbound/timeout_test.go
Normal file
231
internal/outbound/timeout_test.go
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
package outbound
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// scaleTimeouts shrinks the phase budgets so the tests run in milliseconds.
|
||||
func scaleTimeouts(t *testing.T, floor, finalReply time.Duration) {
|
||||
t.Helper()
|
||||
oldFloor, oldRate, oldCap, oldReply := dataFloor, dataRate, dataCap, finalReplyTimeout
|
||||
dataFloor, finalReplyTimeout = floor, finalReply
|
||||
t.Cleanup(func() { dataFloor, dataRate, dataCap, finalReplyTimeout = oldFloor, oldRate, oldCap, oldReply })
|
||||
}
|
||||
|
||||
// slowRelay is a bare SMTP server that waits dotDelay before answering the final dot
|
||||
// (forever when dotDelay is negative).
|
||||
type slowRelay struct {
|
||||
mu sync.Mutex
|
||||
accepted int
|
||||
}
|
||||
|
||||
func startSlowRelay(t *testing.T, dotDelay time.Duration) (*slowRelay, string) {
|
||||
t.Helper()
|
||||
s := &slowRelay{}
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
t.Cleanup(func() { close(done); ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer c.Close()
|
||||
r := bufio.NewReader(c)
|
||||
c.Write([]byte("220 slow ESMTP\r\n"))
|
||||
inData := false
|
||||
for {
|
||||
l, err := r.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
l = strings.TrimRight(l, "\r\n")
|
||||
switch {
|
||||
case inData:
|
||||
if l != "." {
|
||||
continue
|
||||
}
|
||||
inData = false
|
||||
if dotDelay < 0 {
|
||||
<-done
|
||||
return
|
||||
}
|
||||
time.Sleep(dotDelay)
|
||||
s.mu.Lock()
|
||||
s.accepted++
|
||||
s.mu.Unlock()
|
||||
c.Write([]byte("250 2.0.0 queued\r\n"))
|
||||
case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"):
|
||||
c.Write([]byte("250 slow\r\n"))
|
||||
case strings.HasPrefix(l, "MAIL"), strings.HasPrefix(l, "RCPT"):
|
||||
c.Write([]byte("250 2.1.0 ok\r\n"))
|
||||
case l == "DATA":
|
||||
inData = true
|
||||
c.Write([]byte("354 go ahead\r\n"))
|
||||
case l == "QUIT":
|
||||
c.Write([]byte("221 bye\r\n"))
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
return s, ln.Addr().String()
|
||||
}
|
||||
|
||||
func TestASlowReplyToTheFinalDotIsWaitedForAndDeliversOnce(t *testing.T) {
|
||||
// The relay takes longer than the whole setup budget to accept the message (a content filter):
|
||||
// that is not a failure, or the retry would deliver it a second time.
|
||||
scaleTimeouts(t, 100*time.Millisecond, 3*time.Second)
|
||||
s, addr := startSlowRelay(t, 400*time.Millisecond)
|
||||
r := &SMTPRelay{Addr: addr, Timeout: 100 * time.Millisecond}
|
||||
if err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")); err != nil {
|
||||
t.Fatalf("the relay accepted the message: %v", err)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.accepted != 1 {
|
||||
t.Fatalf("accepted %d times", s.accepted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoReplyToTheFinalDotTimesOutAsATemporaryFailure(t *testing.T) {
|
||||
scaleTimeouts(t, 100*time.Millisecond, 300*time.Millisecond)
|
||||
_, addr := startSlowRelay(t, -1)
|
||||
r := &SMTPRelay{Addr: addr, Timeout: 5 * time.Second}
|
||||
start := time.Now()
|
||||
err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n"))
|
||||
var ne net.Error
|
||||
if !errors.As(err, &ne) || !ne.Timeout() {
|
||||
t.Fatalf("want a timeout, got %v", err)
|
||||
}
|
||||
if IsPermanent(err) {
|
||||
t.Fatalf("an unknown outcome is retried, not failed: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "outcome unknown") {
|
||||
t.Fatalf("the error says the relay may have the message: %v", err)
|
||||
}
|
||||
if took := time.Since(start); took < 250*time.Millisecond || took > 3*time.Second {
|
||||
t.Fatalf("the final reply has its own budget, took %v", took)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARelayThatStallsAtTheGreetingTimesOutInTheShortBudget(t *testing.T) {
|
||||
r := &SMTPRelay{Addr: silent(t), Timeout: 150 * time.Millisecond}
|
||||
start := time.Now()
|
||||
err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("x"))
|
||||
var ne net.Error
|
||||
if !errors.As(err, &ne) || !ne.Timeout() {
|
||||
t.Fatalf("want a timeout, got %v", err)
|
||||
}
|
||||
if IsPermanent(err) {
|
||||
t.Fatalf("a timeout is temporary: %v", err)
|
||||
}
|
||||
if took := time.Since(start); took > 2*time.Second {
|
||||
t.Fatalf("took %v", took)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheUploadBudgetGrowsWithTheMessageSize(t *testing.T) {
|
||||
if got := uploadBudget(0); got != time.Minute {
|
||||
t.Fatalf("floor: %v", got)
|
||||
}
|
||||
small, big := uploadBudget(10<<10), uploadBudget(10<<20)
|
||||
if big <= small {
|
||||
t.Fatalf("a big message gets more time: small %v, big %v", small, big)
|
||||
}
|
||||
if want := time.Minute + 320*time.Second; big != want { // 10 MiB at 32 KiB/s
|
||||
t.Fatalf("10 MiB: got %v, want %v", big, want)
|
||||
}
|
||||
if got := uploadBudget(1 << 40); got != 10*time.Minute {
|
||||
t.Fatalf("cap: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancellingTheContextAbortsTheWaitForTheFinalReply(t *testing.T) {
|
||||
_, addr := startSlowRelay(t, -1) // the full 10 minutes apply
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
errc := make(chan error, 1)
|
||||
go func() {
|
||||
errc <- (&SMTPRelay{Addr: addr}).Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n"))
|
||||
}()
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-errc:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("want context.Canceled, got %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Send ignored the cancelled context while waiting for the final reply")
|
||||
}
|
||||
}
|
||||
|
||||
// stopsReadingAtData answers up to DATA, then never reads again.
|
||||
func stopsReadingAtData(t *testing.T) string {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
t.Cleanup(func() { close(done); ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer c.Close()
|
||||
r := bufio.NewReader(c)
|
||||
c.Write([]byte("220 stuck ESMTP\r\n"))
|
||||
for {
|
||||
l, err := r.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
switch l = strings.TrimRight(l, "\r\n"); {
|
||||
case l == "DATA":
|
||||
c.Write([]byte("354 go ahead\r\n"))
|
||||
<-done
|
||||
return
|
||||
case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"):
|
||||
c.Write([]byte("250 stuck\r\n"))
|
||||
default:
|
||||
c.Write([]byte("250 ok\r\n"))
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
return ln.Addr().String()
|
||||
}
|
||||
|
||||
func TestAStalledUploadTimesOutInItsOwnBudget(t *testing.T) {
|
||||
scaleTimeouts(t, 200*time.Millisecond, time.Minute)
|
||||
dataRate = 1 << 40 // the budget is the floor
|
||||
r := &SMTPRelay{Addr: stopsReadingAtData(t), Timeout: time.Minute}
|
||||
big := []byte(strings.Repeat(strings.Repeat("x", 998)+"\r\n", 32<<10)) // 32 MiB, more than the socket buffers
|
||||
start := time.Now()
|
||||
err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, big)
|
||||
var ne net.Error
|
||||
if !errors.As(err, &ne) || !ne.Timeout() || IsPermanent(err) {
|
||||
t.Fatalf("want a temporary timeout, got %v", err)
|
||||
}
|
||||
if took := time.Since(start); took > 5*time.Second {
|
||||
t.Fatalf("the upload has its own budget, took %v", took)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue