diff --git a/internal/outbound/outbound.go b/internal/outbound/outbound.go index 09ebcaa..c2de797 100644 --- a/internal/outbound/outbound.go +++ b/internal/outbound/outbound.go @@ -45,6 +45,59 @@ type SMTPRelay struct { Username string // SMTP AUTH PLAIN; only over TLS or to a loopback address Password string Helo string // EHLO name; "localhost" by default + // Timeout bounds the setup of each transaction: connection, greeting, TLS, AUTH, MAIL FROM, + // RCPT TO and the DATA command. DefaultTimeout if zero. The upload of the message and the + // wait for its acceptance have their own budgets (see uploadBudget and finalReplyTimeout). + Timeout time.Duration +} + +// DefaultTimeout is the default SMTPRelay.Timeout. +const DefaultTimeout = time.Minute + +// Budgets of the later phases of a transaction. Variables so that tests can shrink them. +var ( + dataFloor = time.Minute // least time to upload a message + dataRate = 32 << 10 // slowest upload we wait for, in bytes per second + dataCap = 10 * time.Minute // most time to upload a message + // finalReplyTimeout is how long the relay may take to answer the final dot: RFC 5321 4.5.3.2.6 + // asks for 10 minutes, because content filters run at that point. + finalReplyTimeout = 10 * time.Minute +) + +// uploadBudget is the time allowed to upload a message of size bytes: dataFloor plus the time +// it takes at dataRate, at most dataCap. +func uploadBudget(size int) time.Duration { + b := dataFloor + time.Duration(float64(size)/float64(dataRate)*float64(time.Second)) + return min(b, dataCap) +} + +func (r *SMTPRelay) timeout() time.Duration { + if r.Timeout > 0 { + return r.Timeout + } + return DefaultTimeout +} + +// phasedConn keeps the connection within the budget of the current phase. go-smtp sets its own +// deadline around each command and clears it afterwards; here no deadline it sets, cleared or not, +// may go past the limit of the phase. +type phasedConn struct { + net.Conn + limit time.Time +} + +func (p *phasedConn) SetDeadline(t time.Time) error { + if t.IsZero() || p.limit.Before(t) { + t = p.limit + } + return p.Conn.SetDeadline(t) +} + +// phase gives the connection d to finish the next phase. The context's own deadline and its +// cancellation close the connection instead (see dial), so that the error is the context's. +func (p *phasedConn) phase(d time.Duration) { + p.limit = time.Now().Add(d) + p.Conn.SetDeadline(p.limit) } // Send splits recipients in batches and sends one transaction per batch. @@ -65,20 +118,21 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b return nil } -// dial opens the connection and speaks EHLO, TLS and AUTH as configured. -// The returned release function must be called when the client is done: until then the connection +// dial opens the connection and speaks EHLO, TLS and AUTH as configured. The connection's deadline +// is the end of the setup budget, which starts now. The returned release function must be called when the client is done: until then the connection // is closed if ctx is cancelled, even in the middle of EHLO, STARTTLS or AUTH with a silent relay. -func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), err error) { +func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, conn *phasedConn, release func(), err error) { + setupEnd := time.Now().Add(r.timeout()) host, _, err := net.SplitHostPort(r.Addr) if err != nil { - return nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err) + return nil, nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err) } mode := r.TLS if mode == "" { mode = TLSNone } if r.Username != "" && mode == TLSNone && !isLoopback(host) { - return nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls") + return nil, nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls") } helo := r.Helo if helo == "" { @@ -89,21 +143,20 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e tlsConf.ServerName = r.ServerName } - d := &net.Dialer{Timeout: 10 * time.Second} - var conn net.Conn + d := &net.Dialer{Timeout: 10 * time.Second, Deadline: setupEnd} + var raw net.Conn if mode == TLSImplicit { - conn, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr) + raw, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr) } else { - conn, err = d.DialContext(ctx, "tcp", r.Addr) + raw, err = d.DialContext(ctx, "tcp", r.Addr) } if err != nil { - return nil, nil, err - } - // A relay that stops answering must not hold us past the context: apply its deadline to the - // connection and close the connection if the context is cancelled. - if dl, ok := ctx.Deadline(); ok { - conn.SetDeadline(dl) + return nil, nil, nil, err } + // A relay that stops answering must not hold us past the budget or the context: apply the + // earlier deadline to the connection, and close the connection if the context is cancelled. + conn = &phasedConn{Conn: raw} + conn.phase(time.Until(setupEnd)) stop := make(chan struct{}) go func() { select { @@ -113,10 +166,10 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e } }() release = func() { close(stop) } - fail := func(err error) (*smtp.Client, func(), error) { + fail := func(err error) (*smtp.Client, *phasedConn, func(), error) { release() conn.Close() - return nil, nil, err + return nil, nil, nil, err } if mode == TLSStartTLS { @@ -139,7 +192,7 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e return fail(authError(err)) } } - return c, release, nil + return c, conn, release, nil } func isLoopback(host string) bool { @@ -162,7 +215,7 @@ func authError(err error) error { } func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) (err error) { - c, release, err := r.dial(ctx) + c, conn, release, err := r.dial(ctx) if err != nil { if ctx.Err() != nil { err = ctx.Err() // report the cause, not "use of closed connection" @@ -194,13 +247,23 @@ func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, dat if err != nil { return authError(&PhaseError{Phase: "DATA", Err: err}) } + conn.phase(uploadBudget(len(data))) if _, err := io.Copy(w, bytes.NewReader(data)); err != nil { return &PhaseError{Phase: "DATA", Err: err} } + // Close sends the rest of the message and the final dot, then waits for the verdict. + conn.phase(finalReplyTimeout) if err := w.Close(); err != nil { + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + // Temporary like any timeout, but the relay may have taken the message: a retry can + // deliver it twice. Say so, so the admin can tell. + err = fmt.Errorf("no reply to the end of the message, outcome unknown: %w", err) + } return &PhaseError{Phase: "DATA", Err: err} } // The relay has accepted the message. If QUIT now fails, retrying would deliver it twice. + conn.phase(r.timeout()) c.Quit() return nil } diff --git a/internal/outbound/timeout_test.go b/internal/outbound/timeout_test.go new file mode 100644 index 0000000..b1103d9 --- /dev/null +++ b/internal/outbound/timeout_test.go @@ -0,0 +1,231 @@ +package outbound + +import ( + "bufio" + "context" + "errors" + "net" + "strings" + "sync" + "testing" + "time" +) + +// scaleTimeouts shrinks the phase budgets so the tests run in milliseconds. +func scaleTimeouts(t *testing.T, floor, finalReply time.Duration) { + t.Helper() + oldFloor, oldRate, oldCap, oldReply := dataFloor, dataRate, dataCap, finalReplyTimeout + dataFloor, finalReplyTimeout = floor, finalReply + t.Cleanup(func() { dataFloor, dataRate, dataCap, finalReplyTimeout = oldFloor, oldRate, oldCap, oldReply }) +} + +// slowRelay is a bare SMTP server that waits dotDelay before answering the final dot +// (forever when dotDelay is negative). +type slowRelay struct { + mu sync.Mutex + accepted int +} + +func startSlowRelay(t *testing.T, dotDelay time.Duration) (*slowRelay, string) { + t.Helper() + s := &slowRelay{} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + t.Cleanup(func() { close(done); ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + r := bufio.NewReader(c) + c.Write([]byte("220 slow ESMTP\r\n")) + inData := false + for { + l, err := r.ReadString('\n') + if err != nil { + return + } + l = strings.TrimRight(l, "\r\n") + switch { + case inData: + if l != "." { + continue + } + inData = false + if dotDelay < 0 { + <-done + return + } + time.Sleep(dotDelay) + s.mu.Lock() + s.accepted++ + s.mu.Unlock() + c.Write([]byte("250 2.0.0 queued\r\n")) + case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"): + c.Write([]byte("250 slow\r\n")) + case strings.HasPrefix(l, "MAIL"), strings.HasPrefix(l, "RCPT"): + c.Write([]byte("250 2.1.0 ok\r\n")) + case l == "DATA": + inData = true + c.Write([]byte("354 go ahead\r\n")) + case l == "QUIT": + c.Write([]byte("221 bye\r\n")) + return + } + } + }() + } + }() + return s, ln.Addr().String() +} + +func TestASlowReplyToTheFinalDotIsWaitedForAndDeliversOnce(t *testing.T) { + // The relay takes longer than the whole setup budget to accept the message (a content filter): + // that is not a failure, or the retry would deliver it a second time. + scaleTimeouts(t, 100*time.Millisecond, 3*time.Second) + s, addr := startSlowRelay(t, 400*time.Millisecond) + r := &SMTPRelay{Addr: addr, Timeout: 100 * time.Millisecond} + if err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")); err != nil { + t.Fatalf("the relay accepted the message: %v", err) + } + s.mu.Lock() + defer s.mu.Unlock() + if s.accepted != 1 { + t.Fatalf("accepted %d times", s.accepted) + } +} + +func TestNoReplyToTheFinalDotTimesOutAsATemporaryFailure(t *testing.T) { + scaleTimeouts(t, 100*time.Millisecond, 300*time.Millisecond) + _, addr := startSlowRelay(t, -1) + r := &SMTPRelay{Addr: addr, Timeout: 5 * time.Second} + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() { + t.Fatalf("want a timeout, got %v", err) + } + if IsPermanent(err) { + t.Fatalf("an unknown outcome is retried, not failed: %v", err) + } + if !strings.Contains(err.Error(), "outcome unknown") { + t.Fatalf("the error says the relay may have the message: %v", err) + } + if took := time.Since(start); took < 250*time.Millisecond || took > 3*time.Second { + t.Fatalf("the final reply has its own budget, took %v", took) + } +} + +func TestARelayThatStallsAtTheGreetingTimesOutInTheShortBudget(t *testing.T) { + r := &SMTPRelay{Addr: silent(t), Timeout: 150 * time.Millisecond} + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("x")) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() { + t.Fatalf("want a timeout, got %v", err) + } + if IsPermanent(err) { + t.Fatalf("a timeout is temporary: %v", err) + } + if took := time.Since(start); took > 2*time.Second { + t.Fatalf("took %v", took) + } +} + +func TestTheUploadBudgetGrowsWithTheMessageSize(t *testing.T) { + if got := uploadBudget(0); got != time.Minute { + t.Fatalf("floor: %v", got) + } + small, big := uploadBudget(10<<10), uploadBudget(10<<20) + if big <= small { + t.Fatalf("a big message gets more time: small %v, big %v", small, big) + } + if want := time.Minute + 320*time.Second; big != want { // 10 MiB at 32 KiB/s + t.Fatalf("10 MiB: got %v, want %v", big, want) + } + if got := uploadBudget(1 << 40); got != 10*time.Minute { + t.Fatalf("cap: %v", got) + } +} + +func TestCancellingTheContextAbortsTheWaitForTheFinalReply(t *testing.T) { + _, addr := startSlowRelay(t, -1) // the full 10 minutes apply + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { + errc <- (&SMTPRelay{Addr: addr}).Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")) + }() + time.Sleep(200 * time.Millisecond) + cancel() + select { + case err := <-errc: + if !errors.Is(err, context.Canceled) { + t.Fatalf("want context.Canceled, got %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Send ignored the cancelled context while waiting for the final reply") + } +} + +// stopsReadingAtData answers up to DATA, then never reads again. +func stopsReadingAtData(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + t.Cleanup(func() { close(done); ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + r := bufio.NewReader(c) + c.Write([]byte("220 stuck ESMTP\r\n")) + for { + l, err := r.ReadString('\n') + if err != nil { + return + } + switch l = strings.TrimRight(l, "\r\n"); { + case l == "DATA": + c.Write([]byte("354 go ahead\r\n")) + <-done + return + case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"): + c.Write([]byte("250 stuck\r\n")) + default: + c.Write([]byte("250 ok\r\n")) + } + } + }() + } + }() + return ln.Addr().String() +} + +func TestAStalledUploadTimesOutInItsOwnBudget(t *testing.T) { + scaleTimeouts(t, 200*time.Millisecond, time.Minute) + dataRate = 1 << 40 // the budget is the floor + r := &SMTPRelay{Addr: stopsReadingAtData(t), Timeout: time.Minute} + big := []byte(strings.Repeat(strings.Repeat("x", 998)+"\r\n", 32<<10)) // 32 MiB, more than the socket buffers + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, big) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() || IsPermanent(err) { + t.Fatalf("want a temporary timeout, got %v", err) + } + if took := time.Since(start); took > 5*time.Second { + t.Fatalf("the upload has its own budget, took %v", took) + } +}