diff --git a/internal/outbound/outbound.go b/internal/outbound/outbound.go index 09ebcaa..c2de797 100644 --- a/internal/outbound/outbound.go +++ b/internal/outbound/outbound.go @@ -45,6 +45,59 @@ type SMTPRelay struct { Username string // SMTP AUTH PLAIN; only over TLS or to a loopback address Password string Helo string // EHLO name; "localhost" by default + // Timeout bounds the setup of each transaction: connection, greeting, TLS, AUTH, MAIL FROM, + // RCPT TO and the DATA command. DefaultTimeout if zero. The upload of the message and the + // wait for its acceptance have their own budgets (see uploadBudget and finalReplyTimeout). + Timeout time.Duration +} + +// DefaultTimeout is the default SMTPRelay.Timeout. +const DefaultTimeout = time.Minute + +// Budgets of the later phases of a transaction. Variables so that tests can shrink them. +var ( + dataFloor = time.Minute // least time to upload a message + dataRate = 32 << 10 // slowest upload we wait for, in bytes per second + dataCap = 10 * time.Minute // most time to upload a message + // finalReplyTimeout is how long the relay may take to answer the final dot: RFC 5321 4.5.3.2.6 + // asks for 10 minutes, because content filters run at that point. + finalReplyTimeout = 10 * time.Minute +) + +// uploadBudget is the time allowed to upload a message of size bytes: dataFloor plus the time +// it takes at dataRate, at most dataCap. +func uploadBudget(size int) time.Duration { + b := dataFloor + time.Duration(float64(size)/float64(dataRate)*float64(time.Second)) + return min(b, dataCap) +} + +func (r *SMTPRelay) timeout() time.Duration { + if r.Timeout > 0 { + return r.Timeout + } + return DefaultTimeout +} + +// phasedConn keeps the connection within the budget of the current phase. go-smtp sets its own +// deadline around each command and clears it afterwards; here no deadline it sets, cleared or not, +// may go past the limit of the phase. +type phasedConn struct { + net.Conn + limit time.Time +} + +func (p *phasedConn) SetDeadline(t time.Time) error { + if t.IsZero() || p.limit.Before(t) { + t = p.limit + } + return p.Conn.SetDeadline(t) +} + +// phase gives the connection d to finish the next phase. The context's own deadline and its +// cancellation close the connection instead (see dial), so that the error is the context's. +func (p *phasedConn) phase(d time.Duration) { + p.limit = time.Now().Add(d) + p.Conn.SetDeadline(p.limit) } // Send splits recipients in batches and sends one transaction per batch. @@ -65,20 +118,21 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b return nil } -// dial opens the connection and speaks EHLO, TLS and AUTH as configured. -// The returned release function must be called when the client is done: until then the connection +// dial opens the connection and speaks EHLO, TLS and AUTH as configured. The connection's deadline +// is the end of the setup budget, which starts now. The returned release function must be called when the client is done: until then the connection // is closed if ctx is cancelled, even in the middle of EHLO, STARTTLS or AUTH with a silent relay. -func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), err error) { +func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, conn *phasedConn, release func(), err error) { + setupEnd := time.Now().Add(r.timeout()) host, _, err := net.SplitHostPort(r.Addr) if err != nil { - return nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err) + return nil, nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err) } mode := r.TLS if mode == "" { mode = TLSNone } if r.Username != "" && mode == TLSNone && !isLoopback(host) { - return nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls") + return nil, nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls") } helo := r.Helo if helo == "" { @@ -89,21 +143,20 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e tlsConf.ServerName = r.ServerName } - d := &net.Dialer{Timeout: 10 * time.Second} - var conn net.Conn + d := &net.Dialer{Timeout: 10 * time.Second, Deadline: setupEnd} + var raw net.Conn if mode == TLSImplicit { - conn, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr) + raw, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr) } else { - conn, err = d.DialContext(ctx, "tcp", r.Addr) + raw, err = d.DialContext(ctx, "tcp", r.Addr) } if err != nil { - return nil, nil, err - } - // A relay that stops answering must not hold us past the context: apply its deadline to the - // connection and close the connection if the context is cancelled. - if dl, ok := ctx.Deadline(); ok { - conn.SetDeadline(dl) + return nil, nil, nil, err } + // A relay that stops answering must not hold us past the budget or the context: apply the + // earlier deadline to the connection, and close the connection if the context is cancelled. + conn = &phasedConn{Conn: raw} + conn.phase(time.Until(setupEnd)) stop := make(chan struct{}) go func() { select { @@ -113,10 +166,10 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e } }() release = func() { close(stop) } - fail := func(err error) (*smtp.Client, func(), error) { + fail := func(err error) (*smtp.Client, *phasedConn, func(), error) { release() conn.Close() - return nil, nil, err + return nil, nil, nil, err } if mode == TLSStartTLS { @@ -139,7 +192,7 @@ func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), e return fail(authError(err)) } } - return c, release, nil + return c, conn, release, nil } func isLoopback(host string) bool { @@ -162,7 +215,7 @@ func authError(err error) error { } func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) (err error) { - c, release, err := r.dial(ctx) + c, conn, release, err := r.dial(ctx) if err != nil { if ctx.Err() != nil { err = ctx.Err() // report the cause, not "use of closed connection" @@ -194,13 +247,23 @@ func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, dat if err != nil { return authError(&PhaseError{Phase: "DATA", Err: err}) } + conn.phase(uploadBudget(len(data))) if _, err := io.Copy(w, bytes.NewReader(data)); err != nil { return &PhaseError{Phase: "DATA", Err: err} } + // Close sends the rest of the message and the final dot, then waits for the verdict. + conn.phase(finalReplyTimeout) if err := w.Close(); err != nil { + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + // Temporary like any timeout, but the relay may have taken the message: a retry can + // deliver it twice. Say so, so the admin can tell. + err = fmt.Errorf("no reply to the end of the message, outcome unknown: %w", err) + } return &PhaseError{Phase: "DATA", Err: err} } // The relay has accepted the message. If QUIT now fails, retrying would deliver it twice. + conn.phase(r.timeout()) c.Quit() return nil } diff --git a/internal/outbound/timeout_test.go b/internal/outbound/timeout_test.go new file mode 100644 index 0000000..b1103d9 --- /dev/null +++ b/internal/outbound/timeout_test.go @@ -0,0 +1,231 @@ +package outbound + +import ( + "bufio" + "context" + "errors" + "net" + "strings" + "sync" + "testing" + "time" +) + +// scaleTimeouts shrinks the phase budgets so the tests run in milliseconds. +func scaleTimeouts(t *testing.T, floor, finalReply time.Duration) { + t.Helper() + oldFloor, oldRate, oldCap, oldReply := dataFloor, dataRate, dataCap, finalReplyTimeout + dataFloor, finalReplyTimeout = floor, finalReply + t.Cleanup(func() { dataFloor, dataRate, dataCap, finalReplyTimeout = oldFloor, oldRate, oldCap, oldReply }) +} + +// slowRelay is a bare SMTP server that waits dotDelay before answering the final dot +// (forever when dotDelay is negative). +type slowRelay struct { + mu sync.Mutex + accepted int +} + +func startSlowRelay(t *testing.T, dotDelay time.Duration) (*slowRelay, string) { + t.Helper() + s := &slowRelay{} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + t.Cleanup(func() { close(done); ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + r := bufio.NewReader(c) + c.Write([]byte("220 slow ESMTP\r\n")) + inData := false + for { + l, err := r.ReadString('\n') + if err != nil { + return + } + l = strings.TrimRight(l, "\r\n") + switch { + case inData: + if l != "." { + continue + } + inData = false + if dotDelay < 0 { + <-done + return + } + time.Sleep(dotDelay) + s.mu.Lock() + s.accepted++ + s.mu.Unlock() + c.Write([]byte("250 2.0.0 queued\r\n")) + case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"): + c.Write([]byte("250 slow\r\n")) + case strings.HasPrefix(l, "MAIL"), strings.HasPrefix(l, "RCPT"): + c.Write([]byte("250 2.1.0 ok\r\n")) + case l == "DATA": + inData = true + c.Write([]byte("354 go ahead\r\n")) + case l == "QUIT": + c.Write([]byte("221 bye\r\n")) + return + } + } + }() + } + }() + return s, ln.Addr().String() +} + +func TestASlowReplyToTheFinalDotIsWaitedForAndDeliversOnce(t *testing.T) { + // The relay takes longer than the whole setup budget to accept the message (a content filter): + // that is not a failure, or the retry would deliver it a second time. + scaleTimeouts(t, 100*time.Millisecond, 3*time.Second) + s, addr := startSlowRelay(t, 400*time.Millisecond) + r := &SMTPRelay{Addr: addr, Timeout: 100 * time.Millisecond} + if err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")); err != nil { + t.Fatalf("the relay accepted the message: %v", err) + } + s.mu.Lock() + defer s.mu.Unlock() + if s.accepted != 1 { + t.Fatalf("accepted %d times", s.accepted) + } +} + +func TestNoReplyToTheFinalDotTimesOutAsATemporaryFailure(t *testing.T) { + scaleTimeouts(t, 100*time.Millisecond, 300*time.Millisecond) + _, addr := startSlowRelay(t, -1) + r := &SMTPRelay{Addr: addr, Timeout: 5 * time.Second} + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() { + t.Fatalf("want a timeout, got %v", err) + } + if IsPermanent(err) { + t.Fatalf("an unknown outcome is retried, not failed: %v", err) + } + if !strings.Contains(err.Error(), "outcome unknown") { + t.Fatalf("the error says the relay may have the message: %v", err) + } + if took := time.Since(start); took < 250*time.Millisecond || took > 3*time.Second { + t.Fatalf("the final reply has its own budget, took %v", took) + } +} + +func TestARelayThatStallsAtTheGreetingTimesOutInTheShortBudget(t *testing.T) { + r := &SMTPRelay{Addr: silent(t), Timeout: 150 * time.Millisecond} + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("x")) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() { + t.Fatalf("want a timeout, got %v", err) + } + if IsPermanent(err) { + t.Fatalf("a timeout is temporary: %v", err) + } + if took := time.Since(start); took > 2*time.Second { + t.Fatalf("took %v", took) + } +} + +func TestTheUploadBudgetGrowsWithTheMessageSize(t *testing.T) { + if got := uploadBudget(0); got != time.Minute { + t.Fatalf("floor: %v", got) + } + small, big := uploadBudget(10<<10), uploadBudget(10<<20) + if big <= small { + t.Fatalf("a big message gets more time: small %v, big %v", small, big) + } + if want := time.Minute + 320*time.Second; big != want { // 10 MiB at 32 KiB/s + t.Fatalf("10 MiB: got %v, want %v", big, want) + } + if got := uploadBudget(1 << 40); got != 10*time.Minute { + t.Fatalf("cap: %v", got) + } +} + +func TestCancellingTheContextAbortsTheWaitForTheFinalReply(t *testing.T) { + _, addr := startSlowRelay(t, -1) // the full 10 minutes apply + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { + errc <- (&SMTPRelay{Addr: addr}).Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")) + }() + time.Sleep(200 * time.Millisecond) + cancel() + select { + case err := <-errc: + if !errors.Is(err, context.Canceled) { + t.Fatalf("want context.Canceled, got %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Send ignored the cancelled context while waiting for the final reply") + } +} + +// stopsReadingAtData answers up to DATA, then never reads again. +func stopsReadingAtData(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + t.Cleanup(func() { close(done); ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + r := bufio.NewReader(c) + c.Write([]byte("220 stuck ESMTP\r\n")) + for { + l, err := r.ReadString('\n') + if err != nil { + return + } + switch l = strings.TrimRight(l, "\r\n"); { + case l == "DATA": + c.Write([]byte("354 go ahead\r\n")) + <-done + return + case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"): + c.Write([]byte("250 stuck\r\n")) + default: + c.Write([]byte("250 ok\r\n")) + } + } + }() + } + }() + return ln.Addr().String() +} + +func TestAStalledUploadTimesOutInItsOwnBudget(t *testing.T) { + scaleTimeouts(t, 200*time.Millisecond, time.Minute) + dataRate = 1 << 40 // the budget is the floor + r := &SMTPRelay{Addr: stopsReadingAtData(t), Timeout: time.Minute} + big := []byte(strings.Repeat(strings.Repeat("x", 998)+"\r\n", 32<<10)) // 32 MiB, more than the socket buffers + start := time.Now() + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, big) + var ne net.Error + if !errors.As(err, &ne) || !ne.Timeout() || IsPermanent(err) { + t.Fatalf("want a temporary timeout, got %v", err) + } + if took := time.Since(start); took > 5*time.Second { + t.Fatalf("the upload has its own budget, took %v", took) + } +} diff --git a/internal/queue/queue.go b/internal/queue/queue.go index de5a634..aad7fb6 100644 --- a/internal/queue/queue.go +++ b/internal/queue/queue.go @@ -20,7 +20,6 @@ const MaxAge = 72 * time.Hour const ( defaultConcurrency = 4 defaultBatch = 50 - defaultTimeout = time.Minute defaultInterval = 5 * time.Second // purgeEvery is the least time between two scans that forget old failed deliveries. @@ -55,7 +54,7 @@ type Worker struct { Sender outbound.Sender Concurrency int // simultaneous sends Batch int // deliveries taken per run - Timeout time.Duration // per send + Timeout time.Duration // per send; zero leaves the deadlines to Sender (SMTPRelay bounds each SMTP phase) Interval time.Duration // idle polling, to pick up retries that have become due // Sign, when set, signs the final message (DKIM). On error the message is sent unsigned: // not delivering is worse than a missed signature. @@ -245,15 +244,16 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem } } - timeout := w.Timeout - if timeout <= 0 { - timeout = defaultTimeout - } // Wait for the slot before starting the timeout, so waiting does not eat into the send. if err := w.pace().Wait(ctx); err != nil { return Stats{} // shutting down: the delivery stays queued, unchanged } - sctx, cancel := context.WithTimeout(ctx, timeout) + // No total deadline by default: one that covers the whole transaction cut relays that take + // minutes to accept a message, which was then retried and delivered twice. + sctx, cancel := ctx, context.CancelFunc(func() {}) + if w.Timeout > 0 { + sctx, cancel = context.WithTimeout(ctx, w.Timeout) + } err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg) cancel() diff --git a/internal/queue/queue_test.go b/internal/queue/queue_test.go index 24ae823..abd1c29 100644 --- a/internal/queue/queue_test.go +++ b/internal/queue/queue_test.go @@ -405,6 +405,24 @@ func TestSlowRelayTimesOutAndIsRetried(t *testing.T) { } } +func TestTheQueueLeavesTheSendDeadlinesToTheSender(t *testing.T) { + // One total deadline cut a relay that took over a minute to answer the final dot: the message + // was accepted, then retried and delivered again. The sender bounds each SMTP phase itself. + e := setup(t) + e.enqueue(t, 1) + var hasDeadline bool + e.snd.fn = func(ctx context.Context, _ string, _ []string) error { + _, hasDeadline = ctx.Deadline() + return nil + } + if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 { + t.Fatalf("sent: %+v", st) + } + if hasDeadline { + t.Fatal("the queue must not put a deadline on the whole send") + } +} + func waitFor(t *testing.T, what string, cond func() bool) { t.Helper() deadline := time.Now().Add(3 * time.Second) diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go index 79765bf..1cfc525 100644 --- a/internal/store/m2_test.go +++ b/internal/store/m2_test.go @@ -717,3 +717,25 @@ func TestConfirmPendingIsSingleUseUnderConcurrency(t *testing.T) { } } } + +func TestMigrationAddsTheOutboxPriorityAloneAfterAHalfDoneUpgrade(t *testing.T) { + // A crash between adding post_id and priority leaves outbox with the first and not the second. + path := t.TempDir() + "/old.db" + s, _ := Open(path) + if _, err := s.db.Exec(`DROP INDEX outbox_due_prio`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`ALTER TABLE outbox DROP COLUMN priority`); err != nil { + t.Fatal(err) + } + s.Close() + + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + if ok, err := hasColumn(s2.db, "outbox", "priority"); err != nil || !ok { + t.Fatalf("priority is added back: %v %v", ok, err) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index bfe150e..6f74622 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -325,8 +325,12 @@ func migrate(db *sql.DB) error { } } if has, _ := hasColumn(db, "outbox", "post_id"); !has { - if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN post_id INTEGER NOT NULL DEFAULT 0; - ALTER TABLE outbox ADD COLUMN priority INTEGER NOT NULL DEFAULT 0;`); err != nil { + if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN post_id INTEGER NOT NULL DEFAULT 0`); err != nil { + return err + } + } + if has, _ := hasColumn(db, "outbox", "priority"); !has { + if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN priority INTEGER NOT NULL DEFAULT 0`); err != nil { return err } }