fix(app): let LMTP sessions and HTTP requests finish on Close

Close cut the LMTP connections while their handlers still ran, so a post
committed to the queue could lose its 250 and be sent again by the MTA.

- Close stops accepting, waits for the LMTP Serve loop to return, then
  calls smtp.Server.Shutdown and http.Server.Shutdown with one shared
  deadline (shutdownTimeout, 30 s)
- when the deadline passes, the LMTP connections still open are cut
  through a tracking listener (smtp.Server.Close is a no-op once Shutdown
  began) and the HTTP servers are closed
- the queue worker and the store stop only after that
- tests: a post in flight during Close is queued and gets its 250; a hung
  client is cut after the grace period; the open-connections test uses a
  short grace period
This commit is contained in:
tgiachi 2026-11-10 11:55:45 +01:00
commit 13e3d16723
2 changed files with 207 additions and 15 deletions

View file

@ -26,6 +26,9 @@ import (
"codeberg.org/squidd/parsimony/internal/web"
)
// shutdownTimeout bounds how long Close waits for the sessions in flight before it cuts them.
var shutdownTimeout = 30 * time.Second
// nowFunc is the clock of the store (queue schedules, token expiry). Tests move it.
var nowFunc = time.Now
@ -90,6 +93,51 @@ func lmtpIsOpen(network, addr string) bool {
return ip == nil || !ip.IsLoopback()
}
// connTracker remembers the open connections of a listener, so Close can cut the ones still open
// when the grace period ends: once Shutdown has begun, smtp.Server.Close does nothing.
type connTracker struct {
net.Listener
mu sync.Mutex
conns map[*trackedConn]struct{}
}
func (t *connTracker) Accept() (net.Conn, error) {
c, err := t.Listener.Accept()
if err != nil {
return nil, err
}
tc := &trackedConn{Conn: c, t: t}
t.mu.Lock()
t.conns[tc] = struct{}{}
t.mu.Unlock()
return tc, nil
}
// closeAll cuts every connection still open.
func (t *connTracker) closeAll() {
t.mu.Lock()
open := make([]*trackedConn, 0, len(t.conns))
for c := range t.conns {
open = append(open, c)
}
t.mu.Unlock()
for _, c := range open {
c.Close()
}
}
type trackedConn struct {
net.Conn
t *connTracker
}
func (c *trackedConn) Close() error {
c.t.mu.Lock()
delete(c.t.conns, c)
c.t.mu.Unlock()
return c.Conn.Close()
}
// App is a running daemon.
type App struct {
// LMTPAddr and APIAddr are the addresses actually bound.
@ -98,14 +146,16 @@ type App struct {
APIAddr string
WebAddr string // empty when the public web is disabled
store *store.Store
lmtp *smtp.Server
lns []net.Listener // closed by Close so Serve returns even if Close wins the race with it
web *http.Server
pub *http.Server
wg sync.WaitGroup
stop context.CancelFunc // stops the queue worker
once sync.Once
store *store.Store
lmtp *smtp.Server
lmtpOpen *connTracker // the LMTP connections, cut by Close when the grace period ends
lmtpDone chan struct{} // closed when the LMTP Serve returns
lns []net.Listener // closed by Close so Serve returns even if Close wins the race with it
web *http.Server
pub *http.Server
wg sync.WaitGroup
stop context.CancelFunc // stops the queue worker
once sync.Once
}
// releaser lets the API approve held messages through the router.
@ -214,6 +264,9 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
a.pub = &http.Server{Handler: web.New(st, rt), ReadHeaderTimeout: 10 * time.Second}
}
a.lmtpOpen = &connTracker{Listener: serveLn, conns: map[*trackedConn]struct{}{}}
a.lmtpDone = make(chan struct{})
var workerCtx context.Context
workerCtx, a.stop = context.WithCancel(context.Background())
a.wg.Add(1)
@ -224,7 +277,8 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
a.wg.Add(1)
go func() {
defer a.wg.Done()
if err := a.lmtp.Serve(serveLn); err != nil && !errors.Is(err, smtp.ErrServerClosed) && !errors.Is(err, net.ErrClosed) {
defer close(a.lmtpDone)
if err := a.lmtp.Serve(a.lmtpOpen); err != nil && !errors.Is(err, smtp.ErrServerClosed) && !errors.Is(err, net.ErrClosed) {
log.Printf("lmtp: %v", err)
}
}()
@ -249,18 +303,26 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
return a, nil
}
// Close stops the servers and closes the database. It is safe to call twice.
// Close stops the daemon and closes the database. It is safe to call twice. It stops accepting
// connections, then lets the LMTP sessions and HTTP requests in flight finish, so a post being
// received is queued and gets its 250 (a cut reply makes the MTA send the post again). After
// shutdownTimeout it cuts whatever is still open.
func (a *App) Close() {
a.once.Do(func() {
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()
for _, ln := range a.lns {
ln.Close()
}
a.lmtp.Close()
if a.web != nil {
a.web.Close()
<-a.lmtpDone // Serve has returned: no connection is added while Shutdown waits
if err := a.lmtp.Shutdown(ctx); errors.Is(err, context.DeadlineExceeded) {
log.Printf("lmtp: sessions still open after %v, closing them", shutdownTimeout)
a.lmtpOpen.closeAll()
}
if a.pub != nil {
a.pub.Close()
for _, srv := range []*http.Server{a.web, a.pub} {
if srv != nil && srv.Shutdown(ctx) != nil {
srv.Close()
}
}
a.stop() // in-flight sends are interrupted and stay queued
a.wg.Wait()

View file

@ -353,6 +353,7 @@ func TestWebDisabledWhenListenIsEmpty(t *testing.T) {
}
func TestCloseDropsOpenConnections(t *testing.T) {
setShutdownTimeout(t, 200*time.Millisecond)
relayAddr, _ := startSink(t)
a, err := Start(testConfig(t, relayAddr), "tok")
if err != nil {
@ -432,6 +433,135 @@ func deadAddr(t *testing.T) string {
return addr
}
func setShutdownTimeout(t *testing.T, d time.Duration) {
old := shutdownTimeout
shutdownTimeout = d
t.Cleanup(func() { shutdownTimeout = old })
}
// waitRefused waits until the LMTP listener no longer accepts connections.
func waitRefused(t *testing.T, addr string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for {
c, err := net.Dial("tcp", addr)
if err != nil {
return
}
c.Close()
if time.Now().After(deadline) {
t.Fatal("the LMTP listener still accepts connections")
}
time.Sleep(5 * time.Millisecond)
}
}
func TestCloseLetsAnLMTPDeliveryInFlightFinish(t *testing.T) {
cfg := testConfig(t, deadAddr(t)) // the post stays queued, where it can be counted
a, err := Start(cfg, "tok")
if err != nil {
t.Fatal(err)
}
call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok")
call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok")
// A connection the HTTP client opened and never used would hold the API's shutdown for 5 s.
http.DefaultClient.CloseIdleConnections()
conn, err := net.Dial("tcp", a.LMTPAddr)
if err != nil {
t.Fatal(err)
}
c := smtp.NewClientLMTP(conn)
defer c.Close()
if err := c.Hello("localhost"); err != nil {
t.Fatal(err)
}
if err := c.Mail("alice@example.com", nil); err != nil {
t.Fatal(err)
}
if err := c.Rcpt("dev@example.org", nil); err != nil {
t.Fatal(err)
}
w, err := c.Data()
if err != nil {
t.Fatal(err)
}
w.Write([]byte("From: alice@example.com\r\nSubject: in flight\r\nMessage-ID: <f@x>\r\n\r\nbody\r\n"))
// The daemon stops while the post is being sent.
closed := make(chan struct{})
go func() { a.Close(); close(closed) }()
waitRefused(t, a.LMTPAddr)
if _, err := w.CloseWithLMTPResponse(); err != nil {
t.Fatalf("the post in flight must get its 250: %v", err)
}
if err := c.Quit(); err != nil {
t.Fatalf("quit: %v", err)
}
select {
case <-closed:
case <-time.After(5 * time.Second):
t.Fatal("Close must return once the session is over")
}
peek, err := store.Open(cfg.DB)
if err != nil {
t.Fatal(err)
}
defer peek.Close()
if st, _ := peek.OutboxStats(); st.Pending != 1 {
t.Fatalf("the post must be queued: %+v", st)
}
}
func TestCloseCutsAHungLMTPClientAfterTheGracePeriod(t *testing.T) {
setShutdownTimeout(t, 200*time.Millisecond)
relayAddr, _ := startSink(t)
a, err := Start(testConfig(t, relayAddr), "tok")
if err != nil {
t.Fatal(err)
}
conn, err := net.Dial("tcp", a.LMTPAddr)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
c := smtp.NewClientLMTP(conn)
if err := c.Hello("localhost"); err != nil {
t.Fatal(err)
}
if err := c.Mail("alice@example.com", nil); err != nil {
t.Fatal(err)
}
if err := c.Rcpt("dev@example.org", nil); err != nil {
t.Fatal(err)
}
if _, err := c.Data(); err != nil {
t.Fatal(err)
}
// The client never ends the message.
start := time.Now()
closed := make(chan struct{})
go func() { a.Close(); close(closed) }()
select {
case <-closed:
case <-time.After(5 * time.Second):
t.Fatal("Close must not wait for a hung client beyond the grace period")
}
if d := time.Since(start); d < 150*time.Millisecond {
t.Fatalf("Close returned after %v: it must give the session its grace period", d)
}
conn.SetReadDeadline(time.Now().Add(2 * time.Second))
buf := make([]byte, 512)
for {
if _, err := conn.Read(buf); err != nil {
if ne, ok := err.(net.Error); ok && ne.Timeout() {
t.Fatal("the hung connection is still open after Close")
}
break
}
}
}
func TestQueuedMailSurvivesARestartAndIsRetriedWhenTheRelayReturns(t *testing.T) {
var skew atomic.Int64
nowFunc = func() time.Time { return time.Now().Add(time.Duration(skew.Load())) }