fix(app): let LMTP sessions and HTTP requests finish on Close
Close cut the LMTP connections while their handlers still ran, so a post committed to the queue could lose its 250 and be sent again by the MTA. - Close stops accepting, waits for the LMTP Serve loop to return, then calls smtp.Server.Shutdown and http.Server.Shutdown with one shared deadline (shutdownTimeout, 30 s) - when the deadline passes, the LMTP connections still open are cut through a tracking listener (smtp.Server.Close is a no-op once Shutdown began) and the HTTP servers are closed - the queue worker and the store stop only after that - tests: a post in flight during Close is queued and gets its 250; a hung client is cut after the grace period; the open-connections test uses a short grace period
This commit is contained in:
parent
6a68f316ba
commit
13e3d16723
2 changed files with 207 additions and 15 deletions
|
|
@ -26,6 +26,9 @@ import (
|
|||
"codeberg.org/squidd/parsimony/internal/web"
|
||||
)
|
||||
|
||||
// shutdownTimeout bounds how long Close waits for the sessions in flight before it cuts them.
|
||||
var shutdownTimeout = 30 * time.Second
|
||||
|
||||
// nowFunc is the clock of the store (queue schedules, token expiry). Tests move it.
|
||||
var nowFunc = time.Now
|
||||
|
||||
|
|
@ -90,6 +93,51 @@ func lmtpIsOpen(network, addr string) bool {
|
|||
return ip == nil || !ip.IsLoopback()
|
||||
}
|
||||
|
||||
// connTracker remembers the open connections of a listener, so Close can cut the ones still open
|
||||
// when the grace period ends: once Shutdown has begun, smtp.Server.Close does nothing.
|
||||
type connTracker struct {
|
||||
net.Listener
|
||||
mu sync.Mutex
|
||||
conns map[*trackedConn]struct{}
|
||||
}
|
||||
|
||||
func (t *connTracker) Accept() (net.Conn, error) {
|
||||
c, err := t.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tc := &trackedConn{Conn: c, t: t}
|
||||
t.mu.Lock()
|
||||
t.conns[tc] = struct{}{}
|
||||
t.mu.Unlock()
|
||||
return tc, nil
|
||||
}
|
||||
|
||||
// closeAll cuts every connection still open.
|
||||
func (t *connTracker) closeAll() {
|
||||
t.mu.Lock()
|
||||
open := make([]*trackedConn, 0, len(t.conns))
|
||||
for c := range t.conns {
|
||||
open = append(open, c)
|
||||
}
|
||||
t.mu.Unlock()
|
||||
for _, c := range open {
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
|
||||
type trackedConn struct {
|
||||
net.Conn
|
||||
t *connTracker
|
||||
}
|
||||
|
||||
func (c *trackedConn) Close() error {
|
||||
c.t.mu.Lock()
|
||||
delete(c.t.conns, c)
|
||||
c.t.mu.Unlock()
|
||||
return c.Conn.Close()
|
||||
}
|
||||
|
||||
// App is a running daemon.
|
||||
type App struct {
|
||||
// LMTPAddr and APIAddr are the addresses actually bound.
|
||||
|
|
@ -98,14 +146,16 @@ type App struct {
|
|||
APIAddr string
|
||||
WebAddr string // empty when the public web is disabled
|
||||
|
||||
store *store.Store
|
||||
lmtp *smtp.Server
|
||||
lns []net.Listener // closed by Close so Serve returns even if Close wins the race with it
|
||||
web *http.Server
|
||||
pub *http.Server
|
||||
wg sync.WaitGroup
|
||||
stop context.CancelFunc // stops the queue worker
|
||||
once sync.Once
|
||||
store *store.Store
|
||||
lmtp *smtp.Server
|
||||
lmtpOpen *connTracker // the LMTP connections, cut by Close when the grace period ends
|
||||
lmtpDone chan struct{} // closed when the LMTP Serve returns
|
||||
lns []net.Listener // closed by Close so Serve returns even if Close wins the race with it
|
||||
web *http.Server
|
||||
pub *http.Server
|
||||
wg sync.WaitGroup
|
||||
stop context.CancelFunc // stops the queue worker
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
// releaser lets the API approve held messages through the router.
|
||||
|
|
@ -214,6 +264,9 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
a.pub = &http.Server{Handler: web.New(st, rt), ReadHeaderTimeout: 10 * time.Second}
|
||||
}
|
||||
|
||||
a.lmtpOpen = &connTracker{Listener: serveLn, conns: map[*trackedConn]struct{}{}}
|
||||
a.lmtpDone = make(chan struct{})
|
||||
|
||||
var workerCtx context.Context
|
||||
workerCtx, a.stop = context.WithCancel(context.Background())
|
||||
a.wg.Add(1)
|
||||
|
|
@ -224,7 +277,8 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
a.wg.Add(1)
|
||||
go func() {
|
||||
defer a.wg.Done()
|
||||
if err := a.lmtp.Serve(serveLn); err != nil && !errors.Is(err, smtp.ErrServerClosed) && !errors.Is(err, net.ErrClosed) {
|
||||
defer close(a.lmtpDone)
|
||||
if err := a.lmtp.Serve(a.lmtpOpen); err != nil && !errors.Is(err, smtp.ErrServerClosed) && !errors.Is(err, net.ErrClosed) {
|
||||
log.Printf("lmtp: %v", err)
|
||||
}
|
||||
}()
|
||||
|
|
@ -249,18 +303,26 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
return a, nil
|
||||
}
|
||||
|
||||
// Close stops the servers and closes the database. It is safe to call twice.
|
||||
// Close stops the daemon and closes the database. It is safe to call twice. It stops accepting
|
||||
// connections, then lets the LMTP sessions and HTTP requests in flight finish, so a post being
|
||||
// received is queued and gets its 250 (a cut reply makes the MTA send the post again). After
|
||||
// shutdownTimeout it cuts whatever is still open.
|
||||
func (a *App) Close() {
|
||||
a.once.Do(func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
||||
defer cancel()
|
||||
for _, ln := range a.lns {
|
||||
ln.Close()
|
||||
}
|
||||
a.lmtp.Close()
|
||||
if a.web != nil {
|
||||
a.web.Close()
|
||||
<-a.lmtpDone // Serve has returned: no connection is added while Shutdown waits
|
||||
if err := a.lmtp.Shutdown(ctx); errors.Is(err, context.DeadlineExceeded) {
|
||||
log.Printf("lmtp: sessions still open after %v, closing them", shutdownTimeout)
|
||||
a.lmtpOpen.closeAll()
|
||||
}
|
||||
if a.pub != nil {
|
||||
a.pub.Close()
|
||||
for _, srv := range []*http.Server{a.web, a.pub} {
|
||||
if srv != nil && srv.Shutdown(ctx) != nil {
|
||||
srv.Close()
|
||||
}
|
||||
}
|
||||
a.stop() // in-flight sends are interrupted and stay queued
|
||||
a.wg.Wait()
|
||||
|
|
|
|||
|
|
@ -353,6 +353,7 @@ func TestWebDisabledWhenListenIsEmpty(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestCloseDropsOpenConnections(t *testing.T) {
|
||||
setShutdownTimeout(t, 200*time.Millisecond)
|
||||
relayAddr, _ := startSink(t)
|
||||
a, err := Start(testConfig(t, relayAddr), "tok")
|
||||
if err != nil {
|
||||
|
|
@ -432,6 +433,135 @@ func deadAddr(t *testing.T) string {
|
|||
return addr
|
||||
}
|
||||
|
||||
func setShutdownTimeout(t *testing.T, d time.Duration) {
|
||||
old := shutdownTimeout
|
||||
shutdownTimeout = d
|
||||
t.Cleanup(func() { shutdownTimeout = old })
|
||||
}
|
||||
|
||||
// waitRefused waits until the LMTP listener no longer accepts connections.
|
||||
func waitRefused(t *testing.T, addr string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
c, err := net.Dial("tcp", addr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.Close()
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the LMTP listener still accepts connections")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseLetsAnLMTPDeliveryInFlightFinish(t *testing.T) {
|
||||
cfg := testConfig(t, deadAddr(t)) // the post stays queued, where it can be counted
|
||||
a, err := Start(cfg, "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok")
|
||||
call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok")
|
||||
// A connection the HTTP client opened and never used would hold the API's shutdown for 5 s.
|
||||
http.DefaultClient.CloseIdleConnections()
|
||||
|
||||
conn, err := net.Dial("tcp", a.LMTPAddr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := smtp.NewClientLMTP(conn)
|
||||
defer c.Close()
|
||||
if err := c.Hello("localhost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Mail("alice@example.com", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Rcpt("dev@example.org", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w, err := c.Data()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write([]byte("From: alice@example.com\r\nSubject: in flight\r\nMessage-ID: <f@x>\r\n\r\nbody\r\n"))
|
||||
|
||||
// The daemon stops while the post is being sent.
|
||||
closed := make(chan struct{})
|
||||
go func() { a.Close(); close(closed) }()
|
||||
waitRefused(t, a.LMTPAddr)
|
||||
if _, err := w.CloseWithLMTPResponse(); err != nil {
|
||||
t.Fatalf("the post in flight must get its 250: %v", err)
|
||||
}
|
||||
if err := c.Quit(); err != nil {
|
||||
t.Fatalf("quit: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Close must return once the session is over")
|
||||
}
|
||||
peek, err := store.Open(cfg.DB)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer peek.Close()
|
||||
if st, _ := peek.OutboxStats(); st.Pending != 1 {
|
||||
t.Fatalf("the post must be queued: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseCutsAHungLMTPClientAfterTheGracePeriod(t *testing.T) {
|
||||
setShutdownTimeout(t, 200*time.Millisecond)
|
||||
relayAddr, _ := startSink(t)
|
||||
a, err := Start(testConfig(t, relayAddr), "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn, err := net.Dial("tcp", a.LMTPAddr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
c := smtp.NewClientLMTP(conn)
|
||||
if err := c.Hello("localhost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Mail("alice@example.com", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Rcpt("dev@example.org", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := c.Data(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The client never ends the message.
|
||||
start := time.Now()
|
||||
closed := make(chan struct{})
|
||||
go func() { a.Close(); close(closed) }()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Close must not wait for a hung client beyond the grace period")
|
||||
}
|
||||
if d := time.Since(start); d < 150*time.Millisecond {
|
||||
t.Fatalf("Close returned after %v: it must give the session its grace period", d)
|
||||
}
|
||||
conn.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
buf := make([]byte, 512)
|
||||
for {
|
||||
if _, err := conn.Read(buf); err != nil {
|
||||
if ne, ok := err.(net.Error); ok && ne.Timeout() {
|
||||
t.Fatal("the hung connection is still open after Close")
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueuedMailSurvivesARestartAndIsRetriedWhenTheRelayReturns(t *testing.T) {
|
||||
var skew atomic.Int64
|
||||
nowFunc = func() time.Time { return time.Now().Add(time.Duration(skew.Load())) }
|
||||
|
|
|
|||
Loading…
Reference in a new issue