package app import ( "bufio" "bytes" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/rsa" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/base64" "encoding/pem" "errors" "fmt" "io" "log" "math/big" "net" "net/http" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" "github.com/emersion/go-msgauth/dkim" "github.com/emersion/go-sasl" "github.com/emersion/go-smtp" "codeberg.org/squidd/parsimony/internal/config" "codeberg.org/squidd/parsimony/internal/store" ) // sink is a fake SMTP relay that records what it receives. type sink struct { mu sync.Mutex msgs []sunk } type sunk struct { from string to []string data string } func (s *sink) NewSession(*smtp.Conn) (smtp.Session, error) { return &sinkSession{s: s}, nil } type sinkSession struct { s *sink from string to []string } func (x *sinkSession) Reset() { x.from, x.to = "", nil } func (x *sinkSession) Logout() error { return nil } func (x *sinkSession) Mail(f string, _ *smtp.MailOptions) error { x.from = f return nil } func (x *sinkSession) Rcpt(t string, _ *smtp.RcptOptions) error { x.to = append(x.to, t) return nil } func (x *sinkSession) Data(r io.Reader) error { b, _ := io.ReadAll(r) x.s.mu.Lock() x.s.msgs = append(x.s.msgs, sunk{x.from, x.to, string(b)}) x.s.mu.Unlock() return nil } func (s *sink) count() int { s.mu.Lock() defer s.mu.Unlock() return len(s.msgs) } // waitCount waits until the relay holds n messages. func (s *sink) waitCount(t *testing.T, n int) { t.Helper() deadline := time.Now().Add(5 * time.Second) for s.count() < n { if time.Now().After(deadline) { t.Fatalf("relay has %d messages, want %d", s.count(), n) } time.Sleep(5 * time.Millisecond) } } func (s *sink) all() []sunk { s.mu.Lock() defer s.mu.Unlock() return append([]sunk(nil), s.msgs...) } func startSink(t *testing.T) (string, *sink) { t.Helper() sk := &sink{} srv := smtp.NewServer(sk) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } go srv.Serve(ln) t.Cleanup(func() { srv.Close() }) return ln.Addr().String(), sk } func testConfig(t *testing.T, relay string) config.Config { t.Helper() var c config.Config c.DB = t.TempDir() + "/p.db" c.LMTP.Network = "tcp" c.LMTP.Addr = "127.0.0.1:0" c.Relay.Addr = relay c.API.Listen = "127.0.0.1:0" c.Web.Listen = "127.0.0.1:0" return c } // client opens one connection per request: a pooled connection the transport dialed and never // used would hold an HTTP server's graceful shutdown for 5 s. var client = &http.Client{Transport: &http.Transport{DisableKeepAlives: true}} func call(t *testing.T, a *App, method, path, body, token string) int { t.Helper() req, _ := http.NewRequest(method, "http://"+a.APIAddr+path, strings.NewReader(body)) if token != "" { req.Header.Set("Authorization", "Bearer "+token) } resp, err := client.Do(req) if err != nil { t.Fatal(err) } resp.Body.Close() return resp.StatusCode } func fetch(t *testing.T, method, url string) (int, string) { t.Helper() req, _ := http.NewRequest(method, url, nil) resp, err := client.Do(req) if err != nil { t.Fatal(err) } defer resp.Body.Close() b, _ := io.ReadAll(resp.Body) return resp.StatusCode, string(b) } func lmtpSend(t *testing.T, addr, from, rcpt, msg string) error { t.Helper() conn, err := net.Dial("tcp", addr) if err != nil { t.Fatal(err) } c := smtp.NewClientLMTP(conn) defer c.Close() if err := c.Hello("localhost"); err != nil { t.Fatal(err) } if err := c.Mail(from, nil); err != nil { t.Fatal(err) } if err := c.Rcpt(rcpt, nil); err != nil { t.Fatal(err) } w, err := c.Data() if err != nil { t.Fatal(err) } w.Write([]byte(msg)) _, err = w.CloseWithLMTPResponse() return err } func TestEndToEnd(t *testing.T) { relayAddr, relay := startSink(t) a, err := Start(testConfig(t, relayAddr), "tok") if err != nil { t.Fatalf("start: %v", err) } t.Cleanup(func() { a.Close() }) if code := call(t, a, "GET", "/api/v1/health", "", ""); code != 200 { t.Fatalf("health = %d", code) } if code := call(t, a, "GET", "/api/v1/lists", "", ""); code != 401 { t.Fatalf("unauthenticated = %d", code) } if code := call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok"); code != 201 { t.Fatalf("create list = %d", code) } for _, m := range []string{"alice@example.com", "bob@example.com"} { if code := call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"`+m+`"}`, "tok"); code != 201 { t.Fatalf("add %s = %d", m, code) } } msg := "From: Alice \r\nSubject: hello\r\n\r\nbody\r\n" if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { t.Fatalf("lmtp: %v", err) } relay.waitCount(t, 2) got := relay.all()[0] if !strings.Contains(got.data, "List-Id: Dev ") || !strings.HasPrefix(got.from, "dev+bounce-") { t.Fatalf("unexpected relayed mail: %+v", got) } if err := lmtpSend(t, a.LMTPAddr, "mallory@evil.com", "dev@example.org", msg); err == nil { t.Fatal("non-member must be rejected") } time.Sleep(100 * time.Millisecond) // give a wrongly queued message time to show up if relay.count() != 2 { t.Fatal("rejected mail must not be relayed") } } func TestStartWithoutTokenDisablesAPI(t *testing.T) { relayAddr, _ := startSink(t) a, err := Start(testConfig(t, relayAddr), "") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) if a.APIAddr != "" { t.Fatalf("API must be disabled, listening on %s", a.APIAddr) } if conn, err := net.DialTimeout("tcp", a.LMTPAddr, time.Second); err != nil { t.Fatalf("LMTP must still run: %v", err) } else { conn.Close() } } func TestStartFailsOnBadDatabase(t *testing.T) { c := testConfig(t, "127.0.0.1:1") c.DB = "/nonexistent-dir/x/p.db" if _, err := Start(c, ""); err == nil { t.Fatal("want error") } } func TestStartFailsWhenPortIsBusy(t *testing.T) { ln, _ := net.Listen("tcp", "127.0.0.1:0") defer ln.Close() c := testConfig(t, "127.0.0.1:1") c.LMTP.Addr = ln.Addr().String() if _, err := Start(c, ""); err == nil { t.Fatal("want error") } } func TestCloseIsIdempotentAndStopsListeners(t *testing.T) { relayAddr, _ := startSink(t) a, _ := Start(testConfig(t, relayAddr), "tok") addr := a.LMTPAddr a.Close() a.Close() if c, err := net.DialTimeout("tcp", addr, 200*time.Millisecond); err == nil { c.Close() t.Fatal("LMTP still accepting after Close") } } func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) { relayAddr, relay := startSink(t) cfg := testConfig(t, relayAddr) cfg.BaseURL = "http://placeholder.invalid" a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) if a.WebAddr == "" { t.Fatal("web must be listening") } call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok") for _, m := range []string{"alice@example.com", "bob@example.com"} { call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"`+m+`"}`, "tok") } web := "http://" + a.WebAddr // Private by default: stored but not on the web. msg := "From: Alice \r\nSubject: Quarterly plan\r\nMessage-ID: \r\n\r\nmail me at alice@example.com\r\n" if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { t.Fatal(err) } if code, _ := fetch(t, "GET", web+"/archive/dev@example.org"); code != 404 { t.Fatalf("private archive must be 404, got %d", code) } if code := call(t, a, "GET", "/api/v1/lists/1/archive?q=quarterly", "", "tok"); code != 200 { t.Fatalf("api archive = %d", code) } // Make it public. if code := call(t, a, "PATCH", "/api/v1/lists/1", `{"archive_policy":"public"}`, "tok"); code != 200 { t.Fatalf("patch = %d", code) } code, body := fetch(t, "GET", web+"/archive/dev@example.org?q=quarterly") if code != 200 || !strings.Contains(body, "Quarterly plan") { t.Fatalf("public archive: %d %.200s", code, body) } code, body = fetch(t, "GET", web+"/archive/dev@example.org/1") if code != 200 || !strings.Contains(body, "alice@…") || strings.Contains(body, "alice@example.com") { t.Fatalf("message page must hide addresses: %d %.300s", code, body) } // The relayed copy carries a working one-click link for that recipient. relay.waitCount(t, 2) var link string for _, m := range relay.all() { if len(m.to) == 1 && m.to[0] == "bob@example.com" { i := strings.Index(m.data, "List-Unsubscribe: ")] } } if link == "" { t.Fatal("bob got no copy") } if code, _ := fetch(t, "GET", web+link); code != 200 { t.Fatalf("confirm page = %d", code) } if code, _ := fetch(t, "POST", web+link); code != 200 { t.Fatalf("one-click = %d", code) } before := relay.count() if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", "From: Alice \r\nSubject: Second\r\nMessage-ID: \r\n\r\nhi\r\n"); err != nil { t.Fatal(err) } relay.waitCount(t, before+1) time.Sleep(100 * time.Millisecond) if got := relay.count() - before; got != 1 { t.Fatalf("after bob left only alice should get a copy, got %d", got) } } func TestWebBelievesForwardedForOnlyFromTrustedProxies(t *testing.T) { relayAddr, _ := startSink(t) cfg := testConfig(t, relayAddr) cfg.Web.TrustedProxies = []string{"127.0.0.1"} // the test client is the proxy a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) // The form's rate limit counts every POST, before the list is looked up: an unknown list will do. post := func(xff string) int { req, _ := http.NewRequest("POST", "http://"+a.WebAddr+"/subscribe/ghost@example.org", strings.NewReader("email=a%40x.com")) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") req.Header.Set("X-Forwarded-For", xff) resp, err := client.Do(req) if err != nil { t.Fatal(err) } resp.Body.Close() return resp.StatusCode } for i := 0; i < 10; i++ { post("203.0.113.1") } if c := post("203.0.113.1"); c != 429 { t.Fatalf("one client over its limit: %d, want 429", c) } if c := post("203.0.113.2"); c != 404 { t.Fatalf("another client behind the same proxy: %d, want 404 (not limited)", c) } } func TestPendingPerHourReachesTheStore(t *testing.T) { relayAddr, relay := startSink(t) cfg := testConfig(t, relayAddr) cfg.Limits.PendingPerHour = 1 a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok") for _, from := range []string{"carol@x.com", "dave@x.com"} { if err := lmtpSend(t, a.LMTPAddr, from, "dev+join@example.org", ""); err != nil { t.Fatal(err) } } relay.waitCount(t, 1) time.Sleep(200 * time.Millisecond) if n := len(relay.all()); n != 1 { t.Fatalf("limits.pending_per_hour = 1: %d confirmation mails, want 1", n) } } func TestWebDisabledWhenListenIsEmpty(t *testing.T) { relayAddr, _ := startSink(t) cfg := testConfig(t, relayAddr) cfg.Web.Listen = "" a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) if a.WebAddr != "" { t.Fatalf("web must be off, listening on %s", a.WebAddr) } } func TestCloseDropsOpenConnections(t *testing.T) { setShutdownTimeout(t, 200*time.Millisecond) relayAddr, _ := startSink(t) a, err := Start(testConfig(t, relayAddr), "tok") if err != nil { t.Fatal(err) } conns := map[string]net.Conn{} for name, addr := range map[string]string{"web": a.WebAddr, "api": a.APIAddr, "lmtp": a.LMTPAddr} { c, err := net.Dial("tcp", addr) if err != nil { t.Fatal(err) } defer c.Close() conns[name] = c } time.Sleep(100 * time.Millisecond) // let the servers accept a.Close() for name, c := range conns { c.SetReadDeadline(time.Now().Add(2 * time.Second)) buf := make([]byte, 512) for { if _, err := c.Read(buf); err != nil { if ne, ok := err.(net.Error); ok && ne.Timeout() { t.Errorf("%s: connection still open after Close", name) } break } } } } func TestAdminUIAndAPIShareTheListener(t *testing.T) { relayAddr, _ := startSink(t) a, err := Start(testConfig(t, relayAddr), "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) base := "http://" + a.APIAddr // The API is not shadowed by the UI. if code, body := fetch(t, "GET", base+"/api/v1/health"); code != 200 || !strings.Contains(body, `"status"`) { t.Fatalf("health: %d %q", code, body) } if code, _ := fetch(t, "GET", base+"/api/v1/lists"); code != 401 { t.Fatalf("api must still require a token, got %d", code) } if code, body := fetch(t, "GET", base+"/api/nope"); code == 200 || strings.Contains(strings.ToLower(body), "") { t.Fatalf("%s: not the UI: %q", p, body) } case 503: if !strings.Contains(body, "make admin") { t.Fatalf("%s: 503 without a hint: %q", p, body) } default: t.Fatalf("%s: status %d", p, code) } } } // deadAddr returns an address nothing listens on. func deadAddr(t *testing.T) string { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } addr := ln.Addr().String() ln.Close() return addr } func setShutdownTimeout(t *testing.T, d time.Duration) { old := shutdownTimeout shutdownTimeout = d t.Cleanup(func() { shutdownTimeout = old }) } // waitRefused waits until the LMTP listener no longer accepts connections. func waitRefused(t *testing.T, addr string) { t.Helper() deadline := time.Now().Add(5 * time.Second) for { c, err := net.Dial("tcp", addr) if err != nil { return } c.Close() if time.Now().After(deadline) { t.Fatal("the LMTP listener still accepts connections") } time.Sleep(5 * time.Millisecond) } } func TestCloseLetsAnLMTPDeliveryInFlightFinish(t *testing.T) { cfg := testConfig(t, deadAddr(t)) // the post stays queued, where it can be counted a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok") call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok") conn, err := net.Dial("tcp", a.LMTPAddr) if err != nil { t.Fatal(err) } c := smtp.NewClientLMTP(conn) defer c.Close() if err := c.Hello("localhost"); err != nil { t.Fatal(err) } if err := c.Mail("alice@example.com", nil); err != nil { t.Fatal(err) } if err := c.Rcpt("dev@example.org", nil); err != nil { t.Fatal(err) } w, err := c.Data() if err != nil { t.Fatal(err) } w.Write([]byte("From: alice@example.com\r\nSubject: in flight\r\nMessage-ID: \r\n\r\nbody\r\n")) // The daemon stops while the post is being sent. closed := make(chan struct{}) go func() { a.Close(); close(closed) }() waitRefused(t, a.LMTPAddr) if _, err := w.CloseWithLMTPResponse(); err != nil { t.Fatalf("the post in flight must get its 250: %v", err) } if err := c.Quit(); err != nil { t.Fatalf("quit: %v", err) } select { case <-closed: case <-time.After(5 * time.Second): t.Fatal("Close must return once the session is over") } peek, err := store.Open(cfg.DB) if err != nil { t.Fatal(err) } defer peek.Close() if st, _ := peek.OutboxStats(); st.Pending != 1 { t.Fatalf("the post must be queued: %+v", st) } } func TestCloseCutsAHungLMTPClientAfterTheGracePeriod(t *testing.T) { setShutdownTimeout(t, 200*time.Millisecond) relayAddr, _ := startSink(t) a, err := Start(testConfig(t, relayAddr), "tok") if err != nil { t.Fatal(err) } conn, err := net.Dial("tcp", a.LMTPAddr) if err != nil { t.Fatal(err) } defer conn.Close() c := smtp.NewClientLMTP(conn) if err := c.Hello("localhost"); err != nil { t.Fatal(err) } if err := c.Mail("alice@example.com", nil); err != nil { t.Fatal(err) } if err := c.Rcpt("dev@example.org", nil); err != nil { t.Fatal(err) } if _, err := c.Data(); err != nil { t.Fatal(err) } // The client never ends the message. start := time.Now() closed := make(chan struct{}) go func() { a.Close(); close(closed) }() select { case <-closed: case <-time.After(5 * time.Second): t.Fatal("Close must not wait for a hung client beyond the grace period") } if d := time.Since(start); d < 150*time.Millisecond { t.Fatalf("Close returned after %v: it must give the session its grace period", d) } conn.SetReadDeadline(time.Now().Add(2 * time.Second)) buf := make([]byte, 512) for { if _, err := conn.Read(buf); err != nil { if ne, ok := err.(net.Error); ok && ne.Timeout() { t.Fatal("the hung connection is still open after Close") } break } } } func TestCloseLetsAnAPIRequestInFlightFinish(t *testing.T) { relayAddr, _ := startSink(t) a, err := Start(testConfig(t, relayAddr), "tok") if err != nil { t.Fatal(err) } conn, err := net.Dial("tcp", a.APIAddr) if err != nil { t.Fatal(err) } defer conn.Close() body := `{"name":"dev","domain":"example.org"}` fmt.Fprintf(conn, "POST /api/v1/lists HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer tok\r\nContent-Type: application/json\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s", len(body), body[:10]) time.Sleep(100 * time.Millisecond) // the handler is now reading the body closed := make(chan struct{}) go func() { a.Close(); close(closed) }() waitRefused(t, a.APIAddr) io.WriteString(conn, body[10:]) conn.SetReadDeadline(time.Now().Add(5 * time.Second)) resp, err := http.ReadResponse(bufio.NewReader(conn), nil) if err != nil { t.Fatalf("the request in flight must get its answer: %v", err) } resp.Body.Close() if resp.StatusCode != http.StatusCreated { t.Fatalf("status = %d", resp.StatusCode) } select { case <-closed: case <-time.After(5 * time.Second): t.Fatal("Close must return once the request is over") } } func TestFinishedLMTPSessionsAreForgotten(t *testing.T) { relayAddr, _ := startSink(t) a, err := Start(testConfig(t, relayAddr), "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) for i := 0; i < 3; i++ { lmtpSend(t, a.LMTPAddr, "alice@example.com", "nope@example.org", "Subject: x\r\n\r\nx\r\n") } deadline := time.Now().Add(5 * time.Second) for { a.lmtpOpen.mu.Lock() n := len(a.lmtpOpen.conns) a.lmtpOpen.mu.Unlock() if n == 0 { break } if time.Now().After(deadline) { t.Fatalf("%d closed sessions are still tracked", n) } time.Sleep(5 * time.Millisecond) } } func TestQueuedMailSurvivesARestartAndIsRetriedWhenTheRelayReturns(t *testing.T) { var skew atomic.Int64 nowFunc = func() time.Time { return time.Now().Add(time.Duration(skew.Load())) } t.Cleanup(func() { nowFunc = time.Now }) // 1. The relay is down: the post is still accepted and stays queued. cfg := testConfig(t, deadAddr(t)) a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok") for _, m := range []string{"alice@example.com", "bob@example.com"} { call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"`+m+`"}`, "tok") } msg := "From: Alice \r\nSubject: queued\r\nMessage-ID: \r\n\r\nbody\r\n" if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { t.Fatalf("a post must be accepted while the relay is down: %v", err) } peek, err := store.Open(cfg.DB) if err != nil { t.Fatal(err) } defer peek.Close() deadline := time.Now().Add(5 * time.Second) for { st, _ := peek.OutboxStats() if st.Pending == 2 && st.Retrying == 2 { break } if time.Now().After(deadline) { t.Fatalf("queue after the first attempts: %+v", st) } time.Sleep(10 * time.Millisecond) } // 2. Restart with a working relay, after the backoff has passed. a.Close() relayAddr, relay := startSink(t) cfg.Relay.Addr = relayAddr skew.Store(int64(10 * time.Minute)) b, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { b.Close() }) relay.waitCount(t, 2) got := relay.all() if !strings.Contains(got[0].data, "List-Id: Dev ") || !strings.Contains(got[0].data, "Subject: queued") { t.Fatalf("delivered mail: %q", got[0].data) } deadline = time.Now().Add(5 * time.Second) for { st, _ := peek.OutboxStats() if st.Pending == 0 { break } if time.Now().After(deadline) { t.Fatalf("delivered mail must leave the queue: %+v", st) } time.Sleep(10 * time.Millisecond) } } func TestCloseInterruptsASlowSendWithoutLosingIt(t *testing.T) { // A relay that accepts the connection and never answers. ln, _ := net.Listen("tcp", "127.0.0.1:0") t.Cleanup(func() { ln.Close() }) go func() { for { c, err := ln.Accept() if err != nil { return } defer c.Close() } }() cfg := testConfig(t, ln.Addr().String()) a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok") call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok") msg := "From: Alice \r\nSubject: slow\r\nMessage-ID: \r\n\r\nbody\r\n" if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { t.Fatal(err) } time.Sleep(200 * time.Millisecond) // the worker is now stuck talking to the silent relay closed := make(chan struct{}) go func() { a.Close(); close(closed) }() select { case <-closed: case <-time.After(5 * time.Second): t.Fatal("Close must not wait for a hung relay") } peek, err := store.Open(cfg.DB) if err != nil { t.Fatal(err) } defer peek.Close() if st, _ := peek.OutboxStats(); st.Pending != 1 { t.Fatalf("the interrupted delivery must stay queued: %+v", st) } } // dkimKey writes a fresh RSA key as PKCS#1 PEM and returns the file and the public key. func dkimKey(t *testing.T) (string, *rsa.PublicKey) { t.Helper() k, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { t.Fatal(err) } path := filepath.Join(t.TempDir(), "dkim.pem") if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}), 0o600); err != nil { t.Fatal(err) } return path, &k.PublicKey } func verifyDKIM(t *testing.T, raw string, selector, domain string, pub *rsa.PublicKey) []*dkim.Verification { t.Helper() der, _ := x509.MarshalPKIXPublicKey(pub) vs, err := dkim.VerifyWithOptions(strings.NewReader(raw), &dkim.VerifyOptions{ LookupTXT: func(name string) ([]string, error) { if name == selector+"._domainkey."+domain { return []string{"v=DKIM1; k=rsa; p=" + base64.StdEncoding.EncodeToString(der)}, nil } return nil, errors.New("no record " + name) }, }) if err != nil { t.Fatalf("verify: %v", err) } return vs } func TestOutgoingMailIsSignedWhenAKeyIsConfigured(t *testing.T) { relayAddr, relay := startSink(t) keyFile, pub := dkimKey(t) cfg := testConfig(t, relayAddr) cfg.DKIM = []config.DKIMKey{{Domain: "example.org", Selector: "mail", KeyFile: keyFile}} a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok") call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok") // A post. msg := "From: Alice \r\nSubject: signed post\r\nMessage-ID: \r\n\r\nbody\r\n" if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { t.Fatal(err) } // A confirmation mail (system mail) from the same list domain. if err := lmtpSend(t, a.LMTPAddr, "carol@x.com", "dev+join@example.org", ""); err != nil { t.Fatal(err) } relay.waitCount(t, 2) signed := 0 for _, m := range relay.all() { vs := verifyDKIM(t, m.data, "mail", "example.org", pub) if len(vs) != 1 || vs[0].Err != nil || vs[0].Domain != "example.org" { t.Fatalf("every outgoing message is signed with the list domain: %+v\n%.300s", vs, m.data) } signed++ } if signed != 2 { t.Fatalf("signed %d", signed) } } func TestMailIsUnsignedWithoutKeysAndForOtherDomains(t *testing.T) { relayAddr, relay := startSink(t) keyFile, _ := dkimKey(t) cfg := testConfig(t, relayAddr) cfg.DKIM = []config.DKIMKey{{Domain: "another.test", Selector: "mail", KeyFile: keyFile}} a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok") call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok") msg := "From: Alice \r\nSubject: x\r\nMessage-ID: \r\n\r\nbody\r\n" if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { t.Fatal(err) } relay.waitCount(t, 1) if strings.Contains(relay.all()[0].data, "DKIM-Signature") { t.Fatal("a list on a domain without a key is sent unsigned") } } func TestStartFailsOnAnUnusableDKIMKey(t *testing.T) { relayAddr, _ := startSink(t) bad := filepath.Join(t.TempDir(), "bad.pem") os.WriteFile(bad, []byte("not a key"), 0o600) for name, keyFile := range map[string]string{"missing file": "/nonexistent/dkim.pem", "garbage": bad} { cfg := testConfig(t, relayAddr) cfg.DKIM = []config.DKIMKey{{Domain: "example.org", Selector: "mail", KeyFile: keyFile}} a, err := Start(cfg, "tok") if err == nil { a.Close() t.Fatalf("%s: a broken key must stop the daemon at startup, not later at send time", name) } if !strings.Contains(err.Error(), "dkim") || !strings.Contains(err.Error(), "example.org") { t.Errorf("%s: the error names the key: %v", name, err) } } } // greeting dials LMTP and returns the first line the server says, or an error if it hangs up. func greeting(network, addr string) (string, error) { conn, err := net.DialTimeout(network, addr, 2*time.Second) if err != nil { return "", err } defer conn.Close() conn.SetReadDeadline(time.Now().Add(2 * time.Second)) buf := make([]byte, 128) n, err := conn.Read(buf) return string(buf[:n]), err } func TestLMTPAllowListAdmitsOnlyListedClients(t *testing.T) { relayAddr, _ := startSink(t) allowed := testConfig(t, relayAddr) allowed.LMTP.Allow = []string{"10.255.0.0/16", "127.0.0.0/8"} a, err := Start(allowed, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) if g, err := greeting("tcp", a.LMTPAddr); err != nil || !strings.HasPrefix(g, "220") { t.Fatalf("a listed client is served: %q %v", g, err) } denied := testConfig(t, relayAddr) denied.LMTP.Allow = []string{"10.255.0.0/16"} b, err := Start(denied, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { b.Close() }) if g, err := greeting("tcp", b.LMTPAddr); err == nil || g != "" { t.Fatalf("a client outside the list must be dropped before the greeting: %q %v", g, err) } // The daemon keeps working for the allowed ones after refusing someone. if g, err := greeting("tcp", a.LMTPAddr); err != nil || !strings.HasPrefix(g, "220") { t.Fatalf("still serving: %q %v", g, err) } } func TestLMTPAllowListDoesNotApplyToUnixSockets(t *testing.T) { relayAddr, _ := startSink(t) cfg := testConfig(t, relayAddr) cfg.LMTP.Network = "unix" cfg.LMTP.Addr = filepath.Join(t.TempDir(), "lmtp.sock") cfg.LMTP.Allow = []string{"10.0.0.0/8"} // meaningless for a socket: access is the file's permissions a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) if g, err := greeting("unix", cfg.LMTP.Addr); err != nil || !strings.HasPrefix(g, "220") { t.Fatalf("unix socket: %q %v", g, err) } } func TestStartWarnsAboutAnOpenLMTPPort(t *testing.T) { relayAddr, _ := startSink(t) for name, tc := range map[string]struct { addr string allow []string warn bool }{ "all interfaces, no list": {"0.0.0.0:0", nil, true}, "empty host, no list": {":0", nil, true}, "all interfaces, listed": {"0.0.0.0:0", []string{"127.0.0.0/8"}, false}, "loopback": {"127.0.0.1:0", nil, false}, "localhost": {"localhost:0", nil, false}, } { var buf bytes.Buffer log.SetOutput(&buf) cfg := testConfig(t, relayAddr) cfg.LMTP.Addr = tc.addr cfg.LMTP.Allow = tc.allow a, err := Start(cfg, "tok") log.SetOutput(os.Stderr) if err != nil { t.Fatalf("%s: %v", name, err) } a.Close() if got := strings.Contains(buf.String(), "lmtp.allow"); got != tc.warn { t.Errorf("%s: warning = %v, want %v\n%s", name, got, tc.warn, buf.String()) } } } // --- an external relay: TLS and authentication ------------------------------------------------ type tlsRelay struct { sink *sink addr string caFile string mu sync.Mutex auths []string hellos []string // the EHLO name seen with each message } type tlsRelayBackend struct { r *tlsRelay user, pass string } func (b *tlsRelayBackend) NewSession(c *smtp.Conn) (smtp.Session, error) { return &tlsRelaySession{b: b, conn: c}, nil } type tlsRelaySession struct { b *tlsRelayBackend conn *smtp.Conn authed bool from string to []string } func (s *tlsRelaySession) Reset() { s.from, s.to = "", nil } func (s *tlsRelaySession) Logout() error { return nil } func (s *tlsRelaySession) AuthMechanisms() []string { return []string{sasl.Plain} } func (s *tlsRelaySession) Auth(string) (sasl.Server, error) { return sasl.NewPlainServer(func(_, user, pass string) error { s.b.r.mu.Lock() s.b.r.auths = append(s.b.r.auths, user+":"+pass) s.b.r.mu.Unlock() if user != s.b.user || pass != s.b.pass { return &smtp.SMTPError{Code: 535, EnhancedCode: smtp.EnhancedCode{5, 7, 8}, Message: "bad credentials"} } s.authed = true return nil }), nil } func (s *tlsRelaySession) Mail(from string, _ *smtp.MailOptions) error { if !s.authed { return &smtp.SMTPError{Code: 530, EnhancedCode: smtp.EnhancedCode{5, 7, 0}, Message: "authenticate first"} } if _, ok := s.conn.TLSConnectionState(); !ok { return errors.New("the relay was reached without TLS") } s.from = from s.b.r.mu.Lock() s.b.r.hellos = append(s.b.r.hellos, s.conn.Hostname()) s.b.r.mu.Unlock() return nil } func (s *tlsRelaySession) Rcpt(to string, _ *smtp.RcptOptions) error { s.to = append(s.to, to) return nil } func (s *tlsRelaySession) Data(r io.Reader) error { b, _ := io.ReadAll(r) s.b.r.sink.mu.Lock() s.b.r.sink.msgs = append(s.b.r.sink.msgs, sunk{s.from, s.to, string(b)}) s.b.r.sink.mu.Unlock() return nil } // startTLSRelay starts an authenticating STARTTLS relay and writes the CA that signed its certificate. func startTLSRelay(t *testing.T, user, pass string) *tlsRelay { t.Helper() caKey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) caTmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test CA"}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} caDER, _ := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey) ca, _ := x509.ParseCertificate(caDER) key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "relay"}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}} der, _ := x509.CreateCertificate(rand.Reader, leaf, ca, &key.PublicKey, caKey) r := &tlsRelay{sink: &sink{}} r.caFile = filepath.Join(t.TempDir(), "ca.pem") os.WriteFile(r.caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), 0o600) srv := smtp.NewServer(&tlsRelayBackend{r: r, user: user, pass: pass}) srv.TLSConfig = &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}} ln, _ := net.Listen("tcp", "127.0.0.1:0") r.addr = ln.Addr().String() go srv.Serve(ln) t.Cleanup(func() { srv.Close() }) return r } func relayConfig(t *testing.T, r *tlsRelay, password string) config.Config { t.Helper() pwFile := filepath.Join(t.TempDir(), "relay-password") os.WriteFile(pwFile, []byte(password+"\n"), 0o600) cfg := testConfig(t, r.addr) cfg.Relay.TLS = "starttls" cfg.Relay.CAFile = r.caFile cfg.Relay.Username = "ses-user" cfg.Relay.PasswordFile = pwFile cfg.Relay.Helo = "lists.example.org" return cfg } func postOne(t *testing.T, a *App) { t.Helper() call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok") call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok") msg := "From: Alice \r\nSubject: via a relay\r\nMessage-ID: \r\n\r\nbody\r\n" if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil { t.Fatal(err) } } func TestMailGoesThroughAnAuthenticatingTLSRelay(t *testing.T) { relay := startTLSRelay(t, "ses-user", "p@ss word") a, err := Start(relayConfig(t, relay, "p@ss word"), "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) postOne(t, a) relay.sink.waitCount(t, 1) if !strings.Contains(relay.sink.all()[0].data, "Subject: via a relay") { t.Fatalf("%q", relay.sink.all()[0].data) } relay.mu.Lock() defer relay.mu.Unlock() if len(relay.auths) == 0 || relay.auths[0] != "ses-user:p@ss word" { t.Fatalf("the password comes from the file, without its line ending: %v", relay.auths) } } func TestAWrongRelayPasswordKeepsTheMailQueued(t *testing.T) { relay := startTLSRelay(t, "ses-user", "right") cfg := relayConfig(t, relay, "wrong") a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) postOne(t, a) peek, err := store.Open(cfg.DB) if err != nil { t.Fatal(err) } defer peek.Close() deadline := time.Now().Add(5 * time.Second) for { st, _ := peek.OutboxStats() if st.Pending == 1 && st.Retrying == 1 && st.Failed == 0 { break } if time.Now().After(deadline) { t.Fatalf("the delivery must stay queued and retry, not fail for good: %+v", st) } time.Sleep(10 * time.Millisecond) } list, _ := peek.ListOutbox(10) if !strings.Contains(list[0].LastError, "authentication") || strings.Contains(list[0].LastError, "wrong") { t.Fatalf("the admin sees what is wrong, without the password: %q", list[0].LastError) } if relay.sink.count() != 0 { t.Fatal("nothing was delivered") } } func TestStartFailsOnUnusableRelaySecrets(t *testing.T) { relay := startTLSRelay(t, "ses-user", "p") for name, mutate := range map[string]func(*config.Config){ "missing password file": func(c *config.Config) { c.Relay.PasswordFile = "/nonexistent/pw" }, "empty password file": func(c *config.Config) { os.WriteFile(c.Relay.PasswordFile, []byte("\n"), 0o600) }, "missing CA file": func(c *config.Config) { c.Relay.CAFile = "/nonexistent/ca.pem" }, "CA file without cert": func(c *config.Config) { os.WriteFile(c.Relay.CAFile, []byte("nope"), 0o600) }, } { cfg := relayConfig(t, relay, "p") mutate(&cfg) if a, err := Start(cfg, "tok"); err == nil { a.Close() t.Errorf("%s: the daemon must refuse to start rather than fail at the first send", name) } else if !strings.Contains(err.Error(), "relay") { t.Errorf("%s: the error names the relay setting: %v", name, err) } } } func TestTheRelayAddressIsNotDialledAtStartup(t *testing.T) { // Starting must not depend on the relay being reachable: mail waits in the queue. cfg := testConfig(t, deadAddr(t)) cfg.Relay.TLS = "starttls" a, err := Start(cfg, "tok") if err != nil { t.Fatalf("an unreachable relay is a runtime condition, not a startup error: %v", err) } a.Close() } func TestTheConfiguredEHLONameReachesTheRelay(t *testing.T) { relay := startTLSRelay(t, "ses-user", "p") a, err := Start(relayConfig(t, relay, "p"), "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) postOne(t, a) relay.sink.waitCount(t, 1) relay.mu.Lock() defer relay.mu.Unlock() if len(relay.hellos) != 1 || relay.hellos[0] != "lists.example.org" { t.Fatalf("EHLO names: %v", relay.hellos) } } func TestWithoutAConfiguredNameEHLOIsTheHostName(t *testing.T) { host, err := os.Hostname() if err != nil || host == "" { t.Skip("no host name on this machine") } relay := startTLSRelay(t, "ses-user", "p") cfg := relayConfig(t, relay, "p") cfg.Relay.Helo = "" a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) postOne(t, a) relay.sink.waitCount(t, 1) relay.mu.Lock() defer relay.mu.Unlock() if len(relay.hellos) != 1 || relay.hellos[0] != host { t.Fatalf("EHLO %v, want the host name %q (real relays dislike \"localhost\")", relay.hellos, host) } } func TestTheServerNameSettingReachesTheTLSCheck(t *testing.T) { relay := startTLSRelay(t, "ses-user", "p") // its certificate is for 127.0.0.1 cfg := relayConfig(t, relay, "p") cfg.Relay.ServerName = "other.example" a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) postOne(t, a) peek, _ := store.Open(cfg.DB) defer peek.Close() deadline := time.Now().Add(5 * time.Second) for { list, _ := peek.ListOutbox(10) if len(list) == 1 && list[0].Attempts == 1 { if !strings.Contains(strings.ToLower(list[0].LastError), "certificate") || list[0].Failed { t.Fatalf("a name mismatch is a retryable certificate error: %+v", list[0]) } break } if time.Now().After(deadline) { t.Fatal("the delivery never failed on the certificate name") } time.Sleep(10 * time.Millisecond) } if relay.sink.count() != 0 { t.Fatal("nothing may be sent to a relay whose certificate does not match") } } // shortTimeouts makes the HTTP timeouts small enough to test. func shortTimeouts(t *testing.T) { t.Helper() r, w, i := httpReadTimeout, httpWriteTimeout, httpIdleTimeout httpReadTimeout, httpWriteTimeout, httpIdleTimeout = 300*time.Millisecond, 300*time.Millisecond, 300*time.Millisecond t.Cleanup(func() { httpReadTimeout, httpWriteTimeout, httpIdleTimeout = r, w, i }) } // stalled sends a request that promises a body and never finishes it, and // reports whether the server hung up within a few seconds. func stalled(t *testing.T, addr, req string) bool { t.Helper() c, err := net.Dial("tcp", addr) if err != nil { t.Fatal(err) } defer c.Close() if _, err := c.Write([]byte(req)); err != nil { t.Fatal(err) } c.SetReadDeadline(time.Now().Add(4 * time.Second)) _, err = io.Copy(io.Discard, c) // returns nil on EOF, an error on our deadline return err == nil } func TestSlowRequestsAreCutOnBothServers(t *testing.T) { shortTimeouts(t) relayAddr, _ := startSink(t) a, err := Start(testConfig(t, relayAddr), "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) hdr := "POST %s HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer tok\r\nContent-Type: application/json\r\nContent-Length: 500\r\n\r\n{\"name\":" for name, c := range map[string]struct{ addr, path string }{ "api": {a.APIAddr, "/api/v1/lists"}, "web": {a.WebAddr, "/unsubscribe/tok"}, } { if !stalled(t, c.addr, strings.Replace(hdr, "%s", c.path, 1)) { t.Errorf("%s: a stalled body was not cut", name) } } // An idle keep-alive connection is closed too. for name, addr := range map[string]string{"api": a.APIAddr, "web": a.WebAddr} { if !stalled(t, addr, "GET /nothing HTTP/1.1\r\nHost: x\r\n\r\n") { t.Errorf("%s: an idle connection was kept", name) } } } // slowReader hands out its data in two halves, a pause apart. type slowReader struct { parts [][]byte pause time.Duration } func (s *slowReader) Read(p []byte) (int, error) { if len(s.parts) == 0 { return 0, io.EOF } if len(s.parts) == 1 { time.Sleep(s.pause) } n := copy(p, s.parts[0]) s.parts = s.parts[1:] return n, nil } func TestArchiveImportIsNotCutByTheDefaultTimeouts(t *testing.T) { shortTimeouts(t) relayAddr, _ := startSink(t) a, err := Start(testConfig(t, relayAddr), "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok") m1 := "From a@example.com Tue Jan 2 15:04:05 2024\nFrom: a@example.com\nSubject: one\nMessage-ID: <1@x>\n\nbody one\n\n" m2 := "From a@example.com Tue Jan 2 15:04:06 2024\nFrom: a@example.com\nSubject: two\nMessage-ID: <2@x>\n\nbody two\n" body := &slowReader{parts: [][]byte{[]byte(m1), []byte(m2)}, pause: 3 * httpReadTimeout} req, _ := http.NewRequest("POST", "http://"+a.APIAddr+"/api/v1/lists/1/archive/import", body) req.Header.Set("Authorization", "Bearer tok") req.ContentLength = -1 // chunked: the body is streamed resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("import: %v", err) } defer resp.Body.Close() b, _ := io.ReadAll(resp.Body) if resp.StatusCode != 200 || !strings.Contains(string(b), `"imported":2`) { t.Fatalf("import = %d %s", resp.StatusCode, b) } } func TestHTTPServersCarryTheTimeouts(t *testing.T) { relayAddr, _ := startSink(t) a, err := Start(testConfig(t, relayAddr), "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) if httpReadTimeout != 30*time.Second || httpWriteTimeout != 60*time.Second || httpIdleTimeout != 120*time.Second { t.Fatalf("defaults: %v %v %v", httpReadTimeout, httpWriteTimeout, httpIdleTimeout) } for name, s := range map[string]*http.Server{"api": a.web, "web": a.pub} { if s.ReadTimeout != httpReadTimeout || s.WriteTimeout != httpWriteTimeout || s.IdleTimeout != httpIdleTimeout || s.ReadHeaderTimeout == 0 { t.Errorf("%s: timeouts %v %v %v %v", name, s.ReadHeaderTimeout, s.ReadTimeout, s.WriteTimeout, s.IdleTimeout) } } } func TestTheRelayRateReachesTheWorker(t *testing.T) { relayAddr, relay := startSink(t) cfg := testConfig(t, relayAddr) cfg.Relay.RatePerSecond = 20 // 50 ms between two sends a, err := Start(cfg, "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok") const members = 8 for i := 0; i < members; i++ { call(t, a, "POST", "/api/v1/lists/1/members", fmt.Sprintf(`{"address":"m%d@example.com"}`, i), "tok") } start := time.Now() msg := "From: M0 \r\nSubject: paced\r\nMessage-ID: \r\n\r\nbody\r\n" if err := lmtpSend(t, a.LMTPAddr, "m0@example.com", "dev@example.org", msg); err != nil { t.Fatal(err) } relay.waitCount(t, members) // 8 sends started 50 ms apart: the last one cannot start before 350 ms. if took := time.Since(start); took < 330*time.Millisecond { t.Fatalf("%d deliveries took %v: the rate did not reach the worker", members, took) } } func TestWithoutARateTheRelayIsNotPaced(t *testing.T) { relayAddr, relay := startSink(t) a, err := Start(testConfig(t, relayAddr), "tok") if err != nil { t.Fatal(err) } t.Cleanup(func() { a.Close() }) call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok") const members = 8 for i := 0; i < members; i++ { call(t, a, "POST", "/api/v1/lists/1/members", fmt.Sprintf(`{"address":"m%d@example.com"}`, i), "tok") } start := time.Now() msg := "From: M0 \r\nSubject: free\r\nMessage-ID: \r\n\r\nbody\r\n" if err := lmtpSend(t, a.LMTPAddr, "m0@example.com", "dev@example.org", msg); err != nil { t.Fatal(err) } relay.waitCount(t, members) if took := time.Since(start); took > 3*time.Second { t.Fatalf("%d deliveries took %v without a rate", members, took) } }