Merge pull request 'TLS and authentication towards an external SMTP relay' (#29) from feat/relay-tls into main

This commit is contained in:
squidd 2026-11-10 11:14:08 +01:00
commit f2eb79f56c
9 changed files with 1059 additions and 18 deletions

View file

@ -140,6 +140,36 @@ Use `db: /data/parsimony.db` in the config. Do not publish the LMTP port (2424):
shared with the container, and `lmtp.allow` should list that network's range (see Security notes). A new named volume gets the right ownership by itself;
for a bind mount, make the host directory writable by uid 65532.
## Sending through an external relay
By default parsimony hands mail to an SMTP server on the same machine (`relay.addr: 127.0.0.1:25`). It can also send through
an external relay such as Amazon SES, Mailgun or a company smarthost:
```yaml
relay:
addr: email-smtp.eu-west-1.amazonaws.com:587
tls: starttls # none (default), starttls or tls (port 465)
username: AKIAEXAMPLE
password_file: /run/secrets/relay-password # the password is never in the config
helo: lists.example.org # EHLO name; this machine's host name by default
# ca_file: /etc/parsimony/private-ca.pem # to trust a private CA
# server_name: smtp.example.com # when the certificate is for another name than the host in addr
```
- **TLS is required, never silently skipped.** With `starttls` a relay that does not offer it makes the send fail; the
certificate is always verified (TLS 1.2 or later; system roots, or `ca_file`).
- **Credentials only travel encrypted.** SMTP AUTH PLAIN is refused over a clear connection to a remote relay, at load
time and again at send time. A local relay (127.0.0.1, `::1`, `localhost`) may be used without TLS.
- **A wrong password does not lose mail.** A refused login (or a relay that wants authentication) is treated as a temporary
error: the deliveries stay queued, retry with backoff, and show the reason in the queue (without the password). Fix the
password and restart; the mail goes out. A missing or empty password file, or an unreadable CA file, stops the daemon at
startup.
- With `starttls`, the very first EHLO before the upgrade says `localhost` (the SMTP library offers no way to set it
earlier); the configured `helo` is used right after, over TLS.
- Keep SPF and DKIM right for the path your mail takes (for SES, `include:amazonses.com` in SPF), and check that the provider
accepts the list's per-recipient return addresses (`list+bounce-ID-MAC@domain`) as envelope sender. Bounces come back
to those addresses, so they reach parsimony only if the domain's MX delivers them to its LMTP.
## Delivery and the queue
LMTP answers `250` when the post is in the queue, so a down relay never bounces mail back to Postfix.

2
go.mod
View file

@ -5,6 +5,7 @@ go 1.27.3
require (
github.com/emersion/go-message v0.18.2
github.com/emersion/go-msgauth v0.7.0
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6
github.com/emersion/go-smtp v0.25.1
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.62.0
@ -12,7 +13,6 @@ require (
require (
github.com/dustin/go-humanize v1.1.0 // indirect
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/ncruces/go-strftime v1.1.0 // indirect

View file

@ -29,6 +29,17 @@ import (
// nowFunc is the clock of the store (queue schedules, token expiry). Tests move it.
var nowFunc = time.Now
// heloName is the EHLO name: the configured one, else this machine's host name.
func heloName(configured string) string {
if configured != "" {
return configured
}
if h, err := os.Hostname(); err == nil && h != "" {
return h
}
return "localhost"
}
// allowListener drops connections from clients that are not in the allow list,
// before the LMTP greeting: LMTP has no authentication of its own.
type allowListener struct {
@ -123,6 +134,16 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
sign = signer.Sign
}
// Same for the relay secrets: read them now, not at the first send.
relayPassword, err := cfg.RelayPassword()
if err != nil {
return nil, err
}
relayCAs, err := cfg.RelayRootCAs()
if err != nil {
return nil, err
}
st, err := store.Open(cfg.DB)
if err != nil {
return nil, fmt.Errorf("store: %w", err)
@ -133,8 +154,11 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
rt := router.New(st)
rt.BaseURL = cfg.BaseURL
worker := &queue.Worker{
Store: st,
Sender: &outbound.SMTPRelay{Addr: cfg.Relay.Addr},
Store: st,
Sender: &outbound.SMTPRelay{
Addr: cfg.Relay.Addr, TLS: cfg.Relay.TLS, ServerName: cfg.Relay.ServerName, RootCAs: relayCAs,
Username: cfg.Relay.Username, Password: relayPassword, Helo: heloName(cfg.Relay.Helo),
},
Concurrency: cfg.Relay.Concurrency,
Sign: sign,
}

View file

@ -2,14 +2,19 @@ package app
import (
"bytes"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/base64"
"encoding/pem"
"errors"
"io"
"log"
"math/big"
"net"
"net/http"
"os"
@ -21,6 +26,7 @@ import (
"time"
"github.com/emersion/go-msgauth/dkim"
"github.com/emersion/go-sasl"
"github.com/emersion/go-smtp"
"codeberg.org/squidd/parsimony/internal/config"
@ -728,3 +734,273 @@ func TestStartWarnsAboutAnOpenLMTPPort(t *testing.T) {
}
}
}
// --- an external relay: TLS and authentication ------------------------------------------------
type tlsRelay struct {
sink *sink
addr string
caFile string
mu sync.Mutex
auths []string
hellos []string // the EHLO name seen with each message
}
type tlsRelayBackend struct {
r *tlsRelay
user, pass string
}
func (b *tlsRelayBackend) NewSession(c *smtp.Conn) (smtp.Session, error) {
return &tlsRelaySession{b: b, conn: c}, nil
}
type tlsRelaySession struct {
b *tlsRelayBackend
conn *smtp.Conn
authed bool
from string
to []string
}
func (s *tlsRelaySession) Reset() { s.from, s.to = "", nil }
func (s *tlsRelaySession) Logout() error { return nil }
func (s *tlsRelaySession) AuthMechanisms() []string { return []string{sasl.Plain} }
func (s *tlsRelaySession) Auth(string) (sasl.Server, error) {
return sasl.NewPlainServer(func(_, user, pass string) error {
s.b.r.mu.Lock()
s.b.r.auths = append(s.b.r.auths, user+":"+pass)
s.b.r.mu.Unlock()
if user != s.b.user || pass != s.b.pass {
return &smtp.SMTPError{Code: 535, EnhancedCode: smtp.EnhancedCode{5, 7, 8}, Message: "bad credentials"}
}
s.authed = true
return nil
}), nil
}
func (s *tlsRelaySession) Mail(from string, _ *smtp.MailOptions) error {
if !s.authed {
return &smtp.SMTPError{Code: 530, EnhancedCode: smtp.EnhancedCode{5, 7, 0}, Message: "authenticate first"}
}
if _, ok := s.conn.TLSConnectionState(); !ok {
return errors.New("the relay was reached without TLS")
}
s.from = from
s.b.r.mu.Lock()
s.b.r.hellos = append(s.b.r.hellos, s.conn.Hostname())
s.b.r.mu.Unlock()
return nil
}
func (s *tlsRelaySession) Rcpt(to string, _ *smtp.RcptOptions) error {
s.to = append(s.to, to)
return nil
}
func (s *tlsRelaySession) Data(r io.Reader) error {
b, _ := io.ReadAll(r)
s.b.r.sink.mu.Lock()
s.b.r.sink.msgs = append(s.b.r.sink.msgs, sunk{s.from, s.to, string(b)})
s.b.r.sink.mu.Unlock()
return nil
}
// startTLSRelay starts an authenticating STARTTLS relay and writes the CA that signed its certificate.
func startTLSRelay(t *testing.T, user, pass string) *tlsRelay {
t.Helper()
caKey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
caTmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test CA"}, NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign}
caDER, _ := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey)
ca, _ := x509.ParseCertificate(caDER)
key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "relay"}, NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour), IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}
der, _ := x509.CreateCertificate(rand.Reader, leaf, ca, &key.PublicKey, caKey)
r := &tlsRelay{sink: &sink{}}
r.caFile = filepath.Join(t.TempDir(), "ca.pem")
os.WriteFile(r.caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), 0o600)
srv := smtp.NewServer(&tlsRelayBackend{r: r, user: user, pass: pass})
srv.TLSConfig = &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}}
ln, _ := net.Listen("tcp", "127.0.0.1:0")
r.addr = ln.Addr().String()
go srv.Serve(ln)
t.Cleanup(func() { srv.Close() })
return r
}
func relayConfig(t *testing.T, r *tlsRelay, password string) config.Config {
t.Helper()
pwFile := filepath.Join(t.TempDir(), "relay-password")
os.WriteFile(pwFile, []byte(password+"\n"), 0o600)
cfg := testConfig(t, r.addr)
cfg.Relay.TLS = "starttls"
cfg.Relay.CAFile = r.caFile
cfg.Relay.Username = "ses-user"
cfg.Relay.PasswordFile = pwFile
cfg.Relay.Helo = "lists.example.org"
return cfg
}
func postOne(t *testing.T, a *App) {
t.Helper()
call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok")
call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok")
msg := "From: Alice <alice@example.com>\r\nSubject: via a relay\r\nMessage-ID: <r1@x>\r\n\r\nbody\r\n"
if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil {
t.Fatal(err)
}
}
func TestMailGoesThroughAnAuthenticatingTLSRelay(t *testing.T) {
relay := startTLSRelay(t, "ses-user", "p@ss word")
a, err := Start(relayConfig(t, relay, "p@ss word"), "tok")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { a.Close() })
postOne(t, a)
relay.sink.waitCount(t, 1)
if !strings.Contains(relay.sink.all()[0].data, "Subject: via a relay") {
t.Fatalf("%q", relay.sink.all()[0].data)
}
relay.mu.Lock()
defer relay.mu.Unlock()
if len(relay.auths) == 0 || relay.auths[0] != "ses-user:p@ss word" {
t.Fatalf("the password comes from the file, without its line ending: %v", relay.auths)
}
}
func TestAWrongRelayPasswordKeepsTheMailQueued(t *testing.T) {
relay := startTLSRelay(t, "ses-user", "right")
cfg := relayConfig(t, relay, "wrong")
a, err := Start(cfg, "tok")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { a.Close() })
postOne(t, a)
peek, err := store.Open(cfg.DB)
if err != nil {
t.Fatal(err)
}
defer peek.Close()
deadline := time.Now().Add(5 * time.Second)
for {
st, _ := peek.OutboxStats()
if st.Pending == 1 && st.Retrying == 1 && st.Failed == 0 {
break
}
if time.Now().After(deadline) {
t.Fatalf("the delivery must stay queued and retry, not fail for good: %+v", st)
}
time.Sleep(10 * time.Millisecond)
}
list, _ := peek.ListOutbox(10)
if !strings.Contains(list[0].LastError, "authentication") || strings.Contains(list[0].LastError, "wrong") {
t.Fatalf("the admin sees what is wrong, without the password: %q", list[0].LastError)
}
if relay.sink.count() != 0 {
t.Fatal("nothing was delivered")
}
}
func TestStartFailsOnUnusableRelaySecrets(t *testing.T) {
relay := startTLSRelay(t, "ses-user", "p")
for name, mutate := range map[string]func(*config.Config){
"missing password file": func(c *config.Config) { c.Relay.PasswordFile = "/nonexistent/pw" },
"empty password file": func(c *config.Config) { os.WriteFile(c.Relay.PasswordFile, []byte("\n"), 0o600) },
"missing CA file": func(c *config.Config) { c.Relay.CAFile = "/nonexistent/ca.pem" },
"CA file without cert": func(c *config.Config) { os.WriteFile(c.Relay.CAFile, []byte("nope"), 0o600) },
} {
cfg := relayConfig(t, relay, "p")
mutate(&cfg)
if a, err := Start(cfg, "tok"); err == nil {
a.Close()
t.Errorf("%s: the daemon must refuse to start rather than fail at the first send", name)
} else if !strings.Contains(err.Error(), "relay") {
t.Errorf("%s: the error names the relay setting: %v", name, err)
}
}
}
func TestTheRelayAddressIsNotDialledAtStartup(t *testing.T) {
// Starting must not depend on the relay being reachable: mail waits in the queue.
cfg := testConfig(t, deadAddr(t))
cfg.Relay.TLS = "starttls"
a, err := Start(cfg, "tok")
if err != nil {
t.Fatalf("an unreachable relay is a runtime condition, not a startup error: %v", err)
}
a.Close()
}
func TestTheConfiguredEHLONameReachesTheRelay(t *testing.T) {
relay := startTLSRelay(t, "ses-user", "p")
a, err := Start(relayConfig(t, relay, "p"), "tok")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { a.Close() })
postOne(t, a)
relay.sink.waitCount(t, 1)
relay.mu.Lock()
defer relay.mu.Unlock()
if len(relay.hellos) != 1 || relay.hellos[0] != "lists.example.org" {
t.Fatalf("EHLO names: %v", relay.hellos)
}
}
func TestWithoutAConfiguredNameEHLOIsTheHostName(t *testing.T) {
host, err := os.Hostname()
if err != nil || host == "" {
t.Skip("no host name on this machine")
}
relay := startTLSRelay(t, "ses-user", "p")
cfg := relayConfig(t, relay, "p")
cfg.Relay.Helo = ""
a, err := Start(cfg, "tok")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { a.Close() })
postOne(t, a)
relay.sink.waitCount(t, 1)
relay.mu.Lock()
defer relay.mu.Unlock()
if len(relay.hellos) != 1 || relay.hellos[0] != host {
t.Fatalf("EHLO %v, want the host name %q (real relays dislike \"localhost\")", relay.hellos, host)
}
}
func TestTheServerNameSettingReachesTheTLSCheck(t *testing.T) {
relay := startTLSRelay(t, "ses-user", "p") // its certificate is for 127.0.0.1
cfg := relayConfig(t, relay, "p")
cfg.Relay.ServerName = "other.example"
a, err := Start(cfg, "tok")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { a.Close() })
postOne(t, a)
peek, _ := store.Open(cfg.DB)
defer peek.Close()
deadline := time.Now().Add(5 * time.Second)
for {
list, _ := peek.ListOutbox(10)
if len(list) == 1 && list[0].Attempts == 1 {
if !strings.Contains(strings.ToLower(list[0].LastError), "certificate") || list[0].Failed {
t.Fatalf("a name mismatch is a retryable certificate error: %+v", list[0])
}
break
}
if time.Now().After(deadline) {
t.Fatal("the delivery never failed on the certificate name")
}
time.Sleep(10 * time.Millisecond)
}
if relay.sink.count() != 0 {
t.Fatal("nothing may be sent to a relay whose certificate does not match")
}
}

View file

@ -3,6 +3,8 @@ package config
import (
"bytes"
"crypto/x509"
"errors"
"fmt"
"net"
"os"
@ -33,6 +35,16 @@ type Config struct {
Relay struct {
Addr string `yaml:"addr"` // host:port of the outgoing SMTP relay
Concurrency int `yaml:"concurrency"` // simultaneous deliveries
// TLS is "none" (default: a local Postfix), "starttls" (required, never silently skipped)
// or "tls" (encrypted from the first byte, port 465).
TLS string `yaml:"tls"`
ServerName string `yaml:"server_name"` // name the certificate must match; the host of addr by default
CAFile string `yaml:"ca_file"` // PEM bundle of the CAs to trust; the system's by default
Helo string `yaml:"helo"` // EHLO name; this machine's host name by default
// Username and the content of PasswordFile are the SMTP AUTH PLAIN credentials. The password
// is never in the config itself.
Username string `yaml:"username"`
PasswordFile string `yaml:"password_file"`
} `yaml:"relay"`
}
@ -72,7 +84,71 @@ func parseNet(s string) (*net.IPNet, error) {
return &net.IPNet{IP: ip, Mask: net.CIDRMask(bits, bits)}, nil
}
func isLoopbackHost(host string) bool {
if host == "localhost" {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
// RelayPassword reads the relay password from its file (only the line ending is removed). It is
// empty when no username is configured.
func (c Config) RelayPassword() (string, error) {
if c.Relay.Username == "" {
return "", nil
}
b, err := os.ReadFile(c.Relay.PasswordFile)
if err != nil {
return "", fmt.Errorf("relay.password_file: %w", err)
}
pw := strings.TrimRight(string(b), "\r\n")
if pw == "" {
return "", fmt.Errorf("relay.password_file %s is empty", c.Relay.PasswordFile)
}
return pw, nil
}
// RelayRootCAs reads relay.ca_file. It returns nil when none is set: the system roots apply.
func (c Config) RelayRootCAs() (*x509.CertPool, error) {
if c.Relay.CAFile == "" {
return nil, nil
}
b, err := os.ReadFile(c.Relay.CAFile)
if err != nil {
return nil, fmt.Errorf("relay.ca_file: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(b) {
return nil, fmt.Errorf("relay.ca_file %s contains no certificate", c.Relay.CAFile)
}
return pool, nil
}
func (c Config) validateRelay() error {
r := c.Relay
host, _, err := net.SplitHostPort(r.Addr)
if err != nil {
return fmt.Errorf("relay.addr %q must be host:port", r.Addr)
}
switch r.TLS {
case "", "none", "starttls", "tls":
default:
return fmt.Errorf("relay.tls must be none, starttls or tls, not %q", r.TLS)
}
if (r.Username == "") != (r.PasswordFile == "") {
return errors.New("relay.username and relay.password_file go together")
}
if r.Username != "" && (r.TLS == "" || r.TLS == "none") && !isLoopbackHost(host) {
return fmt.Errorf("relay.username is set but relay.tls is not: the password would cross the network in clear to %s; set relay.tls to starttls or tls", host)
}
return nil
}
func (c Config) validate() error {
if err := c.validateRelay(); err != nil {
return err
}
for i, a := range c.LMTP.Allow {
if _, err := parseNet(a); err != nil {
return fmt.Errorf("lmtp.allow[%d]: %w", i, err)

View file

@ -1,11 +1,19 @@
package config
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func write(t *testing.T, content string) string {
@ -126,3 +134,115 @@ func TestLoadRejectsABadLMTPAllowEntry(t *testing.T) {
}
}
}
func TestLoadRelayTLSSettings(t *testing.T) {
c, err := Load(write(t, "relay:\n addr: email-smtp.eu-west-1.amazonaws.com:587\n tls: starttls\n username: AKIAEXAMPLE\n password_file: /run/secrets/relay\n ca_file: /etc/ssl/private-ca.pem\n server_name: smtp.example.com\n helo: lists.example.org\n concurrency: 8\n"))
if err != nil {
t.Fatal(err)
}
r := c.Relay
if r.TLS != "starttls" || r.Username != "AKIAEXAMPLE" || r.PasswordFile != "/run/secrets/relay" || r.CAFile != "/etc/ssl/private-ca.pem" ||
r.ServerName != "smtp.example.com" || r.Helo != "lists.example.org" || r.Concurrency != 8 {
t.Fatalf("relay: %+v", r)
}
if d, _ := Load(write(t, "{}\n")); d.Relay.TLS != "" || d.Relay.Username != "" {
t.Fatalf("defaults must keep the plain local relay: %+v", d.Relay)
}
}
func TestLoadRejectsBadRelaySettings(t *testing.T) {
for name, y := range map[string]string{
"unknown tls mode": "relay:\n tls: ssl\n",
"username without password file": "relay:\n addr: smtp.example.com:587\n tls: starttls\n username: u\n",
"password file without username": "relay:\n addr: smtp.example.com:587\n tls: starttls\n password_file: /p\n",
"credentials in clear to a host": "relay:\n addr: smtp.example.com:587\n username: u\n password_file: /p\n",
"credentials, tls none, a remote": "relay:\n addr: smtp.example.com:587\n tls: none\n username: u\n password_file: /p\n",
"inline password": "relay:\n addr: smtp.example.com:587\n tls: starttls\n username: u\n password: hunter2\n",
"relay address without a port": "relay:\n addr: smtp.example.com\n",
} {
if _, err := Load(write(t, y)); err == nil {
t.Errorf("%s: want an error", name)
}
}
_, err := Load(write(t, "relay:\n addr: smtp.example.com:587\n username: u\n password_file: /p\n"))
if err == nil || !strings.Contains(err.Error(), "relay.tls") {
t.Fatalf("the error says what to set: %v", err)
}
}
func TestCredentialsToALoopbackRelayNeedNoTLS(t *testing.T) {
for _, addr := range []string{"127.0.0.1:587", "localhost:587", "[::1]:587"} {
if _, err := Load(write(t, "relay:\n addr: \""+addr+"\"\n username: u\n password_file: /p\n")); err != nil {
t.Errorf("%s: a local relay with SASL is fine in clear: %v", addr, err)
}
}
for _, mode := range []string{"starttls", "tls"} {
if _, err := Load(write(t, "relay:\n addr: smtp.example.com:587\n tls: "+mode+"\n username: u\n password_file: /p\n")); err != nil {
t.Errorf("%s: %v", mode, err)
}
}
}
func TestRelayPasswordIsReadFromItsFile(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "relay-password")
os.WriteFile(path, []byte(" s3cret with spaces \n"), 0o600)
c, _ := Load(write(t, "relay:\n addr: smtp.example.com:587\n tls: starttls\n username: u\n password_file: "+path+"\n"))
pw, err := c.RelayPassword()
if err != nil || pw != " s3cret with spaces " {
t.Fatalf("only the line ending is removed: %q %v", pw, err)
}
os.WriteFile(path, []byte("with-crlf\r\n"), 0o600)
if pw, _ := c.RelayPassword(); pw != "with-crlf" {
t.Fatalf("%q", pw)
}
os.WriteFile(path, []byte("\n"), 0o600)
if _, err := c.RelayPassword(); err == nil || !strings.Contains(err.Error(), "empty") {
t.Fatalf("an empty password file is an error: %v", err)
}
os.Remove(path)
if _, err := c.RelayPassword(); err == nil {
t.Fatal("a missing password file is an error")
}
if none, _ := Load(write(t, "{}\n")); true {
if pw, err := none.RelayPassword(); err != nil || pw != "" {
t.Fatalf("no username, no password: %q %v", pw, err)
}
}
}
func TestRelayCAFile(t *testing.T) {
good := filepath.Join(t.TempDir(), "ca.pem")
os.WriteFile(good, testCAPEM(t), 0o600)
c, _ := Load(write(t, "relay:\n ca_file: "+good+"\n"))
pool, err := c.RelayRootCAs()
if err != nil || pool == nil {
t.Fatalf("%v %v", pool, err)
}
bad := filepath.Join(t.TempDir(), "bad.pem")
os.WriteFile(bad, []byte("not a certificate"), 0o600)
c, _ = Load(write(t, "relay:\n ca_file: "+bad+"\n"))
if _, err := c.RelayRootCAs(); err == nil || !strings.Contains(err.Error(), "certificate") {
t.Fatalf("a CA file without a certificate is an error: %v", err)
}
c, _ = Load(write(t, "relay:\n ca_file: /nonexistent.pem\n"))
if _, err := c.RelayRootCAs(); err == nil {
t.Fatal("a missing CA file is an error")
}
if none, _ := Load(write(t, "{}\n")); true {
if pool, err := none.RelayRootCAs(); err != nil || pool != nil {
t.Fatalf("without ca_file the system roots are used: %v %v", pool, err)
}
}
}
func testCAPEM(t *testing.T) []byte {
t.Helper()
key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
}

View file

@ -4,12 +4,16 @@ package outbound
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"net"
"strings"
"sync"
"time"
"github.com/emersion/go-sasl"
"github.com/emersion/go-smtp"
)
@ -22,10 +26,24 @@ type Sender interface {
// DefaultBatchSize is the number of recipients per SMTP transaction.
const DefaultBatchSize = 50
// SMTPRelay sends through a plain SMTP relay such as a local Postfix.
// TLS modes of SMTPRelay.
const (
TLSNone = "none" // plain connection (a local Postfix); the default
TLSStartTLS = "starttls" // upgrade the connection; fail if the relay does not offer it
TLSImplicit = "tls" // encrypted from the first byte (port 465)
)
// SMTPRelay sends through an SMTP relay: a local Postfix, or an external one such as SES.
type SMTPRelay struct {
Addr string // host:port
BatchSize int // recipients per transaction; DefaultBatchSize if zero
TLS string // TLSNone (default), TLSStartTLS or TLSImplicit
ServerName string // name the certificate must match; the host of Addr by default
RootCAs *x509.CertPool // trusted CAs; the system's by default
Username string // SMTP AUTH PLAIN; only over TLS or to a loopback address
Password string
Helo string // EHLO name; "localhost" by default
}
// Send splits recipients in batches and sends one transaction per batch.
@ -46,19 +64,46 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b
return nil
}
func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) (err error) {
d := net.Dialer{Timeout: 10 * time.Second}
conn, err := d.DialContext(ctx, "tcp", r.Addr)
// dial opens the connection and speaks EHLO, TLS and AUTH as configured.
// The returned release function must be called when the client is done: until then the connection
// is closed if ctx is cancelled, even in the middle of EHLO, STARTTLS or AUTH with a silent relay.
func (r *SMTPRelay) dial(ctx context.Context) (c *smtp.Client, release func(), err error) {
host, _, err := net.SplitHostPort(r.Addr)
if err != nil {
return err
return nil, nil, fmt.Errorf("relay address %q: %w", r.Addr, err)
}
// A relay that stops answering must not hold us past the context: apply its
// deadline to the connection and close the connection if it is cancelled.
mode := r.TLS
if mode == "" {
mode = TLSNone
}
if r.Username != "" && mode == TLSNone && !isLoopback(host) {
return nil, nil, errors.New("refusing to send credentials in clear to a remote relay: set relay.tls to starttls or tls")
}
helo := r.Helo
if helo == "" {
helo = "localhost"
}
tlsConf := &tls.Config{ServerName: host, RootCAs: r.RootCAs, MinVersion: tls.VersionTLS12}
if r.ServerName != "" {
tlsConf.ServerName = r.ServerName
}
d := &net.Dialer{Timeout: 10 * time.Second}
var conn net.Conn
if mode == TLSImplicit {
conn, err = (&tls.Dialer{NetDialer: d, Config: tlsConf}).DialContext(ctx, "tcp", r.Addr)
} else {
conn, err = d.DialContext(ctx, "tcp", r.Addr)
}
if err != nil {
return nil, nil, err
}
// A relay that stops answering must not hold us past the context: apply its deadline to the
// connection and close the connection if the context is cancelled.
if dl, ok := ctx.Deadline(); ok {
conn.SetDeadline(dl)
}
stop := make(chan struct{})
defer close(stop)
go func() {
select {
case <-ctx.Done():
@ -66,19 +111,73 @@ func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, dat
case <-stop:
}
}()
release = func() { close(stop) }
fail := func(err error) (*smtp.Client, func(), error) {
release()
conn.Close()
return nil, nil, err
}
if mode == TLSStartTLS {
// go-smtp performs EHLO and STARTTLS itself and fails when STARTTLS is not offered, so a
// downgrade is impossible. Its first EHLO says "localhost"; ours follows over TLS below.
c, err = smtp.NewClientStartTLS(conn, tlsConf)
if err != nil {
return fail(err)
}
} else {
c = smtp.NewClient(conn)
}
if err := c.Hello(helo); err != nil {
c.Close()
return fail(err)
}
if r.Username != "" {
if err := c.Auth(sasl.NewPlainClient("", r.Username, r.Password)); err != nil {
c.Close()
return fail(authError(err))
}
}
return c, release, nil
}
func isLoopback(host string) bool {
if host == "localhost" {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
// authError turns a refusal of our credentials into a temporary error. A wrong password or a
// relay that wants authentication is a configuration problem: if it were permanent every message
// would be failed for good, instead of waiting in the queue until the admin fixes it.
func authError(err error) error {
var se *smtp.SMTPError
if errors.As(err, &se) && (se.Code == 530 || se.Code == 534 || se.Code == 535 || se.Code == 538) {
return fmt.Errorf("relay authentication failed (%d %s)", se.Code, strings.TrimSpace(se.Message))
}
return err
}
func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) (err error) {
c, release, err := r.dial(ctx)
if err != nil {
if ctx.Err() != nil {
err = ctx.Err() // report the cause, not "use of closed connection"
}
return err
}
defer release()
defer c.Close()
defer func() {
if err != nil && ctx.Err() != nil {
err = ctx.Err() // report the cause, not "use of closed connection"
err = ctx.Err()
}
}()
c := smtp.NewClient(conn)
defer c.Close()
if err := c.Hello("localhost"); err != nil {
return err
}
if err := c.SendMail(from, to, bytes.NewReader(data)); err != nil {
return err
return authError(err)
}
return c.Quit()
}

View file

@ -0,0 +1,411 @@
package outbound
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"errors"
"io"
"math/big"
"net"
"strings"
"sync"
"testing"
"time"
"github.com/emersion/go-sasl"
"github.com/emersion/go-smtp"
)
// --- a throwaway CA and certificates -------------------------------------------------------
type pki struct {
pool *x509.CertPool
good tls.Certificate // valid for 127.0.0.1 and localhost
wrong tls.Certificate // signed by the CA but for another name
rogue tls.Certificate // valid for 127.0.0.1 but signed by an unknown CA
}
func newPKI(t *testing.T) *pki {
t.Helper()
mk := func(parent *x509.Certificate, parentKey *ecdsa.PrivateKey, tmpl *x509.Certificate) (*x509.Certificate, *ecdsa.PrivateKey, []byte) {
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
if parent == nil {
parent, parentKey = tmpl, key
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, parent, &key.PublicKey, parentKey)
if err != nil {
t.Fatal(err)
}
c, _ := x509.ParseCertificate(der)
return c, key, der
}
serial := int64(1)
tmpl := func(cn string, ca bool, dns []string, ips []net.IP) *x509.Certificate {
serial++
return &x509.Certificate{
SerialNumber: big.NewInt(serial), Subject: pkix.Name{CommonName: cn},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour),
IsCA: ca, BasicConstraintsValid: true,
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
DNSNames: dns, IPAddresses: ips,
}
}
ca, caKey, _ := mk(nil, nil, tmpl("test CA", true, nil, nil))
rogueCA, rogueKey, _ := mk(nil, nil, tmpl("rogue CA", true, nil, nil))
leaf := func(parent *x509.Certificate, pk *ecdsa.PrivateKey, dns []string, ips []net.IP) tls.Certificate {
_, key, der := mk(parent, pk, tmpl("relay", false, dns, ips))
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
}
pool := x509.NewCertPool()
pool.AddCert(ca)
loop := []net.IP{net.ParseIP("127.0.0.1")}
return &pki{
pool: pool,
good: leaf(ca, caKey, []string{"localhost"}, loop),
wrong: leaf(ca, caKey, []string{"other.example"}, nil),
rogue: leaf(rogueCA, rogueKey, []string{"localhost"}, loop),
}
}
// --- an SMTP server with TLS and AUTH --------------------------------------------------------
type relayOpts struct {
cert *tls.Certificate // nil: no TLS at all
implicit bool // TLS from the first byte (port 465 style)
user, pass string // when set, AUTH PLAIN must match
requireAuth bool // answer 530 to MAIL until authenticated
}
type testRelay struct {
mu sync.Mutex
addr string
messages []captured
auths []string // "user:pass" seen
hellos []string // EHLO names seen at MAIL time
tlsSeen []bool // whether each message arrived over TLS
}
type relayBackend struct {
r *testRelay
o relayOpts
}
func (b *relayBackend) NewSession(c *smtp.Conn) (smtp.Session, error) {
return &relaySession{b: b, conn: c}, nil
}
type relaySession struct {
b *relayBackend
conn *smtp.Conn
authed bool
from string
to []string
}
func (s *relaySession) Reset() { s.from, s.to = "", nil }
func (s *relaySession) Logout() error { return nil }
func (s *relaySession) AuthMechanisms() []string {
if s.b.o.user == "" {
return nil
}
return []string{sasl.Plain}
}
func (s *relaySession) Auth(mech string) (sasl.Server, error) {
return sasl.NewPlainServer(func(identity, username, password string) error {
s.b.r.mu.Lock()
s.b.r.auths = append(s.b.r.auths, username+":"+password)
s.b.r.mu.Unlock()
if username != s.b.o.user || password != s.b.o.pass {
return &smtp.SMTPError{Code: 535, EnhancedCode: smtp.EnhancedCode{5, 7, 8}, Message: "Authentication credentials invalid"}
}
s.authed = true
return nil
}), nil
}
func (s *relaySession) Mail(from string, _ *smtp.MailOptions) error {
if s.b.o.requireAuth && !s.authed {
return &smtp.SMTPError{Code: 530, EnhancedCode: smtp.EnhancedCode{5, 7, 0}, Message: "Authentication required"}
}
s.from = from
s.b.r.mu.Lock()
s.b.r.hellos = append(s.b.r.hellos, s.conn.Hostname())
s.b.r.mu.Unlock()
return nil
}
func (s *relaySession) Rcpt(to string, _ *smtp.RcptOptions) error {
s.to = append(s.to, to)
return nil
}
func (s *relaySession) Data(r io.Reader) error {
b, _ := io.ReadAll(r)
_, isTLS := s.conn.TLSConnectionState()
s.b.r.mu.Lock()
s.b.r.messages = append(s.b.r.messages, captured{s.from, s.to, string(b)})
s.b.r.tlsSeen = append(s.b.r.tlsSeen, isTLS)
s.b.r.mu.Unlock()
return nil
}
func startRelay(t *testing.T, o relayOpts) *testRelay {
t.Helper()
r := &testRelay{}
srv := smtp.NewServer(&relayBackend{r: r, o: o})
srv.AllowInsecureAuth = false // credentials are only accepted over TLS
if o.cert != nil {
srv.TLSConfig = &tls.Config{Certificates: []tls.Certificate{*o.cert}}
}
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
r.addr = ln.Addr().String()
serveLn := ln
if o.implicit {
serveLn = nil
go srv.Serve(tls.NewListener(ln, srv.TLSConfig))
} else {
go srv.Serve(ln)
}
_ = serveLn
t.Cleanup(func() { srv.Close() })
return r
}
func (r *testRelay) count() int {
r.mu.Lock()
defer r.mu.Unlock()
return len(r.messages)
}
func send(r *SMTPRelay) error {
return r.Send(context.Background(), "bounce@example.org", []string{"a@x.com"}, []byte("Subject: hi\r\n\r\nbody\r\n"))
}
// --- STARTTLS ---------------------------------------------------------------------------------
func TestStartTLSDeliversOverAnEncryptedConnection(t *testing.T) {
p := newPKI(t)
srv := startRelay(t, relayOpts{cert: &p.good})
r := &SMTPRelay{Addr: srv.addr, TLS: "starttls", RootCAs: p.pool}
if err := send(r); err != nil {
t.Fatal(err)
}
if srv.count() != 1 || !srv.tlsSeen[0] {
t.Fatalf("the message must arrive over TLS: %d %v", srv.count(), srv.tlsSeen)
}
}
func TestStartTLSIsRequiredNotOptional(t *testing.T) {
srv := startRelay(t, relayOpts{}) // a server that does not offer STARTTLS
err := send(&SMTPRelay{Addr: srv.addr, TLS: "starttls"})
if err == nil || !strings.Contains(err.Error(), "STARTTLS") {
t.Fatalf("want a STARTTLS error, got %v", err)
}
if srv.count() != 0 {
t.Fatal("a downgrade attack must not make us send the message in clear")
}
if IsPermanent(err) {
t.Fatalf("a relay that cannot do TLS is a configuration problem to fix, not a reason to drop mail: %v", err)
}
}
func TestImplicitTLS(t *testing.T) {
p := newPKI(t)
srv := startRelay(t, relayOpts{cert: &p.good, implicit: true})
if err := send(&SMTPRelay{Addr: srv.addr, TLS: "tls", RootCAs: p.pool}); err != nil {
t.Fatal(err)
}
if srv.count() != 1 || !srv.tlsSeen[0] {
t.Fatalf("%d %v", srv.count(), srv.tlsSeen)
}
}
func TestCertificateProblemsAreRefusedAndAreNotPermanent(t *testing.T) {
p := newPKI(t)
for name, tc := range map[string]struct {
cert *tls.Certificate
mode string
impl bool
}{
"wrong name, starttls": {&p.wrong, "starttls", false},
"wrong name, implicit": {&p.wrong, "tls", true},
"unknown CA, starttls": {&p.rogue, "starttls", false},
"unknown CA, implicit": {&p.rogue, "tls", true},
} {
srv := startRelay(t, relayOpts{cert: tc.cert, implicit: tc.impl})
err := send(&SMTPRelay{Addr: srv.addr, TLS: tc.mode, RootCAs: p.pool})
if err == nil {
t.Errorf("%s: an untrusted relay must be refused", name)
continue
}
if !strings.Contains(strings.ToLower(err.Error()), "certificate") {
t.Errorf("%s: say it is a certificate problem: %v", name, err)
}
if srv.count() != 0 {
t.Errorf("%s: nothing may be sent to an untrusted relay", name)
}
if IsPermanent(err) {
t.Errorf("%s: a certificate problem must not drop the mail: %v", name, err)
}
}
}
func TestServerNameOverridesTheNameToVerify(t *testing.T) {
p := newPKI(t)
srv := startRelay(t, relayOpts{cert: &p.wrong}) // the certificate is for other.example
if err := send(&SMTPRelay{Addr: srv.addr, TLS: "starttls", RootCAs: p.pool, ServerName: "other.example"}); err != nil {
t.Fatalf("server_name is the name the certificate must match: %v", err)
}
}
// --- AUTH ---------------------------------------------------------------------------------------
func TestAuthPlainOverStartTLS(t *testing.T) {
p := newPKI(t)
srv := startRelay(t, relayOpts{cert: &p.good, user: "ses-user", pass: "s3cret pass", requireAuth: true})
r := &SMTPRelay{Addr: srv.addr, TLS: "starttls", RootCAs: p.pool, Username: "ses-user", Password: "s3cret pass"}
if err := send(r); err != nil {
t.Fatal(err)
}
if srv.count() != 1 || len(srv.auths) != 1 || srv.auths[0] != "ses-user:s3cret pass" {
t.Fatalf("messages=%d auths=%v", srv.count(), srv.auths)
}
}
func TestAuthPlainOverImplicitTLS(t *testing.T) {
p := newPKI(t)
srv := startRelay(t, relayOpts{cert: &p.good, implicit: true, user: "u", pass: "p", requireAuth: true})
if err := send(&SMTPRelay{Addr: srv.addr, TLS: "tls", RootCAs: p.pool, Username: "u", Password: "p"}); err != nil {
t.Fatal(err)
}
if srv.count() != 1 {
t.Fatal("not delivered")
}
}
func TestCredentialsAreNeverSentInClearToARemoteRelay(t *testing.T) {
// 192.0.2.1 is reserved for documentation: the client must refuse before it even connects.
for _, mode := range []string{"", "none"} {
start := time.Now()
err := send(&SMTPRelay{Addr: "192.0.2.1:25", TLS: mode, Username: "u", Password: "p"})
if err == nil || !strings.Contains(err.Error(), "clear") {
t.Fatalf("mode %q: want a refusal to send credentials in clear, got %v", mode, err)
}
if time.Since(start) > time.Second {
t.Fatalf("mode %q: the refusal must not wait for a connection", mode)
}
}
// A clear connection to a relay that does not offer AUTH without TLS: nothing reaches the server.
srv := startRelay(t, relayOpts{user: "u", pass: "p", requireAuth: true})
if err := send(&SMTPRelay{Addr: srv.addr, TLS: "none", Username: "u", Password: "p"}); err == nil {
t.Fatal("the relay cannot authenticate us in clear")
}
if len(srv.auths) != 0 {
t.Fatalf("the password reached the server: %v", srv.auths)
}
}
func TestAWrongPasswordKeepsTheMailQueuedAndLeaksNothing(t *testing.T) {
p := newPKI(t)
srv := startRelay(t, relayOpts{cert: &p.good, user: "u", pass: "right", requireAuth: true})
err := send(&SMTPRelay{Addr: srv.addr, TLS: "starttls", RootCAs: p.pool, Username: "u", Password: "Wr0ngPassw0rd"})
if err == nil {
t.Fatal("want an authentication error")
}
if IsPermanent(err) {
t.Fatalf("a wrong password must not fail every message for good: the admin fixes it and the mail goes out: %v", err)
}
if !strings.Contains(err.Error(), "relay authentication failed (535") {
t.Fatalf("say what went wrong, with the reply code: %v", err)
}
if strings.Contains(err.Error(), "Wr0ngPassw0rd") {
t.Fatal("never put the password in an error")
}
if srv.count() != 0 {
t.Fatal("nothing delivered")
}
}
func TestARelayThatWantsAuthenticationIsATemporaryProblem(t *testing.T) {
p := newPKI(t)
srv := startRelay(t, relayOpts{cert: &p.good, user: "u", pass: "p", requireAuth: true})
// We forgot to configure credentials: the relay answers 530 at MAIL FROM.
err := send(&SMTPRelay{Addr: srv.addr, TLS: "starttls", RootCAs: p.pool})
if err == nil || IsPermanent(err) {
t.Fatalf("530 means credentials are missing: retry after the admin fixes it, got %v", err)
}
if !strings.Contains(err.Error(), "relay authentication failed (530") {
t.Fatalf("%v", err)
}
}
// --- EHLO name ----------------------------------------------------------------------------------
func TestHeloNameIsUsed(t *testing.T) {
p := newPKI(t)
srv := startRelay(t, relayOpts{cert: &p.good})
if err := send(&SMTPRelay{Addr: srv.addr, TLS: "starttls", RootCAs: p.pool, Helo: "lists.example.org"}); err != nil {
t.Fatal(err)
}
if len(srv.hellos) != 1 || srv.hellos[0] != "lists.example.org" {
t.Fatalf("EHLO names: %v", srv.hellos)
}
plain := startRelay(t, relayOpts{})
if err := send(&SMTPRelay{Addr: plain.addr, Helo: "lists.example.org"}); err != nil {
t.Fatal(err)
}
if plain.hellos[0] != "lists.example.org" {
t.Fatalf("EHLO over a plain connection: %v", plain.hellos)
}
}
func TestInvalidHeloNamesAreRejectedBeforeConnecting(t *testing.T) {
srv := startRelay(t, relayOpts{})
if err := send(&SMTPRelay{Addr: srv.addr, Helo: "bad name\r\nMAIL FROM:<x@y>"}); err == nil {
t.Fatal("a name with a line break must never reach the wire")
}
}
// --- unchanged behaviour -------------------------------------------------------------------------
func TestPlainRelayStillWorksWithoutAnyTLSSetting(t *testing.T) {
srv := startRelay(t, relayOpts{})
if err := send(&SMTPRelay{Addr: srv.addr}); err != nil {
t.Fatal(err)
}
if srv.count() != 1 || srv.tlsSeen[0] {
t.Fatalf("the default is the plain connection used so far: %v", srv.tlsSeen)
}
}
func TestAuthOverPlainConnectionToLoopbackIsAllowedWhenTheServerOffersIt(t *testing.T) {
// A local Postfix with SASL on 127.0.0.1: no TLS needed to reach it.
srv := startPlainAuthRelay(t)
err := send(&SMTPRelay{Addr: srv, TLS: "none", Username: "u", Password: "p"})
if err != nil {
t.Fatalf("%v", err)
}
}
// startPlainAuthRelay is a loopback relay that accepts AUTH without TLS.
func startPlainAuthRelay(t *testing.T) string {
t.Helper()
r := &testRelay{}
srv := smtp.NewServer(&relayBackend{r: r, o: relayOpts{user: "u", pass: "p", requireAuth: true}})
srv.AllowInsecureAuth = true
ln, _ := net.Listen("tcp", "127.0.0.1:0")
go srv.Serve(ln)
t.Cleanup(func() { srv.Close() })
return ln.Addr().String()
}
var _ = errors.New

View file

@ -6,6 +6,11 @@ lmtp:
allow: [172.16.0.0/12] # LMTP has no authentication: list who may connect (here: the Docker networks)
relay:
addr: postfix:25
# To send through an external relay (SES, Mailgun, a smarthost) instead:
# addr: smtp.example.com:587
# tls: starttls
# username: user
# password_file: /run/secrets/relay-password
concurrency: 4 # simultaneous deliveries
web:
listen: 0.0.0.0:8081 # public archive and unsubscribe pages; "" disables them