parsimony/internal/router/router.go
tgiachi 0dc9c03bae feat(router): send newsletter deliveries with priority 1
- Deliver and Release (through outbound) queue the deliveries of a list
  whose post policy is publishers with priority 1, everything else 0;
  confirmation mail stays 0
- tests for Deliver, Release, confirmation mail and the recorded post
2026-11-10 12:39:34 +01:00

354 lines
12 KiB
Go

// Package router decides what happens to an incoming message.
package router
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"log"
"net/mail"
"strings"
"sync"
"time"
"codeberg.org/squidd/parsimony/internal/archive"
"codeberg.org/squidd/parsimony/internal/store"
)
var (
// ErrUnknownList means the recipient address is not a list or a list command.
ErrUnknownList = errors.New("unknown list")
// ErrDiscarded means the message was dropped on purpose (a loop or an automatic reply). It is
// acknowledged to the MTA: refusing it would make the sender's server bounce it, starting a new loop.
ErrDiscarded = errors.New("discarded")
// ErrHeldFull means the list's moderation queue is full: refuse for now, the sender may retry.
ErrHeldFull = store.ErrHeldFull
// ErrHeldTooLarge means the message is too big to hold for moderation on a moderated list.
ErrHeldTooLarge = store.ErrHeldTooLarge
// ErrNotMember means the sender may not post to the list.
ErrNotMember = errors.New("sender is not a member")
// ErrHeld means the message was queued for moderation. It is not a failure.
ErrHeld = errors.New("held for moderation")
// ErrBadToken means a confirmation or bounce token is invalid, used or expired.
ErrBadToken = errors.New("invalid or expired token")
// ErrGone means the member behind a valid token is already unsubscribed.
ErrGone = errors.New("already unsubscribed")
// ErrJoinClosed means the list does not accept self-service subscriptions.
ErrJoinClosed = errors.New("list is closed")
// ErrDuplicate means the list already queued the same post less than an hour ago: the MTA sent
// it again because our reply was lost. It is acknowledged and not queued again.
ErrDuplicate = store.ErrDuplicate
)
// Recipient is a subscriber that receives a message.
type Recipient struct {
ID int64 // member id, used for the VERP bounce address
Address string
}
// Result describes how to deliver an accepted message.
type Result struct {
List store.List
Recipients []Recipient
Data []byte // message to send, with list headers
}
// Router applies list policy to incoming messages.
type Router struct {
store *store.Store
// BaseURL is the public web URL; enables one-click unsubscribe when set.
BaseURL string
// OnEnqueue, when set, is called after mail is queued (to wake the sender).
OnEnqueue func()
secretOnce sync.Once
secretVal []byte
secretErr error
}
// New creates a Router backed by s.
func New(s *store.Store) *Router { return &Router{store: s} }
// Route validates a message from sender to the list address rcpt.
// Moderated lists queue the message and return ErrHeld.
func (r *Router) Route(sender, rcpt string, data []byte) (Result, error) {
l, err := r.store.FindList(rcpt)
if errors.Is(err, store.ErrNotFound) {
return Result{}, ErrUnknownList
}
if err != nil {
return Result{}, err
}
if why, drop := discardReason(l, data); drop {
log.Printf("discard post to %s from %s: %s", l.Address(), sender, why)
return Result{}, ErrDiscarded
}
if l.PostPolicy == store.PostModerated {
return Result{}, r.hold(l, sender, data, "the list is moderated")
}
ok, err := r.store.IsMember(l.ID, sender)
if err != nil {
return Result{}, err
}
if !ok {
return Result{}, ErrNotMember
}
// Membership is judged on the envelope sender, which anyone can forge, while the archive, the
// rewritten From and Reply-To come from the From header. Only trust a post whose header names
// the same address: otherwise a member could pass as someone else. An admin decides on the rest.
if from, ok := fromHeaderAddress(data); !ok || !strings.EqualFold(strings.TrimSpace(sender), from) {
return Result{}, r.hold(l, sender, data, "the From header does not match the sender of the message ("+displayFrom(data, from, ok)+")")
}
return r.prepare(l, data)
}
// hold queues a message for the admin and reports it as held.
func (r *Router) hold(l store.List, sender string, data []byte, reason string) error {
if _, err := r.store.HoldMessage(l.ID, sender, data, reason); err != nil {
return err
}
return ErrHeld
}
// prepare builds the delivery plan for a list.
func (r *Router) prepare(l store.List, data []byte) (Result, error) {
members, err := r.store.ActiveMembers(l.ID)
if err != nil {
return Result{}, err
}
rcpts := make([]Recipient, len(members))
for i, m := range members {
rcpts[i] = Recipient{ID: m.ID, Address: m.Address}
}
r.archive(l, data)
out, err := rewrite(l, data)
if err != nil {
return Result{}, err
}
return Result{List: l, Recipients: rcpts, Data: out}, nil
}
// Deliver queues the message for every recipient, each with its own signed
// bounce address (VERP) and unsubscribe headers. The queue write is atomic: on
// error nothing was queued and the caller may safely retry. A post the list
// already queued in the last hour is not queued again: Deliver returns ErrDuplicate.
func (r *Router) Deliver(res Result) error {
items, err := r.outbound(res)
if err != nil {
return err
}
id := postID(res.Data)
if err := r.store.EnqueuePost(res.List.ID, id, res.Data, items); err != nil {
if errors.Is(err, ErrDuplicate) {
log.Printf("duplicate post to %s not delivered again: %s", res.List.Address(), id)
}
return err
}
if r.OnEnqueue != nil && len(items) > 0 {
r.OnEnqueue()
}
return nil
}
// postID identifies a post for the duplicate check: its Message-ID, or a hash of the whole
// message when it has none.
func postID(data []byte) string {
if m, err := mail.ReadMessage(bytes.NewReader(data)); err == nil {
if id := m.Header.Get("Message-ID"); id != "" {
return "mid:" + id
}
}
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])
}
// outbound builds one queue entry per recipient. A newsletter (post policy publishers) is sent
// with priority 1, so that its many deliveries never delay discussion mail.
func (r *Router) outbound(res Result) ([]store.NewOutbound, error) {
priority := 0
if res.List.PostPolicy == store.PostPublishers {
priority = 1
}
items := make([]store.NewOutbound, 0, len(res.Recipients))
for _, rc := range res.Recipients {
from := r.bounceAddress(res.List, rc.ID)
if from == "" {
return nil, errors.New("cannot create bounce address")
}
items = append(items, store.NewOutbound{
From: from, Rcpt: rc.Address, MemberID: rc.ID, Prefix: r.unsubscribeHeaders(res.List, rc.ID), Priority: priority,
})
}
return items, nil
}
// enqueue stores deliveries and wakes the sender.
func (r *Router) enqueue(data []byte, items []store.NewOutbound) error {
if err := r.store.Enqueue(data, items); err != nil {
return err
}
if r.OnEnqueue != nil && len(items) > 0 {
r.OnEnqueue()
}
return nil
}
// unsubscribeHeaders builds the per-recipient List-Unsubscribe headers: a signed
// one-click link (RFC 8058) when BaseURL is set, and the +leave address.
func (r *Router) unsubscribeHeaders(l store.List, memberID int64) []byte {
value := "<mailto:" + l.Name + "+leave@" + l.Domain + ">"
var b strings.Builder
if tok := r.UnsubscribeToken(memberID); r.BaseURL != "" && tok != "" {
value = "<" + strings.TrimRight(r.BaseURL, "/") + "/unsubscribe/" + tok + ">, " + value
b.WriteString("List-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n")
}
b.WriteString("List-Unsubscribe: " + value + "\r\n")
return []byte(b.String())
}
// archive stores the original message. Failures are logged, never fatal: the
// post must be delivered even if the archive is unavailable.
func (r *Router) archive(l store.List, data []byte) {
if l.ArchivePolicy == store.ArchiveOff {
return
}
en, err := archive.Parse(data, time.Now())
if err != nil {
log.Printf("archive %s: %v", l.Address(), err)
return
}
_, _, err = r.store.ArchiveMessage(l.ID, store.Message{
MessageID: en.MessageID, InReplyTo: en.InReplyTo, Subject: en.Subject,
Sender: en.Sender, Date: en.Date, Body: en.Body,
})
if err != nil {
log.Printf("archive %s: %v", l.Address(), err)
}
}
// Release approves a held message: it leaves the moderation queue and is queued for delivery in
// one transaction, so simultaneous approvals deliver it once (the others get store.ErrNotFound).
func (r *Router) Release(heldID int64) error {
h, err := r.store.GetHeld(heldID)
if err != nil {
return err
}
l, err := r.store.GetList(h.ListID)
if err != nil {
return err
}
res, err := r.prepare(l, h.Data)
if err != nil {
return err
}
items, err := r.outbound(res)
if err != nil {
return err
}
if err := r.store.EnqueueFromHeld(heldID, postID(res.Data), res.Data, items); err != nil {
return err
}
if r.OnEnqueue != nil && len(items) > 0 {
r.OnEnqueue()
}
return nil
}
// fromHeaderAddress returns the lowercase address of the From header when there is exactly one.
func fromHeaderAddress(data []byte) (string, bool) {
m, err := mail.ReadMessage(bytes.NewReader(data))
if err != nil {
return "", false
}
addrs, err := m.Header.AddressList("From")
if err != nil || len(addrs) != 1 || addrs[0].Address == "" {
return "", false
}
return strings.ToLower(addrs[0].Address), true
}
// displayFrom describes the From header for the held-message reason.
func displayFrom(data []byte, addr string, ok bool) string {
if ok {
return "From: " + addr
}
if m, err := mail.ReadMessage(bytes.NewReader(data)); err == nil {
if raw := strings.TrimSpace(m.Header.Get("From")); raw != "" {
if len(raw) > 80 {
raw = raw[:80] + "…"
}
return "From: " + raw
}
}
return "no usable From header"
}
// ownListID reports whether the message carries this list's own List-Id: it is list mail coming back.
func ownListID(l store.List, h mail.Header) bool {
own := l.Name + "." + l.Domain
for _, v := range h["List-Id"] {
if i, j := strings.LastIndex(v, "<"), strings.LastIndex(v, ">"); i >= 0 && j > i && strings.EqualFold(strings.TrimSpace(v[i+1:j]), own) {
return true
}
}
return false
}
// autoSubmitted returns the lowercase Auto-Submitted keyword (RFC 3834), or "" when the header is absent.
func autoSubmitted(h mail.Header) string {
if f := strings.Fields(h.Get("Auto-Submitted")); len(f) > 0 {
return strings.ToLower(f[0])
}
return ""
}
// precedence returns the lowercase Precedence value.
func precedence(h mail.Header) string {
return strings.ToLower(strings.TrimSpace(h.Get("Precedence")))
}
// discardReason spots mail that must not be distributed: our own list mail coming back (a loop)
// and genuine automatic replies such as vacation messages (RFC 3834). Other machine-generated
// mail (Auto-Submitted: auto-generated, cron, CI, monitoring) is a legitimate post and goes
// through the normal checks.
func discardReason(l store.List, data []byte) (string, bool) {
m, err := mail.ReadMessage(bytes.NewReader(data))
if err != nil {
return "", false
}
if ownListID(l, m.Header) {
return "it carries this list's own List-Id: a mail loop", true
}
if autoSubmitted(m.Header) == "auto-replied" {
return "Auto-Submitted: auto-replied", true
}
if precedence(m.Header) == "auto_reply" {
return "Precedence: auto_reply", true
}
return "", false
}
// robotReason spots a request mail (join, leave, confirm) that a person did not write: a null
// sender, any Auto-Submitted other than "no", a bulk, junk or auto_reply Precedence, or the list's
// own List-Id. A robot must not drive a subscription change or get a confirmation mail.
func robotReason(l store.List, sender string, data []byte) (string, bool) {
if strings.TrimSpace(sender) == "" {
return "null sender", true
}
m, err := mail.ReadMessage(bytes.NewReader(data))
if err != nil {
return "", false
}
if ownListID(l, m.Header) {
return "it carries this list's own List-Id", true
}
if kind := autoSubmitted(m.Header); kind != "" && kind != "no" {
return "Auto-Submitted: " + kind, true
}
switch p := precedence(m.Header); p {
case "auto_reply", "bulk", "junk":
return "Precedence: " + p, true
}
return "", false
}