parsimony/internal/store/store.go
tgiachi 2abd435494 feat(store): record posts and count their deliveries
- posts table: one row per post queued to a list (message id, subject,
  time, recipients, sent, failed, dropped), deleted with the list and
  forgotten after 400 days when a new post is recorded
- EnqueuePost and EnqueueFromHeld record the post in the same
  transaction and stamp every delivery with its id; a refused duplicate
  records nothing; other mail keeps post_id 0
- CompleteOutbox counts sent, the new DropOutbox counts dropped and
  FailOutbox counts failed, each in the transaction that changes the
  delivery; a delivery that already failed is never counted twice
- outbox priority column; DueOutbox orders by priority, next_at, id
- ListPosts newest first (default 50, at most 200) with Pending derived
- additive migration of the outbox columns, tested on an old database
2026-11-10 12:37:52 +01:00

1117 lines
36 KiB
Go

// Package store persists lists, members and moderation state in SQLite.
package store
import (
"bytes"
"crypto/rand"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"mime"
"net/mail"
"strings"
"time"
"unicode"
_ "modernc.org/sqlite"
)
var (
// ErrNotFound is returned when a record does not exist.
ErrNotFound = errors.New("not found")
// ErrHeldFull is returned when a list's moderation queue has no more room.
ErrHeldFull = errors.New("moderation queue is full")
// ErrHeldTooLarge is returned for a message too big to be held for moderation.
ErrHeldTooLarge = errors.New("message too large to hold for moderation")
// ErrExists is returned when creating something that already exists.
ErrExists = errors.New("already exists")
// ErrDuplicate is returned when the same post was already queued for the list within postMemory.
ErrDuplicate = errors.New("post already delivered")
)
// Policy values.
const (
PostMembers = "members" // only members may post, others are rejected
PostModerated = "moderated" // every post waits for approval
PostPublishers = "publishers" // newsletter: only members flagged as publishers may post
JoinOpen = "open" // anyone can subscribe (with confirmation)
JoinClosed = "closed" // only the admin adds members
)
// Archive policy values.
const (
ArchivePublic = "public" // browsable on the web
ArchivePrivate = "private" // stored, readable through the API only
ArchiveOff = "off" // not stored
)
// Pending actions.
const (
ActionJoin = "join"
ActionLeave = "leave"
)
// Limits on the moderation queue of one list. They are variables so the tests can lower them.
var (
heldMessageLimit = 10 << 20 // bytes in one held message
heldBytesLimit = 200 << 20 // bytes held for one list, all messages together
)
// BounceThreshold is the number of hard bounces after which a member is disabled.
const BounceThreshold = 3
// BounceWindow is how long a hard bounce counts: a bounce after a quiet window restarts the count.
const BounceWindow = 30 * 24 * time.Hour
const (
pendingTTL = 72 * time.Hour
pendingRateLimit = time.Hour
// Caps on confirmation mail, per hour. Anyone can name any address in a join request, so the
// daemon must not mail one victim many times (maxPendingPerAddress, across all lists) or send
// an unbounded amount in total (maxPendingPerHour).
// A moderated list could be flooded with mail that nobody reads: the queue per list is bounded,
// and a message nobody moderated for a month is dropped.
maxHeldPerList = 500
heldTTL = 30 * 24 * time.Hour
maxPendingPerAddress = 3
maxPendingPerHour = 300
)
// List is a mailing list.
type List struct {
ID int64
Name string // local part, lowercase
Domain string // lowercase
Title string
PostPolicy string
JoinPolicy string
ArchivePolicy string
}
// Address returns the list posting address.
func (l List) Address() string { return l.Name + "@" + l.Domain }
// Member is a list subscriber.
type Member struct {
ID int64
ListID int64
Address string // lowercase
DisplayName string
Disabled bool // too many hard bounces: no more delivery
BounceCount int
}
// Held is a message waiting for moderation.
type Held struct {
ID int64
ListID int64
Sender string // envelope sender
Data []byte
CreatedAt time.Time
Reason string // why the message was held, for the admin
// What a listing needs, stored when the message is held so that a listing never has to read a body.
Subject string // decoded, at most 200 characters
FromHeader string // the From header as received, at most 200 characters
Size int // bytes of the message
}
// Pending is an unconfirmed subscribe or unsubscribe request.
type Pending struct {
Token string
ListID int64
Address string
Action string
}
// Store wraps the SQLite database.
type Store struct {
db *sql.DB
// Now returns the current time; tests replace it.
Now func() time.Time
}
const schema = `
CREATE TABLE IF NOT EXISTS lists (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
domain TEXT NOT NULL,
title TEXT NOT NULL DEFAULT '',
post_policy TEXT NOT NULL DEFAULT 'members',
join_policy TEXT NOT NULL DEFAULT 'open',
archive_policy TEXT NOT NULL DEFAULT 'private',
UNIQUE(name, domain)
);
CREATE TABLE IF NOT EXISTS members (
id INTEGER PRIMARY KEY AUTOINCREMENT,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
address TEXT NOT NULL,
display_name TEXT NOT NULL DEFAULT '',
disabled INTEGER NOT NULL DEFAULT 0,
bounce_count INTEGER NOT NULL DEFAULT 0,
bounce_at INTEGER NOT NULL DEFAULT 0,
UNIQUE(list_id, address)
);
CREATE TABLE IF NOT EXISTS held (
id INTEGER PRIMARY KEY AUTOINCREMENT,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
sender TEXT NOT NULL,
data BLOB NOT NULL,
created_at INTEGER NOT NULL,
reason TEXT NOT NULL DEFAULT '',
subject TEXT NOT NULL DEFAULT '',
from_header TEXT NOT NULL DEFAULT '',
size INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS pending (
token TEXT PRIMARY KEY,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
address TEXT NOT NULL,
action TEXT NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS archive (
id INTEGER PRIMARY KEY AUTOINCREMENT,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
thread_id INTEGER NOT NULL DEFAULT 0,
message_id TEXT NOT NULL,
in_reply_to TEXT NOT NULL DEFAULT '',
subject TEXT NOT NULL DEFAULT '',
sender TEXT NOT NULL DEFAULT '',
date INTEGER NOT NULL,
body TEXT NOT NULL DEFAULT '',
size INTEGER NOT NULL DEFAULT 0,
UNIQUE(list_id, message_id)
);
CREATE INDEX IF NOT EXISTS archive_list_date ON archive(list_id, date DESC, id DESC);
CREATE INDEX IF NOT EXISTS archive_thread ON archive(list_id, thread_id);
CREATE VIRTUAL TABLE IF NOT EXISTS archive_fts USING fts5(
subject, body, sender, content='archive', content_rowid='id'
);
CREATE TRIGGER IF NOT EXISTS archive_ai AFTER INSERT ON archive BEGIN
INSERT INTO archive_fts(rowid, subject, body, sender) VALUES (new.id, new.subject, new.body, new.sender);
END;
CREATE TRIGGER IF NOT EXISTS archive_ad AFTER DELETE ON archive BEGIN
INSERT INTO archive_fts(archive_fts, rowid, subject, body, sender) VALUES ('delete', old.id, old.subject, old.body, old.sender);
END;
CREATE TABLE IF NOT EXISTS outbox_msgs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
data BLOB NOT NULL
);
CREATE TABLE IF NOT EXISTS outbox (
id INTEGER PRIMARY KEY AUTOINCREMENT,
msg_id INTEGER NOT NULL REFERENCES outbox_msgs(id) ON DELETE CASCADE,
envelope_from TEXT NOT NULL,
rcpt TEXT NOT NULL,
member_id INTEGER NOT NULL DEFAULT 0,
prefix BLOB NOT NULL DEFAULT x'',
attempts INTEGER NOT NULL DEFAULT 0,
next_at INTEGER NOT NULL,
last_error TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL,
failed INTEGER NOT NULL DEFAULT 0,
post_id INTEGER NOT NULL DEFAULT 0,
priority INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id);
CREATE INDEX IF NOT EXISTS outbox_due_prio ON outbox(priority, next_at, id);
CREATE TABLE IF NOT EXISTS posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
message_id TEXT NOT NULL DEFAULT '',
subject TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL,
recipients INTEGER NOT NULL DEFAULT 0,
sent INTEGER NOT NULL DEFAULT 0,
failed INTEGER NOT NULL DEFAULT 0,
dropped INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS posts_list ON posts(list_id, created_at, id);
CREATE TABLE IF NOT EXISTS delivered (
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
post_id TEXT NOT NULL,
created_at INTEGER NOT NULL,
PRIMARY KEY (list_id, post_id)
);
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);`
// Open opens (and migrates) the database at path. Use ":memory:" for tests.
func Open(path string) (*Store, error) {
// The pragmas are in the DSN so that every connection of the pool has them.
sep := "?"
if strings.Contains(path, "?") {
sep = "&"
}
dsn := path + sep + "_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)"
db, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, err
}
// An in-memory database is per-connection: keep a single one.
db.SetMaxOpenConns(1)
if err := migrate(db); err != nil {
db.Close()
return nil, fmt.Errorf("migrate: %w", err)
}
return &Store{db: db, Now: time.Now}, nil
}
func hasColumn(db *sql.DB, table, column string) (bool, error) {
var n int
err := db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info(?) WHERE name=?`, table, column).Scan(&n)
return n > 0, err
}
// migrate upgrades a database created by milestone 1, then ensures the schema.
func migrate(db *sql.DB) error {
if ok, err := hasColumn(db, "members", "list_id"); err != nil {
return err
} else if ok {
if hasID, err := hasColumn(db, "members", "id"); err != nil {
return err
} else if !hasID {
if err := migrateMembersV1(db); err != nil {
return err
}
}
}
if ok, err := hasColumn(db, "lists", "name"); err != nil {
return err
} else if ok {
if has, _ := hasColumn(db, "lists", "post_policy"); !has {
if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN post_policy TEXT NOT NULL DEFAULT 'members';
ALTER TABLE lists ADD COLUMN join_policy TEXT NOT NULL DEFAULT 'open';`); err != nil {
return err
}
}
}
if ok, err := hasColumn(db, "lists", "name"); err != nil {
return err
} else if ok {
if has, _ := hasColumn(db, "lists", "archive_policy"); !has {
if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN archive_policy TEXT NOT NULL DEFAULT 'private'`); err != nil {
return err
}
}
}
if ok, err := hasColumn(db, "members", "id"); err != nil {
return err
} else if ok {
if has, _ := hasColumn(db, "members", "bounce_at"); !has {
if _, err := db.Exec(`ALTER TABLE members ADD COLUMN bounce_at INTEGER NOT NULL DEFAULT 0`); err != nil {
return err
}
}
}
if ok, err := hasColumn(db, "outbox", "msg_id"); err != nil {
return err
} else if ok {
if has, _ := hasColumn(db, "outbox", "failed"); !has {
if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN failed INTEGER NOT NULL DEFAULT 0`); err != nil {
return err
}
}
if has, _ := hasColumn(db, "outbox", "post_id"); !has {
if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN post_id INTEGER NOT NULL DEFAULT 0;
ALTER TABLE outbox ADD COLUMN priority INTEGER NOT NULL DEFAULT 0;`); err != nil {
return err
}
}
}
if ok, err := hasColumn(db, "held", "list_id"); err != nil {
return err
} else if ok {
if has, _ := hasColumn(db, "held", "reason"); !has {
if _, err := db.Exec(`ALTER TABLE held ADD COLUMN reason TEXT NOT NULL DEFAULT ''`); err != nil {
return err
}
}
if has, _ := hasColumn(db, "held", "size"); !has {
if err := addHeldSummary(db); err != nil {
return err
}
}
}
_, err := db.Exec(schema)
return err
}
// migrateMembersV1 replaces the milestone 1 members table (no id column) in one transaction, so a
// failure or a crash in the middle leaves the old table exactly as it was.
func migrateMembersV1(db *sql.DB) error {
tx, err := db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.Exec(`ALTER TABLE members RENAME TO members_v1`); err != nil {
return err
}
if _, err := tx.Exec(schema); err != nil {
return err
}
if _, err := tx.Exec(`INSERT INTO members(list_id, address, display_name)
SELECT list_id, address, display_name FROM members_v1 ORDER BY list_id, address;
DROP TABLE members_v1;`); err != nil {
return err
}
return tx.Commit()
}
// addHeldSummary adds the summary columns of the held queue and fills them for the messages
// already there, one message at a time so that a big queue is not loaded into memory.
func addHeldSummary(db *sql.DB) error {
for _, stmt := range []string{
`ALTER TABLE held ADD COLUMN subject TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE held ADD COLUMN from_header TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE held ADD COLUMN size INTEGER NOT NULL DEFAULT 0`,
`UPDATE held SET size=length(data)`,
} {
if _, err := db.Exec(stmt); err != nil {
return err
}
}
rows, err := db.Query(`SELECT id FROM held`)
if err != nil {
return err
}
var ids []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
rows.Close()
return err
}
ids = append(ids, id)
}
rows.Close()
for _, id := range ids {
var data []byte
if err := db.QueryRow(`SELECT data FROM held WHERE id=?`, id).Scan(&data); err != nil {
return err
}
subject, from := summarize(data)
if _, err := db.Exec(`UPDATE held SET subject=?, from_header=? WHERE id=?`, subject, from, id); err != nil {
return err
}
}
return nil
}
// summarize extracts what a moderation listing shows: the decoded subject and the From header, both
// without control characters and cut to 200 characters.
func summarize(data []byte) (subject, from string) {
m, err := mail.ReadMessage(bytes.NewReader(data))
if err != nil {
return "", ""
}
raw := m.Header.Get("Subject")
if dec, err := new(mime.WordDecoder).DecodeHeader(raw); err == nil {
raw = dec
}
return cleanSummary(raw), cleanSummary(m.Header.Get("From"))
}
// cleanSummary makes a header value fit for a listing: one line without control characters, at most 200 characters.
func cleanSummary(s string) string {
s = strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return -1
}
return r
}, strings.Join(strings.Fields(s), " "))
if r := []rune(s); len(r) > 200 {
s = string(r[:200])
}
return s
}
// Close closes the database.
func (s *Store) Close() error { return s.db.Close() }
func norm(addr string) string { return strings.ToLower(strings.TrimSpace(addr)) }
func isUnique(err error) bool { return err != nil && strings.Contains(err.Error(), "UNIQUE") }
func affected(res sql.Result, err error) error {
if err != nil {
return err
}
if n, _ := res.RowsAffected(); n == 0 {
return ErrNotFound
}
return nil
}
// Lists.
const listCols = `id, name, domain, title, post_policy, join_policy, archive_policy`
func scanList(row interface{ Scan(...any) error }) (List, error) {
var l List
err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy, &l.ArchivePolicy)
if errors.Is(err, sql.ErrNoRows) {
return List{}, ErrNotFound
}
return l, err
}
// CreateList adds a new list with default policies.
func (s *Store) CreateList(name, domain, title string) (List, error) {
res, err := s.db.Exec(`INSERT INTO lists(name, domain, title) VALUES(?,?,?)`,
strings.ToLower(name), strings.ToLower(domain), title)
if isUnique(err) {
return List{}, ErrExists
}
if err != nil {
return List{}, err
}
id, _ := res.LastInsertId()
return s.GetList(id)
}
// FindList resolves a posting address like "dev@example.org".
func (s *Store) FindList(addr string) (List, error) {
name, domain, ok := strings.Cut(norm(addr), "@")
if !ok {
return List{}, ErrNotFound
}
return scanList(s.db.QueryRow(`SELECT `+listCols+` FROM lists WHERE name=? AND domain=?`, name, domain))
}
// GetList returns a list by id.
func (s *Store) GetList(id int64) (List, error) {
return scanList(s.db.QueryRow(`SELECT `+listCols+` FROM lists WHERE id=?`, id))
}
// ListLists returns every list ordered by address.
func (s *Store) ListLists() ([]List, error) {
rows, err := s.db.Query(`SELECT ` + listCols + ` FROM lists ORDER BY domain, name`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []List
for rows.Next() {
l, err := scanList(rows)
if err != nil {
return nil, err
}
out = append(out, l)
}
return out, rows.Err()
}
// UpdateList changes the title of a list.
func (s *Store) UpdateList(id int64, title string) error {
return affected(s.db.Exec(`UPDATE lists SET title=? WHERE id=?`, title, id))
}
// SetPolicies changes the post, join and archive policies of a list.
func (s *Store) SetPolicies(id int64, post, join, archive string) error {
if post != PostMembers && post != PostModerated {
return fmt.Errorf("invalid post policy %q", post)
}
if join != JoinOpen && join != JoinClosed {
return fmt.Errorf("invalid join policy %q", join)
}
if archive != ArchivePublic && archive != ArchivePrivate && archive != ArchiveOff {
return fmt.Errorf("invalid archive policy %q", archive)
}
return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=?, archive_policy=? WHERE id=?`, post, join, archive, id))
}
// DeleteList removes a list with its members, held messages and pending requests.
func (s *Store) DeleteList(id int64) error {
return affected(s.db.Exec(`DELETE FROM lists WHERE id=?`, id))
}
// Members.
const memberCols = `id, list_id, address, display_name, disabled, bounce_count`
func scanMember(row interface{ Scan(...any) error }) (Member, error) {
var m Member
var dis int
err := row.Scan(&m.ID, &m.ListID, &m.Address, &m.DisplayName, &dis, &m.BounceCount)
if errors.Is(err, sql.ErrNoRows) {
return Member{}, ErrNotFound
}
m.Disabled = dis != 0
return m, err
}
// AddMember subscribes an address to a list. Re-adding keeps the member id and
// clears the bounce state.
func (s *Store) AddMember(listID int64, addr, displayName string) error {
return addMember(s.db, listID, addr, displayName)
}
func addMember(q querier, listID int64, addr, displayName string) error {
_, err := q.Exec(`INSERT INTO members(list_id, address, display_name) VALUES(?,?,?)
ON CONFLICT(list_id, address) DO UPDATE SET display_name=excluded.display_name, disabled=0, bounce_count=0`,
listID, norm(addr), displayName)
return err
}
// IsMember reports whether addr is subscribed to the list.
func (s *Store) IsMember(listID int64, addr string) (bool, error) {
var n int
err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=?`, listID, norm(addr)).Scan(&n)
return n > 0, err
}
// IsActiveMember reports whether addr is subscribed and not disabled by bounces.
func (s *Store) IsActiveMember(listID int64, addr string) (bool, error) {
var n int
err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=? AND disabled=0`, listID, norm(addr)).Scan(&n)
return n > 0, err
}
func (s *Store) queryMembers(where string, args ...any) ([]Member, error) {
rows, err := s.db.Query(`SELECT `+memberCols+` FROM members WHERE `+where+` ORDER BY address`, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Member
for rows.Next() {
m, err := scanMember(rows)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}
// Members returns all subscribers of a list, including disabled ones.
func (s *Store) Members(listID int64) ([]Member, error) {
return s.queryMembers(`list_id=?`, listID)
}
// ActiveMembers returns the subscribers that should receive mail.
func (s *Store) ActiveMembers(listID int64) ([]Member, error) {
return s.queryMembers(`list_id=? AND disabled=0`, listID)
}
// MemberByID returns a member by its id.
func (s *Store) MemberByID(id int64) (Member, error) {
return scanMember(s.db.QueryRow(`SELECT `+memberCols+` FROM members WHERE id=?`, id))
}
// RemoveMember unsubscribes an address from a list.
func (s *Store) RemoveMember(listID int64, addr string) error {
return removeMember(s.db, listID, addr)
}
func removeMember(q querier, listID int64, addr string) error {
return affected(q.Exec(`DELETE FROM members WHERE list_id=? AND address=?`, listID, norm(addr)))
}
// DeleteMember unsubscribes a member by id.
func (s *Store) DeleteMember(id int64) error {
return affected(s.db.Exec(`DELETE FROM members WHERE id=?`, id))
}
// RecordHardBounce counts a hard bounce and disables the member once the count
// reaches threshold. The count holds recent bounces only: when the last one is
// older than BounceWindow it restarts at 1. It reports whether the member is now disabled.
func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) {
now := s.Now()
if err := affected(s.db.Exec(`UPDATE members SET
bounce_count=CASE WHEN bounce_at<? THEN 1 ELSE bounce_count+1 END, bounce_at=? WHERE id=?`,
now.Add(-BounceWindow).Unix(), now.Unix(), memberID)); err != nil {
return false, err
}
if _, err := s.db.Exec(`UPDATE members SET disabled=1 WHERE id=? AND bounce_count>=?`, memberID, threshold); err != nil {
return false, err
}
m, err := s.MemberByID(memberID)
return m.Disabled, err
}
// Held messages.
// HoldMessage queues a message for moderation and returns its id. reason tells the admin why. A list
// holds a bounded amount: ErrHeldTooLarge for one message over heldMessageLimit, ErrHeldFull when the
// number of messages or the bytes held for the list would be exceeded.
func (s *Store) HoldMessage(listID int64, sender string, data []byte, reason string) (int64, error) {
if data == nil {
data = []byte{} // the column is NOT NULL
}
if len(data) > heldMessageLimit {
return 0, ErrHeldTooLarge
}
now := s.Now()
if _, err := s.db.Exec(`DELETE FROM held WHERE list_id=? AND created_at<?`, listID, now.Add(-heldTTL).Unix()); err != nil {
return 0, err
}
var n, bytesHeld int
if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(size),0) FROM held WHERE list_id=?`, listID).Scan(&n, &bytesHeld); err != nil {
return 0, err
}
if n >= maxHeldPerList || bytesHeld+len(data) > heldBytesLimit {
return 0, ErrHeldFull
}
subject, from := summarize(data)
res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at, reason, subject, from_header, size) VALUES(?,?,?,?,?,?,?,?)`,
listID, norm(sender), data, now.Unix(), reason, subject, from, len(data))
if err != nil {
return 0, err
}
return res.LastInsertId()
}
const heldCols = `id, list_id, sender, created_at, reason, subject, from_header, size`
func scanHeld(row interface{ Scan(...any) error }, withData bool) (Held, error) {
var h Held
var ts int64
var err error
if withData {
err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size, &h.Data)
} else {
err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size)
}
if errors.Is(err, sql.ErrNoRows) {
return Held{}, ErrNotFound
}
h.CreatedAt = time.Unix(ts, 0)
return h, err
}
// ListHeld returns the held messages of a list, oldest first. It carries the summary of each message
// and never its body (Data is nil): a listing must not read up to 25 MiB per message into memory.
func (s *Store) ListHeld(listID int64) ([]Held, error) {
rows, err := s.db.Query(`SELECT `+heldCols+` FROM held WHERE list_id=? ORDER BY id`, listID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Held
for rows.Next() {
h, err := scanHeld(rows, false)
if err != nil {
return nil, err
}
out = append(out, h)
}
return out, rows.Err()
}
// GetHeld returns one held message with its body.
func (s *Store) GetHeld(id int64) (Held, error) {
return scanHeld(s.db.QueryRow(`SELECT `+heldCols+`, data FROM held WHERE id=?`, id), true)
}
// DeleteHeld discards a held message.
func (s *Store) DeleteHeld(id int64) error {
return affected(s.db.Exec(`DELETE FROM held WHERE id=?`, id))
}
// Pending confirmations.
func randomHex(n int) (string, error) {
b := make([]byte, n)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
// CreatePending records a subscribe or unsubscribe request and returns its
// token. created is false when an identical request was made in the last hour:
// this limits how often a third party can trigger confirmation mail.
func (s *Store) CreatePending(listID int64, addr, action string) (token string, created bool, err error) {
addr = norm(addr)
now := s.Now()
// Expired requests are dead weight (a token older than the TTL is refused): drop them here, so
// the table stays small without a background job.
if _, err = s.db.Exec(`DELETE FROM pending WHERE created_at<?`, now.Add(-pendingTTL).Unix()); err != nil {
return "", false, err
}
var n int
err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE list_id=? AND address=? AND action=? AND created_at>?`,
listID, addr, action, now.Add(-pendingRateLimit).Unix()).Scan(&n)
if err != nil || n > 0 {
return "", false, err
}
since := now.Add(-pendingRateLimit).Unix()
if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE address=? AND created_at>?`, addr, since).Scan(&n); err != nil || n >= maxPendingPerAddress {
return "", false, err
}
if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE created_at>?`, since).Scan(&n); err != nil || n >= maxPendingPerHour {
return "", false, err
}
// 80 bits: unguessable within the 72 h life of a token, and short enough that
// name(<=30)+"+confirm-"+token fits the 64 characters of a local part.
token, err = randomHex(10)
if err != nil {
return "", false, err
}
_, err = s.db.Exec(`INSERT INTO pending(token, list_id, address, action, created_at) VALUES(?,?,?,?,?)`,
token, listID, addr, action, now.Unix())
if err != nil {
return "", false, err
}
return token, true, nil
}
// ConfirmPending applies the request behind a token and consumes the token, in one transaction: a
// token is only used up when its action happened, and two confirmations of the same token cannot
// both apply (the first one to claim the row wins, the other gets ErrNotFound).
//
// Unknown, used, expired tokens and tokens of another list give ErrNotFound (the last two are left
// alone). check, when not nil, runs before anything changes; its error is returned and the token
// stays valid. A "join" subscribes the address (re-enabling a disabled member), a "leave"
// unsubscribes it (already gone is fine).
func (s *Store) ConfirmPending(token string, listID int64, check func(Pending) error) (Pending, error) {
tx, err := s.db.Begin()
if err != nil {
return Pending{}, err
}
defer tx.Rollback()
var p Pending
var ts int64
// DELETE ... RETURNING takes the write lock first, so concurrent confirmations queue up here.
err = tx.QueryRow(`DELETE FROM pending WHERE token=? AND list_id=? RETURNING token, list_id, address, action, created_at`,
token, listID).Scan(&p.Token, &p.ListID, &p.Address, &p.Action, &ts)
if errors.Is(err, sql.ErrNoRows) {
return Pending{}, ErrNotFound
}
if err != nil {
return Pending{}, err
}
if s.Now().Sub(time.Unix(ts, 0)) > pendingTTL {
// Expired: the row is gone for good.
if err := tx.Commit(); err != nil {
return Pending{}, err
}
return Pending{}, ErrNotFound
}
if check != nil {
if err := check(p); err != nil {
return Pending{}, err
}
}
switch p.Action {
case ActionJoin:
err = addMember(tx, p.ListID, p.Address, "")
case ActionLeave:
if err = removeMember(tx, p.ListID, p.Address); errors.Is(err, ErrNotFound) {
err = nil
}
}
if err != nil {
return Pending{}, err
}
if err := tx.Commit(); err != nil {
return Pending{}, err
}
return p, nil
}
// Secret returns the HMAC secret, generating and saving it on first use.
func (s *Store) Secret() ([]byte, error) {
var v string
err := s.db.QueryRow(`SELECT value FROM settings WHERE key='secret'`).Scan(&v)
if errors.Is(err, sql.ErrNoRows) {
v, err = randomHex(32)
if err != nil {
return nil, err
}
if _, err = s.db.Exec(`INSERT INTO settings(key, value) VALUES('secret', ?)`, v); err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return hex.DecodeString(v)
}
// Archive.
// Message is an archived post.
type Message struct {
ID int64
ListID int64
ThreadID int64
MessageID string // Message-ID header, unique per list
InReplyTo string
Subject string
Sender string // display name of the original author
Date time.Time
Body string // plain text; empty in listings
Size int
}
const msgCols = `id, list_id, thread_id, message_id, in_reply_to, subject, sender, date, size`
func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, error) {
var m Message
var ts int64
var err error
if withBody {
err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size, &m.Body)
} else {
err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size)
}
if errors.Is(err, sql.ErrNoRows) {
return Message{}, ErrNotFound
}
m.Date = time.Unix(ts, 0)
return m, err
}
// querier is what archiveOne needs from a *sql.DB or a *sql.Tx.
type querier interface {
Exec(query string, args ...any) (sql.Result, error)
QueryRow(query string, args ...any) *sql.Row
}
// ArchiveResult is the outcome for one message of ArchiveBatch.
type ArchiveResult struct {
ID int64
Created bool
}
// ArchiveMessage stores a post. A message already archived in the same list
// (same Message-ID) is not stored again: created is false and the existing id
// is returned, so MTA retries are harmless.
func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) {
return archiveOne(s.db, listID, m)
}
// ArchiveBatch stores many messages in one transaction: all or none. It is much
// faster than ArchiveMessage in a loop, because SQLite syncs once per commit.
func (s *Store) ArchiveBatch(listID int64, ms []Message) ([]ArchiveResult, error) {
if len(ms) == 0 {
return nil, nil
}
tx, err := s.db.Begin()
if err != nil {
return nil, err
}
defer tx.Rollback()
out := make([]ArchiveResult, 0, len(ms))
for _, m := range ms {
id, created, err := archiveOne(tx, listID, m)
if err != nil {
return nil, err
}
out = append(out, ArchiveResult{ID: id, Created: created})
}
return out, tx.Commit()
}
func archiveOne(q querier, listID int64, m Message) (id int64, created bool, err error) {
if m.MessageID == "" {
return 0, false, errors.New("message id is required")
}
err = q.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
if err == nil {
return id, false, nil
}
if !errors.Is(err, sql.ErrNoRows) {
return 0, false, err
}
var thread int64
if m.InReplyTo != "" {
e := q.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
if e != nil && !errors.Is(e, sql.ErrNoRows) {
return 0, false, e
}
}
res, err := q.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
VALUES(?,?,?,?,?,?,?,?,?)`,
listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body))
if err != nil {
return 0, false, err
}
id, _ = res.LastInsertId()
if thread == 0 {
if _, err := q.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
return 0, false, err
}
}
return id, true, nil
}
// threadRoots maps every id to the root of its reply chain. parent[id] is the message id replies to,
// when that message exists. A cycle of replies has the smallest id of the cycle as root. Each chain
// is walked once, so the cost is linear in the number of messages.
func threadRoots(ids []int64, parent map[int64]int64) map[int64]int64 {
roots := make(map[int64]int64, len(ids))
for _, start := range ids {
if _, done := roots[start]; done {
continue
}
path := []int64{start}
seen := map[int64]int{start: 0}
cur := start
var result int64
for {
p, ok := parent[cur]
if !ok {
result = cur
break
}
if r, known := roots[p]; known {
result = r
break
}
if at, loop := seen[p]; loop { // a cycle: its smallest id is the root
result = path[at]
for _, id := range path[at:] {
result = min(result, id)
}
break
}
seen[p] = len(path)
path = append(path, p)
cur = p
}
for _, id := range path {
roots[id] = result
}
}
return roots
}
// Rethread recomputes the thread of every message of a list from the In-Reply-To
// links, and returns how many messages changed thread. Insertion order does not
// matter: a message imported before its parent ends up in the parent's thread.
// A cycle of replies becomes one thread whose id is the smallest id in the cycle.
func (s *Store) Rethread(listID int64) (int, error) {
rows, err := s.db.Query(`SELECT id, message_id, in_reply_to, thread_id FROM archive WHERE list_id=?`, listID)
if err != nil {
return 0, err
}
type row struct {
id, thread int64
replyTo string
}
var all []row
byMsgID := map[string]int64{}
for rows.Next() {
var r row
var msgID string
if err := rows.Scan(&r.id, &msgID, &r.replyTo, &r.thread); err != nil {
rows.Close()
return 0, err
}
all = append(all, r)
byMsgID[msgID] = r.id
}
rows.Close()
if err := rows.Err(); err != nil {
return 0, err
}
parent := map[int64]int64{}
ids := make([]int64, 0, len(all))
for _, r := range all {
ids = append(ids, r.id)
if p, ok := byMsgID[r.replyTo]; ok && r.replyTo != "" {
parent[r.id] = p
}
}
roots := threadRoots(ids, parent)
tx, err := s.db.Begin()
if err != nil {
return 0, err
}
defer tx.Rollback()
changed := 0
for _, r := range all {
if want := roots[r.id]; want != r.thread {
if _, err := tx.Exec(`UPDATE archive SET thread_id=? WHERE id=?`, want, r.id); err != nil {
return 0, err
}
changed++
}
}
return changed, tx.Commit()
}
// GetMessage returns an archived message with its body.
func (s *Store) GetMessage(id int64) (Message, error) {
return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true)
}
// DeleteMessage removes an archived message and its search entry.
func (s *Store) DeleteMessage(id int64) error {
return affected(s.db.Exec(`DELETE FROM archive WHERE id=?`, id))
}
func (s *Store) queryMessages(q string, args ...any) ([]Message, error) {
rows, err := s.db.Query(q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Message
for rows.Next() {
m, err := scanMessage(rows, false)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}
// ListMessages returns a page of a list's archive, newest first, and the total
// count. Bodies are not loaded.
func (s *Store) ListMessages(listID int64, limit, offset int) ([]Message, int, error) {
var total int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive WHERE list_id=?`, listID).Scan(&total); err != nil {
return nil, 0, err
}
out, err := s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? ORDER BY date DESC, id DESC LIMIT ? OFFSET ?`,
listID, limit, offset)
return out, total, err
}
// ftsQuery turns free text into an FTS5 query where every word is a quoted
// literal (AND semantics), so user input can never be read as FTS syntax.
func ftsQuery(q string) string {
words := strings.Fields(q)
for i, w := range words {
words[i] = `"` + strings.ReplaceAll(w, `"`, `""`) + `"`
}
return strings.Join(words, " ")
}
// SearchMessages runs a full-text search over subject, body and sender of one
// list, newest first. Bodies are not loaded.
func (s *Store) SearchMessages(listID int64, query string, limit, offset int) ([]Message, int, error) {
match := ftsQuery(query)
if match == "" {
return nil, 0, nil
}
var total int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid
WHERE archive_fts MATCH ? AND a.list_id=?`, match, listID).Scan(&total); err != nil {
return nil, 0, err
}
out, err := s.queryMessages(`SELECT a.id, a.list_id, a.thread_id, a.message_id, a.in_reply_to, a.subject, a.sender, a.date, a.size
FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid
WHERE archive_fts MATCH ? AND a.list_id=? ORDER BY a.date DESC, a.id DESC LIMIT ? OFFSET ?`,
match, listID, limit, offset)
return out, total, err
}
// ThreadMessages returns every message of a thread, oldest first, without bodies: a thread listing
// shows subjects, and a thread can hold many messages of up to 1 MiB of text each.
func (s *Store) ThreadMessages(listID, threadID int64) ([]Message, error) {
return s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID)
}