parsimony/internal/store/store.go
tgiachi e7e1afbde9 fix: approve a held message once, however many approvals race
- Release removes the held message and queues its deliveries in one transaction
- Of several simultaneous approvals one wins, the others get 404
- Extract the thread-root computation and test its linearity without the database (fast under -race)
2026-11-09 20:29:42 +01:00

894 lines
27 KiB
Go

// Package store persists lists, members and moderation state in SQLite.
package store
import (
"crypto/rand"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"strings"
"time"
_ "modernc.org/sqlite"
)
var (
// ErrNotFound is returned when a record does not exist.
ErrNotFound = errors.New("not found")
// ErrHeldFull is returned when a list's moderation queue has no more room.
ErrHeldFull = errors.New("moderation queue is full")
// ErrExists is returned when creating something that already exists.
ErrExists = errors.New("already exists")
)
// Policy values.
const (
PostMembers = "members" // only members may post, others are rejected
PostModerated = "moderated" // every post waits for approval
JoinOpen = "open" // anyone can subscribe (with confirmation)
JoinClosed = "closed" // only the admin adds members
)
// Archive policy values.
const (
ArchivePublic = "public" // browsable on the web
ArchivePrivate = "private" // stored, readable through the API only
ArchiveOff = "off" // not stored
)
// Pending actions.
const (
ActionJoin = "join"
ActionLeave = "leave"
)
// BounceThreshold is the number of hard bounces after which a member is disabled.
const BounceThreshold = 3
const (
pendingTTL = 72 * time.Hour
pendingRateLimit = time.Hour
// Caps on confirmation mail, per hour. Anyone can name any address in a join request, so the
// daemon must not mail one victim many times (maxPendingPerAddress, across all lists) or send
// an unbounded amount in total (maxPendingPerHour).
// A moderated list could be flooded with mail that nobody reads: the queue per list is bounded,
// and a message nobody moderated for a month is dropped.
maxHeldPerList = 500
heldTTL = 30 * 24 * time.Hour
maxPendingPerAddress = 3
maxPendingPerHour = 300
)
// List is a mailing list.
type List struct {
ID int64
Name string // local part, lowercase
Domain string // lowercase
Title string
PostPolicy string
JoinPolicy string
ArchivePolicy string
}
// Address returns the list posting address.
func (l List) Address() string { return l.Name + "@" + l.Domain }
// Member is a list subscriber.
type Member struct {
ID int64
ListID int64
Address string // lowercase
DisplayName string
Disabled bool // too many hard bounces: no more delivery
BounceCount int
}
// Held is a message waiting for moderation.
type Held struct {
ID int64
ListID int64
Sender string // envelope sender
Data []byte
CreatedAt time.Time
Reason string // why the message was held, for the admin
}
// Pending is an unconfirmed subscribe or unsubscribe request.
type Pending struct {
Token string
ListID int64
Address string
Action string
}
// Store wraps the SQLite database.
type Store struct {
db *sql.DB
// Now returns the current time; tests replace it.
Now func() time.Time
}
const schema = `
CREATE TABLE IF NOT EXISTS lists (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
domain TEXT NOT NULL,
title TEXT NOT NULL DEFAULT '',
post_policy TEXT NOT NULL DEFAULT 'members',
join_policy TEXT NOT NULL DEFAULT 'open',
archive_policy TEXT NOT NULL DEFAULT 'private',
UNIQUE(name, domain)
);
CREATE TABLE IF NOT EXISTS members (
id INTEGER PRIMARY KEY AUTOINCREMENT,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
address TEXT NOT NULL,
display_name TEXT NOT NULL DEFAULT '',
disabled INTEGER NOT NULL DEFAULT 0,
bounce_count INTEGER NOT NULL DEFAULT 0,
UNIQUE(list_id, address)
);
CREATE TABLE IF NOT EXISTS held (
id INTEGER PRIMARY KEY AUTOINCREMENT,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
sender TEXT NOT NULL,
data BLOB NOT NULL,
created_at INTEGER NOT NULL,
reason TEXT NOT NULL DEFAULT ''
);
CREATE TABLE IF NOT EXISTS pending (
token TEXT PRIMARY KEY,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
address TEXT NOT NULL,
action TEXT NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS archive (
id INTEGER PRIMARY KEY AUTOINCREMENT,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
thread_id INTEGER NOT NULL DEFAULT 0,
message_id TEXT NOT NULL,
in_reply_to TEXT NOT NULL DEFAULT '',
subject TEXT NOT NULL DEFAULT '',
sender TEXT NOT NULL DEFAULT '',
date INTEGER NOT NULL,
body TEXT NOT NULL DEFAULT '',
size INTEGER NOT NULL DEFAULT 0,
UNIQUE(list_id, message_id)
);
CREATE INDEX IF NOT EXISTS archive_list_date ON archive(list_id, date DESC, id DESC);
CREATE INDEX IF NOT EXISTS archive_thread ON archive(list_id, thread_id);
CREATE VIRTUAL TABLE IF NOT EXISTS archive_fts USING fts5(
subject, body, sender, content='archive', content_rowid='id'
);
CREATE TRIGGER IF NOT EXISTS archive_ai AFTER INSERT ON archive BEGIN
INSERT INTO archive_fts(rowid, subject, body, sender) VALUES (new.id, new.subject, new.body, new.sender);
END;
CREATE TRIGGER IF NOT EXISTS archive_ad AFTER DELETE ON archive BEGIN
INSERT INTO archive_fts(archive_fts, rowid, subject, body, sender) VALUES ('delete', old.id, old.subject, old.body, old.sender);
END;
CREATE TABLE IF NOT EXISTS outbox_msgs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
data BLOB NOT NULL
);
CREATE TABLE IF NOT EXISTS outbox (
id INTEGER PRIMARY KEY AUTOINCREMENT,
msg_id INTEGER NOT NULL REFERENCES outbox_msgs(id) ON DELETE CASCADE,
envelope_from TEXT NOT NULL,
rcpt TEXT NOT NULL,
member_id INTEGER NOT NULL DEFAULT 0,
prefix BLOB NOT NULL DEFAULT x'',
attempts INTEGER NOT NULL DEFAULT 0,
next_at INTEGER NOT NULL,
last_error TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id);
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);`
// Open opens (and migrates) the database at path. Use ":memory:" for tests.
func Open(path string) (*Store, error) {
db, err := sql.Open("sqlite", path)
if err != nil {
return nil, err
}
// An in-memory database is per-connection: keep a single one.
db.SetMaxOpenConns(1)
if _, err := db.Exec("PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL;"); err != nil {
db.Close()
return nil, err
}
if err := migrate(db); err != nil {
db.Close()
return nil, fmt.Errorf("migrate: %w", err)
}
return &Store{db: db, Now: time.Now}, nil
}
func hasColumn(db *sql.DB, table, column string) (bool, error) {
var n int
err := db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info(?) WHERE name=?`, table, column).Scan(&n)
return n > 0, err
}
// migrate upgrades a database created by milestone 1, then ensures the schema.
func migrate(db *sql.DB) error {
if ok, err := hasColumn(db, "members", "list_id"); err != nil {
return err
} else if ok {
if hasID, err := hasColumn(db, "members", "id"); err != nil {
return err
} else if !hasID {
if _, err := db.Exec(`ALTER TABLE members RENAME TO members_v1`); err != nil {
return err
}
if _, err := db.Exec(schema); err != nil {
return err
}
if _, err := db.Exec(`INSERT INTO members(list_id, address, display_name)
SELECT list_id, address, display_name FROM members_v1 ORDER BY list_id, address;
DROP TABLE members_v1;`); err != nil {
return err
}
}
}
if ok, err := hasColumn(db, "lists", "name"); err != nil {
return err
} else if ok {
if has, _ := hasColumn(db, "lists", "post_policy"); !has {
if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN post_policy TEXT NOT NULL DEFAULT 'members';
ALTER TABLE lists ADD COLUMN join_policy TEXT NOT NULL DEFAULT 'open';`); err != nil {
return err
}
}
}
if ok, err := hasColumn(db, "lists", "name"); err != nil {
return err
} else if ok {
if has, _ := hasColumn(db, "lists", "archive_policy"); !has {
if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN archive_policy TEXT NOT NULL DEFAULT 'private'`); err != nil {
return err
}
}
}
if ok, err := hasColumn(db, "held", "list_id"); err != nil {
return err
} else if ok {
if has, _ := hasColumn(db, "held", "reason"); !has {
if _, err := db.Exec(`ALTER TABLE held ADD COLUMN reason TEXT NOT NULL DEFAULT ''`); err != nil {
return err
}
}
}
_, err := db.Exec(schema)
return err
}
// Close closes the database.
func (s *Store) Close() error { return s.db.Close() }
func norm(addr string) string { return strings.ToLower(strings.TrimSpace(addr)) }
func isUnique(err error) bool { return err != nil && strings.Contains(err.Error(), "UNIQUE") }
func affected(res sql.Result, err error) error {
if err != nil {
return err
}
if n, _ := res.RowsAffected(); n == 0 {
return ErrNotFound
}
return nil
}
// Lists.
const listCols = `id, name, domain, title, post_policy, join_policy, archive_policy`
func scanList(row interface{ Scan(...any) error }) (List, error) {
var l List
err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy, &l.ArchivePolicy)
if errors.Is(err, sql.ErrNoRows) {
return List{}, ErrNotFound
}
return l, err
}
// CreateList adds a new list with default policies.
func (s *Store) CreateList(name, domain, title string) (List, error) {
res, err := s.db.Exec(`INSERT INTO lists(name, domain, title) VALUES(?,?,?)`,
strings.ToLower(name), strings.ToLower(domain), title)
if isUnique(err) {
return List{}, ErrExists
}
if err != nil {
return List{}, err
}
id, _ := res.LastInsertId()
return s.GetList(id)
}
// FindList resolves a posting address like "dev@example.org".
func (s *Store) FindList(addr string) (List, error) {
name, domain, ok := strings.Cut(norm(addr), "@")
if !ok {
return List{}, ErrNotFound
}
return scanList(s.db.QueryRow(`SELECT `+listCols+` FROM lists WHERE name=? AND domain=?`, name, domain))
}
// GetList returns a list by id.
func (s *Store) GetList(id int64) (List, error) {
return scanList(s.db.QueryRow(`SELECT `+listCols+` FROM lists WHERE id=?`, id))
}
// ListLists returns every list ordered by address.
func (s *Store) ListLists() ([]List, error) {
rows, err := s.db.Query(`SELECT ` + listCols + ` FROM lists ORDER BY domain, name`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []List
for rows.Next() {
l, err := scanList(rows)
if err != nil {
return nil, err
}
out = append(out, l)
}
return out, rows.Err()
}
// UpdateList changes the title of a list.
func (s *Store) UpdateList(id int64, title string) error {
return affected(s.db.Exec(`UPDATE lists SET title=? WHERE id=?`, title, id))
}
// SetPolicies changes the post, join and archive policies of a list.
func (s *Store) SetPolicies(id int64, post, join, archive string) error {
if post != PostMembers && post != PostModerated {
return fmt.Errorf("invalid post policy %q", post)
}
if join != JoinOpen && join != JoinClosed {
return fmt.Errorf("invalid join policy %q", join)
}
if archive != ArchivePublic && archive != ArchivePrivate && archive != ArchiveOff {
return fmt.Errorf("invalid archive policy %q", archive)
}
return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=?, archive_policy=? WHERE id=?`, post, join, archive, id))
}
// DeleteList removes a list with its members, held messages and pending requests.
func (s *Store) DeleteList(id int64) error {
return affected(s.db.Exec(`DELETE FROM lists WHERE id=?`, id))
}
// Members.
const memberCols = `id, list_id, address, display_name, disabled, bounce_count`
func scanMember(row interface{ Scan(...any) error }) (Member, error) {
var m Member
var dis int
err := row.Scan(&m.ID, &m.ListID, &m.Address, &m.DisplayName, &dis, &m.BounceCount)
if errors.Is(err, sql.ErrNoRows) {
return Member{}, ErrNotFound
}
m.Disabled = dis != 0
return m, err
}
// AddMember subscribes an address to a list. Re-adding keeps the member id and
// clears the bounce state.
func (s *Store) AddMember(listID int64, addr, displayName string) error {
_, err := s.db.Exec(`INSERT INTO members(list_id, address, display_name) VALUES(?,?,?)
ON CONFLICT(list_id, address) DO UPDATE SET display_name=excluded.display_name, disabled=0, bounce_count=0`,
listID, norm(addr), displayName)
return err
}
// IsMember reports whether addr is subscribed to the list.
func (s *Store) IsMember(listID int64, addr string) (bool, error) {
var n int
err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=?`, listID, norm(addr)).Scan(&n)
return n > 0, err
}
func (s *Store) queryMembers(where string, args ...any) ([]Member, error) {
rows, err := s.db.Query(`SELECT `+memberCols+` FROM members WHERE `+where+` ORDER BY address`, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Member
for rows.Next() {
m, err := scanMember(rows)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}
// Members returns all subscribers of a list, including disabled ones.
func (s *Store) Members(listID int64) ([]Member, error) {
return s.queryMembers(`list_id=?`, listID)
}
// ActiveMembers returns the subscribers that should receive mail.
func (s *Store) ActiveMembers(listID int64) ([]Member, error) {
return s.queryMembers(`list_id=? AND disabled=0`, listID)
}
// MemberByID returns a member by its id.
func (s *Store) MemberByID(id int64) (Member, error) {
return scanMember(s.db.QueryRow(`SELECT `+memberCols+` FROM members WHERE id=?`, id))
}
// RemoveMember unsubscribes an address from a list.
func (s *Store) RemoveMember(listID int64, addr string) error {
return affected(s.db.Exec(`DELETE FROM members WHERE list_id=? AND address=?`, listID, norm(addr)))
}
// DeleteMember unsubscribes a member by id.
func (s *Store) DeleteMember(id int64) error {
return affected(s.db.Exec(`DELETE FROM members WHERE id=?`, id))
}
// RecordHardBounce counts a hard bounce and disables the member once the count
// reaches threshold. It reports whether the member is now disabled.
func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) {
if err := affected(s.db.Exec(`UPDATE members SET bounce_count=bounce_count+1 WHERE id=?`, memberID)); err != nil {
return false, err
}
if _, err := s.db.Exec(`UPDATE members SET disabled=1 WHERE id=? AND bounce_count>=?`, memberID, threshold); err != nil {
return false, err
}
m, err := s.MemberByID(memberID)
return m.Disabled, err
}
// Held messages.
// HoldMessage queues a message for moderation and returns its id. reason tells the admin why.
func (s *Store) HoldMessage(listID int64, sender string, data []byte, reason string) (int64, error) {
if data == nil {
data = []byte{} // the column is NOT NULL
}
now := s.Now()
if _, err := s.db.Exec(`DELETE FROM held WHERE list_id=? AND created_at<?`, listID, now.Add(-heldTTL).Unix()); err != nil {
return 0, err
}
var n int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM held WHERE list_id=?`, listID).Scan(&n); err != nil {
return 0, err
}
if n >= maxHeldPerList {
return 0, ErrHeldFull
}
res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at, reason) VALUES(?,?,?,?,?)`,
listID, norm(sender), data, now.Unix(), reason)
if err != nil {
return 0, err
}
return res.LastInsertId()
}
func scanHeld(row interface{ Scan(...any) error }) (Held, error) {
var h Held
var ts int64
err := row.Scan(&h.ID, &h.ListID, &h.Sender, &h.Data, &ts, &h.Reason)
if errors.Is(err, sql.ErrNoRows) {
return Held{}, ErrNotFound
}
h.CreatedAt = time.Unix(ts, 0)
return h, err
}
// ListHeld returns the held messages of a list, oldest first.
func (s *Store) ListHeld(listID int64) ([]Held, error) {
rows, err := s.db.Query(`SELECT id, list_id, sender, data, created_at, reason FROM held WHERE list_id=? ORDER BY id`, listID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Held
for rows.Next() {
h, err := scanHeld(rows)
if err != nil {
return nil, err
}
out = append(out, h)
}
return out, rows.Err()
}
// GetHeld returns a held message by id.
func (s *Store) GetHeld(id int64) (Held, error) {
return scanHeld(s.db.QueryRow(`SELECT id, list_id, sender, data, created_at, reason FROM held WHERE id=?`, id))
}
// DeleteHeld discards a held message.
func (s *Store) DeleteHeld(id int64) error {
return affected(s.db.Exec(`DELETE FROM held WHERE id=?`, id))
}
// Pending confirmations.
func randomHex(n int) (string, error) {
b := make([]byte, n)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
// CreatePending records a subscribe or unsubscribe request and returns its
// token. created is false when an identical request was made in the last hour:
// this limits how often a third party can trigger confirmation mail.
func (s *Store) CreatePending(listID int64, addr, action string) (token string, created bool, err error) {
addr = norm(addr)
now := s.Now()
var n int
err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE list_id=? AND address=? AND action=? AND created_at>?`,
listID, addr, action, now.Add(-pendingRateLimit).Unix()).Scan(&n)
if err != nil || n > 0 {
return "", false, err
}
since := now.Add(-pendingRateLimit).Unix()
if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE address=? AND created_at>?`, addr, since).Scan(&n); err != nil || n >= maxPendingPerAddress {
return "", false, err
}
if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE created_at>?`, since).Scan(&n); err != nil || n >= maxPendingPerHour {
return "", false, err
}
token, err = randomHex(16)
if err != nil {
return "", false, err
}
_, err = s.db.Exec(`INSERT INTO pending(token, list_id, address, action, created_at) VALUES(?,?,?,?,?)`,
token, listID, addr, action, now.Unix())
if err != nil {
return "", false, err
}
return token, true, nil
}
// ConfirmPending consumes a token. Unknown, used and expired tokens give ErrNotFound.
func (s *Store) ConfirmPending(token string) (Pending, error) {
var p Pending
var ts int64
err := s.db.QueryRow(`SELECT token, list_id, address, action, created_at FROM pending WHERE token=?`, token).
Scan(&p.Token, &p.ListID, &p.Address, &p.Action, &ts)
if errors.Is(err, sql.ErrNoRows) {
return Pending{}, ErrNotFound
}
if err != nil {
return Pending{}, err
}
if _, err := s.db.Exec(`DELETE FROM pending WHERE token=?`, token); err != nil {
return Pending{}, err
}
if s.Now().Sub(time.Unix(ts, 0)) > pendingTTL {
return Pending{}, ErrNotFound
}
return p, nil
}
// Secret returns the HMAC secret, generating and saving it on first use.
func (s *Store) Secret() ([]byte, error) {
var v string
err := s.db.QueryRow(`SELECT value FROM settings WHERE key='secret'`).Scan(&v)
if errors.Is(err, sql.ErrNoRows) {
v, err = randomHex(32)
if err != nil {
return nil, err
}
if _, err = s.db.Exec(`INSERT INTO settings(key, value) VALUES('secret', ?)`, v); err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return hex.DecodeString(v)
}
// Archive.
// Message is an archived post.
type Message struct {
ID int64
ListID int64
ThreadID int64
MessageID string // Message-ID header, unique per list
InReplyTo string
Subject string
Sender string // display name of the original author
Date time.Time
Body string // plain text; empty in listings
Size int
}
const msgCols = `id, list_id, thread_id, message_id, in_reply_to, subject, sender, date, size`
func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, error) {
var m Message
var ts int64
var err error
if withBody {
err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size, &m.Body)
} else {
err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size)
}
if errors.Is(err, sql.ErrNoRows) {
return Message{}, ErrNotFound
}
m.Date = time.Unix(ts, 0)
return m, err
}
// querier is what archiveOne needs from a *sql.DB or a *sql.Tx.
type querier interface {
Exec(query string, args ...any) (sql.Result, error)
QueryRow(query string, args ...any) *sql.Row
}
// ArchiveResult is the outcome for one message of ArchiveBatch.
type ArchiveResult struct {
ID int64
Created bool
}
// ArchiveMessage stores a post. A message already archived in the same list
// (same Message-ID) is not stored again: created is false and the existing id
// is returned, so MTA retries are harmless.
func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) {
return archiveOne(s.db, listID, m)
}
// ArchiveBatch stores many messages in one transaction: all or none. It is much
// faster than ArchiveMessage in a loop, because SQLite syncs once per commit.
func (s *Store) ArchiveBatch(listID int64, ms []Message) ([]ArchiveResult, error) {
if len(ms) == 0 {
return nil, nil
}
tx, err := s.db.Begin()
if err != nil {
return nil, err
}
defer tx.Rollback()
out := make([]ArchiveResult, 0, len(ms))
for _, m := range ms {
id, created, err := archiveOne(tx, listID, m)
if err != nil {
return nil, err
}
out = append(out, ArchiveResult{ID: id, Created: created})
}
return out, tx.Commit()
}
func archiveOne(q querier, listID int64, m Message) (id int64, created bool, err error) {
if m.MessageID == "" {
return 0, false, errors.New("message id is required")
}
err = q.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
if err == nil {
return id, false, nil
}
if !errors.Is(err, sql.ErrNoRows) {
return 0, false, err
}
var thread int64
if m.InReplyTo != "" {
e := q.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
if e != nil && !errors.Is(e, sql.ErrNoRows) {
return 0, false, e
}
}
res, err := q.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
VALUES(?,?,?,?,?,?,?,?,?)`,
listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body))
if err != nil {
return 0, false, err
}
id, _ = res.LastInsertId()
if thread == 0 {
if _, err := q.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
return 0, false, err
}
}
return id, true, nil
}
// threadRoots maps every id to the root of its reply chain. parent[id] is the message id replies to,
// when that message exists. A cycle of replies has the smallest id of the cycle as root. Each chain
// is walked once, so the cost is linear in the number of messages.
func threadRoots(ids []int64, parent map[int64]int64) map[int64]int64 {
roots := make(map[int64]int64, len(ids))
for _, start := range ids {
if _, done := roots[start]; done {
continue
}
path := []int64{start}
seen := map[int64]int{start: 0}
cur := start
var result int64
for {
p, ok := parent[cur]
if !ok {
result = cur
break
}
if r, known := roots[p]; known {
result = r
break
}
if at, loop := seen[p]; loop { // a cycle: its smallest id is the root
result = path[at]
for _, id := range path[at:] {
result = min(result, id)
}
break
}
seen[p] = len(path)
path = append(path, p)
cur = p
}
for _, id := range path {
roots[id] = result
}
}
return roots
}
// Rethread recomputes the thread of every message of a list from the In-Reply-To
// links, and returns how many messages changed thread. Insertion order does not
// matter: a message imported before its parent ends up in the parent's thread.
// A cycle of replies becomes one thread whose id is the smallest id in the cycle.
func (s *Store) Rethread(listID int64) (int, error) {
rows, err := s.db.Query(`SELECT id, message_id, in_reply_to, thread_id FROM archive WHERE list_id=?`, listID)
if err != nil {
return 0, err
}
type row struct {
id, thread int64
replyTo string
}
var all []row
byMsgID := map[string]int64{}
for rows.Next() {
var r row
var msgID string
if err := rows.Scan(&r.id, &msgID, &r.replyTo, &r.thread); err != nil {
rows.Close()
return 0, err
}
all = append(all, r)
byMsgID[msgID] = r.id
}
rows.Close()
if err := rows.Err(); err != nil {
return 0, err
}
parent := map[int64]int64{}
ids := make([]int64, 0, len(all))
for _, r := range all {
ids = append(ids, r.id)
if p, ok := byMsgID[r.replyTo]; ok && r.replyTo != "" {
parent[r.id] = p
}
}
roots := threadRoots(ids, parent)
tx, err := s.db.Begin()
if err != nil {
return 0, err
}
defer tx.Rollback()
changed := 0
for _, r := range all {
if want := roots[r.id]; want != r.thread {
if _, err := tx.Exec(`UPDATE archive SET thread_id=? WHERE id=?`, want, r.id); err != nil {
return 0, err
}
changed++
}
}
return changed, tx.Commit()
}
// GetMessage returns an archived message with its body.
func (s *Store) GetMessage(id int64) (Message, error) {
return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true)
}
// DeleteMessage removes an archived message and its search entry.
func (s *Store) DeleteMessage(id int64) error {
return affected(s.db.Exec(`DELETE FROM archive WHERE id=?`, id))
}
func (s *Store) queryMessages(q string, args ...any) ([]Message, error) {
rows, err := s.db.Query(q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Message
for rows.Next() {
m, err := scanMessage(rows, false)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}
// ListMessages returns a page of a list's archive, newest first, and the total
// count. Bodies are not loaded.
func (s *Store) ListMessages(listID int64, limit, offset int) ([]Message, int, error) {
var total int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive WHERE list_id=?`, listID).Scan(&total); err != nil {
return nil, 0, err
}
out, err := s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? ORDER BY date DESC, id DESC LIMIT ? OFFSET ?`,
listID, limit, offset)
return out, total, err
}
// ftsQuery turns free text into an FTS5 query where every word is a quoted
// literal (AND semantics), so user input can never be read as FTS syntax.
func ftsQuery(q string) string {
words := strings.Fields(q)
for i, w := range words {
words[i] = `"` + strings.ReplaceAll(w, `"`, `""`) + `"`
}
return strings.Join(words, " ")
}
// SearchMessages runs a full-text search over subject, body and sender of one
// list, newest first. Bodies are not loaded.
func (s *Store) SearchMessages(listID int64, query string, limit, offset int) ([]Message, int, error) {
match := ftsQuery(query)
if match == "" {
return nil, 0, nil
}
var total int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid
WHERE archive_fts MATCH ? AND a.list_id=?`, match, listID).Scan(&total); err != nil {
return nil, 0, err
}
out, err := s.queryMessages(`SELECT a.id, a.list_id, a.thread_id, a.message_id, a.in_reply_to, a.subject, a.sender, a.date, a.size
FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid
WHERE archive_fts MATCH ? AND a.list_id=? ORDER BY a.date DESC, a.id DESC LIMIT ? OFFSET ?`,
match, listID, limit, offset)
return out, total, err
}
// ThreadMessages returns every message of a thread, oldest first, with bodies.
func (s *Store) ThreadMessages(listID, threadID int64) ([]Message, error) {
rows, err := s.db.Query(`SELECT `+msgCols+`, body FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Message
for rows.Next() {
m, err := scanMessage(rows, true)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}