1094 lines
35 KiB
Go
1094 lines
35 KiB
Go
// Package store persists lists, members and moderation state in SQLite.
|
|
package store
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/rand"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"mime"
|
|
"net/mail"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
var (
|
|
// ErrNotFound is returned when a record does not exist.
|
|
ErrNotFound = errors.New("not found")
|
|
// ErrHeldFull is returned when a list's moderation queue has no more room.
|
|
ErrHeldFull = errors.New("moderation queue is full")
|
|
// ErrHeldTooLarge is returned for a message too big to be held for moderation.
|
|
ErrHeldTooLarge = errors.New("message too large to hold for moderation")
|
|
// ErrExists is returned when creating something that already exists.
|
|
ErrExists = errors.New("already exists")
|
|
// ErrDuplicate is returned when the same post was already queued for the list within postMemory.
|
|
ErrDuplicate = errors.New("post already delivered")
|
|
)
|
|
|
|
// Policy values.
|
|
const (
|
|
PostMembers = "members" // only members may post, others are rejected
|
|
PostModerated = "moderated" // every post waits for approval
|
|
PostPublishers = "publishers" // newsletter: only members flagged as publishers may post
|
|
JoinOpen = "open" // anyone can subscribe (with confirmation)
|
|
JoinClosed = "closed" // only the admin adds members
|
|
)
|
|
|
|
// Archive policy values.
|
|
const (
|
|
ArchivePublic = "public" // browsable on the web
|
|
ArchivePrivate = "private" // stored, readable through the API only
|
|
ArchiveOff = "off" // not stored
|
|
)
|
|
|
|
// Pending actions.
|
|
const (
|
|
ActionJoin = "join"
|
|
ActionLeave = "leave"
|
|
)
|
|
|
|
// Limits on the moderation queue of one list. They are variables so the tests can lower them.
|
|
var (
|
|
heldMessageLimit = 10 << 20 // bytes in one held message
|
|
heldBytesLimit = 200 << 20 // bytes held for one list, all messages together
|
|
)
|
|
|
|
// BounceThreshold is the number of hard bounces after which a member is disabled.
|
|
const BounceThreshold = 3
|
|
|
|
// BounceWindow is how long a hard bounce counts: a bounce after a quiet window restarts the count.
|
|
const BounceWindow = 30 * 24 * time.Hour
|
|
|
|
const (
|
|
pendingTTL = 72 * time.Hour
|
|
pendingRateLimit = time.Hour
|
|
|
|
// Caps on confirmation mail, per hour. Anyone can name any address in a join request, so the
|
|
// daemon must not mail one victim many times (maxPendingPerAddress, across all lists) or send
|
|
// an unbounded amount in total (maxPendingPerHour).
|
|
// A moderated list could be flooded with mail that nobody reads: the queue per list is bounded,
|
|
// and a message nobody moderated for a month is dropped.
|
|
maxHeldPerList = 500
|
|
heldTTL = 30 * 24 * time.Hour
|
|
|
|
maxPendingPerAddress = 3
|
|
maxPendingPerHour = 300
|
|
)
|
|
|
|
// List is a mailing list.
|
|
type List struct {
|
|
ID int64
|
|
Name string // local part, lowercase
|
|
Domain string // lowercase
|
|
Title string
|
|
PostPolicy string
|
|
JoinPolicy string
|
|
ArchivePolicy string
|
|
}
|
|
|
|
// Address returns the list posting address.
|
|
func (l List) Address() string { return l.Name + "@" + l.Domain }
|
|
|
|
// Member is a list subscriber.
|
|
type Member struct {
|
|
ID int64
|
|
ListID int64
|
|
Address string // lowercase
|
|
DisplayName string
|
|
Disabled bool // too many hard bounces: no more delivery
|
|
BounceCount int
|
|
}
|
|
|
|
// Held is a message waiting for moderation.
|
|
type Held struct {
|
|
ID int64
|
|
ListID int64
|
|
Sender string // envelope sender
|
|
Data []byte
|
|
CreatedAt time.Time
|
|
Reason string // why the message was held, for the admin
|
|
|
|
// What a listing needs, stored when the message is held so that a listing never has to read a body.
|
|
Subject string // decoded, at most 200 characters
|
|
FromHeader string // the From header as received, at most 200 characters
|
|
Size int // bytes of the message
|
|
}
|
|
|
|
// Pending is an unconfirmed subscribe or unsubscribe request.
|
|
type Pending struct {
|
|
Token string
|
|
ListID int64
|
|
Address string
|
|
Action string
|
|
}
|
|
|
|
// Store wraps the SQLite database.
|
|
type Store struct {
|
|
db *sql.DB
|
|
// Now returns the current time; tests replace it.
|
|
Now func() time.Time
|
|
}
|
|
|
|
const schema = `
|
|
CREATE TABLE IF NOT EXISTS lists (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL,
|
|
domain TEXT NOT NULL,
|
|
title TEXT NOT NULL DEFAULT '',
|
|
post_policy TEXT NOT NULL DEFAULT 'members',
|
|
join_policy TEXT NOT NULL DEFAULT 'open',
|
|
archive_policy TEXT NOT NULL DEFAULT 'private',
|
|
UNIQUE(name, domain)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS members (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
|
|
address TEXT NOT NULL,
|
|
display_name TEXT NOT NULL DEFAULT '',
|
|
disabled INTEGER NOT NULL DEFAULT 0,
|
|
bounce_count INTEGER NOT NULL DEFAULT 0,
|
|
bounce_at INTEGER NOT NULL DEFAULT 0,
|
|
UNIQUE(list_id, address)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS held (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
|
|
sender TEXT NOT NULL,
|
|
data BLOB NOT NULL,
|
|
created_at INTEGER NOT NULL,
|
|
reason TEXT NOT NULL DEFAULT '',
|
|
subject TEXT NOT NULL DEFAULT '',
|
|
from_header TEXT NOT NULL DEFAULT '',
|
|
size INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
CREATE TABLE IF NOT EXISTS pending (
|
|
token TEXT PRIMARY KEY,
|
|
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
|
|
address TEXT NOT NULL,
|
|
action TEXT NOT NULL,
|
|
created_at INTEGER NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS archive (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
|
|
thread_id INTEGER NOT NULL DEFAULT 0,
|
|
message_id TEXT NOT NULL,
|
|
in_reply_to TEXT NOT NULL DEFAULT '',
|
|
subject TEXT NOT NULL DEFAULT '',
|
|
sender TEXT NOT NULL DEFAULT '',
|
|
date INTEGER NOT NULL,
|
|
body TEXT NOT NULL DEFAULT '',
|
|
size INTEGER NOT NULL DEFAULT 0,
|
|
UNIQUE(list_id, message_id)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS archive_list_date ON archive(list_id, date DESC, id DESC);
|
|
CREATE INDEX IF NOT EXISTS archive_thread ON archive(list_id, thread_id);
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS archive_fts USING fts5(
|
|
subject, body, sender, content='archive', content_rowid='id'
|
|
);
|
|
CREATE TRIGGER IF NOT EXISTS archive_ai AFTER INSERT ON archive BEGIN
|
|
INSERT INTO archive_fts(rowid, subject, body, sender) VALUES (new.id, new.subject, new.body, new.sender);
|
|
END;
|
|
CREATE TRIGGER IF NOT EXISTS archive_ad AFTER DELETE ON archive BEGIN
|
|
INSERT INTO archive_fts(archive_fts, rowid, subject, body, sender) VALUES ('delete', old.id, old.subject, old.body, old.sender);
|
|
END;
|
|
CREATE TABLE IF NOT EXISTS outbox_msgs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
data BLOB NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS outbox (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
msg_id INTEGER NOT NULL REFERENCES outbox_msgs(id) ON DELETE CASCADE,
|
|
envelope_from TEXT NOT NULL,
|
|
rcpt TEXT NOT NULL,
|
|
member_id INTEGER NOT NULL DEFAULT 0,
|
|
prefix BLOB NOT NULL DEFAULT x'',
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
next_at INTEGER NOT NULL,
|
|
last_error TEXT NOT NULL DEFAULT '',
|
|
created_at INTEGER NOT NULL,
|
|
failed INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id);
|
|
CREATE TABLE IF NOT EXISTS delivered (
|
|
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
|
|
post_id TEXT NOT NULL,
|
|
created_at INTEGER NOT NULL,
|
|
PRIMARY KEY (list_id, post_id)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS settings (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL
|
|
);`
|
|
|
|
// Open opens (and migrates) the database at path. Use ":memory:" for tests.
|
|
func Open(path string) (*Store, error) {
|
|
// The pragmas are in the DSN so that every connection of the pool has them.
|
|
sep := "?"
|
|
if strings.Contains(path, "?") {
|
|
sep = "&"
|
|
}
|
|
dsn := path + sep + "_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)"
|
|
db, err := sql.Open("sqlite", dsn)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// An in-memory database is per-connection: keep a single one.
|
|
db.SetMaxOpenConns(1)
|
|
if err := migrate(db); err != nil {
|
|
db.Close()
|
|
return nil, fmt.Errorf("migrate: %w", err)
|
|
}
|
|
return &Store{db: db, Now: time.Now}, nil
|
|
}
|
|
|
|
func hasColumn(db *sql.DB, table, column string) (bool, error) {
|
|
var n int
|
|
err := db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info(?) WHERE name=?`, table, column).Scan(&n)
|
|
return n > 0, err
|
|
}
|
|
|
|
// migrate upgrades a database created by milestone 1, then ensures the schema.
|
|
func migrate(db *sql.DB) error {
|
|
if ok, err := hasColumn(db, "members", "list_id"); err != nil {
|
|
return err
|
|
} else if ok {
|
|
if hasID, err := hasColumn(db, "members", "id"); err != nil {
|
|
return err
|
|
} else if !hasID {
|
|
if err := migrateMembersV1(db); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if ok, err := hasColumn(db, "lists", "name"); err != nil {
|
|
return err
|
|
} else if ok {
|
|
if has, _ := hasColumn(db, "lists", "post_policy"); !has {
|
|
if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN post_policy TEXT NOT NULL DEFAULT 'members';
|
|
ALTER TABLE lists ADD COLUMN join_policy TEXT NOT NULL DEFAULT 'open';`); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if ok, err := hasColumn(db, "lists", "name"); err != nil {
|
|
return err
|
|
} else if ok {
|
|
if has, _ := hasColumn(db, "lists", "archive_policy"); !has {
|
|
if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN archive_policy TEXT NOT NULL DEFAULT 'private'`); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if ok, err := hasColumn(db, "members", "id"); err != nil {
|
|
return err
|
|
} else if ok {
|
|
if has, _ := hasColumn(db, "members", "bounce_at"); !has {
|
|
if _, err := db.Exec(`ALTER TABLE members ADD COLUMN bounce_at INTEGER NOT NULL DEFAULT 0`); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if ok, err := hasColumn(db, "outbox", "msg_id"); err != nil {
|
|
return err
|
|
} else if ok {
|
|
if has, _ := hasColumn(db, "outbox", "failed"); !has {
|
|
if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN failed INTEGER NOT NULL DEFAULT 0`); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if ok, err := hasColumn(db, "held", "list_id"); err != nil {
|
|
return err
|
|
} else if ok {
|
|
if has, _ := hasColumn(db, "held", "reason"); !has {
|
|
if _, err := db.Exec(`ALTER TABLE held ADD COLUMN reason TEXT NOT NULL DEFAULT ''`); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if has, _ := hasColumn(db, "held", "size"); !has {
|
|
if err := addHeldSummary(db); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
_, err := db.Exec(schema)
|
|
return err
|
|
}
|
|
|
|
// migrateMembersV1 replaces the milestone 1 members table (no id column) in one transaction, so a
|
|
// failure or a crash in the middle leaves the old table exactly as it was.
|
|
func migrateMembersV1(db *sql.DB) error {
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if _, err := tx.Exec(`ALTER TABLE members RENAME TO members_v1`); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.Exec(schema); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.Exec(`INSERT INTO members(list_id, address, display_name)
|
|
SELECT list_id, address, display_name FROM members_v1 ORDER BY list_id, address;
|
|
DROP TABLE members_v1;`); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// addHeldSummary adds the summary columns of the held queue and fills them for the messages
|
|
// already there, one message at a time so that a big queue is not loaded into memory.
|
|
func addHeldSummary(db *sql.DB) error {
|
|
for _, stmt := range []string{
|
|
`ALTER TABLE held ADD COLUMN subject TEXT NOT NULL DEFAULT ''`,
|
|
`ALTER TABLE held ADD COLUMN from_header TEXT NOT NULL DEFAULT ''`,
|
|
`ALTER TABLE held ADD COLUMN size INTEGER NOT NULL DEFAULT 0`,
|
|
`UPDATE held SET size=length(data)`,
|
|
} {
|
|
if _, err := db.Exec(stmt); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
rows, err := db.Query(`SELECT id FROM held`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var ids []int64
|
|
for rows.Next() {
|
|
var id int64
|
|
if err := rows.Scan(&id); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
ids = append(ids, id)
|
|
}
|
|
rows.Close()
|
|
for _, id := range ids {
|
|
var data []byte
|
|
if err := db.QueryRow(`SELECT data FROM held WHERE id=?`, id).Scan(&data); err != nil {
|
|
return err
|
|
}
|
|
subject, from := summarize(data)
|
|
if _, err := db.Exec(`UPDATE held SET subject=?, from_header=? WHERE id=?`, subject, from, id); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// summarize extracts what a moderation listing shows: the decoded subject and the From header, both
|
|
// without control characters and cut to 200 characters.
|
|
func summarize(data []byte) (subject, from string) {
|
|
m, err := mail.ReadMessage(bytes.NewReader(data))
|
|
if err != nil {
|
|
return "", ""
|
|
}
|
|
clean := func(s string) string {
|
|
s = strings.Map(func(r rune) rune {
|
|
if unicode.IsControl(r) {
|
|
return -1
|
|
}
|
|
return r
|
|
}, strings.Join(strings.Fields(s), " "))
|
|
if r := []rune(s); len(r) > 200 {
|
|
s = string(r[:200])
|
|
}
|
|
return s
|
|
}
|
|
raw := m.Header.Get("Subject")
|
|
if dec, err := new(mime.WordDecoder).DecodeHeader(raw); err == nil {
|
|
raw = dec
|
|
}
|
|
return clean(raw), clean(m.Header.Get("From"))
|
|
}
|
|
|
|
// Close closes the database.
|
|
func (s *Store) Close() error { return s.db.Close() }
|
|
|
|
func norm(addr string) string { return strings.ToLower(strings.TrimSpace(addr)) }
|
|
|
|
func isUnique(err error) bool { return err != nil && strings.Contains(err.Error(), "UNIQUE") }
|
|
|
|
func affected(res sql.Result, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
return ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Lists.
|
|
|
|
const listCols = `id, name, domain, title, post_policy, join_policy, archive_policy`
|
|
|
|
func scanList(row interface{ Scan(...any) error }) (List, error) {
|
|
var l List
|
|
err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy, &l.ArchivePolicy)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return List{}, ErrNotFound
|
|
}
|
|
return l, err
|
|
}
|
|
|
|
// CreateList adds a new list with default policies.
|
|
func (s *Store) CreateList(name, domain, title string) (List, error) {
|
|
res, err := s.db.Exec(`INSERT INTO lists(name, domain, title) VALUES(?,?,?)`,
|
|
strings.ToLower(name), strings.ToLower(domain), title)
|
|
if isUnique(err) {
|
|
return List{}, ErrExists
|
|
}
|
|
if err != nil {
|
|
return List{}, err
|
|
}
|
|
id, _ := res.LastInsertId()
|
|
return s.GetList(id)
|
|
}
|
|
|
|
// FindList resolves a posting address like "dev@example.org".
|
|
func (s *Store) FindList(addr string) (List, error) {
|
|
name, domain, ok := strings.Cut(norm(addr), "@")
|
|
if !ok {
|
|
return List{}, ErrNotFound
|
|
}
|
|
return scanList(s.db.QueryRow(`SELECT `+listCols+` FROM lists WHERE name=? AND domain=?`, name, domain))
|
|
}
|
|
|
|
// GetList returns a list by id.
|
|
func (s *Store) GetList(id int64) (List, error) {
|
|
return scanList(s.db.QueryRow(`SELECT `+listCols+` FROM lists WHERE id=?`, id))
|
|
}
|
|
|
|
// ListLists returns every list ordered by address.
|
|
func (s *Store) ListLists() ([]List, error) {
|
|
rows, err := s.db.Query(`SELECT ` + listCols + ` FROM lists ORDER BY domain, name`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []List
|
|
for rows.Next() {
|
|
l, err := scanList(rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, l)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// UpdateList changes the title of a list.
|
|
func (s *Store) UpdateList(id int64, title string) error {
|
|
return affected(s.db.Exec(`UPDATE lists SET title=? WHERE id=?`, title, id))
|
|
}
|
|
|
|
// SetPolicies changes the post, join and archive policies of a list.
|
|
func (s *Store) SetPolicies(id int64, post, join, archive string) error {
|
|
if post != PostMembers && post != PostModerated {
|
|
return fmt.Errorf("invalid post policy %q", post)
|
|
}
|
|
if join != JoinOpen && join != JoinClosed {
|
|
return fmt.Errorf("invalid join policy %q", join)
|
|
}
|
|
if archive != ArchivePublic && archive != ArchivePrivate && archive != ArchiveOff {
|
|
return fmt.Errorf("invalid archive policy %q", archive)
|
|
}
|
|
return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=?, archive_policy=? WHERE id=?`, post, join, archive, id))
|
|
}
|
|
|
|
// DeleteList removes a list with its members, held messages and pending requests.
|
|
func (s *Store) DeleteList(id int64) error {
|
|
return affected(s.db.Exec(`DELETE FROM lists WHERE id=?`, id))
|
|
}
|
|
|
|
// Members.
|
|
|
|
const memberCols = `id, list_id, address, display_name, disabled, bounce_count`
|
|
|
|
func scanMember(row interface{ Scan(...any) error }) (Member, error) {
|
|
var m Member
|
|
var dis int
|
|
err := row.Scan(&m.ID, &m.ListID, &m.Address, &m.DisplayName, &dis, &m.BounceCount)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Member{}, ErrNotFound
|
|
}
|
|
m.Disabled = dis != 0
|
|
return m, err
|
|
}
|
|
|
|
// AddMember subscribes an address to a list. Re-adding keeps the member id and
|
|
// clears the bounce state.
|
|
func (s *Store) AddMember(listID int64, addr, displayName string) error {
|
|
return addMember(s.db, listID, addr, displayName)
|
|
}
|
|
|
|
func addMember(q querier, listID int64, addr, displayName string) error {
|
|
_, err := q.Exec(`INSERT INTO members(list_id, address, display_name) VALUES(?,?,?)
|
|
ON CONFLICT(list_id, address) DO UPDATE SET display_name=excluded.display_name, disabled=0, bounce_count=0`,
|
|
listID, norm(addr), displayName)
|
|
return err
|
|
}
|
|
|
|
// IsMember reports whether addr is subscribed to the list.
|
|
func (s *Store) IsMember(listID int64, addr string) (bool, error) {
|
|
var n int
|
|
err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=?`, listID, norm(addr)).Scan(&n)
|
|
return n > 0, err
|
|
}
|
|
|
|
// IsActiveMember reports whether addr is subscribed and not disabled by bounces.
|
|
func (s *Store) IsActiveMember(listID int64, addr string) (bool, error) {
|
|
var n int
|
|
err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=? AND disabled=0`, listID, norm(addr)).Scan(&n)
|
|
return n > 0, err
|
|
}
|
|
|
|
func (s *Store) queryMembers(where string, args ...any) ([]Member, error) {
|
|
rows, err := s.db.Query(`SELECT `+memberCols+` FROM members WHERE `+where+` ORDER BY address`, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []Member
|
|
for rows.Next() {
|
|
m, err := scanMember(rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, m)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// Members returns all subscribers of a list, including disabled ones.
|
|
func (s *Store) Members(listID int64) ([]Member, error) {
|
|
return s.queryMembers(`list_id=?`, listID)
|
|
}
|
|
|
|
// ActiveMembers returns the subscribers that should receive mail.
|
|
func (s *Store) ActiveMembers(listID int64) ([]Member, error) {
|
|
return s.queryMembers(`list_id=? AND disabled=0`, listID)
|
|
}
|
|
|
|
// MemberByID returns a member by its id.
|
|
func (s *Store) MemberByID(id int64) (Member, error) {
|
|
return scanMember(s.db.QueryRow(`SELECT `+memberCols+` FROM members WHERE id=?`, id))
|
|
}
|
|
|
|
// RemoveMember unsubscribes an address from a list.
|
|
func (s *Store) RemoveMember(listID int64, addr string) error {
|
|
return removeMember(s.db, listID, addr)
|
|
}
|
|
|
|
func removeMember(q querier, listID int64, addr string) error {
|
|
return affected(q.Exec(`DELETE FROM members WHERE list_id=? AND address=?`, listID, norm(addr)))
|
|
}
|
|
|
|
// DeleteMember unsubscribes a member by id.
|
|
func (s *Store) DeleteMember(id int64) error {
|
|
return affected(s.db.Exec(`DELETE FROM members WHERE id=?`, id))
|
|
}
|
|
|
|
// RecordHardBounce counts a hard bounce and disables the member once the count
|
|
// reaches threshold. The count holds recent bounces only: when the last one is
|
|
// older than BounceWindow it restarts at 1. It reports whether the member is now disabled.
|
|
func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) {
|
|
now := s.Now()
|
|
if err := affected(s.db.Exec(`UPDATE members SET
|
|
bounce_count=CASE WHEN bounce_at<? THEN 1 ELSE bounce_count+1 END, bounce_at=? WHERE id=?`,
|
|
now.Add(-BounceWindow).Unix(), now.Unix(), memberID)); err != nil {
|
|
return false, err
|
|
}
|
|
if _, err := s.db.Exec(`UPDATE members SET disabled=1 WHERE id=? AND bounce_count>=?`, memberID, threshold); err != nil {
|
|
return false, err
|
|
}
|
|
m, err := s.MemberByID(memberID)
|
|
return m.Disabled, err
|
|
}
|
|
|
|
// Held messages.
|
|
|
|
// HoldMessage queues a message for moderation and returns its id. reason tells the admin why. A list
|
|
// holds a bounded amount: ErrHeldTooLarge for one message over heldMessageLimit, ErrHeldFull when the
|
|
// number of messages or the bytes held for the list would be exceeded.
|
|
func (s *Store) HoldMessage(listID int64, sender string, data []byte, reason string) (int64, error) {
|
|
if data == nil {
|
|
data = []byte{} // the column is NOT NULL
|
|
}
|
|
if len(data) > heldMessageLimit {
|
|
return 0, ErrHeldTooLarge
|
|
}
|
|
now := s.Now()
|
|
if _, err := s.db.Exec(`DELETE FROM held WHERE list_id=? AND created_at<?`, listID, now.Add(-heldTTL).Unix()); err != nil {
|
|
return 0, err
|
|
}
|
|
var n, bytesHeld int
|
|
if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(size),0) FROM held WHERE list_id=?`, listID).Scan(&n, &bytesHeld); err != nil {
|
|
return 0, err
|
|
}
|
|
if n >= maxHeldPerList || bytesHeld+len(data) > heldBytesLimit {
|
|
return 0, ErrHeldFull
|
|
}
|
|
subject, from := summarize(data)
|
|
res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at, reason, subject, from_header, size) VALUES(?,?,?,?,?,?,?,?)`,
|
|
listID, norm(sender), data, now.Unix(), reason, subject, from, len(data))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return res.LastInsertId()
|
|
}
|
|
|
|
const heldCols = `id, list_id, sender, created_at, reason, subject, from_header, size`
|
|
|
|
func scanHeld(row interface{ Scan(...any) error }, withData bool) (Held, error) {
|
|
var h Held
|
|
var ts int64
|
|
var err error
|
|
if withData {
|
|
err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size, &h.Data)
|
|
} else {
|
|
err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size)
|
|
}
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Held{}, ErrNotFound
|
|
}
|
|
h.CreatedAt = time.Unix(ts, 0)
|
|
return h, err
|
|
}
|
|
|
|
// ListHeld returns the held messages of a list, oldest first. It carries the summary of each message
|
|
// and never its body (Data is nil): a listing must not read up to 25 MiB per message into memory.
|
|
func (s *Store) ListHeld(listID int64) ([]Held, error) {
|
|
rows, err := s.db.Query(`SELECT `+heldCols+` FROM held WHERE list_id=? ORDER BY id`, listID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []Held
|
|
for rows.Next() {
|
|
h, err := scanHeld(rows, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, h)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// GetHeld returns one held message with its body.
|
|
func (s *Store) GetHeld(id int64) (Held, error) {
|
|
return scanHeld(s.db.QueryRow(`SELECT `+heldCols+`, data FROM held WHERE id=?`, id), true)
|
|
}
|
|
|
|
// DeleteHeld discards a held message.
|
|
func (s *Store) DeleteHeld(id int64) error {
|
|
return affected(s.db.Exec(`DELETE FROM held WHERE id=?`, id))
|
|
}
|
|
|
|
// Pending confirmations.
|
|
|
|
func randomHex(n int) (string, error) {
|
|
b := make([]byte, n)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(b), nil
|
|
}
|
|
|
|
// CreatePending records a subscribe or unsubscribe request and returns its
|
|
// token. created is false when an identical request was made in the last hour:
|
|
// this limits how often a third party can trigger confirmation mail.
|
|
func (s *Store) CreatePending(listID int64, addr, action string) (token string, created bool, err error) {
|
|
addr = norm(addr)
|
|
now := s.Now()
|
|
// Expired requests are dead weight (a token older than the TTL is refused): drop them here, so
|
|
// the table stays small without a background job.
|
|
if _, err = s.db.Exec(`DELETE FROM pending WHERE created_at<?`, now.Add(-pendingTTL).Unix()); err != nil {
|
|
return "", false, err
|
|
}
|
|
var n int
|
|
err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE list_id=? AND address=? AND action=? AND created_at>?`,
|
|
listID, addr, action, now.Add(-pendingRateLimit).Unix()).Scan(&n)
|
|
if err != nil || n > 0 {
|
|
return "", false, err
|
|
}
|
|
since := now.Add(-pendingRateLimit).Unix()
|
|
if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE address=? AND created_at>?`, addr, since).Scan(&n); err != nil || n >= maxPendingPerAddress {
|
|
return "", false, err
|
|
}
|
|
if err = s.db.QueryRow(`SELECT COUNT(*) FROM pending WHERE created_at>?`, since).Scan(&n); err != nil || n >= maxPendingPerHour {
|
|
return "", false, err
|
|
}
|
|
// 80 bits: unguessable within the 72 h life of a token, and short enough that
|
|
// name(<=30)+"+confirm-"+token fits the 64 characters of a local part.
|
|
token, err = randomHex(10)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
_, err = s.db.Exec(`INSERT INTO pending(token, list_id, address, action, created_at) VALUES(?,?,?,?,?)`,
|
|
token, listID, addr, action, now.Unix())
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
return token, true, nil
|
|
}
|
|
|
|
// ConfirmPending applies the request behind a token and consumes the token, in one transaction: a
|
|
// token is only used up when its action happened, and two confirmations of the same token cannot
|
|
// both apply (the first one to claim the row wins, the other gets ErrNotFound).
|
|
//
|
|
// Unknown, used, expired tokens and tokens of another list give ErrNotFound (the last two are left
|
|
// alone). check, when not nil, runs before anything changes; its error is returned and the token
|
|
// stays valid. A "join" subscribes the address (re-enabling a disabled member), a "leave"
|
|
// unsubscribes it (already gone is fine).
|
|
func (s *Store) ConfirmPending(token string, listID int64, check func(Pending) error) (Pending, error) {
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return Pending{}, err
|
|
}
|
|
defer tx.Rollback()
|
|
var p Pending
|
|
var ts int64
|
|
// DELETE ... RETURNING takes the write lock first, so concurrent confirmations queue up here.
|
|
err = tx.QueryRow(`DELETE FROM pending WHERE token=? AND list_id=? RETURNING token, list_id, address, action, created_at`,
|
|
token, listID).Scan(&p.Token, &p.ListID, &p.Address, &p.Action, &ts)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Pending{}, ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return Pending{}, err
|
|
}
|
|
if s.Now().Sub(time.Unix(ts, 0)) > pendingTTL {
|
|
// Expired: the row is gone for good.
|
|
if err := tx.Commit(); err != nil {
|
|
return Pending{}, err
|
|
}
|
|
return Pending{}, ErrNotFound
|
|
}
|
|
if check != nil {
|
|
if err := check(p); err != nil {
|
|
return Pending{}, err
|
|
}
|
|
}
|
|
switch p.Action {
|
|
case ActionJoin:
|
|
err = addMember(tx, p.ListID, p.Address, "")
|
|
case ActionLeave:
|
|
if err = removeMember(tx, p.ListID, p.Address); errors.Is(err, ErrNotFound) {
|
|
err = nil
|
|
}
|
|
}
|
|
if err != nil {
|
|
return Pending{}, err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return Pending{}, err
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
// Secret returns the HMAC secret, generating and saving it on first use.
|
|
func (s *Store) Secret() ([]byte, error) {
|
|
var v string
|
|
err := s.db.QueryRow(`SELECT value FROM settings WHERE key='secret'`).Scan(&v)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
v, err = randomHex(32)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err = s.db.Exec(`INSERT INTO settings(key, value) VALUES('secret', ?)`, v); err != nil {
|
|
return nil, err
|
|
}
|
|
} else if err != nil {
|
|
return nil, err
|
|
}
|
|
return hex.DecodeString(v)
|
|
}
|
|
|
|
// Archive.
|
|
|
|
// Message is an archived post.
|
|
type Message struct {
|
|
ID int64
|
|
ListID int64
|
|
ThreadID int64
|
|
MessageID string // Message-ID header, unique per list
|
|
InReplyTo string
|
|
Subject string
|
|
Sender string // display name of the original author
|
|
Date time.Time
|
|
Body string // plain text; empty in listings
|
|
Size int
|
|
}
|
|
|
|
const msgCols = `id, list_id, thread_id, message_id, in_reply_to, subject, sender, date, size`
|
|
|
|
func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, error) {
|
|
var m Message
|
|
var ts int64
|
|
var err error
|
|
if withBody {
|
|
err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size, &m.Body)
|
|
} else {
|
|
err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size)
|
|
}
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Message{}, ErrNotFound
|
|
}
|
|
m.Date = time.Unix(ts, 0)
|
|
return m, err
|
|
}
|
|
|
|
// querier is what archiveOne needs from a *sql.DB or a *sql.Tx.
|
|
type querier interface {
|
|
Exec(query string, args ...any) (sql.Result, error)
|
|
QueryRow(query string, args ...any) *sql.Row
|
|
}
|
|
|
|
// ArchiveResult is the outcome for one message of ArchiveBatch.
|
|
type ArchiveResult struct {
|
|
ID int64
|
|
Created bool
|
|
}
|
|
|
|
// ArchiveMessage stores a post. A message already archived in the same list
|
|
// (same Message-ID) is not stored again: created is false and the existing id
|
|
// is returned, so MTA retries are harmless.
|
|
func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) {
|
|
return archiveOne(s.db, listID, m)
|
|
}
|
|
|
|
// ArchiveBatch stores many messages in one transaction: all or none. It is much
|
|
// faster than ArchiveMessage in a loop, because SQLite syncs once per commit.
|
|
func (s *Store) ArchiveBatch(listID int64, ms []Message) ([]ArchiveResult, error) {
|
|
if len(ms) == 0 {
|
|
return nil, nil
|
|
}
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer tx.Rollback()
|
|
out := make([]ArchiveResult, 0, len(ms))
|
|
for _, m := range ms {
|
|
id, created, err := archiveOne(tx, listID, m)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, ArchiveResult{ID: id, Created: created})
|
|
}
|
|
return out, tx.Commit()
|
|
}
|
|
|
|
func archiveOne(q querier, listID int64, m Message) (id int64, created bool, err error) {
|
|
if m.MessageID == "" {
|
|
return 0, false, errors.New("message id is required")
|
|
}
|
|
err = q.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
|
|
if err == nil {
|
|
return id, false, nil
|
|
}
|
|
if !errors.Is(err, sql.ErrNoRows) {
|
|
return 0, false, err
|
|
}
|
|
var thread int64
|
|
if m.InReplyTo != "" {
|
|
e := q.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
|
|
if e != nil && !errors.Is(e, sql.ErrNoRows) {
|
|
return 0, false, e
|
|
}
|
|
}
|
|
res, err := q.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
|
|
VALUES(?,?,?,?,?,?,?,?,?)`,
|
|
listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body))
|
|
if err != nil {
|
|
return 0, false, err
|
|
}
|
|
id, _ = res.LastInsertId()
|
|
if thread == 0 {
|
|
if _, err := q.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
|
|
return 0, false, err
|
|
}
|
|
}
|
|
return id, true, nil
|
|
}
|
|
|
|
// threadRoots maps every id to the root of its reply chain. parent[id] is the message id replies to,
|
|
// when that message exists. A cycle of replies has the smallest id of the cycle as root. Each chain
|
|
// is walked once, so the cost is linear in the number of messages.
|
|
func threadRoots(ids []int64, parent map[int64]int64) map[int64]int64 {
|
|
roots := make(map[int64]int64, len(ids))
|
|
for _, start := range ids {
|
|
if _, done := roots[start]; done {
|
|
continue
|
|
}
|
|
path := []int64{start}
|
|
seen := map[int64]int{start: 0}
|
|
cur := start
|
|
var result int64
|
|
for {
|
|
p, ok := parent[cur]
|
|
if !ok {
|
|
result = cur
|
|
break
|
|
}
|
|
if r, known := roots[p]; known {
|
|
result = r
|
|
break
|
|
}
|
|
if at, loop := seen[p]; loop { // a cycle: its smallest id is the root
|
|
result = path[at]
|
|
for _, id := range path[at:] {
|
|
result = min(result, id)
|
|
}
|
|
break
|
|
}
|
|
seen[p] = len(path)
|
|
path = append(path, p)
|
|
cur = p
|
|
}
|
|
for _, id := range path {
|
|
roots[id] = result
|
|
}
|
|
}
|
|
return roots
|
|
}
|
|
|
|
// Rethread recomputes the thread of every message of a list from the In-Reply-To
|
|
// links, and returns how many messages changed thread. Insertion order does not
|
|
// matter: a message imported before its parent ends up in the parent's thread.
|
|
// A cycle of replies becomes one thread whose id is the smallest id in the cycle.
|
|
func (s *Store) Rethread(listID int64) (int, error) {
|
|
rows, err := s.db.Query(`SELECT id, message_id, in_reply_to, thread_id FROM archive WHERE list_id=?`, listID)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
type row struct {
|
|
id, thread int64
|
|
replyTo string
|
|
}
|
|
var all []row
|
|
byMsgID := map[string]int64{}
|
|
for rows.Next() {
|
|
var r row
|
|
var msgID string
|
|
if err := rows.Scan(&r.id, &msgID, &r.replyTo, &r.thread); err != nil {
|
|
rows.Close()
|
|
return 0, err
|
|
}
|
|
all = append(all, r)
|
|
byMsgID[msgID] = r.id
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
parent := map[int64]int64{}
|
|
ids := make([]int64, 0, len(all))
|
|
for _, r := range all {
|
|
ids = append(ids, r.id)
|
|
if p, ok := byMsgID[r.replyTo]; ok && r.replyTo != "" {
|
|
parent[r.id] = p
|
|
}
|
|
}
|
|
roots := threadRoots(ids, parent)
|
|
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer tx.Rollback()
|
|
changed := 0
|
|
for _, r := range all {
|
|
if want := roots[r.id]; want != r.thread {
|
|
if _, err := tx.Exec(`UPDATE archive SET thread_id=? WHERE id=?`, want, r.id); err != nil {
|
|
return 0, err
|
|
}
|
|
changed++
|
|
}
|
|
}
|
|
return changed, tx.Commit()
|
|
}
|
|
|
|
// GetMessage returns an archived message with its body.
|
|
func (s *Store) GetMessage(id int64) (Message, error) {
|
|
return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true)
|
|
}
|
|
|
|
// DeleteMessage removes an archived message and its search entry.
|
|
func (s *Store) DeleteMessage(id int64) error {
|
|
return affected(s.db.Exec(`DELETE FROM archive WHERE id=?`, id))
|
|
}
|
|
|
|
func (s *Store) queryMessages(q string, args ...any) ([]Message, error) {
|
|
rows, err := s.db.Query(q, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []Message
|
|
for rows.Next() {
|
|
m, err := scanMessage(rows, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, m)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// ListMessages returns a page of a list's archive, newest first, and the total
|
|
// count. Bodies are not loaded.
|
|
func (s *Store) ListMessages(listID int64, limit, offset int) ([]Message, int, error) {
|
|
var total int
|
|
if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive WHERE list_id=?`, listID).Scan(&total); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
out, err := s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? ORDER BY date DESC, id DESC LIMIT ? OFFSET ?`,
|
|
listID, limit, offset)
|
|
return out, total, err
|
|
}
|
|
|
|
// ftsQuery turns free text into an FTS5 query where every word is a quoted
|
|
// literal (AND semantics), so user input can never be read as FTS syntax.
|
|
func ftsQuery(q string) string {
|
|
words := strings.Fields(q)
|
|
for i, w := range words {
|
|
words[i] = `"` + strings.ReplaceAll(w, `"`, `""`) + `"`
|
|
}
|
|
return strings.Join(words, " ")
|
|
}
|
|
|
|
// SearchMessages runs a full-text search over subject, body and sender of one
|
|
// list, newest first. Bodies are not loaded.
|
|
func (s *Store) SearchMessages(listID int64, query string, limit, offset int) ([]Message, int, error) {
|
|
match := ftsQuery(query)
|
|
if match == "" {
|
|
return nil, 0, nil
|
|
}
|
|
var total int
|
|
if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid
|
|
WHERE archive_fts MATCH ? AND a.list_id=?`, match, listID).Scan(&total); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
out, err := s.queryMessages(`SELECT a.id, a.list_id, a.thread_id, a.message_id, a.in_reply_to, a.subject, a.sender, a.date, a.size
|
|
FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid
|
|
WHERE archive_fts MATCH ? AND a.list_id=? ORDER BY a.date DESC, a.id DESC LIMIT ? OFFSET ?`,
|
|
match, listID, limit, offset)
|
|
return out, total, err
|
|
}
|
|
|
|
// ThreadMessages returns every message of a thread, oldest first, without bodies: a thread listing
|
|
// shows subjects, and a thread can hold many messages of up to 1 MiB of text each.
|
|
func (s *Store) ThreadMessages(listID, threadID int64) ([]Message, error) {
|
|
return s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID)
|
|
}
|