parsimony/internal/store/outbox.go
tgiachi 2abd435494 feat(store): record posts and count their deliveries
- posts table: one row per post queued to a list (message id, subject,
  time, recipients, sent, failed, dropped), deleted with the list and
  forgotten after 400 days when a new post is recorded
- EnqueuePost and EnqueueFromHeld record the post in the same
  transaction and stamp every delivery with its id; a refused duplicate
  records nothing; other mail keeps post_id 0
- CompleteOutbox counts sent, the new DropOutbox counts dropped and
  FailOutbox counts failed, each in the transaction that changes the
  delivery; a delivery that already failed is never counted twice
- outbox priority column; DueOutbox orders by priority, next_at, id
- ListPosts newest first (default 50, at most 200) with Pending derived
- additive migration of the outbox columns, tested on an old database
2026-11-10 12:37:52 +01:00

312 lines
11 KiB
Go

package store
import (
"database/sql"
"errors"
"time"
)
// NewOutbound is one recipient to queue.
type NewOutbound struct {
From string // envelope sender (VERP address)
Rcpt string
MemberID int64 // 0 for system mail
Prefix []byte // per-recipient headers, sent before the shared message
Priority int // lower goes first among due deliveries: 0 for discussion and system mail, 1 for a newsletter
}
// OutboxItem is a queued delivery to one recipient.
type OutboxItem struct {
ID int64
MsgID int64
From string
Rcpt string
MemberID int64
Prefix []byte
Attempts int
NextAt time.Time
LastError string
CreatedAt time.Time
// Failed deliveries are never retried (the relay refused them for good, or they are too old).
// They stay listed for a week so the admin can see what was lost and why.
Failed bool
Priority int
}
// OutboxStats summarises the queue.
type OutboxStats struct {
Pending int // deliveries still waiting to be sent
Retrying int // of those, the ones that already failed at least once
Failed int // given up: kept for a week for the admin to see
Oldest time.Time // oldest waiting delivery; zero when none waits
}
// failedTTL is how long a failed delivery stays listed.
const failedTTL = 7 * 24 * time.Hour
// Enqueue stores a message and one delivery per recipient, all or nothing.
func (s *Store) Enqueue(msg []byte, items []NewOutbound) error {
if len(items) == 0 {
return nil
}
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
if err := enqueueTx(tx, s.Now().Unix(), 0, msg, items); err != nil {
return err
}
return tx.Commit()
}
// postMemory is how long a post queued for a list is remembered, to refuse it if the MTA delivers it again.
const postMemory = time.Hour
// EnqueuePost queues a post to a list, like Enqueue, and records postID (the identity of the post)
// in the same transaction, so a crash never leaves one without the other. If the list already
// queued the same postID within postMemory it queues nothing and returns ErrDuplicate: the MTA
// delivers the post again when our reply to the first delivery was lost.
// It also records the post (a Post, with its counters) and stamps every delivery with its id;
// a refused duplicate records nothing.
func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []NewOutbound) error {
now := s.Now()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.Exec(`DELETE FROM delivered WHERE created_at<?`, now.Add(-postMemory).Unix()); err != nil {
return err
}
if _, err := tx.Exec(`INSERT INTO delivered(list_id, post_id, created_at) VALUES(?,?,?)`, listID, postID, now.Unix()); err != nil {
if isUnique(err) {
return ErrDuplicate
}
return err
}
post, err := recordPostTx(tx, now, listID, msg, len(items))
if err != nil {
return err
}
if len(items) > 0 {
if err := enqueueTx(tx, now.Unix(), post, msg, items); err != nil {
return err
}
}
return tx.Commit()
}
// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues
// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the
// others get ErrNotFound, so a message is never delivered twice. With no recipients the message
// is still released. An approval always delivers, even a post already seen, and records postID
// as EnqueuePost does, so a later copy of the same post is refused. It records the post as EnqueuePost does.
func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items []NewOutbound) error {
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
var listID int64
if err := tx.QueryRow(`SELECT list_id FROM held WHERE id=?`, heldID).Scan(&listID); errors.Is(err, sql.ErrNoRows) {
return ErrNotFound
} else if err != nil {
return err
}
if err := affected(tx.Exec(`DELETE FROM held WHERE id=?`, heldID)); err != nil {
return err
}
if _, err := tx.Exec(`INSERT INTO delivered(list_id, post_id, created_at) VALUES(?,?,?)
ON CONFLICT(list_id, post_id) DO UPDATE SET created_at=excluded.created_at`, listID, postID, s.Now().Unix()); err != nil {
return err
}
post, err := recordPostTx(tx, s.Now(), listID, msg, len(items))
if err != nil {
return err
}
if len(items) > 0 {
if err := enqueueTx(tx, s.Now().Unix(), post, msg, items); err != nil {
return err
}
}
return tx.Commit()
}
// enqueueTx stores msg and its deliveries; postID is the posts row they count for, 0 for other mail.
func enqueueTx(tx *sql.Tx, now, postID int64, msg []byte, items []NewOutbound) error {
res, err := tx.Exec(`INSERT INTO outbox_msgs(data) VALUES(?)`, msg)
if err != nil {
return err
}
msgID, _ := res.LastInsertId()
for _, it := range items {
prefix := it.Prefix
if prefix == nil {
prefix = []byte{}
}
if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at, post_id, priority)
VALUES(?,?,?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now, postID, it.Priority); err != nil {
return err
}
}
return nil
}
const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed, priority`
func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) {
var it OutboxItem
var next, created, failed int64
err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed, &it.Priority)
it.NextAt, it.CreatedAt, it.Failed = time.Unix(next, 0), time.Unix(created, 0), failed != 0
return it, err
}
func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) {
rows, err := s.db.Query(q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []OutboxItem
for rows.Next() {
it, err := scanOutbox(rows)
if err != nil {
return nil, err
}
out = append(out, it)
}
return out, rows.Err()
}
// DueOutbox returns up to limit deliveries whose time has come: lowest priority number first (so a
// big newsletter never delays discussion mail), then longest waiting first.
func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) {
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY priority, next_at, id LIMIT ?`,
s.Now().Unix(), limit)
}
// ListOutbox returns up to limit deliveries for display, failing ones first.
func (s *Store) ListOutbox(limit int) ([]OutboxItem, error) {
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox ORDER BY failed DESC, attempts DESC, next_at, id LIMIT ?`, limit)
}
// OutboxData returns the shared message of a queued delivery.
func (s *Store) OutboxData(msgID int64) ([]byte, error) {
var b []byte
err := s.db.QueryRow(`SELECT data FROM outbox_msgs WHERE id=?`, msgID).Scan(&b)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
return b, err
}
// CompleteOutbox removes a delivery the relay accepted, and the shared message once no recipient
// needs it. A delivery of a post counts as sent, in the same transaction.
func (s *Store) CompleteOutbox(id int64) error { return s.removeOutbox(id, "sent") }
// DropOutbox removes a delivery that will never be made (the mailbox does not exist, or the admin
// discarded it), like CompleteOutbox. A delivery of a post counts as dropped, unless it had
// already failed: it is counted as failed already.
func (s *Store) DropOutbox(id int64) error { return s.removeOutbox(id, "dropped") }
// removeOutbox deletes a delivery and adds one to counter (a column of posts, never user input)
// of its post, when it has one and it had not failed.
func (s *Store) removeOutbox(id int64, counter string) error {
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
var msgID, postID int64
var failed bool
err = tx.QueryRow(`SELECT msg_id, post_id, failed FROM outbox WHERE id=?`, id).Scan(&msgID, &postID, &failed)
if errors.Is(err, sql.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return err
}
if _, err := tx.Exec(`DELETE FROM outbox WHERE id=?`, id); err != nil {
return err
}
if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id=? AND NOT EXISTS (SELECT 1 FROM outbox WHERE msg_id=?)`, msgID, msgID); err != nil {
return err
}
if postID != 0 && !failed {
if _, err := tx.Exec(`UPDATE posts SET `+counter+`=`+counter+`+1 WHERE id=?`, postID); err != nil {
return err
}
}
return tx.Commit()
}
// RetryOutbox records a failed attempt and schedules the next one.
func (s *Store) RetryOutbox(id int64, next time.Time, lastError string) error {
return affected(s.db.Exec(`UPDATE outbox SET attempts=attempts+1, next_at=?, last_error=? WHERE id=?`,
next.Unix(), lastError, id))
}
// OutboxStats returns queue totals.
func (s *Store) OutboxStats() (OutboxStats, error) {
var st OutboxStats
var oldest sql.NullInt64
err := s.db.QueryRow(`SELECT COALESCE(SUM(failed=0),0), COALESCE(SUM(failed=0 AND attempts>0),0), COALESCE(SUM(failed),0),
MIN(CASE WHEN failed=0 THEN created_at END) FROM outbox`).Scan(&st.Pending, &st.Retrying, &st.Failed, &oldest)
if oldest.Valid {
st.Oldest = time.Unix(oldest.Int64, 0)
}
return st, err
}
// FailOutbox gives up on a delivery for good, keeping it listed with the reason. Its time is the
// moment of the failure, from which PurgeFailedOutbox counts. A delivery of a post counts as failed,
// in the same transaction.
func (s *Store) FailOutbox(id int64, reason string) error {
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
var postID int64
var failed bool
err = tx.QueryRow(`SELECT post_id, failed FROM outbox WHERE id=?`, id).Scan(&postID, &failed)
if errors.Is(err, sql.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return err
}
if _, err := tx.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`,
reason, s.Now().Unix(), id); err != nil {
return err
}
if postID != 0 && !failed {
if _, err := tx.Exec(`UPDATE posts SET failed=failed+1 WHERE id=?`, postID); err != nil {
return err
}
}
return tx.Commit()
}
// PurgeFailedOutbox forgets failed deliveries older than a week, and the stored messages nobody
// needs any more. It returns how many deliveries were removed.
func (s *Store) PurgeFailedOutbox() (int, error) {
tx, err := s.db.Begin()
if err != nil {
return 0, err
}
defer tx.Rollback()
res, err := tx.Exec(`DELETE FROM outbox WHERE failed=1 AND next_at<?`, s.Now().Add(-failedTTL).Unix())
if err != nil {
return 0, err
}
n, _ := res.RowsAffected()
if n > 0 {
if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id NOT IN (SELECT msg_id FROM outbox)`); err != nil {
return 0, err
}
}
return int(n), tx.Commit()
}