- posts table: one row per post queued to a list (message id, subject, time, recipients, sent, failed, dropped), deleted with the list and forgotten after 400 days when a new post is recorded - EnqueuePost and EnqueueFromHeld record the post in the same transaction and stamp every delivery with its id; a refused duplicate records nothing; other mail keeps post_id 0 - CompleteOutbox counts sent, the new DropOutbox counts dropped and FailOutbox counts failed, each in the transaction that changes the delivery; a delivery that already failed is never counted twice - outbox priority column; DueOutbox orders by priority, next_at, id - ListPosts newest first (default 50, at most 200) with Pending derived - additive migration of the outbox columns, tested on an old database
312 lines
11 KiB
Go
312 lines
11 KiB
Go
package store
|
|
|
|
import (
|
|
"database/sql"
|
|
"errors"
|
|
"time"
|
|
)
|
|
|
|
// NewOutbound is one recipient to queue.
|
|
type NewOutbound struct {
|
|
From string // envelope sender (VERP address)
|
|
Rcpt string
|
|
MemberID int64 // 0 for system mail
|
|
Prefix []byte // per-recipient headers, sent before the shared message
|
|
Priority int // lower goes first among due deliveries: 0 for discussion and system mail, 1 for a newsletter
|
|
}
|
|
|
|
// OutboxItem is a queued delivery to one recipient.
|
|
type OutboxItem struct {
|
|
ID int64
|
|
MsgID int64
|
|
From string
|
|
Rcpt string
|
|
MemberID int64
|
|
Prefix []byte
|
|
Attempts int
|
|
NextAt time.Time
|
|
LastError string
|
|
CreatedAt time.Time
|
|
// Failed deliveries are never retried (the relay refused them for good, or they are too old).
|
|
// They stay listed for a week so the admin can see what was lost and why.
|
|
Failed bool
|
|
Priority int
|
|
}
|
|
|
|
// OutboxStats summarises the queue.
|
|
type OutboxStats struct {
|
|
Pending int // deliveries still waiting to be sent
|
|
Retrying int // of those, the ones that already failed at least once
|
|
Failed int // given up: kept for a week for the admin to see
|
|
Oldest time.Time // oldest waiting delivery; zero when none waits
|
|
}
|
|
|
|
// failedTTL is how long a failed delivery stays listed.
|
|
const failedTTL = 7 * 24 * time.Hour
|
|
|
|
// Enqueue stores a message and one delivery per recipient, all or nothing.
|
|
func (s *Store) Enqueue(msg []byte, items []NewOutbound) error {
|
|
if len(items) == 0 {
|
|
return nil
|
|
}
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if err := enqueueTx(tx, s.Now().Unix(), 0, msg, items); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// postMemory is how long a post queued for a list is remembered, to refuse it if the MTA delivers it again.
|
|
const postMemory = time.Hour
|
|
|
|
// EnqueuePost queues a post to a list, like Enqueue, and records postID (the identity of the post)
|
|
// in the same transaction, so a crash never leaves one without the other. If the list already
|
|
// queued the same postID within postMemory it queues nothing and returns ErrDuplicate: the MTA
|
|
// delivers the post again when our reply to the first delivery was lost.
|
|
// It also records the post (a Post, with its counters) and stamps every delivery with its id;
|
|
// a refused duplicate records nothing.
|
|
func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []NewOutbound) error {
|
|
now := s.Now()
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if _, err := tx.Exec(`DELETE FROM delivered WHERE created_at<?`, now.Add(-postMemory).Unix()); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.Exec(`INSERT INTO delivered(list_id, post_id, created_at) VALUES(?,?,?)`, listID, postID, now.Unix()); err != nil {
|
|
if isUnique(err) {
|
|
return ErrDuplicate
|
|
}
|
|
return err
|
|
}
|
|
post, err := recordPostTx(tx, now, listID, msg, len(items))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(items) > 0 {
|
|
if err := enqueueTx(tx, now.Unix(), post, msg, items); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues
|
|
// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the
|
|
// others get ErrNotFound, so a message is never delivered twice. With no recipients the message
|
|
// is still released. An approval always delivers, even a post already seen, and records postID
|
|
// as EnqueuePost does, so a later copy of the same post is refused. It records the post as EnqueuePost does.
|
|
func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items []NewOutbound) error {
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
var listID int64
|
|
if err := tx.QueryRow(`SELECT list_id FROM held WHERE id=?`, heldID).Scan(&listID); errors.Is(err, sql.ErrNoRows) {
|
|
return ErrNotFound
|
|
} else if err != nil {
|
|
return err
|
|
}
|
|
if err := affected(tx.Exec(`DELETE FROM held WHERE id=?`, heldID)); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.Exec(`INSERT INTO delivered(list_id, post_id, created_at) VALUES(?,?,?)
|
|
ON CONFLICT(list_id, post_id) DO UPDATE SET created_at=excluded.created_at`, listID, postID, s.Now().Unix()); err != nil {
|
|
return err
|
|
}
|
|
post, err := recordPostTx(tx, s.Now(), listID, msg, len(items))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(items) > 0 {
|
|
if err := enqueueTx(tx, s.Now().Unix(), post, msg, items); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// enqueueTx stores msg and its deliveries; postID is the posts row they count for, 0 for other mail.
|
|
func enqueueTx(tx *sql.Tx, now, postID int64, msg []byte, items []NewOutbound) error {
|
|
res, err := tx.Exec(`INSERT INTO outbox_msgs(data) VALUES(?)`, msg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
msgID, _ := res.LastInsertId()
|
|
for _, it := range items {
|
|
prefix := it.Prefix
|
|
if prefix == nil {
|
|
prefix = []byte{}
|
|
}
|
|
if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at, post_id, priority)
|
|
VALUES(?,?,?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now, postID, it.Priority); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed, priority`
|
|
|
|
func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) {
|
|
var it OutboxItem
|
|
var next, created, failed int64
|
|
err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed, &it.Priority)
|
|
it.NextAt, it.CreatedAt, it.Failed = time.Unix(next, 0), time.Unix(created, 0), failed != 0
|
|
return it, err
|
|
}
|
|
|
|
func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) {
|
|
rows, err := s.db.Query(q, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []OutboxItem
|
|
for rows.Next() {
|
|
it, err := scanOutbox(rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, it)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// DueOutbox returns up to limit deliveries whose time has come: lowest priority number first (so a
|
|
// big newsletter never delays discussion mail), then longest waiting first.
|
|
func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) {
|
|
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY priority, next_at, id LIMIT ?`,
|
|
s.Now().Unix(), limit)
|
|
}
|
|
|
|
// ListOutbox returns up to limit deliveries for display, failing ones first.
|
|
func (s *Store) ListOutbox(limit int) ([]OutboxItem, error) {
|
|
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox ORDER BY failed DESC, attempts DESC, next_at, id LIMIT ?`, limit)
|
|
}
|
|
|
|
// OutboxData returns the shared message of a queued delivery.
|
|
func (s *Store) OutboxData(msgID int64) ([]byte, error) {
|
|
var b []byte
|
|
err := s.db.QueryRow(`SELECT data FROM outbox_msgs WHERE id=?`, msgID).Scan(&b)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return nil, ErrNotFound
|
|
}
|
|
return b, err
|
|
}
|
|
|
|
// CompleteOutbox removes a delivery the relay accepted, and the shared message once no recipient
|
|
// needs it. A delivery of a post counts as sent, in the same transaction.
|
|
func (s *Store) CompleteOutbox(id int64) error { return s.removeOutbox(id, "sent") }
|
|
|
|
// DropOutbox removes a delivery that will never be made (the mailbox does not exist, or the admin
|
|
// discarded it), like CompleteOutbox. A delivery of a post counts as dropped, unless it had
|
|
// already failed: it is counted as failed already.
|
|
func (s *Store) DropOutbox(id int64) error { return s.removeOutbox(id, "dropped") }
|
|
|
|
// removeOutbox deletes a delivery and adds one to counter (a column of posts, never user input)
|
|
// of its post, when it has one and it had not failed.
|
|
func (s *Store) removeOutbox(id int64, counter string) error {
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
var msgID, postID int64
|
|
var failed bool
|
|
err = tx.QueryRow(`SELECT msg_id, post_id, failed FROM outbox WHERE id=?`, id).Scan(&msgID, &postID, &failed)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.Exec(`DELETE FROM outbox WHERE id=?`, id); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id=? AND NOT EXISTS (SELECT 1 FROM outbox WHERE msg_id=?)`, msgID, msgID); err != nil {
|
|
return err
|
|
}
|
|
if postID != 0 && !failed {
|
|
if _, err := tx.Exec(`UPDATE posts SET `+counter+`=`+counter+`+1 WHERE id=?`, postID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// RetryOutbox records a failed attempt and schedules the next one.
|
|
func (s *Store) RetryOutbox(id int64, next time.Time, lastError string) error {
|
|
return affected(s.db.Exec(`UPDATE outbox SET attempts=attempts+1, next_at=?, last_error=? WHERE id=?`,
|
|
next.Unix(), lastError, id))
|
|
}
|
|
|
|
// OutboxStats returns queue totals.
|
|
func (s *Store) OutboxStats() (OutboxStats, error) {
|
|
var st OutboxStats
|
|
var oldest sql.NullInt64
|
|
err := s.db.QueryRow(`SELECT COALESCE(SUM(failed=0),0), COALESCE(SUM(failed=0 AND attempts>0),0), COALESCE(SUM(failed),0),
|
|
MIN(CASE WHEN failed=0 THEN created_at END) FROM outbox`).Scan(&st.Pending, &st.Retrying, &st.Failed, &oldest)
|
|
if oldest.Valid {
|
|
st.Oldest = time.Unix(oldest.Int64, 0)
|
|
}
|
|
return st, err
|
|
}
|
|
|
|
// FailOutbox gives up on a delivery for good, keeping it listed with the reason. Its time is the
|
|
// moment of the failure, from which PurgeFailedOutbox counts. A delivery of a post counts as failed,
|
|
// in the same transaction.
|
|
func (s *Store) FailOutbox(id int64, reason string) error {
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
var postID int64
|
|
var failed bool
|
|
err = tx.QueryRow(`SELECT post_id, failed FROM outbox WHERE id=?`, id).Scan(&postID, &failed)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`,
|
|
reason, s.Now().Unix(), id); err != nil {
|
|
return err
|
|
}
|
|
if postID != 0 && !failed {
|
|
if _, err := tx.Exec(`UPDATE posts SET failed=failed+1 WHERE id=?`, postID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// PurgeFailedOutbox forgets failed deliveries older than a week, and the stored messages nobody
|
|
// needs any more. It returns how many deliveries were removed.
|
|
func (s *Store) PurgeFailedOutbox() (int, error) {
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer tx.Rollback()
|
|
res, err := tx.Exec(`DELETE FROM outbox WHERE failed=1 AND next_at<?`, s.Now().Add(-failedTTL).Unix())
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
n, _ := res.RowsAffected()
|
|
if n > 0 {
|
|
if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id NOT IN (SELECT msg_id FROM outbox)`); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
return int(n), tx.Commit()
|
|
}
|