feat(store): durable outbox with shared bodies and retry scheduling
- Add outbox_msgs and outbox tables - Enqueue every recipient of a post in one transaction - Add due, list, complete, retry and stats queries - Delete the shared body with the last recipient
This commit is contained in:
parent
16124bbaf3
commit
092cf4c350
3 changed files with 363 additions and 0 deletions
156
internal/store/outbox.go
Normal file
156
internal/store/outbox.go
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NewOutbound is one recipient to queue.
|
||||
type NewOutbound struct {
|
||||
From string // envelope sender (VERP address)
|
||||
Rcpt string
|
||||
MemberID int64 // 0 for system mail
|
||||
Prefix []byte // per-recipient headers, sent before the shared message
|
||||
}
|
||||
|
||||
// OutboxItem is a queued delivery to one recipient.
|
||||
type OutboxItem struct {
|
||||
ID int64
|
||||
MsgID int64
|
||||
From string
|
||||
Rcpt string
|
||||
MemberID int64
|
||||
Prefix []byte
|
||||
Attempts int
|
||||
NextAt time.Time
|
||||
LastError string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
// OutboxStats summarises the queue.
|
||||
type OutboxStats struct {
|
||||
Pending int // every queued delivery
|
||||
Retrying int // deliveries that already failed at least once
|
||||
Oldest time.Time // zero when the queue is empty
|
||||
}
|
||||
|
||||
// Enqueue stores a message and one delivery per recipient, all or nothing.
|
||||
func (s *Store) Enqueue(msg []byte, items []NewOutbound) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
res, err := tx.Exec(`INSERT INTO outbox_msgs(data) VALUES(?)`, msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msgID, _ := res.LastInsertId()
|
||||
now := s.Now().Unix()
|
||||
for _, it := range items {
|
||||
prefix := it.Prefix
|
||||
if prefix == nil {
|
||||
prefix = []byte{}
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at)
|
||||
VALUES(?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at`
|
||||
|
||||
func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) {
|
||||
var it OutboxItem
|
||||
var next, created int64
|
||||
err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created)
|
||||
it.NextAt, it.CreatedAt = time.Unix(next, 0), time.Unix(created, 0)
|
||||
return it, err
|
||||
}
|
||||
|
||||
func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) {
|
||||
rows, err := s.db.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []OutboxItem
|
||||
for rows.Next() {
|
||||
it, err := scanOutbox(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, it)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// DueOutbox returns up to limit deliveries whose time has come, longest waiting first.
|
||||
func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) {
|
||||
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE next_at<=? ORDER BY next_at, id LIMIT ?`,
|
||||
s.Now().Unix(), limit)
|
||||
}
|
||||
|
||||
// ListOutbox returns up to limit deliveries for display, failing ones first.
|
||||
func (s *Store) ListOutbox(limit int) ([]OutboxItem, error) {
|
||||
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox ORDER BY attempts DESC, next_at, id LIMIT ?`, limit)
|
||||
}
|
||||
|
||||
// OutboxData returns the shared message of a queued delivery.
|
||||
func (s *Store) OutboxData(msgID int64) ([]byte, error) {
|
||||
var b []byte
|
||||
err := s.db.QueryRow(`SELECT data FROM outbox_msgs WHERE id=?`, msgID).Scan(&b)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return b, err
|
||||
}
|
||||
|
||||
// CompleteOutbox removes a delivery (sent, dropped or discarded) and the shared
|
||||
// message once no recipient needs it.
|
||||
func (s *Store) CompleteOutbox(id int64) error {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var msgID int64
|
||||
err = tx.QueryRow(`SELECT msg_id FROM outbox WHERE id=?`, id).Scan(&msgID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM outbox WHERE id=?`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id=? AND NOT EXISTS (SELECT 1 FROM outbox WHERE msg_id=?)`, msgID, msgID); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// RetryOutbox records a failed attempt and schedules the next one.
|
||||
func (s *Store) RetryOutbox(id int64, next time.Time, lastError string) error {
|
||||
return affected(s.db.Exec(`UPDATE outbox SET attempts=attempts+1, next_at=?, last_error=? WHERE id=?`,
|
||||
next.Unix(), lastError, id))
|
||||
}
|
||||
|
||||
// OutboxStats returns queue totals.
|
||||
func (s *Store) OutboxStats() (OutboxStats, error) {
|
||||
var st OutboxStats
|
||||
var oldest sql.NullInt64
|
||||
err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(attempts>0),0), MIN(created_at) FROM outbox`).
|
||||
Scan(&st.Pending, &st.Retrying, &oldest)
|
||||
if oldest.Valid {
|
||||
st.Oldest = time.Unix(oldest.Int64, 0)
|
||||
}
|
||||
return st, err
|
||||
}
|
||||
190
internal/store/outbox_test.go
Normal file
190
internal/store/outbox_test.go
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func items(n int) []NewOutbound {
|
||||
out := make([]NewOutbound, n)
|
||||
for i := range out {
|
||||
out[i] = NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com", MemberID: int64(i + 1), Prefix: []byte("List-Unsubscribe: <u>\r\n")}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestEnqueueAndDue(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
if err := s.Enqueue([]byte("Subject: x\r\n\r\nbody"), items(3)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
due, err := s.DueOutbox(10)
|
||||
if err != nil || len(due) != 3 {
|
||||
t.Fatalf("due: %d %v", len(due), err)
|
||||
}
|
||||
d := due[0]
|
||||
if d.From != "bounce@x.org" || d.Rcpt != "a@x.com" || d.MemberID != 1 || string(d.Prefix) != "List-Unsubscribe: <u>\r\n" || d.Attempts != 0 {
|
||||
t.Fatalf("item: %+v", d)
|
||||
}
|
||||
data, err := s.OutboxData(d.MsgID)
|
||||
if err != nil || string(data) != "Subject: x\r\n\r\nbody" {
|
||||
t.Fatalf("data: %q %v", data, err)
|
||||
}
|
||||
if due[0].MsgID != due[2].MsgID {
|
||||
t.Fatal("recipients of one post must share the stored body")
|
||||
}
|
||||
if got, _ := s.DueOutbox(2); len(got) != 2 {
|
||||
t.Fatalf("limit not applied: %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueueNothingStoresNothing(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
if err := s.Enqueue([]byte("x"), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var n int
|
||||
s.db.QueryRow(`SELECT COUNT(*) FROM outbox_msgs`).Scan(&n)
|
||||
if n != 0 {
|
||||
t.Fatalf("orphan message stored: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteDeletesTheBodyWithTheLastRecipient(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s.Enqueue([]byte("body"), items(2))
|
||||
due, _ := s.DueOutbox(10)
|
||||
if err := s.CompleteOutbox(due[0].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.OutboxData(due[0].MsgID); err != nil {
|
||||
t.Fatalf("body must stay while another recipient waits: %v", err)
|
||||
}
|
||||
if err := s.CompleteOutbox(due[1].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("body must go with the last recipient: %v", err)
|
||||
}
|
||||
if err := s.CompleteOutbox(due[1].ID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryDelaysAndCounts(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
s.Enqueue([]byte("body"), items(2))
|
||||
due, _ := s.DueOutbox(10)
|
||||
next := now.Add(5 * time.Minute)
|
||||
if err := s.RetryOutbox(due[0].ID, next, "connection refused"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := s.DueOutbox(10)
|
||||
if len(got) != 1 || got[0].ID != due[1].ID {
|
||||
t.Fatalf("the retried item must wait: %+v", got)
|
||||
}
|
||||
now = next.Add(-time.Second)
|
||||
if got, _ := s.DueOutbox(10); len(got) != 1 {
|
||||
t.Fatal("not due one second early")
|
||||
}
|
||||
now = next
|
||||
got, _ = s.DueOutbox(10)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("due again at next_at: %d", len(got))
|
||||
}
|
||||
var retried OutboxItem
|
||||
for _, g := range got {
|
||||
if g.ID == due[0].ID {
|
||||
retried = g
|
||||
}
|
||||
}
|
||||
if retried.Attempts != 1 || retried.LastError != "connection refused" || !retried.NextAt.Equal(next) {
|
||||
t.Fatalf("retried: %+v", retried)
|
||||
}
|
||||
if err := s.RetryOutbox(999, next, "x"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDueIsOrderedByNextAttempt(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
s.Enqueue([]byte("body"), items(3))
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.RetryOutbox(due[0].ID, now.Add(-time.Hour), "e") // oldest next_at
|
||||
now = now.Add(time.Minute)
|
||||
got, _ := s.DueOutbox(10)
|
||||
if got[0].ID != due[0].ID {
|
||||
t.Fatalf("the longest waiting item goes first: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutboxStats(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
st, err := s.OutboxStats()
|
||||
if err != nil || st.Pending != 0 || st.Retrying != 0 || !st.Oldest.IsZero() {
|
||||
t.Fatalf("empty: %+v %v", st, err)
|
||||
}
|
||||
s.Enqueue([]byte("body"), items(3))
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.RetryOutbox(due[0].ID, now.Add(time.Minute), "boom")
|
||||
now = now.Add(10 * time.Second)
|
||||
st, _ = s.OutboxStats()
|
||||
if st.Pending != 3 || st.Retrying != 1 || !st.Oldest.Equal(time.Unix(1_000_000, 0)) {
|
||||
t.Fatalf("stats: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListOutbox(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s.Enqueue([]byte("body"), items(5))
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.RetryOutbox(due[3].ID, time.Now().Add(time.Hour), "refused")
|
||||
got, err := s.ListOutbox(3)
|
||||
if err != nil || len(got) != 3 {
|
||||
t.Fatalf("list: %d %v", len(got), err)
|
||||
}
|
||||
failing, _ := s.ListOutbox(10)
|
||||
if failing[0].ID != due[3].ID {
|
||||
t.Fatalf("items that failed come first: %+v", failing[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscardOutbox(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s.Enqueue([]byte("body"), items(1))
|
||||
due, _ := s.DueOutbox(10)
|
||||
if err := s.CompleteOutbox(due[0].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.CompleteOutbox(due[0].ID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueSurvivesReopen(t *testing.T) {
|
||||
path := t.TempDir() + "/q.db"
|
||||
s, _ := Open(path)
|
||||
s.Enqueue([]byte("persist me"), items(2))
|
||||
s.Close()
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s2.Close()
|
||||
due, _ := s2.DueOutbox(10)
|
||||
if len(due) != 2 {
|
||||
t.Fatalf("queue lost on restart: %d", len(due))
|
||||
}
|
||||
if data, _ := s2.OutboxData(due[0].MsgID); string(data) != "persist me" {
|
||||
t.Fatalf("data: %q", data)
|
||||
}
|
||||
}
|
||||
|
|
@ -152,6 +152,23 @@ END;
|
|||
CREATE TRIGGER IF NOT EXISTS archive_ad AFTER DELETE ON archive BEGIN
|
||||
INSERT INTO archive_fts(archive_fts, rowid, subject, body, sender) VALUES ('delete', old.id, old.subject, old.body, old.sender);
|
||||
END;
|
||||
CREATE TABLE IF NOT EXISTS outbox_msgs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
data BLOB NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS outbox (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
msg_id INTEGER NOT NULL REFERENCES outbox_msgs(id) ON DELETE CASCADE,
|
||||
envelope_from TEXT NOT NULL,
|
||||
rcpt TEXT NOT NULL,
|
||||
member_id INTEGER NOT NULL DEFAULT 0,
|
||||
prefix BLOB NOT NULL DEFAULT x'',
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
next_at INTEGER NOT NULL,
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id);
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
|
|
|
|||
Loading…
Reference in a new issue