diff --git a/internal/store/outbox.go b/internal/store/outbox.go new file mode 100644 index 0000000..37790c5 --- /dev/null +++ b/internal/store/outbox.go @@ -0,0 +1,156 @@ +package store + +import ( + "database/sql" + "errors" + "time" +) + +// NewOutbound is one recipient to queue. +type NewOutbound struct { + From string // envelope sender (VERP address) + Rcpt string + MemberID int64 // 0 for system mail + Prefix []byte // per-recipient headers, sent before the shared message +} + +// OutboxItem is a queued delivery to one recipient. +type OutboxItem struct { + ID int64 + MsgID int64 + From string + Rcpt string + MemberID int64 + Prefix []byte + Attempts int + NextAt time.Time + LastError string + CreatedAt time.Time +} + +// OutboxStats summarises the queue. +type OutboxStats struct { + Pending int // every queued delivery + Retrying int // deliveries that already failed at least once + Oldest time.Time // zero when the queue is empty +} + +// Enqueue stores a message and one delivery per recipient, all or nothing. +func (s *Store) Enqueue(msg []byte, items []NewOutbound) error { + if len(items) == 0 { + return nil + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + res, err := tx.Exec(`INSERT INTO outbox_msgs(data) VALUES(?)`, msg) + if err != nil { + return err + } + msgID, _ := res.LastInsertId() + now := s.Now().Unix() + for _, it := range items { + prefix := it.Prefix + if prefix == nil { + prefix = []byte{} + } + if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at) + VALUES(?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now); err != nil { + return err + } + } + return tx.Commit() +} + +const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at` + +func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) { + var it OutboxItem + var next, created int64 + err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created) + it.NextAt, it.CreatedAt = time.Unix(next, 0), time.Unix(created, 0) + return it, err +} + +func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) { + rows, err := s.db.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []OutboxItem + for rows.Next() { + it, err := scanOutbox(rows) + if err != nil { + return nil, err + } + out = append(out, it) + } + return out, rows.Err() +} + +// DueOutbox returns up to limit deliveries whose time has come, longest waiting first. +func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) { + return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE next_at<=? ORDER BY next_at, id LIMIT ?`, + s.Now().Unix(), limit) +} + +// ListOutbox returns up to limit deliveries for display, failing ones first. +func (s *Store) ListOutbox(limit int) ([]OutboxItem, error) { + return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox ORDER BY attempts DESC, next_at, id LIMIT ?`, limit) +} + +// OutboxData returns the shared message of a queued delivery. +func (s *Store) OutboxData(msgID int64) ([]byte, error) { + var b []byte + err := s.db.QueryRow(`SELECT data FROM outbox_msgs WHERE id=?`, msgID).Scan(&b) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + return b, err +} + +// CompleteOutbox removes a delivery (sent, dropped or discarded) and the shared +// message once no recipient needs it. +func (s *Store) CompleteOutbox(id int64) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var msgID int64 + err = tx.QueryRow(`SELECT msg_id FROM outbox WHERE id=?`, id).Scan(&msgID) + if errors.Is(err, sql.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM outbox WHERE id=?`, id); err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id=? AND NOT EXISTS (SELECT 1 FROM outbox WHERE msg_id=?)`, msgID, msgID); err != nil { + return err + } + return tx.Commit() +} + +// RetryOutbox records a failed attempt and schedules the next one. +func (s *Store) RetryOutbox(id int64, next time.Time, lastError string) error { + return affected(s.db.Exec(`UPDATE outbox SET attempts=attempts+1, next_at=?, last_error=? WHERE id=?`, + next.Unix(), lastError, id)) +} + +// OutboxStats returns queue totals. +func (s *Store) OutboxStats() (OutboxStats, error) { + var st OutboxStats + var oldest sql.NullInt64 + err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(attempts>0),0), MIN(created_at) FROM outbox`). + Scan(&st.Pending, &st.Retrying, &oldest) + if oldest.Valid { + st.Oldest = time.Unix(oldest.Int64, 0) + } + return st, err +} diff --git a/internal/store/outbox_test.go b/internal/store/outbox_test.go new file mode 100644 index 0000000..9e88886 --- /dev/null +++ b/internal/store/outbox_test.go @@ -0,0 +1,190 @@ +package store + +import ( + "errors" + "testing" + "time" +) + +func items(n int) []NewOutbound { + out := make([]NewOutbound, n) + for i := range out { + out[i] = NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com", MemberID: int64(i + 1), Prefix: []byte("List-Unsubscribe: \r\n")} + } + return out +} + +func TestEnqueueAndDue(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + if err := s.Enqueue([]byte("Subject: x\r\n\r\nbody"), items(3)); err != nil { + t.Fatal(err) + } + due, err := s.DueOutbox(10) + if err != nil || len(due) != 3 { + t.Fatalf("due: %d %v", len(due), err) + } + d := due[0] + if d.From != "bounce@x.org" || d.Rcpt != "a@x.com" || d.MemberID != 1 || string(d.Prefix) != "List-Unsubscribe: \r\n" || d.Attempts != 0 { + t.Fatalf("item: %+v", d) + } + data, err := s.OutboxData(d.MsgID) + if err != nil || string(data) != "Subject: x\r\n\r\nbody" { + t.Fatalf("data: %q %v", data, err) + } + if due[0].MsgID != due[2].MsgID { + t.Fatal("recipients of one post must share the stored body") + } + if got, _ := s.DueOutbox(2); len(got) != 2 { + t.Fatalf("limit not applied: %d", len(got)) + } +} + +func TestEnqueueNothingStoresNothing(t *testing.T) { + s := newTestStore(t) + if err := s.Enqueue([]byte("x"), nil); err != nil { + t.Fatal(err) + } + var n int + s.db.QueryRow(`SELECT COUNT(*) FROM outbox_msgs`).Scan(&n) + if n != 0 { + t.Fatalf("orphan message stored: %d", n) + } +} + +func TestCompleteDeletesTheBodyWithTheLastRecipient(t *testing.T) { + s := newTestStore(t) + s.Enqueue([]byte("body"), items(2)) + due, _ := s.DueOutbox(10) + if err := s.CompleteOutbox(due[0].ID); err != nil { + t.Fatal(err) + } + if _, err := s.OutboxData(due[0].MsgID); err != nil { + t.Fatalf("body must stay while another recipient waits: %v", err) + } + if err := s.CompleteOutbox(due[1].ID); err != nil { + t.Fatal(err) + } + if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) { + t.Fatalf("body must go with the last recipient: %v", err) + } + if err := s.CompleteOutbox(due[1].ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestRetryDelaysAndCounts(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + s.Enqueue([]byte("body"), items(2)) + due, _ := s.DueOutbox(10) + next := now.Add(5 * time.Minute) + if err := s.RetryOutbox(due[0].ID, next, "connection refused"); err != nil { + t.Fatal(err) + } + got, _ := s.DueOutbox(10) + if len(got) != 1 || got[0].ID != due[1].ID { + t.Fatalf("the retried item must wait: %+v", got) + } + now = next.Add(-time.Second) + if got, _ := s.DueOutbox(10); len(got) != 1 { + t.Fatal("not due one second early") + } + now = next + got, _ = s.DueOutbox(10) + if len(got) != 2 { + t.Fatalf("due again at next_at: %d", len(got)) + } + var retried OutboxItem + for _, g := range got { + if g.ID == due[0].ID { + retried = g + } + } + if retried.Attempts != 1 || retried.LastError != "connection refused" || !retried.NextAt.Equal(next) { + t.Fatalf("retried: %+v", retried) + } + if err := s.RetryOutbox(999, next, "x"); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestDueIsOrderedByNextAttempt(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + s.Enqueue([]byte("body"), items(3)) + due, _ := s.DueOutbox(10) + s.RetryOutbox(due[0].ID, now.Add(-time.Hour), "e") // oldest next_at + now = now.Add(time.Minute) + got, _ := s.DueOutbox(10) + if got[0].ID != due[0].ID { + t.Fatalf("the longest waiting item goes first: %+v", got) + } +} + +func TestOutboxStats(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + st, err := s.OutboxStats() + if err != nil || st.Pending != 0 || st.Retrying != 0 || !st.Oldest.IsZero() { + t.Fatalf("empty: %+v %v", st, err) + } + s.Enqueue([]byte("body"), items(3)) + due, _ := s.DueOutbox(10) + s.RetryOutbox(due[0].ID, now.Add(time.Minute), "boom") + now = now.Add(10 * time.Second) + st, _ = s.OutboxStats() + if st.Pending != 3 || st.Retrying != 1 || !st.Oldest.Equal(time.Unix(1_000_000, 0)) { + t.Fatalf("stats: %+v", st) + } +} + +func TestListOutbox(t *testing.T) { + s := newTestStore(t) + s.Enqueue([]byte("body"), items(5)) + due, _ := s.DueOutbox(10) + s.RetryOutbox(due[3].ID, time.Now().Add(time.Hour), "refused") + got, err := s.ListOutbox(3) + if err != nil || len(got) != 3 { + t.Fatalf("list: %d %v", len(got), err) + } + failing, _ := s.ListOutbox(10) + if failing[0].ID != due[3].ID { + t.Fatalf("items that failed come first: %+v", failing[0]) + } +} + +func TestDiscardOutbox(t *testing.T) { + s := newTestStore(t) + s.Enqueue([]byte("body"), items(1)) + due, _ := s.DueOutbox(10) + if err := s.CompleteOutbox(due[0].ID); err != nil { + t.Fatal(err) + } + if err := s.CompleteOutbox(due[0].ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } +} + +func TestQueueSurvivesReopen(t *testing.T) { + path := t.TempDir() + "/q.db" + s, _ := Open(path) + s.Enqueue([]byte("persist me"), items(2)) + s.Close() + s2, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer s2.Close() + due, _ := s2.DueOutbox(10) + if len(due) != 2 { + t.Fatalf("queue lost on restart: %d", len(due)) + } + if data, _ := s2.OutboxData(due[0].MsgID); string(data) != "persist me" { + t.Fatalf("data: %q", data) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 4572fc4..e7fee0f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -152,6 +152,23 @@ END; CREATE TRIGGER IF NOT EXISTS archive_ad AFTER DELETE ON archive BEGIN INSERT INTO archive_fts(archive_fts, rowid, subject, body, sender) VALUES ('delete', old.id, old.subject, old.body, old.sender); END; +CREATE TABLE IF NOT EXISTS outbox_msgs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + data BLOB NOT NULL +); +CREATE TABLE IF NOT EXISTS outbox ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + msg_id INTEGER NOT NULL REFERENCES outbox_msgs(id) ON DELETE CASCADE, + envelope_from TEXT NOT NULL, + rcpt TEXT NOT NULL, + member_id INTEGER NOT NULL DEFAULT 0, + prefix BLOB NOT NULL DEFAULT x'', + attempts INTEGER NOT NULL DEFAULT 0, + next_at INTEGER NOT NULL, + last_error TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id); CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL