package store import ( "database/sql" "errors" "time" ) // NewOutbound is one recipient to queue. type NewOutbound struct { From string // envelope sender (VERP address) Rcpt string MemberID int64 // 0 for system mail Prefix []byte // per-recipient headers, sent before the shared message Priority int // lower goes first among due deliveries: 0 for discussion and system mail, 1 for a newsletter } // OutboxItem is a queued delivery to one recipient. type OutboxItem struct { ID int64 MsgID int64 From string Rcpt string MemberID int64 Prefix []byte Attempts int NextAt time.Time LastError string CreatedAt time.Time // Failed deliveries are never retried (the relay refused them for good, or they are too old). // They stay listed for a week so the admin can see what was lost and why. Failed bool Priority int } // OutboxStats summarises the queue. type OutboxStats struct { Pending int // deliveries still waiting to be sent Retrying int // of those, the ones that already failed at least once Failed int // given up: kept for a week for the admin to see Oldest time.Time // oldest waiting delivery; zero when none waits } // failedTTL is how long a failed delivery stays listed. const failedTTL = 7 * 24 * time.Hour // Enqueue stores a message and one delivery per recipient, all or nothing. func (s *Store) Enqueue(msg []byte, items []NewOutbound) error { if len(items) == 0 { return nil } tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() if err := enqueueTx(tx, s.Now().Unix(), 0, msg, items); err != nil { return err } return tx.Commit() } // postMemory is how long a post queued for a list is remembered, to refuse it if the MTA delivers it again. const postMemory = time.Hour // EnqueuePost queues a post to a list, like Enqueue, and records postID (the identity of the post) // in the same transaction, so a crash never leaves one without the other. If the list already // queued the same postID within postMemory it queues nothing and returns ErrDuplicate: the MTA // delivers the post again when our reply to the first delivery was lost. // It also records the post (a Post, with its counters) and stamps every delivery with its id; // a refused duplicate records nothing. func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []NewOutbound) error { now := s.Now() tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() if _, err := tx.Exec(`DELETE FROM delivered WHERE created_at 0 { if err := enqueueTx(tx, now.Unix(), post, msg, items); err != nil { return err } } return tx.Commit() } // EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues // its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the // others get ErrNotFound, so a message is never delivered twice. With no recipients the message // is still released. An approval always delivers, even a post already seen, and records postID // as EnqueuePost does, so a later copy of the same post is refused. It records the post as EnqueuePost does. func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items []NewOutbound) error { tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() var listID int64 if err := tx.QueryRow(`SELECT list_id FROM held WHERE id=?`, heldID).Scan(&listID); errors.Is(err, sql.ErrNoRows) { return ErrNotFound } else if err != nil { return err } if err := affected(tx.Exec(`DELETE FROM held WHERE id=?`, heldID)); err != nil { return err } if _, err := tx.Exec(`INSERT INTO delivered(list_id, post_id, created_at) VALUES(?,?,?) ON CONFLICT(list_id, post_id) DO UPDATE SET created_at=excluded.created_at`, listID, postID, s.Now().Unix()); err != nil { return err } post, err := recordPostTx(tx, s.Now(), listID, msg, len(items)) if err != nil { return err } if len(items) > 0 { if err := enqueueTx(tx, s.Now().Unix(), post, msg, items); err != nil { return err } } return tx.Commit() } // enqueueTx stores msg and its deliveries; postID is the posts row they count for, 0 for other mail. func enqueueTx(tx *sql.Tx, now, postID int64, msg []byte, items []NewOutbound) error { res, err := tx.Exec(`INSERT INTO outbox_msgs(data) VALUES(?)`, msg) if err != nil { return err } msgID, _ := res.LastInsertId() for _, it := range items { prefix := it.Prefix if prefix == nil { prefix = []byte{} } if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at, post_id, priority) VALUES(?,?,?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now, postID, it.Priority); err != nil { return err } } return nil } const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed, priority` func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) { var it OutboxItem var next, created, failed int64 err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed, &it.Priority) it.NextAt, it.CreatedAt, it.Failed = time.Unix(next, 0), time.Unix(created, 0), failed != 0 return it, err } func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) { rows, err := s.db.Query(q, args...) if err != nil { return nil, err } defer rows.Close() var out []OutboxItem for rows.Next() { it, err := scanOutbox(rows) if err != nil { return nil, err } out = append(out, it) } return out, rows.Err() } // DueOutbox returns up to limit deliveries whose time has come: lowest priority number first (so a // big newsletter never delays discussion mail), then longest waiting first. func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) { return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY priority, next_at, id LIMIT ?`, s.Now().Unix(), limit) } // ListOutbox returns up to limit deliveries for display, failing ones first. func (s *Store) ListOutbox(limit int) ([]OutboxItem, error) { return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox ORDER BY failed DESC, attempts DESC, next_at, id LIMIT ?`, limit) } // OutboxData returns the shared message of a queued delivery. func (s *Store) OutboxData(msgID int64) ([]byte, error) { var b []byte err := s.db.QueryRow(`SELECT data FROM outbox_msgs WHERE id=?`, msgID).Scan(&b) if errors.Is(err, sql.ErrNoRows) { return nil, ErrNotFound } return b, err } // CompleteOutbox removes a delivery the relay accepted, and the shared message once no recipient // needs it. A delivery of a post counts as sent, in the same transaction. func (s *Store) CompleteOutbox(id int64) error { return s.removeOutbox(id, "sent") } // DropOutbox removes a delivery that will never be made (the mailbox does not exist, or the admin // discarded it), like CompleteOutbox. A delivery of a post counts as dropped, unless it had // already failed: it is counted as failed already. func (s *Store) DropOutbox(id int64) error { return s.removeOutbox(id, "dropped") } // removeOutbox deletes a delivery and adds one to counter (a column of posts, never user input) // of its post, when it has one and it had not failed. func (s *Store) removeOutbox(id int64, counter string) error { tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() var msgID, postID int64 var failed bool err = tx.QueryRow(`SELECT msg_id, post_id, failed FROM outbox WHERE id=?`, id).Scan(&msgID, &postID, &failed) if errors.Is(err, sql.ErrNoRows) { return ErrNotFound } if err != nil { return err } if _, err := tx.Exec(`DELETE FROM outbox WHERE id=?`, id); err != nil { return err } if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id=? AND NOT EXISTS (SELECT 1 FROM outbox WHERE msg_id=?)`, msgID, msgID); err != nil { return err } if postID != 0 && !failed { if _, err := tx.Exec(`UPDATE posts SET `+counter+`=`+counter+`+1 WHERE id=?`, postID); err != nil { return err } } return tx.Commit() } // RetryOutbox records a failed attempt and schedules the next one. func (s *Store) RetryOutbox(id int64, next time.Time, lastError string) error { return affected(s.db.Exec(`UPDATE outbox SET attempts=attempts+1, next_at=?, last_error=? WHERE id=?`, next.Unix(), lastError, id)) } // OutboxStats returns queue totals. func (s *Store) OutboxStats() (OutboxStats, error) { var st OutboxStats var oldest sql.NullInt64 err := s.db.QueryRow(`SELECT COALESCE(SUM(failed=0),0), COALESCE(SUM(failed=0 AND attempts>0),0), COALESCE(SUM(failed),0), MIN(CASE WHEN failed=0 THEN created_at END) FROM outbox`).Scan(&st.Pending, &st.Retrying, &st.Failed, &oldest) if oldest.Valid { st.Oldest = time.Unix(oldest.Int64, 0) } return st, err } // FailOutbox gives up on a delivery for good, keeping it listed with the reason. Its time is the // moment of the failure, from which PurgeFailedOutbox counts. A delivery of a post counts as failed, // in the same transaction. func (s *Store) FailOutbox(id int64, reason string) error { tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() var postID int64 var failed bool err = tx.QueryRow(`SELECT post_id, failed FROM outbox WHERE id=?`, id).Scan(&postID, &failed) if errors.Is(err, sql.ErrNoRows) { return ErrNotFound } if err != nil { return err } if _, err := tx.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`, reason, s.Now().Unix(), id); err != nil { return err } if postID != 0 && !failed { if _, err := tx.Exec(`UPDATE posts SET failed=failed+1 WHERE id=?`, postID); err != nil { return err } } return tx.Commit() } // PurgeFailedOutbox forgets failed deliveries older than a week, and the stored messages nobody // needs any more. It returns how many deliveries were removed. func (s *Store) PurgeFailedOutbox() (int, error) { tx, err := s.db.Begin() if err != nil { return 0, err } defer tx.Rollback() res, err := tx.Exec(`DELETE FROM outbox WHERE failed=1 AND next_at 0 { if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id NOT IN (SELECT msg_id FROM outbox)`); err != nil { return 0, err } } return int(n), tx.Commit() }