Merge branch 'feat/nl-posts-stats' into feat/newsletter
This commit is contained in:
commit
34f461bb6b
13 changed files with 1423 additions and 42 deletions
|
|
@ -76,6 +76,7 @@ func (s *server) routes() []route {
|
|||
{"GET /api/v1/queue", s.auth(s.getQueue)},
|
||||
{"DELETE /api/v1/queue/{id}", s.auth(s.discardQueued)},
|
||||
{"GET /api/v1/lists/{id}/held", s.auth(s.listHeld)},
|
||||
{"GET /api/v1/lists/{id}/posts", s.auth(s.listPosts)},
|
||||
{"POST /api/v1/held/{id}/approve", s.auth(s.approveHeld)},
|
||||
{"DELETE /api/v1/held/{id}", s.auth(s.discardHeld)},
|
||||
}
|
||||
|
|
@ -667,7 +668,7 @@ func (s *server) discardQueued(w http.ResponseWriter, r *http.Request) {
|
|||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := s.store.CompleteOutbox(id); err != nil {
|
||||
if err := s.store.DropOutbox(id); err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
|
|
@ -701,3 +702,45 @@ func (s *server) importArchive(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
writeJSON(w, http.StatusOK, sum)
|
||||
}
|
||||
|
||||
// Handlers: posts.
|
||||
|
||||
type postDTO struct {
|
||||
ID int64 `json:"id"`
|
||||
MessageID string `json:"message_id"`
|
||||
Subject string `json:"subject"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Recipients int `json:"recipients"`
|
||||
Sent int `json:"sent"`
|
||||
Failed int `json:"failed"`
|
||||
Dropped int `json:"dropped"`
|
||||
Pending int `json:"pending"`
|
||||
}
|
||||
|
||||
// listPosts returns the posts queued to a list with their delivery counters, newest first.
|
||||
func (s *server) listPosts(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := listID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
limit, ok := intParam(r, "limit", 0) // 0: the store's default
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadRequest, "limit must be a non-negative integer")
|
||||
return
|
||||
}
|
||||
if _, err := s.store.GetList(id); err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
ps, err := s.store.ListPosts(id, limit)
|
||||
if err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
out := make([]postDTO, 0, len(ps))
|
||||
for _, p := range ps {
|
||||
out = append(out, postDTO{ID: p.ID, MessageID: p.MessageID, Subject: p.Subject, CreatedAt: p.CreatedAt,
|
||||
Recipients: p.Recipients, Sent: p.Sent, Failed: p.Failed, Dropped: p.Dropped, Pending: p.Pending})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
|
|
|||
103
internal/api/posts_test.go
Normal file
103
internal/api/posts_test.go
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
type postJSON struct {
|
||||
ID int64 `json:"id"`
|
||||
MessageID string `json:"message_id"`
|
||||
Subject string `json:"subject"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Recipients int `json:"recipients"`
|
||||
Sent int `json:"sent"`
|
||||
Failed int `json:"failed"`
|
||||
Dropped int `json:"dropped"`
|
||||
Pending int `json:"pending"`
|
||||
}
|
||||
|
||||
func TestListPosts(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
now := time.Unix(1_700_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
rcpts := []store.NewOutbound{{From: "b@x.org", Rcpt: "a@x.com"}, {From: "b@x.org", Rcpt: "b@x.com"}, {From: "b@x.org", Rcpt: "c@x.com"}}
|
||||
s.EnqueuePost(l.ID, "mid:<1@x>", []byte("Message-ID: <1@x>\r\nSubject: first\r\n\r\nsecret body"), rcpts)
|
||||
now = now.Add(time.Minute)
|
||||
s.EnqueuePost(l.ID, "mid:<2@x>", []byte("Message-ID: <2@x>\r\nSubject: second\r\n\r\nsecret body"), rcpts[:1])
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.CompleteOutbox(due[0].ID)
|
||||
s.FailOutbox(due[1].ID, "no")
|
||||
|
||||
rec := do(t, h, "GET", fmt.Sprintf("/api/v1/lists/%d/posts", l.ID), "", true)
|
||||
expect(t, rec, 200)
|
||||
var ps []postJSON
|
||||
decode(t, rec, &ps)
|
||||
if len(ps) != 2 || ps[0].Subject != "second" || ps[1].Subject != "first" {
|
||||
t.Fatalf("newest first: %+v", ps)
|
||||
}
|
||||
p := ps[1]
|
||||
if p.ID == 0 || p.MessageID != "<1@x>" || !p.CreatedAt.Equal(now.Add(-time.Minute)) || p.Recipients != 3 ||
|
||||
p.Sent != 1 || p.Failed != 1 || p.Dropped != 0 || p.Pending != 1 {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "secret body") {
|
||||
t.Fatal("bodies are never listed")
|
||||
}
|
||||
|
||||
decode(t, do(t, h, "GET", fmt.Sprintf("/api/v1/lists/%d/posts?limit=1", l.ID), "", true), &ps)
|
||||
if len(ps) != 1 || ps[0].Subject != "second" {
|
||||
t.Fatalf("limit=1: %+v", ps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListPostsEmptyIsAnArray(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
rec := do(t, h, "GET", fmt.Sprintf("/api/v1/lists/%d/posts", l.ID), "", true)
|
||||
expect(t, rec, 200)
|
||||
if strings.TrimSpace(rec.Body.String()) != "[]" {
|
||||
t.Fatalf("body %q", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestListPostsLimit(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
for i := 0; i < 60; i++ {
|
||||
s.EnqueuePost(l.ID, fmt.Sprintf("mid:%d", i), []byte("Subject: x\r\n\r\n"), nil)
|
||||
}
|
||||
path := fmt.Sprintf("/api/v1/lists/%d/posts", l.ID)
|
||||
var ps []postJSON
|
||||
decode(t, do(t, h, "GET", path, "", true), &ps)
|
||||
if len(ps) != 50 {
|
||||
t.Fatalf("default 50: %d", len(ps))
|
||||
}
|
||||
for _, bad := range []string{"-1", "x", "1.5"} {
|
||||
expect(t, do(t, h, "GET", path+"?limit="+bad, "", true), 400)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListPostsUnknownListAndAuth(t *testing.T) {
|
||||
h, _ := queueServer(t)
|
||||
expect(t, do(t, h, "GET", "/api/v1/lists/99/posts", "", true), 404)
|
||||
expect(t, do(t, h, "GET", "/api/v1/lists/abc/posts", "", true), 404)
|
||||
expect(t, do(t, h, "GET", "/api/v1/lists/99/posts", "", false), 401)
|
||||
}
|
||||
|
||||
func TestDiscardingADeliveryOfAPostCountsAsDropped(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.EnqueuePost(l.ID, "mid:<1@x>", []byte("Subject: x\r\n\r\n"), []store.NewOutbound{{From: "b@x.org", Rcpt: "a@x.com"}})
|
||||
due, _ := s.DueOutbox(10)
|
||||
expect(t, do(t, h, "DELETE", fmt.Sprintf("/api/v1/queue/%d", due[0].ID), "", true), 204)
|
||||
ps, _ := s.ListPosts(l.ID, 0)
|
||||
if len(ps) != 1 || ps[0].Dropped != 1 || ps[0].Sent != 0 || ps[0].Pending != 0 {
|
||||
t.Fatalf("posts: %+v", ps)
|
||||
}
|
||||
}
|
||||
|
|
@ -54,6 +54,8 @@ Administration (talks to the REST API):
|
|||
archive import <list> <file|-> add the messages of an mbox file (Mailman export);
|
||||
safe to repeat, threads are rebuilt
|
||||
|
||||
posts ls <list> [--limit N] posts sent to a list with their delivery counts
|
||||
|
||||
queue ls outbound queue
|
||||
queue rm <id> discard one queued delivery
|
||||
|
||||
|
|
@ -110,6 +112,7 @@ func Run(args []string, io IO, getenv func(string) string) int {
|
|||
"held": {"ls": c.heldLs, "approve": c.heldApprove, "discard": c.heldDiscard},
|
||||
"archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm, "import": c.archiveImport},
|
||||
"queue": {"ls": c.queueLs, "rm": c.queueRm},
|
||||
"posts": {"ls": c.postsLs},
|
||||
"dkim": {"keygen": c.dkimKeygen, "record": c.dkimRecord, "check": c.dkimCheck},
|
||||
}
|
||||
group, ok := commands[rest[0]]
|
||||
|
|
|
|||
|
|
@ -754,3 +754,56 @@ func (c *cmd) queueRm(args []string) int {
|
|||
return err
|
||||
}, "Discarded queued delivery %d")
|
||||
}
|
||||
|
||||
// postsLs shows the posts queued to a list, newest first, with how far their delivery got.
|
||||
func (c *cmd) postsLs(args []string) int {
|
||||
fs := newFlagSet("posts ls")
|
||||
limit := fs.Int("limit", 50, "")
|
||||
pos, err := parseArgs(fs, args)
|
||||
if err != nil {
|
||||
return c.usageError("%v", err)
|
||||
}
|
||||
if len(pos) != 1 {
|
||||
return c.usageError("usage: parsimony posts ls <list> [--limit N]")
|
||||
}
|
||||
l, err := c.resolveList(pos[0])
|
||||
if err != nil {
|
||||
return c.fail(err)
|
||||
}
|
||||
cl, _ := c.client()
|
||||
raw, err := cl.do("GET", fmt.Sprintf("/lists/%d/posts?limit=%d", l.ID, *limit), nil)
|
||||
if err != nil {
|
||||
return c.fail(err)
|
||||
}
|
||||
if c.json {
|
||||
return c.printJSON(raw)
|
||||
}
|
||||
var ps []struct {
|
||||
ID int64 `json:"id"`
|
||||
Subject string `json:"subject"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Recipients int `json:"recipients"`
|
||||
Sent int `json:"sent"`
|
||||
Failed int `json:"failed"`
|
||||
Dropped int `json:"dropped"`
|
||||
Pending int `json:"pending"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &ps); err != nil {
|
||||
return c.fail(err)
|
||||
}
|
||||
if len(ps) == 0 {
|
||||
fmt.Fprintln(c.io.Out, "No posts yet.")
|
||||
return 0
|
||||
}
|
||||
rows := [][]string{{"ID", "DATE", "SUBJECT", "RECIPIENTS", "SENT", "PENDING", "FAILED", "DROPPED"}}
|
||||
for _, p := range ps {
|
||||
subject := p.Subject
|
||||
if subject == "" {
|
||||
subject = "(no subject)"
|
||||
}
|
||||
rows = append(rows, []string{strconv.FormatInt(p.ID, 10), formatDate(p.CreatedAt), subject, strconv.Itoa(p.Recipients),
|
||||
strconv.Itoa(p.Sent), strconv.Itoa(p.Pending), strconv.Itoa(p.Failed), strconv.Itoa(p.Dropped)})
|
||||
}
|
||||
c.table(rows)
|
||||
return 0
|
||||
}
|
||||
|
|
|
|||
95
internal/cli/posts_test.go
Normal file
95
internal/cli/posts_test.go
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
func TestPostsLs(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
l, _ := e.s.FindList("dev@example.org")
|
||||
if out := e.ok(t, "posts", "ls", "dev@example.org"); !strings.Contains(out, "No posts yet.") {
|
||||
t.Fatalf("empty: %q", out)
|
||||
}
|
||||
now := time.Date(2026, 3, 1, 9, 30, 0, 0, time.UTC)
|
||||
e.s.Now = func() time.Time { return now }
|
||||
var rcpts []store.NewOutbound
|
||||
for i := 0; i < 7; i++ {
|
||||
rcpts = append(rcpts, store.NewOutbound{From: "b@x.org", Rcpt: fmt.Sprintf("u%d@x.com", i)})
|
||||
}
|
||||
e.s.EnqueuePost(l.ID, "mid:<1@x>", []byte("Message-ID: <1@x>\r\nSubject: March issue\r\n\r\nb"), rcpts)
|
||||
due, _ := e.s.DueOutbox(10)
|
||||
for _, d := range due[:4] {
|
||||
e.s.CompleteOutbox(d.ID)
|
||||
}
|
||||
e.s.FailOutbox(due[4].ID, "no")
|
||||
now = now.Add(time.Hour)
|
||||
e.s.EnqueuePost(l.ID, "mid:<2@x>", []byte("Message-ID: <2@x>\r\n\r\nb"), nil)
|
||||
|
||||
out := e.ok(t, "posts", "ls", fmt.Sprint(l.ID))
|
||||
lines := strings.Split(strings.TrimSpace(out), "\n")
|
||||
if len(lines) != 3 {
|
||||
t.Fatalf("posts ls:\n%s", out)
|
||||
}
|
||||
if f := strings.Fields(lines[0]); strings.Join(f, " ") != "ID DATE SUBJECT RECIPIENTS SENT PENDING FAILED DROPPED" {
|
||||
t.Fatalf("header: %q", lines[0])
|
||||
}
|
||||
if f := strings.Fields(lines[1]); strings.Join(f, " ") != "2 2026-03-01 10:30 (no subject) 0 0 0 0 0" {
|
||||
t.Fatalf("newest first: %q", lines[1])
|
||||
}
|
||||
if f := strings.Fields(lines[2]); strings.Join(f, " ") != "1 2026-03-01 09:30 March issue 7 4 2 1 0" {
|
||||
t.Fatalf("row: %q", lines[2])
|
||||
}
|
||||
|
||||
if out := e.ok(t, "posts", "ls", "dev@example.org", "--limit", "1"); strings.Contains(out, "March issue") || !strings.Contains(out, "(no subject)") {
|
||||
t.Fatalf("--limit 1:\n%s", out)
|
||||
}
|
||||
var js []map[string]any
|
||||
if err := json.Unmarshal([]byte(e.ok(t, "--json", "posts", "ls", "dev@example.org")), &js); err != nil || len(js) != 2 ||
|
||||
js[1]["subject"] != "March issue" || js[1]["pending"] != float64(2) || js[1]["message_id"] != "<1@x>" {
|
||||
t.Fatalf("json: %v %v", js, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostsLsErrors(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.fails(t, 2, "usage: parsimony posts ls", "posts", "ls")
|
||||
e.fails(t, 2, "usage: parsimony posts ls", "posts", "ls", "a@x", "b@x")
|
||||
e.fails(t, 2, "flag provided but not defined", "posts", "ls", "a@x", "--nope")
|
||||
e.fails(t, 1, "no such list", "posts", "ls", "nope@example.org")
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
e.fails(t, 1, "limit must be", "posts", "ls", "dev@example.org", "--limit", "-1")
|
||||
if out := e.ok(t, "help"); !strings.Contains(out, "posts ls <list> [--limit N]") {
|
||||
t.Fatalf("help lacks posts ls:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostsLsEscapesControlCharacters(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/lists/1":
|
||||
fmt.Fprint(w, `{"id":1,"address":"dev@example.org"}`)
|
||||
case "/api/v1/lists/1/posts":
|
||||
b, _ := json.Marshal([]map[string]any{{"id": 1, "subject": "s " + esc, "created_at": time.Unix(0, 0), "recipients": 1}})
|
||||
w.Write(b)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
e := &env{srv: srv}
|
||||
out := e.ok(t, "posts", "ls", "1")
|
||||
noControl(t, "posts ls", out)
|
||||
if !strings.Contains(out, `\x1b[31mRED`) {
|
||||
t.Fatalf("the escape sequence should stay visible as text:\n%s", out)
|
||||
}
|
||||
}
|
||||
114
internal/queue/posts_test.go
Normal file
114
internal/queue/posts_test.go
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/emersion/go-smtp"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
// post queues a post of n recipients (a@x.com, b@x.com...) to a new list and returns the list.
|
||||
func (e *env) post(t *testing.T, n int) store.List {
|
||||
t.Helper()
|
||||
l, err := e.s.CreateList("dev", "example.org", "Dev")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var items []store.NewOutbound
|
||||
for i := 0; i < n; i++ {
|
||||
items = append(items, store.NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com"})
|
||||
}
|
||||
if err := e.s.EnqueuePost(l.ID, "mid:<p@x>", []byte("Message-ID: <p@x>\r\nSubject: hi\r\n\r\nbody\r\n"), items); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func (e *env) onlyPost(t *testing.T, l store.List) store.Post {
|
||||
t.Helper()
|
||||
posts, err := e.s.ListPosts(l.ID, 0)
|
||||
if err != nil || len(posts) != 1 {
|
||||
t.Fatalf("posts: %+v %v", posts, err)
|
||||
}
|
||||
return posts[0]
|
||||
}
|
||||
|
||||
func TestTheWorkerCountsEachOutcomeOfAPost(t *testing.T) {
|
||||
e := setup(t)
|
||||
l := e.post(t, 4)
|
||||
e.snd.fn = func(_ context.Context, _ string, to []string) error {
|
||||
switch to[0] {
|
||||
case "b@x.com":
|
||||
return errors.New("connection refused")
|
||||
case "c@x.com":
|
||||
return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "policy"}
|
||||
case "d@x.com":
|
||||
return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
st, err := e.w.RunOnce(context.Background())
|
||||
if err != nil || st.Sent != 1 || st.Retried != 1 || st.Failed != 1 || st.Dropped != 1 {
|
||||
t.Fatalf("run: %+v %v", st, err)
|
||||
}
|
||||
p := e.onlyPost(t, l)
|
||||
if p.Recipients != 4 || p.Sent != 1 || p.Failed != 1 || p.Dropped != 1 || p.Pending != 1 {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
// The retried delivery goes out later.
|
||||
e.snd.fn = nil
|
||||
e.clk.set(t0.Add(Backoff(1)))
|
||||
e.w.RunOnce(context.Background())
|
||||
if p := e.onlyPost(t, l); p.Sent != 2 || p.Pending != 0 {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGivingUpAfterMaxAgeCountsAsFailed(t *testing.T) {
|
||||
e := setup(t)
|
||||
l := e.post(t, 1)
|
||||
e.snd.fn = func(context.Context, string, []string) error { return errors.New("down") }
|
||||
e.clk.set(t0.Add(MaxAge + 1))
|
||||
if st, _ := e.w.RunOnce(context.Background()); st.Failed != 1 {
|
||||
t.Fatalf("stats: %+v", st)
|
||||
}
|
||||
if p := e.onlyPost(t, l); p.Failed != 1 || p.Pending != 0 {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMissingBodyCountsAsDropped(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.w.readBody = func(int64) ([]byte, error) { return nil, store.ErrNotFound }
|
||||
l := e.post(t, 2)
|
||||
e.w.RunOnce(context.Background())
|
||||
if p := e.onlyPost(t, l); p.Dropped != 2 || p.Sent != 0 || p.Pending != 0 {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscussionMailIsSentBeforeABigNewsletterQueuedEarlier(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.w.Concurrency = 1
|
||||
var news []store.NewOutbound
|
||||
for i := 0; i < 120; i++ {
|
||||
news = append(news, store.NewOutbound{From: "b@x.org", Rcpt: "reader@x.com", Priority: 1})
|
||||
}
|
||||
if err := e.s.Enqueue([]byte("Subject: newsletter\r\n\r\nx"), news); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e.clk.set(t0.Add(1))
|
||||
e.s.Enqueue([]byte("Subject: talk\r\n\r\nx"), []store.NewOutbound{
|
||||
{From: "b@x.org", Rcpt: "first@x.com"}, {From: "b@x.org", Rcpt: "second@x.com"},
|
||||
})
|
||||
if _, err := e.w.RunOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e.snd.sent[0].To[0] != "first@x.com" || e.snd.sent[1].To[0] != "second@x.com" || e.snd.sent[2].To[0] != "reader@x.com" {
|
||||
t.Fatalf("order: %v %v %v", e.snd.sent[0].To, e.snd.sent[1].To, e.snd.sent[2].To)
|
||||
}
|
||||
}
|
||||
|
|
@ -219,7 +219,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
|
|||
func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem) Stats {
|
||||
data, err := cache.get(it.MsgID)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
w.complete(it)
|
||||
w.drop(it)
|
||||
return Stats{Dropped: 1}
|
||||
}
|
||||
if err != nil {
|
||||
|
|
@ -259,7 +259,7 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem
|
|||
}
|
||||
}
|
||||
log.Printf("queue: drop %s: %v", it.Rcpt, err)
|
||||
w.complete(it)
|
||||
w.drop(it)
|
||||
return out
|
||||
case outbound.IsPermanent(err):
|
||||
// The relay refused the message itself (policy, size, line length...): nobody gets it, so
|
||||
|
|
@ -292,3 +292,10 @@ func (w *Worker) complete(it store.OutboxItem) {
|
|||
log.Printf("queue: remove %d: %v", it.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// drop removes a delivery that will never be made; a delivery of a post counts as dropped.
|
||||
func (w *Worker) drop(it store.OutboxItem) {
|
||||
if err := w.Store.DropOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) {
|
||||
log.Printf("queue: drop %d: %v", it.ID, err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
127
internal/router/priority_test.go
Normal file
127
internal/router/priority_test.go
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
package router
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
// newsletterEnv is newEnv on a database file whose list has the publishers post policy, set
|
||||
// directly in the database.
|
||||
func newsletterEnv(t *testing.T) *env {
|
||||
t.Helper()
|
||||
path := t.TempDir() + "/p.db"
|
||||
s, err := store.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.AddMember(l.ID, "alice@example.com", "Alice")
|
||||
s.AddMember(l.ID, "bob@example.com", "Bob")
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err := db.Exec(`UPDATE lists SET post_policy=? WHERE id=?`, store.PostPublishers, l.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l, _ = s.GetList(l.ID)
|
||||
return &env{r: New(s), s: s, list: l}
|
||||
}
|
||||
|
||||
func duePriorities(t *testing.T, s *store.Store) []int {
|
||||
t.Helper()
|
||||
due, err := s.DueOutbox(100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []int
|
||||
for _, d := range due {
|
||||
out = append(out, d.Priority)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func allEqual(ps []int, want int, n int) bool {
|
||||
if len(ps) != n {
|
||||
return false
|
||||
}
|
||||
for _, p := range ps {
|
||||
if p != want {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
const samplePost = "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: <p@x>\r\n\r\nbody\r\n"
|
||||
|
||||
func TestANewsletterIsDeliveredWithPriorityOne(t *testing.T) {
|
||||
e := newsletterEnv(t)
|
||||
res, err := e.r.prepare(e.list, []byte(samplePost))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.r.Deliver(res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ps := duePriorities(t, e.s); !allEqual(ps, 1, 2) {
|
||||
t.Fatalf("priorities: %v", ps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADiscussionPostIsDeliveredWithPriorityZero(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
if err := e.post(t, samplePost); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ps := duePriorities(t, e.s); !allEqual(ps, 0, 2) {
|
||||
t.Fatalf("priorities: %v", ps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReleasedNewsletterHasPriorityOne(t *testing.T) {
|
||||
e := newsletterEnv(t)
|
||||
id, _ := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(samplePost), "r")
|
||||
if err := e.r.Release(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ps := duePriorities(t, e.s); !allEqual(ps, 1, 2) {
|
||||
t.Fatalf("priorities: %v", ps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReleasedDiscussionPostHasPriorityZero(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
id, _ := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(samplePost), "r")
|
||||
if err := e.r.Release(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ps := duePriorities(t, e.s); !allEqual(ps, 0, 2) {
|
||||
t.Fatalf("priorities: %v", ps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfirmationMailOfANewsletterHasPriorityZero(t *testing.T) {
|
||||
e := newsletterEnv(t)
|
||||
if handled, err := e.r.Command("carol@x.com", "dev+join@example.org", nil); !handled || err != nil {
|
||||
t.Fatalf("join: %v %v", handled, err)
|
||||
}
|
||||
if ps := duePriorities(t, e.s); !allEqual(ps, 0, 1) {
|
||||
t.Fatalf("priorities: %v", ps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPostIsRecordedWithItsRecipients(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
if err := e.post(t, samplePost); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
posts, err := e.s.ListPosts(e.list.ID, 0)
|
||||
if err != nil || len(posts) != 1 || posts[0].Recipients != 2 || posts[0].MessageID != "<p@x>" || posts[0].Subject == "" {
|
||||
t.Fatalf("posts: %+v %v", posts, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -176,8 +176,13 @@ func postID(data []byte) string {
|
|||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// outbound builds one queue entry per recipient.
|
||||
// outbound builds one queue entry per recipient. A newsletter (post policy publishers) is sent
|
||||
// with priority 1, so that its many deliveries never delay discussion mail.
|
||||
func (r *Router) outbound(res Result) ([]store.NewOutbound, error) {
|
||||
priority := 0
|
||||
if res.List.PostPolicy == store.PostPublishers {
|
||||
priority = 1
|
||||
}
|
||||
items := make([]store.NewOutbound, 0, len(res.Recipients))
|
||||
for _, rc := range res.Recipients {
|
||||
from := r.bounceAddress(res.List, rc.ID)
|
||||
|
|
@ -185,7 +190,7 @@ func (r *Router) outbound(res Result) ([]store.NewOutbound, error) {
|
|||
return nil, errors.New("cannot create bounce address")
|
||||
}
|
||||
items = append(items, store.NewOutbound{
|
||||
From: from, Rcpt: rc.Address, MemberID: rc.ID, Prefix: r.unsubscribeHeaders(res.List, rc.ID),
|
||||
From: from, Rcpt: rc.Address, MemberID: rc.ID, Prefix: r.unsubscribeHeaders(res.List, rc.ID), Priority: priority,
|
||||
})
|
||||
}
|
||||
return items, nil
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ type NewOutbound struct {
|
|||
Rcpt string
|
||||
MemberID int64 // 0 for system mail
|
||||
Prefix []byte // per-recipient headers, sent before the shared message
|
||||
Priority int // lower goes first among due deliveries: 0 for discussion and system mail, 1 for a newsletter
|
||||
}
|
||||
|
||||
// OutboxItem is a queued delivery to one recipient.
|
||||
|
|
@ -28,7 +29,8 @@ type OutboxItem struct {
|
|||
CreatedAt time.Time
|
||||
// Failed deliveries are never retried (the relay refused them for good, or they are too old).
|
||||
// They stay listed for a week so the admin can see what was lost and why.
|
||||
Failed bool
|
||||
Failed bool
|
||||
Priority int
|
||||
}
|
||||
|
||||
// OutboxStats summarises the queue.
|
||||
|
|
@ -52,7 +54,7 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error {
|
|||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil {
|
||||
if err := enqueueTx(tx, s.Now().Unix(), 0, msg, items); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
|
|
@ -65,6 +67,8 @@ const postMemory = time.Hour
|
|||
// in the same transaction, so a crash never leaves one without the other. If the list already
|
||||
// queued the same postID within postMemory it queues nothing and returns ErrDuplicate: the MTA
|
||||
// delivers the post again when our reply to the first delivery was lost.
|
||||
// It also records the post (a Post, with its counters) and stamps every delivery with its id;
|
||||
// a refused duplicate records nothing.
|
||||
func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []NewOutbound) error {
|
||||
now := s.Now()
|
||||
tx, err := s.db.Begin()
|
||||
|
|
@ -81,8 +85,12 @@ func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []New
|
|||
}
|
||||
return err
|
||||
}
|
||||
post, err := recordPostTx(tx, now, listID, msg, len(items))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(items) > 0 {
|
||||
if err := enqueueTx(tx, now.Unix(), msg, items); err != nil {
|
||||
if err := enqueueTx(tx, now.Unix(), post, msg, items); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
@ -93,7 +101,7 @@ func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []New
|
|||
// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the
|
||||
// others get ErrNotFound, so a message is never delivered twice. With no recipients the message
|
||||
// is still released. An approval always delivers, even a post already seen, and records postID
|
||||
// as EnqueuePost does, so a later copy of the same post is refused.
|
||||
// as EnqueuePost does, so a later copy of the same post is refused. It records the post as EnqueuePost does.
|
||||
func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items []NewOutbound) error {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
|
|
@ -113,15 +121,20 @@ func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items [
|
|||
ON CONFLICT(list_id, post_id) DO UPDATE SET created_at=excluded.created_at`, listID, postID, s.Now().Unix()); err != nil {
|
||||
return err
|
||||
}
|
||||
post, err := recordPostTx(tx, s.Now(), listID, msg, len(items))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(items) > 0 {
|
||||
if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil {
|
||||
if err := enqueueTx(tx, s.Now().Unix(), post, msg, items); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error {
|
||||
// enqueueTx stores msg and its deliveries; postID is the posts row they count for, 0 for other mail.
|
||||
func enqueueTx(tx *sql.Tx, now, postID int64, msg []byte, items []NewOutbound) error {
|
||||
res, err := tx.Exec(`INSERT INTO outbox_msgs(data) VALUES(?)`, msg)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -132,20 +145,20 @@ func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error {
|
|||
if prefix == nil {
|
||||
prefix = []byte{}
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at)
|
||||
VALUES(?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now); err != nil {
|
||||
if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at, post_id, priority)
|
||||
VALUES(?,?,?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now, postID, it.Priority); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed`
|
||||
const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed, priority`
|
||||
|
||||
func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) {
|
||||
var it OutboxItem
|
||||
var next, created, failed int64
|
||||
err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed)
|
||||
err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed, &it.Priority)
|
||||
it.NextAt, it.CreatedAt, it.Failed = time.Unix(next, 0), time.Unix(created, 0), failed != 0
|
||||
return it, err
|
||||
}
|
||||
|
|
@ -167,9 +180,10 @@ func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) {
|
|||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// DueOutbox returns up to limit deliveries whose time has come, longest waiting first.
|
||||
// DueOutbox returns up to limit deliveries whose time has come: lowest priority number first (so a
|
||||
// big newsletter never delays discussion mail), then longest waiting first.
|
||||
func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) {
|
||||
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY next_at, id LIMIT ?`,
|
||||
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY priority, next_at, id LIMIT ?`,
|
||||
s.Now().Unix(), limit)
|
||||
}
|
||||
|
||||
|
|
@ -188,16 +202,26 @@ func (s *Store) OutboxData(msgID int64) ([]byte, error) {
|
|||
return b, err
|
||||
}
|
||||
|
||||
// CompleteOutbox removes a delivery (sent, dropped or discarded) and the shared
|
||||
// message once no recipient needs it.
|
||||
func (s *Store) CompleteOutbox(id int64) error {
|
||||
// CompleteOutbox removes a delivery the relay accepted, and the shared message once no recipient
|
||||
// needs it. A delivery of a post counts as sent, in the same transaction.
|
||||
func (s *Store) CompleteOutbox(id int64) error { return s.removeOutbox(id, "sent") }
|
||||
|
||||
// DropOutbox removes a delivery that will never be made (the mailbox does not exist, or the admin
|
||||
// discarded it), like CompleteOutbox. A delivery of a post counts as dropped, unless it had
|
||||
// already failed: it is counted as failed already.
|
||||
func (s *Store) DropOutbox(id int64) error { return s.removeOutbox(id, "dropped") }
|
||||
|
||||
// removeOutbox deletes a delivery and adds one to counter (a column of posts, never user input)
|
||||
// of its post, when it has one and it had not failed.
|
||||
func (s *Store) removeOutbox(id int64, counter string) error {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var msgID int64
|
||||
err = tx.QueryRow(`SELECT msg_id FROM outbox WHERE id=?`, id).Scan(&msgID)
|
||||
var msgID, postID int64
|
||||
var failed bool
|
||||
err = tx.QueryRow(`SELECT msg_id, post_id, failed FROM outbox WHERE id=?`, id).Scan(&msgID, &postID, &failed)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
|
|
@ -210,6 +234,11 @@ func (s *Store) CompleteOutbox(id int64) error {
|
|||
if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id=? AND NOT EXISTS (SELECT 1 FROM outbox WHERE msg_id=?)`, msgID, msgID); err != nil {
|
||||
return err
|
||||
}
|
||||
if postID != 0 && !failed {
|
||||
if _, err := tx.Exec(`UPDATE posts SET `+counter+`=`+counter+`+1 WHERE id=?`, postID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
|
|
@ -232,10 +261,33 @@ func (s *Store) OutboxStats() (OutboxStats, error) {
|
|||
}
|
||||
|
||||
// FailOutbox gives up on a delivery for good, keeping it listed with the reason. Its time is the
|
||||
// moment of the failure, from which PurgeFailedOutbox counts.
|
||||
// moment of the failure, from which PurgeFailedOutbox counts. A delivery of a post counts as failed,
|
||||
// in the same transaction.
|
||||
func (s *Store) FailOutbox(id int64, reason string) error {
|
||||
return affected(s.db.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`,
|
||||
reason, s.Now().Unix(), id))
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var postID int64
|
||||
var failed bool
|
||||
err = tx.QueryRow(`SELECT post_id, failed FROM outbox WHERE id=?`, id).Scan(&postID, &failed)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`,
|
||||
reason, s.Now().Unix(), id); err != nil {
|
||||
return err
|
||||
}
|
||||
if postID != 0 && !failed {
|
||||
if _, err := tx.Exec(`UPDATE posts SET failed=failed+1 WHERE id=?`, postID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// PurgeFailedOutbox forgets failed deliveries older than a week, and the stored messages nobody
|
||||
|
|
|
|||
78
internal/store/posts.go
Normal file
78
internal/store/posts.go
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"net/mail"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Post is a post queued to a list, with the outcome of its deliveries so far.
|
||||
type Post struct {
|
||||
ID int64
|
||||
ListID int64
|
||||
MessageID string // the Message-ID header, at most 200 characters
|
||||
Subject string // decoded, at most 200 characters
|
||||
CreatedAt time.Time
|
||||
Recipients int // deliveries queued
|
||||
Sent int // accepted by the relay
|
||||
Failed int // given up for good
|
||||
Dropped int // the mailbox does not exist, or discarded by the admin
|
||||
Pending int // still in the queue: Recipients - Sent - Failed - Dropped
|
||||
}
|
||||
|
||||
// postTTL is how long a post and its counters are kept.
|
||||
const postTTL = 400 * 24 * time.Hour
|
||||
|
||||
// Limits of ListPosts.
|
||||
const (
|
||||
defaultPostsLimit = 50
|
||||
maxPostsLimit = 200
|
||||
)
|
||||
|
||||
// recordPostTx stores the posts row of a post queued to a list and returns its id. It forgets
|
||||
// posts older than postTTL on the way, so the table never grows without bound.
|
||||
func recordPostTx(tx *sql.Tx, now time.Time, listID int64, msg []byte, recipients int) (int64, error) {
|
||||
if _, err := tx.Exec(`DELETE FROM posts WHERE created_at<?`, now.Add(-postTTL).Unix()); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
subject, _ := summarize(msg)
|
||||
var messageID string
|
||||
if m, err := mail.ReadMessage(bytes.NewReader(msg)); err == nil {
|
||||
messageID = cleanSummary(m.Header.Get("Message-ID"))
|
||||
}
|
||||
res, err := tx.Exec(`INSERT INTO posts(list_id, message_id, subject, created_at, recipients) VALUES(?,?,?,?,?)`,
|
||||
listID, messageID, subject, now.Unix(), recipients)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.LastInsertId()
|
||||
}
|
||||
|
||||
// ListPosts returns the posts of a list, newest first: up to limit (50 when limit <= 0, at most 200).
|
||||
func (s *Store) ListPosts(listID int64, limit int) ([]Post, error) {
|
||||
if limit <= 0 {
|
||||
limit = defaultPostsLimit
|
||||
}
|
||||
if limit > maxPostsLimit {
|
||||
limit = maxPostsLimit
|
||||
}
|
||||
rows, err := s.db.Query(`SELECT id, list_id, message_id, subject, created_at, recipients, sent, failed, dropped
|
||||
FROM posts WHERE list_id=? ORDER BY created_at DESC, id DESC LIMIT ?`, listID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Post
|
||||
for rows.Next() {
|
||||
var p Post
|
||||
var created int64
|
||||
if err := rows.Scan(&p.ID, &p.ListID, &p.MessageID, &p.Subject, &created, &p.Recipients, &p.Sent, &p.Failed, &p.Dropped); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.CreatedAt = time.Unix(created, 0)
|
||||
p.Pending = p.Recipients - p.Sent - p.Failed - p.Dropped
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
424
internal/store/posts_test.go
Normal file
424
internal/store/posts_test.go
Normal file
|
|
@ -0,0 +1,424 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const postMsg = "Message-ID: <a@x>\r\nSubject: =?utf-8?q?Caff=C3=A8?= news\x07\r\n\r\nbody"
|
||||
|
||||
func onlyPost(t *testing.T, s *Store, listID int64) Post {
|
||||
t.Helper()
|
||||
posts, err := s.ListPosts(listID, 0)
|
||||
if err != nil || len(posts) != 1 {
|
||||
t.Fatalf("posts: %+v %v", posts, err)
|
||||
}
|
||||
return posts[0]
|
||||
}
|
||||
|
||||
func outboxPostIDs(t *testing.T, s *Store) []int64 {
|
||||
t.Helper()
|
||||
rows, err := s.db.Query(`SELECT post_id FROM outbox ORDER BY id`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var ids []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
rows.Scan(&id)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func TestEnqueuePostRecordsThePost(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(3)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := onlyPost(t, s, l.ID)
|
||||
if p.ID == 0 || p.ListID != l.ID || p.MessageID != "<a@x>" || p.Subject != "Caffè news" || !p.CreatedAt.Equal(now) ||
|
||||
p.Recipients != 3 || p.Sent != 0 || p.Failed != 0 || p.Dropped != 0 || p.Pending != 3 {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
for _, id := range outboxPostIDs(t, s) {
|
||||
if id != p.ID {
|
||||
t.Fatalf("every delivery carries the post id: %d != %d", id, p.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueuePostRecordsAPostRowWithNoRecipients(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p := onlyPost(t, s, l.ID); p.Recipients != 0 || p.Pending != 0 {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefusedDuplicateRecordsNoPost(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1))
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1)); !errors.Is(err, ErrDuplicate) {
|
||||
t.Fatalf("want ErrDuplicate, got %v", err)
|
||||
}
|
||||
onlyPost(t, s, l.ID)
|
||||
}
|
||||
|
||||
func TestAFailedQueueWriteRecordsNoPost(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON outbox BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(2)); err == nil {
|
||||
t.Fatal("want the queue error")
|
||||
}
|
||||
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
|
||||
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte(postMsg), items(2)); err == nil {
|
||||
t.Fatal("want the queue error")
|
||||
}
|
||||
if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 {
|
||||
t.Fatalf("posts: %+v", posts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleasingAHeldPostRecordsItOnce(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
|
||||
if err := s.EnqueueFromHeld(hid, "mid:<a@x>", []byte(postMsg), items(2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.EnqueueFromHeld(hid, "mid:<a@x>", []byte(postMsg), items(2)) // a second approval: ErrNotFound
|
||||
p := onlyPost(t, s, l.ID)
|
||||
if p.Recipients != 2 || p.Subject != "Caffè news" {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
for _, id := range outboxPostIDs(t, s) {
|
||||
if id != p.ID {
|
||||
t.Fatalf("delivery post id %d != %d", id, p.ID)
|
||||
}
|
||||
}
|
||||
hid2, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
|
||||
if err := s.EnqueueFromHeld(hid2, "mid:<b@x>", []byte(postMsg), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 2 || posts[0].Recipients != 0 {
|
||||
t.Fatalf("a released post with no recipients has a row too: %+v", posts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOtherMailIsNotAPost(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
if err := s.Enqueue([]byte(postMsg), items(2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 {
|
||||
t.Fatalf("posts: %+v", posts)
|
||||
}
|
||||
for _, id := range outboxPostIDs(t, s) {
|
||||
if id != 0 {
|
||||
t.Fatalf("post id %d", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostSubjectIsCutAndClean(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
msg := "Message-ID: <a\x1b@x>\r\nSubject: " + strings.Repeat("é", 300) + "\r\n\r\nb"
|
||||
s.EnqueuePost(l.ID, "mid:x", []byte(msg), items(1))
|
||||
p := onlyPost(t, s, l.ID)
|
||||
if p.Subject != strings.Repeat("é", 200) || p.MessageID != "<a@x>" {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountersFollowTheDeliveries(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(6))
|
||||
due, _ := s.DueOutbox(10)
|
||||
|
||||
if err := s.RetryOutbox(due[0].ID, now.Add(time.Minute), "down"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p := onlyPost(t, s, l.ID); p.Sent+p.Failed+p.Dropped != 0 || p.Pending != 6 {
|
||||
t.Fatalf("a retry changes nothing: %+v", p)
|
||||
}
|
||||
if err := s.CompleteOutbox(due[1].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.FailOutbox(due[2].ID, "refused"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.DropOutbox(due[3].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.DropOutbox(due[4].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := onlyPost(t, s, l.ID)
|
||||
if p.Sent != 1 || p.Failed != 1 || p.Dropped != 2 || p.Pending != 2 {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
// Discarding a failed delivery: it is already counted as failed.
|
||||
if err := s.DropOutbox(due[2].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.CompleteOutbox(due[2].ID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
if err := s.DropOutbox(due[2].ID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
if p2 := onlyPost(t, s, l.ID); p2 != p {
|
||||
t.Fatalf("discarding a failed delivery changes nothing: %+v", p2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompletingAFailedDeliveryDoesNotCountItAsSent(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1))
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.FailOutbox(due[0].ID, "refused")
|
||||
if err := s.CompleteOutbox(due[0].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p := onlyPost(t, s, l.ID); p.Sent != 0 || p.Failed != 1 || p.Pending != 0 {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDropOutboxRemovesTheDeliveryAndTheBody(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s.Enqueue([]byte("body"), items(1))
|
||||
due, _ := s.DueOutbox(10)
|
||||
if err := s.DropOutbox(due[0].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st, _ := s.OutboxStats(); st.Pending != 0 || st.Failed != 0 {
|
||||
t.Fatalf("stats: %+v", st)
|
||||
}
|
||||
if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("body: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPurgingOldFailuresKeepsTheCounters(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(2))
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.FailOutbox(due[0].ID, "refused")
|
||||
now = now.Add(8 * 24 * time.Hour)
|
||||
if n, err := s.PurgeFailedOutbox(); err != nil || n != 1 {
|
||||
t.Fatalf("purged %d %v", n, err)
|
||||
}
|
||||
if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 1 {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListPostsNewestFirstWithALimit(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
dev, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
ops, _ := s.CreateList("ops", "example.org", "Ops")
|
||||
for i := 0; i < 210; i++ {
|
||||
now = now.Add(time.Second)
|
||||
if err := s.EnqueuePost(dev.ID, "mid:"+string(rune(0x100+i)), []byte(postMsg), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
s.EnqueuePost(ops.ID, "mid:ops", []byte(postMsg), items(1))
|
||||
all, _ := s.ListPosts(dev.ID, 1000)
|
||||
if len(all) != 200 {
|
||||
t.Fatalf("capped at 200: %d", len(all))
|
||||
}
|
||||
for i := 1; i < len(all); i++ {
|
||||
if !all[i-1].CreatedAt.After(all[i].CreatedAt) {
|
||||
t.Fatalf("not newest first at %d", i)
|
||||
}
|
||||
}
|
||||
if def, _ := s.ListPosts(dev.ID, 0); len(def) != 50 || def[0].ID != all[0].ID {
|
||||
t.Fatalf("default 50: %d", len(def))
|
||||
}
|
||||
if neg, _ := s.ListPosts(dev.ID, -1); len(neg) != 50 {
|
||||
t.Fatalf("default 50: %d", len(neg))
|
||||
}
|
||||
if some, _ := s.ListPosts(dev.ID, 7); len(some) != 7 {
|
||||
t.Fatalf("limit 7: %d", len(some))
|
||||
}
|
||||
if over, _ := s.ListPosts(dev.ID, 201); len(over) != 200 {
|
||||
t.Fatalf("limit 201: %d", len(over))
|
||||
}
|
||||
if exact, _ := s.ListPosts(dev.ID, 200); len(exact) != 200 {
|
||||
t.Fatalf("limit 200: %d", len(exact))
|
||||
}
|
||||
if o, _ := s.ListPosts(ops.ID, 0); len(o) != 1 || o[0].Recipients != 1 {
|
||||
t.Fatalf("other list: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListPostsBreaksTiesByID(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.Now = func() time.Time { return time.Unix(1_000_000, 0) }
|
||||
s.EnqueuePost(l.ID, "mid:1", []byte(postMsg), nil)
|
||||
s.EnqueuePost(l.ID, "mid:2", []byte(postMsg), nil)
|
||||
posts, _ := s.ListPosts(l.ID, 0)
|
||||
if len(posts) != 2 || posts[0].ID < posts[1].ID {
|
||||
t.Fatalf("the later post first: %+v", posts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOldPostsArePurgedWhenANewOneIsRecorded(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(100_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
dev, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
ops, _ := s.CreateList("ops", "example.org", "Ops")
|
||||
s.EnqueuePost(dev.ID, "mid:old", []byte(postMsg), nil)
|
||||
now = now.Add(time.Second)
|
||||
s.EnqueuePost(dev.ID, "mid:kept", []byte(postMsg), nil)
|
||||
kept := now
|
||||
now = now.Add(400 * 24 * time.Hour) // the first is now older than 400 days, the second exactly 400 days old
|
||||
hid, _ := s.HoldMessage(ops.ID, "a@x.com", []byte("x"), "r")
|
||||
if err := s.EnqueueFromHeld(hid, "mid:new", []byte(postMsg), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
posts, _ := s.ListPosts(dev.ID, 0)
|
||||
if len(posts) != 1 || !posts[0].CreatedAt.Equal(kept) {
|
||||
t.Fatalf("posts: %+v", posts)
|
||||
}
|
||||
now = now.Add(400 * 24 * time.Hour)
|
||||
s.EnqueuePost(ops.ID, "mid:newer", []byte(postMsg), nil)
|
||||
if posts, _ := s.ListPosts(dev.ID, 0); len(posts) != 0 {
|
||||
t.Fatalf("EnqueuePost purges too: %+v", posts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeletingAListDeletesItsPosts(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), nil)
|
||||
if err := s.DeleteList(l.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var n int
|
||||
if err := s.db.QueryRow(`SELECT COUNT(*) FROM posts`).Scan(&n); err != nil || n != 0 {
|
||||
t.Fatalf("posts left: %d %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationAddsPostAndPriorityColumns(t *testing.T) {
|
||||
path := t.TempDir() + "/old.db"
|
||||
s, _ := Open(path)
|
||||
s.Enqueue([]byte("x"), items(1))
|
||||
for _, q := range []string{`DROP INDEX IF EXISTS outbox_due_prio`, `ALTER TABLE outbox DROP COLUMN post_id`,
|
||||
`ALTER TABLE outbox DROP COLUMN priority`, `DROP TABLE posts`} {
|
||||
if _, err := s.db.Exec(q); err != nil {
|
||||
t.Fatalf("%s: %v", q, err)
|
||||
}
|
||||
}
|
||||
s.Close()
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
due, _ := s2.DueOutbox(10)
|
||||
if len(due) != 1 {
|
||||
t.Fatalf("an existing delivery stays waiting: %+v", due)
|
||||
}
|
||||
if ids := outboxPostIDs(t, s2); len(ids) != 1 || ids[0] != 0 {
|
||||
t.Fatalf("post ids: %v", ids)
|
||||
}
|
||||
l, _ := s2.CreateList("dev", "example.org", "Dev")
|
||||
if err := s2.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
onlyPost(t, s2, l.ID)
|
||||
}
|
||||
|
||||
func TestDueOutboxSendsLowerPriorityNumbersFirst(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
news := items(3)
|
||||
for i := range news {
|
||||
news[i].Priority = 1
|
||||
}
|
||||
s.Enqueue([]byte("newsletter"), news)
|
||||
now = now.Add(time.Minute)
|
||||
s.Enqueue([]byte("discussion"), items(2))
|
||||
due, _ := s.DueOutbox(10)
|
||||
if len(due) != 5 {
|
||||
t.Fatalf("due: %d", len(due))
|
||||
}
|
||||
for i, d := range due {
|
||||
if d.Priority != map[bool]int{true: 0, false: 1}[i < 2] {
|
||||
t.Fatalf("priority 0 first: %+v", due)
|
||||
}
|
||||
}
|
||||
for i := 1; i < len(due); i++ {
|
||||
if due[i].Priority == due[i-1].Priority && due[i].ID < due[i-1].ID {
|
||||
t.Fatalf("FIFO within a priority: %+v", due)
|
||||
}
|
||||
}
|
||||
if got, _ := s.DueOutbox(2); got[0].Priority != 0 || got[1].Priority != 0 {
|
||||
t.Fatalf("the limit takes the first in priority: %+v", got)
|
||||
}
|
||||
// A priority-0 delivery that is not due yet does not jump ahead.
|
||||
s.RetryOutbox(due[0].ID, now.Add(time.Hour), "down")
|
||||
s.RetryOutbox(due[1].ID, now.Add(time.Hour), "down")
|
||||
if got, _ := s.DueOutbox(10); len(got) != 3 || got[0].Priority != 1 {
|
||||
t.Fatalf("due: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithinAPriorityTheLongestWaitingGoesFirst(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
s.Enqueue([]byte("body"), items(3))
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.RetryOutbox(due[2].ID, now.Add(-time.Hour), "e")
|
||||
got, _ := s.DueOutbox(10)
|
||||
if got[0].ID != due[2].ID || got[1].ID != due[0].ID || got[2].ID != due[1].ID {
|
||||
t.Fatalf("order: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailingAFailedDeliveryAgainCountsItOnce(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1))
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.FailOutbox(due[0].ID, "refused")
|
||||
if err := s.FailOutbox(due[0].ID, "refused again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 0 {
|
||||
t.Fatalf("post: %+v", p)
|
||||
}
|
||||
}
|
||||
|
|
@ -214,9 +214,24 @@ CREATE TABLE IF NOT EXISTS outbox (
|
|||
next_at INTEGER NOT NULL,
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL,
|
||||
failed INTEGER NOT NULL DEFAULT 0
|
||||
failed INTEGER NOT NULL DEFAULT 0,
|
||||
post_id INTEGER NOT NULL DEFAULT 0,
|
||||
priority INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id);
|
||||
CREATE INDEX IF NOT EXISTS outbox_due_prio ON outbox(priority, next_at, id);
|
||||
CREATE TABLE IF NOT EXISTS posts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
|
||||
message_id TEXT NOT NULL DEFAULT '',
|
||||
subject TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL,
|
||||
recipients INTEGER NOT NULL DEFAULT 0,
|
||||
sent INTEGER NOT NULL DEFAULT 0,
|
||||
failed INTEGER NOT NULL DEFAULT 0,
|
||||
dropped INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS posts_list ON posts(list_id, created_at, id);
|
||||
CREATE TABLE IF NOT EXISTS delivered (
|
||||
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
|
||||
post_id TEXT NOT NULL,
|
||||
|
|
@ -309,6 +324,12 @@ func migrate(db *sql.DB) error {
|
|||
return err
|
||||
}
|
||||
}
|
||||
if has, _ := hasColumn(db, "outbox", "post_id"); !has {
|
||||
if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN post_id INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE outbox ADD COLUMN priority INTEGER NOT NULL DEFAULT 0;`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if ok, err := hasColumn(db, "held", "list_id"); err != nil {
|
||||
return err
|
||||
|
|
@ -397,23 +418,25 @@ func summarize(data []byte) (subject, from string) {
|
|||
if err != nil {
|
||||
return "", ""
|
||||
}
|
||||
clean := func(s string) string {
|
||||
s = strings.Map(func(r rune) rune {
|
||||
if unicode.IsControl(r) {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, strings.Join(strings.Fields(s), " "))
|
||||
if r := []rune(s); len(r) > 200 {
|
||||
s = string(r[:200])
|
||||
}
|
||||
return s
|
||||
}
|
||||
raw := m.Header.Get("Subject")
|
||||
if dec, err := new(mime.WordDecoder).DecodeHeader(raw); err == nil {
|
||||
raw = dec
|
||||
}
|
||||
return clean(raw), clean(m.Header.Get("From"))
|
||||
return cleanSummary(raw), cleanSummary(m.Header.Get("From"))
|
||||
}
|
||||
|
||||
// cleanSummary makes a header value fit for a listing: one line without control characters, at most 200 characters.
|
||||
func cleanSummary(s string) string {
|
||||
s = strings.Map(func(r rune) rune {
|
||||
if unicode.IsControl(r) {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, strings.Join(strings.Fields(s), " "))
|
||||
if r := []rune(s); len(r) > 200 {
|
||||
s = string(r[:200])
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Close closes the database.
|
||||
|
|
|
|||
Loading…
Reference in a new issue