feat(store,api): archive with full-text search, threading and archive policy

- Add archive table with FTS5 index kept in sync by triggers
- Deduplicate by Message-ID per list so MTA retries do not repeat posts
- Thread messages through In-Reply-To within the same list
- Quote every search word so user input is never FTS syntax
- Add archive_policy (public, private, off) with migration
- Add API to list, search, read and delete archived messages
This commit is contained in:
tgiachi 2026-11-09 15:17:13 +01:00
commit d8275aaa1f
7 changed files with 672 additions and 33 deletions

View file

@ -45,6 +45,9 @@ func New(s *store.Store, token string, rel Releaser) http.Handler {
mux.Handle("GET /api/v1/lists/{id}/members", srv.auth(srv.listMembers))
mux.Handle("POST /api/v1/lists/{id}/members", srv.auth(srv.addMember))
mux.Handle("DELETE /api/v1/lists/{id}/members/{address}", srv.auth(srv.removeMember))
mux.Handle("GET /api/v1/lists/{id}/archive", srv.auth(srv.listArchive))
mux.Handle("GET /api/v1/archive/{id}", srv.auth(srv.getArchived))
mux.Handle("DELETE /api/v1/archive/{id}", srv.auth(srv.deleteArchived))
mux.Handle("GET /api/v1/lists/{id}/held", srv.auth(srv.listHeld))
mux.Handle("POST /api/v1/held/{id}/approve", srv.auth(srv.approveHeld))
mux.Handle("DELETE /api/v1/held/{id}", srv.auth(srv.discardHeld))
@ -72,13 +75,14 @@ type listDTO struct {
Title string `json:"title"`
Address string `json:"address"`
PostPolicy string `json:"post_policy"`
JoinPolicy string `json:"join_policy"`
PostPolicy string `json:"post_policy"`
JoinPolicy string `json:"join_policy"`
ArchivePolicy string `json:"archive_policy"`
}
func toListDTO(l store.List) listDTO {
return listDTO{ID: l.ID, Name: l.Name, Domain: l.Domain, Title: l.Title, Address: l.Address(),
PostPolicy: l.PostPolicy, JoinPolicy: l.JoinPolicy}
PostPolicy: l.PostPolicy, JoinPolicy: l.JoinPolicy, ArchivePolicy: l.ArchivePolicy}
}
type memberDTO struct {
@ -195,11 +199,13 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) {
Title *string `json:"title"`
PostPolicy *string `json:"post_policy"`
JoinPolicy *string `json:"join_policy"`
ArchivePolicy *string `json:"archive_policy"`
}
if !decodeBody(w, r, &in) {
return
}
if in.Title == nil && in.PostPolicy == nil && in.JoinPolicy == nil {
if in.Title == nil && in.PostPolicy == nil && in.JoinPolicy == nil && in.ArchivePolicy == nil {
writeError(w, http.StatusBadRequest, "nothing to update")
return
}
@ -222,17 +228,21 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) {
if in.JoinPolicy != nil {
l.JoinPolicy = *in.JoinPolicy
}
if in.ArchivePolicy != nil {
l.ArchivePolicy = *in.ArchivePolicy
}
// Validate before writing anything.
if (l.PostPolicy != store.PostMembers && l.PostPolicy != store.PostModerated) ||
(l.JoinPolicy != store.JoinOpen && l.JoinPolicy != store.JoinClosed) {
writeError(w, http.StatusBadRequest, "post_policy must be members or moderated; join_policy must be open or closed")
(l.JoinPolicy != store.JoinOpen && l.JoinPolicy != store.JoinClosed) ||
(l.ArchivePolicy != store.ArchivePublic && l.ArchivePolicy != store.ArchivePrivate && l.ArchivePolicy != store.ArchiveOff) {
writeError(w, http.StatusBadRequest, "post_policy must be members or moderated; join_policy open or closed; archive_policy public, private or off")
return
}
if err := s.store.UpdateList(id, l.Title); err != nil {
storeError(w, err)
return
}
if err := s.store.SetPolicies(id, l.PostPolicy, l.JoinPolicy); err != nil {
if err := s.store.SetPolicies(id, l.PostPolicy, l.JoinPolicy, l.ArchivePolicy); err != nil {
storeError(w, err)
return
}
@ -385,3 +395,99 @@ func (s *server) discardHeld(w http.ResponseWriter, r *http.Request) {
}
w.WriteHeader(http.StatusNoContent)
}
// Handlers: archive.
const (
defaultPageSize = 50
maxPageSize = 200
)
type messageDTO struct {
ID int64 `json:"id"`
ThreadID int64 `json:"thread_id"`
MessageID string `json:"message_id"`
InReplyTo string `json:"in_reply_to,omitempty"`
Subject string `json:"subject"`
Sender string `json:"sender"`
Date time.Time `json:"date"`
Size int `json:"size"`
Body string `json:"body,omitempty"`
}
func toMessageDTO(m store.Message) messageDTO {
return messageDTO{ID: m.ID, ThreadID: m.ThreadID, MessageID: m.MessageID, InReplyTo: m.InReplyTo,
Subject: m.Subject, Sender: m.Sender, Date: m.Date, Size: m.Size, Body: m.Body}
}
// intParam reads a non-negative integer query parameter.
func intParam(r *http.Request, name string, def int) (int, bool) {
v := r.URL.Query().Get(name)
if v == "" {
return def, true
}
n, err := strconv.Atoi(v)
return n, err == nil && n >= 0
}
func (s *server) listArchive(w http.ResponseWriter, r *http.Request) {
id, ok := listID(w, r)
if !ok {
return
}
limit, ok1 := intParam(r, "limit", defaultPageSize)
offset, ok2 := intParam(r, "offset", 0)
if !ok1 || !ok2 {
writeError(w, http.StatusBadRequest, "limit and offset must be non-negative integers")
return
}
limit = min(max(limit, 1), maxPageSize)
if _, err := s.store.GetList(id); err != nil {
storeError(w, err)
return
}
var (
ms []store.Message
total int
err error
)
if q := strings.TrimSpace(r.URL.Query().Get("q")); q != "" {
ms, total, err = s.store.SearchMessages(id, q, limit, offset)
} else {
ms, total, err = s.store.ListMessages(id, limit, offset)
}
if err != nil {
storeError(w, err)
return
}
out := make([]messageDTO, 0, len(ms))
for _, m := range ms {
out = append(out, toMessageDTO(m))
}
writeJSON(w, http.StatusOK, map[string]any{"total": total, "messages": out})
}
func (s *server) getArchived(w http.ResponseWriter, r *http.Request) {
id, ok := listID(w, r)
if !ok {
return
}
m, err := s.store.GetMessage(id)
if err != nil {
storeError(w, err)
return
}
writeJSON(w, http.StatusOK, toMessageDTO(m))
}
func (s *server) deleteArchived(w http.ResponseWriter, r *http.Request) {
id, ok := listID(w, r)
if !ok {
return
}
if err := s.store.DeleteMessage(id); err != nil {
storeError(w, err)
return
}
w.WriteHeader(http.StatusNoContent)
}

View file

@ -3,10 +3,12 @@ package api
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"codeberg.org/squidd/parsimony/internal/store"
)
@ -78,13 +80,14 @@ func TestEmptyTokenRejectsEverything(t *testing.T) {
}
type listJSON struct {
ID int64 `json:"id"`
Name string `json:"name"`
Domain string `json:"domain"`
Title string `json:"title"`
Address string `json:"address"`
PostPolicy string `json:"post_policy"`
JoinPolicy string `json:"join_policy"`
ID int64 `json:"id"`
Name string `json:"name"`
Domain string `json:"domain"`
Title string `json:"title"`
Address string `json:"address"`
PostPolicy string `json:"post_policy"`
JoinPolicy string `json:"join_policy"`
ArchivePolicy string `json:"archive_policy"`
}
func createList(t *testing.T, h http.Handler) listJSON {
@ -315,3 +318,103 @@ func TestApproveUnknownHeld(t *testing.T) {
type errReleaser struct{ err error }
func (e errReleaser) Release(context.Context, int64) error { return e.err }
func archiveServer(t *testing.T) http.Handler {
t.Helper()
s, _ := store.Open(":memory:")
t.Cleanup(func() { s.Close() })
l, _ := s.CreateList("dev", "example.org", "Dev")
for i, subj := range []string{"First post", "Second post", "Lunch"} {
s.ArchiveMessage(l.ID, store.Message{
MessageID: fmt.Sprintf("<%d@x>", i), Subject: subj, Sender: "Alice",
Date: time.Unix(1_700_000_000+int64(i), 0), Body: "body of " + subj,
})
}
return New(s, token, nil)
}
type archiveJSON struct {
Total int `json:"total"`
Messages []struct {
ID int64 `json:"id"`
ThreadID int64 `json:"thread_id"`
Subject string `json:"subject"`
Sender string `json:"sender"`
Body string `json:"body"`
} `json:"messages"`
}
func TestArchivePolicyViaAPI(t *testing.T) {
h := newServer(t)
l := createList(t, h)
if l.ArchivePolicy != "private" {
t.Fatalf("default = %q", l.ArchivePolicy)
}
rec := do(t, h, "PATCH", "/api/v1/lists/1", `{"archive_policy":"public"}`, true)
expect(t, rec, 200)
var u listJSON
decode(t, rec, &u)
if u.ArchivePolicy != "public" || u.PostPolicy != "members" {
t.Fatalf("patched: %+v", u)
}
expect(t, do(t, h, "PATCH", "/api/v1/lists/1", `{"archive_policy":"bogus"}`, true), 400)
}
func TestArchiveList(t *testing.T) {
h := archiveServer(t)
rec := do(t, h, "GET", "/api/v1/lists/1/archive", "", true)
expect(t, rec, 200)
var out archiveJSON
decode(t, rec, &out)
if out.Total != 3 || len(out.Messages) != 3 || out.Messages[0].Subject != "Lunch" {
t.Fatalf("archive: %+v", out)
}
if out.Messages[0].Body != "" {
t.Fatal("listings must not include bodies")
}
rec = do(t, h, "GET", "/api/v1/lists/1/archive?limit=2&offset=2", "", true)
decode(t, rec, &out)
if out.Total != 3 || len(out.Messages) != 1 || out.Messages[0].Subject != "First post" {
t.Fatalf("page: %+v", out)
}
rec = do(t, h, "GET", "/api/v1/lists/1/archive?q=post", "", true)
decode(t, rec, &out)
if out.Total != 2 || len(out.Messages) != 2 {
t.Fatalf("search: %+v", out)
}
rec = do(t, h, "GET", "/api/v1/lists/1/archive?q=nothingmatches", "", true)
decode(t, rec, &out)
if out.Total != 0 || out.Messages == nil || len(out.Messages) != 0 {
t.Fatalf("empty search must be an empty array: %+v", out)
}
expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?limit=abc", "", true), 400)
expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?offset=-1", "", true), 400)
expect(t, do(t, h, "GET", "/api/v1/lists/99/archive", "", true), 404)
expect(t, do(t, h, "GET", "/api/v1/lists/1/archive", "", false), 401)
}
func TestArchiveLimitIsCapped(t *testing.T) {
h := archiveServer(t)
expect(t, do(t, h, "GET", "/api/v1/lists/1/archive?limit=100000", "", true), 200)
}
func TestArchiveGetAndDelete(t *testing.T) {
h := archiveServer(t)
rec := do(t, h, "GET", "/api/v1/archive/1", "", true)
expect(t, rec, 200)
var m struct {
Subject string `json:"subject"`
Body string `json:"body"`
}
decode(t, rec, &m)
if m.Subject != "First post" || m.Body != "body of First post" {
t.Fatalf("message: %+v", m)
}
expect(t, do(t, h, "GET", "/api/v1/archive/99", "", true), 404)
expect(t, do(t, h, "DELETE", "/api/v1/archive/1", "", true), 204)
expect(t, do(t, h, "GET", "/api/v1/archive/1", "", true), 404)
expect(t, do(t, h, "DELETE", "/api/v1/archive/1", "", true), 404)
expect(t, do(t, h, "DELETE", "/api/v1/archive/2", "", false), 401)
}

View file

@ -180,7 +180,7 @@ func TestModeratedPostIsAcceptedButNotRelayed(t *testing.T) {
t.Cleanup(func() { s.Close() })
l, _ := s.CreateList("dev", "example.org", "Dev")
s.AddMember(l.ID, "alice@example.com", "Alice")
s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen)
s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
relay := &outbound.Fake{}
srv := NewServer(router.New(s), relay)
ln, _ := net.Listen("tcp", "127.0.0.1:0")

View file

@ -85,7 +85,7 @@ func TestDisabledMembersAreSkipped(t *testing.T) {
func TestModeratedListHoldsPosts(t *testing.T) {
e := newEnv(t)
e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen)
e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
for _, sender := range []string{"alice@example.com", "stranger@x.com"} {
_, err := e.r.Route(sender, "dev@example.org", []byte(sample))
if !errors.Is(err, ErrHeld) {
@ -103,7 +103,7 @@ func TestModeratedListHoldsPosts(t *testing.T) {
func TestReleaseHeld(t *testing.T) {
e := newEnv(t)
e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen)
e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
e.r.Route("stranger@x.com", "dev@example.org", []byte(sample))
held, _ := e.s.ListHeld(e.list.ID)
@ -182,7 +182,7 @@ func TestJoinRateLimitedAndSilentForMembers(t *testing.T) {
func TestJoinClosed(t *testing.T) {
e := newEnv(t)
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed)
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate)
_, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
if !errors.Is(err, ErrJoinClosed) {
t.Fatalf("want ErrJoinClosed, got %v", err)
@ -196,7 +196,7 @@ func TestConfirmAfterListClosed(t *testing.T) {
e := newEnv(t)
e.r.Command(e.ctx, e.out, "carol@x.com", "dev+join@example.org", nil)
tok := token(t, lastMail(t, e.out))
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed)
e.s.SetPolicies(e.list.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate)
_, err := e.r.Command(e.ctx, e.out, "carol@x.com", "dev+confirm-"+tok+"@example.org", nil)
if !errors.Is(err, ErrJoinClosed) {
t.Fatalf("want ErrJoinClosed, got %v", err)

View file

@ -0,0 +1,229 @@
package store
import (
"errors"
"strings"
"testing"
"time"
)
func msg(id, subject, body string) Message {
return Message{MessageID: id, Subject: subject, Sender: "Alice", Date: time.Unix(1_700_000_000, 0), Body: body}
}
func TestArchivePolicyDefaultsAndUpdate(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
if l.ArchivePolicy != ArchivePrivate {
t.Fatalf("default = %q", l.ArchivePolicy)
}
if err := s.SetPolicies(l.ID, PostMembers, JoinOpen, ArchivePublic); err != nil {
t.Fatal(err)
}
got, _ := s.GetList(l.ID)
if got.ArchivePolicy != ArchivePublic {
t.Fatalf("policy = %q", got.ArchivePolicy)
}
if err := s.SetPolicies(l.ID, PostMembers, JoinOpen, "bogus"); err == nil {
t.Fatal("invalid archive policy must be rejected")
}
}
func TestArchiveAndFetch(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
m := msg("<1@x>", "Hello", "first body")
id, created, err := s.ArchiveMessage(l.ID, m)
if err != nil || !created || id == 0 {
t.Fatalf("archive: %d %v %v", id, created, err)
}
got, err := s.GetMessage(id)
if err != nil {
t.Fatal(err)
}
if got.Subject != "Hello" || got.Body != "first body" || got.ListID != l.ID || got.ThreadID != id || got.Size != len("first body") {
t.Fatalf("got %+v", got)
}
if !got.Date.Equal(m.Date) {
t.Fatalf("date = %v", got.Date)
}
if _, err := s.GetMessage(999); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestArchiveDeduplicatesByMessageID(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
id1, _, _ := s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "a"))
id2, created, err := s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "a"))
if err != nil || created || id2 != id1 {
t.Fatalf("retry must not duplicate: %d %v %v", id2, created, err)
}
other, _ := s.CreateList("ops", "example.org", "Ops")
if _, created, _ := s.ArchiveMessage(other.ID, msg("<1@x>", "Hello", "a")); !created {
t.Fatal("same Message-ID in another list is a different message")
}
}
func TestThreading(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
root, _, _ := s.ArchiveMessage(l.ID, msg("<root@x>", "Topic", "r"))
reply := msg("<r1@x>", "Re: Topic", "r1")
reply.InReplyTo = "<root@x>"
r1, _, _ := s.ArchiveMessage(l.ID, reply)
nested := msg("<r2@x>", "Re: Topic", "r2")
nested.InReplyTo = "<r1@x>"
r2, _, _ := s.ArchiveMessage(l.ID, nested)
orphan := msg("<o@x>", "Other", "o")
orphan.InReplyTo = "<missing@x>"
o, _, _ := s.ArchiveMessage(l.ID, orphan)
for _, id := range []int64{r1, r2} {
m, _ := s.GetMessage(id)
if m.ThreadID != root {
t.Fatalf("message %d thread = %d, want %d", id, m.ThreadID, root)
}
}
if m, _ := s.GetMessage(o); m.ThreadID != o {
t.Fatalf("orphan must start its own thread, got %d", m.ThreadID)
}
th, err := s.ThreadMessages(l.ID, root)
if err != nil || len(th) != 3 {
t.Fatalf("thread: %d %v", len(th), err)
}
}
func TestThreadingIgnoresOtherLists(t *testing.T) {
s := newTestStore(t)
a, _ := s.CreateList("a", "example.org", "A")
b, _ := s.CreateList("b", "example.org", "B")
s.ArchiveMessage(a.ID, msg("<root@x>", "Topic", "r"))
reply := msg("<r@x>", "Re: Topic", "r")
reply.InReplyTo = "<root@x>"
id, _, _ := s.ArchiveMessage(b.ID, reply)
if m, _ := s.GetMessage(id); m.ThreadID != id {
t.Fatal("a parent in another list must not be used")
}
}
func TestListMessagesNewestFirstWithPaging(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
for i := 0; i < 5; i++ {
m := msg("<m"+string(rune('a'+i))+"@x>", "S"+string(rune('a'+i)), "body")
m.Date = time.Unix(1_700_000_000+int64(i), 0)
s.ArchiveMessage(l.ID, m)
}
got, total, err := s.ListMessages(l.ID, 2, 0)
if err != nil || total != 5 || len(got) != 2 {
t.Fatalf("page 1: %d %d %v", len(got), total, err)
}
if got[0].Subject != "Se" || got[1].Subject != "Sd" {
t.Fatalf("order: %s %s", got[0].Subject, got[1].Subject)
}
if got[0].Body != "" {
t.Fatal("listings must not load bodies")
}
got, _, _ = s.ListMessages(l.ID, 2, 4)
if len(got) != 1 || got[0].Subject != "Sa" {
t.Fatalf("last page: %+v", got)
}
}
func TestSearch(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
other, _ := s.CreateList("ops", "example.org", "Ops")
s.ArchiveMessage(l.ID, msg("<1@x>", "Deploy failed", "the kubernetes rollout broke"))
s.ArchiveMessage(l.ID, msg("<2@x>", "Lunch", "pizza on friday"))
s.ArchiveMessage(other.ID, msg("<3@x>", "Deploy ok", "kubernetes is fine here"))
got, total, err := s.SearchMessages(l.ID, "kubernetes", 10, 0)
if err != nil || total != 1 || len(got) != 1 || got[0].Subject != "Deploy failed" {
t.Fatalf("search: %+v %d %v", got, total, err)
}
// Subject, sender and multiple terms (AND).
if got, _, _ := s.SearchMessages(l.ID, "lunch pizza", 10, 0); len(got) != 1 {
t.Fatalf("subject+body: %+v", got)
}
if got, _, _ := s.SearchMessages(l.ID, "alice", 10, 0); len(got) != 2 {
t.Fatalf("sender: %+v", got)
}
if got, _, _ := s.SearchMessages(l.ID, "lunch kubernetes", 10, 0); len(got) != 0 {
t.Fatalf("AND semantics: %+v", got)
}
}
func TestSearchIsSafeAgainstSyntax(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "world"))
for _, q := range []string{`"`, `foo AND`, `NEAR(`, `*`, `a OR b`, `col:x`, `)(`, "", " "} {
if _, _, err := s.SearchMessages(l.ID, q, 10, 0); err != nil {
t.Errorf("query %q must not error: %v", q, err)
}
}
// FTS operators are plain words: "world OR zzz" needs both terms.
if got, _, _ := s.SearchMessages(l.ID, "world OR zzz", 10, 0); len(got) != 0 {
t.Fatalf("operators must be literal: %+v", got)
}
}
func TestDeleteMessageRemovesFromSearch(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
id, _, _ := s.ArchiveMessage(l.ID, msg("<1@x>", "Secret", "leaked password"))
if err := s.DeleteMessage(id); err != nil {
t.Fatal(err)
}
if got, _, _ := s.SearchMessages(l.ID, "password", 10, 0); len(got) != 0 {
t.Fatalf("deleted message still searchable: %+v", got)
}
if err := s.DeleteMessage(id); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestDeleteListCleansArchiveAndIndex(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.ArchiveMessage(l.ID, msg("<1@x>", "Hello", "uniqueword"))
if err := s.DeleteList(l.ID); err != nil {
t.Fatal(err)
}
var n int
s.db.QueryRow(`SELECT COUNT(*) FROM archive_fts WHERE archive_fts MATCH 'uniqueword'`).Scan(&n)
if n != 0 {
t.Fatal("search index must follow cascade deletes")
}
}
func TestMigrationAddsArchiveColumn(t *testing.T) {
// A database from milestone 2 has no archive_policy column.
path := t.TempDir() + "/m2.db"
s, _ := Open(path)
s.CreateList("dev", "example.org", "Dev")
s.db.Exec(`ALTER TABLE lists DROP COLUMN archive_policy`)
s.Close()
s2, err := Open(path)
if err != nil {
t.Fatalf("migrate: %v", err)
}
defer s2.Close()
l, err := s2.FindList("dev@example.org")
if err != nil || l.ArchivePolicy != ArchivePrivate {
t.Fatalf("list: %+v %v", l, err)
}
}
func TestLongBodyIsStoredAsIs(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
body := strings.Repeat("x", 100_000)
id, _, _ := s.ArchiveMessage(l.ID, msg("<big@x>", "Big", body))
if got, _ := s.GetMessage(id); len(got.Body) != len(body) {
t.Fatalf("body len %d", len(got.Body))
}
}

View file

@ -14,17 +14,17 @@ func TestListPolicies(t *testing.T) {
if l.PostPolicy != "members" || l.JoinPolicy != "open" {
t.Fatalf("defaults: %+v", l)
}
if err := s.SetPolicies(l.ID, "moderated", "closed"); err != nil {
if err := s.SetPolicies(l.ID, "moderated", "closed", "private"); err != nil {
t.Fatal(err)
}
got, _ := s.GetList(l.ID)
if got.PostPolicy != "moderated" || got.JoinPolicy != "closed" {
t.Fatalf("policies: %+v", got)
}
if err := s.SetPolicies(l.ID, "bogus", "open"); err == nil {
if err := s.SetPolicies(l.ID, "bogus", "open", "private"); err == nil {
t.Fatal("invalid policy must be rejected")
}
if err := s.SetPolicies(999, "members", "open"); !errors.Is(err, ErrNotFound) {
if err := s.SetPolicies(999, "members", "open", "private"); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}

View file

@ -28,6 +28,13 @@ const (
JoinClosed = "closed" // only the admin adds members
)
// Archive policy values.
const (
ArchivePublic = "public" // browsable on the web
ArchivePrivate = "private" // stored, readable through the API only
ArchiveOff = "off" // not stored
)
// Pending actions.
const (
ActionJoin = "join"
@ -41,12 +48,13 @@ const (
// List is a mailing list.
type List struct {
ID int64
Name string // local part, lowercase
Domain string // lowercase
Title string
PostPolicy string
JoinPolicy string
ID int64
Name string // local part, lowercase
Domain string // lowercase
Title string
PostPolicy string
JoinPolicy string
ArchivePolicy string
}
// Address returns the list posting address.
@ -94,6 +102,7 @@ CREATE TABLE IF NOT EXISTS lists (
title TEXT NOT NULL DEFAULT '',
post_policy TEXT NOT NULL DEFAULT 'members',
join_policy TEXT NOT NULL DEFAULT 'open',
archive_policy TEXT NOT NULL DEFAULT 'private',
UNIQUE(name, domain)
);
CREATE TABLE IF NOT EXISTS members (
@ -119,6 +128,30 @@ CREATE TABLE IF NOT EXISTS pending (
action TEXT NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS archive (
id INTEGER PRIMARY KEY AUTOINCREMENT,
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
thread_id INTEGER NOT NULL DEFAULT 0,
message_id TEXT NOT NULL,
in_reply_to TEXT NOT NULL DEFAULT '',
subject TEXT NOT NULL DEFAULT '',
sender TEXT NOT NULL DEFAULT '',
date INTEGER NOT NULL,
body TEXT NOT NULL DEFAULT '',
size INTEGER NOT NULL DEFAULT 0,
UNIQUE(list_id, message_id)
);
CREATE INDEX IF NOT EXISTS archive_list_date ON archive(list_id, date DESC, id DESC);
CREATE INDEX IF NOT EXISTS archive_thread ON archive(list_id, thread_id);
CREATE VIRTUAL TABLE IF NOT EXISTS archive_fts USING fts5(
subject, body, sender, content='archive', content_rowid='id'
);
CREATE TRIGGER IF NOT EXISTS archive_ai AFTER INSERT ON archive BEGIN
INSERT INTO archive_fts(rowid, subject, body, sender) VALUES (new.id, new.subject, new.body, new.sender);
END;
CREATE TRIGGER IF NOT EXISTS archive_ad AFTER DELETE ON archive BEGIN
INSERT INTO archive_fts(archive_fts, rowid, subject, body, sender) VALUES ('delete', old.id, old.subject, old.body, old.sender);
END;
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
@ -180,6 +213,15 @@ func migrate(db *sql.DB) error {
}
}
}
if ok, err := hasColumn(db, "lists", "name"); err != nil {
return err
} else if ok {
if has, _ := hasColumn(db, "lists", "archive_policy"); !has {
if _, err := db.Exec(`ALTER TABLE lists ADD COLUMN archive_policy TEXT NOT NULL DEFAULT 'private'`); err != nil {
return err
}
}
}
_, err := db.Exec(schema)
return err
}
@ -203,11 +245,11 @@ func affected(res sql.Result, err error) error {
// Lists.
const listCols = `id, name, domain, title, post_policy, join_policy`
const listCols = `id, name, domain, title, post_policy, join_policy, archive_policy`
func scanList(row interface{ Scan(...any) error }) (List, error) {
var l List
err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy)
err := row.Scan(&l.ID, &l.Name, &l.Domain, &l.Title, &l.PostPolicy, &l.JoinPolicy, &l.ArchivePolicy)
if errors.Is(err, sql.ErrNoRows) {
return List{}, ErrNotFound
}
@ -265,15 +307,18 @@ func (s *Store) UpdateList(id int64, title string) error {
return affected(s.db.Exec(`UPDATE lists SET title=? WHERE id=?`, title, id))
}
// SetPolicies changes the post and join policies of a list.
func (s *Store) SetPolicies(id int64, post, join string) error {
// SetPolicies changes the post, join and archive policies of a list.
func (s *Store) SetPolicies(id int64, post, join, archive string) error {
if post != PostMembers && post != PostModerated {
return fmt.Errorf("invalid post policy %q", post)
}
if join != JoinOpen && join != JoinClosed {
return fmt.Errorf("invalid join policy %q", join)
}
return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=? WHERE id=?`, post, join, id))
if archive != ArchivePublic && archive != ArchivePrivate && archive != ArchiveOff {
return fmt.Errorf("invalid archive policy %q", archive)
}
return affected(s.db.Exec(`UPDATE lists SET post_policy=?, join_policy=?, archive_policy=? WHERE id=?`, post, join, archive, id))
}
// DeleteList removes a list with its members, held messages and pending requests.
@ -485,3 +530,159 @@ func (s *Store) Secret() ([]byte, error) {
}
return hex.DecodeString(v)
}
// Archive.
// Message is an archived post.
type Message struct {
ID int64
ListID int64
ThreadID int64
MessageID string // Message-ID header, unique per list
InReplyTo string
Subject string
Sender string // display name of the original author
Date time.Time
Body string // plain text; empty in listings
Size int
}
const msgCols = `id, list_id, thread_id, message_id, in_reply_to, subject, sender, date, size`
func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, error) {
var m Message
var ts int64
var err error
if withBody {
err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size, &m.Body)
} else {
err = row.Scan(&m.ID, &m.ListID, &m.ThreadID, &m.MessageID, &m.InReplyTo, &m.Subject, &m.Sender, &ts, &m.Size)
}
if errors.Is(err, sql.ErrNoRows) {
return Message{}, ErrNotFound
}
m.Date = time.Unix(ts, 0)
return m, err
}
// ArchiveMessage stores a post. A message already archived in the same list
// (same Message-ID) is not stored again: created is false and the existing id
// is returned, so MTA retries are harmless.
func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) {
if m.MessageID == "" {
return 0, false, errors.New("message id is required")
}
err = s.db.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
if err == nil {
return id, false, nil
}
if !errors.Is(err, sql.ErrNoRows) {
return 0, false, err
}
var thread int64
if m.InReplyTo != "" {
e := s.db.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
if e != nil && !errors.Is(e, sql.ErrNoRows) {
return 0, false, e
}
}
res, err := s.db.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
VALUES(?,?,?,?,?,?,?,?,?)`,
listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body))
if err != nil {
return 0, false, err
}
id, _ = res.LastInsertId()
if thread == 0 {
if _, err := s.db.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
return 0, false, err
}
}
return id, true, nil
}
// GetMessage returns an archived message with its body.
func (s *Store) GetMessage(id int64) (Message, error) {
return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true)
}
// DeleteMessage removes an archived message and its search entry.
func (s *Store) DeleteMessage(id int64) error {
return affected(s.db.Exec(`DELETE FROM archive WHERE id=?`, id))
}
func (s *Store) queryMessages(q string, args ...any) ([]Message, error) {
rows, err := s.db.Query(q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Message
for rows.Next() {
m, err := scanMessage(rows, false)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}
// ListMessages returns a page of a list's archive, newest first, and the total
// count. Bodies are not loaded.
func (s *Store) ListMessages(listID int64, limit, offset int) ([]Message, int, error) {
var total int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive WHERE list_id=?`, listID).Scan(&total); err != nil {
return nil, 0, err
}
out, err := s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? ORDER BY date DESC, id DESC LIMIT ? OFFSET ?`,
listID, limit, offset)
return out, total, err
}
// ftsQuery turns free text into an FTS5 query where every word is a quoted
// literal (AND semantics), so user input can never be read as FTS syntax.
func ftsQuery(q string) string {
words := strings.Fields(q)
for i, w := range words {
words[i] = `"` + strings.ReplaceAll(w, `"`, `""`) + `"`
}
return strings.Join(words, " ")
}
// SearchMessages runs a full-text search over subject, body and sender of one
// list, newest first. Bodies are not loaded.
func (s *Store) SearchMessages(listID int64, query string, limit, offset int) ([]Message, int, error) {
match := ftsQuery(query)
if match == "" {
return nil, 0, nil
}
var total int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid
WHERE archive_fts MATCH ? AND a.list_id=?`, match, listID).Scan(&total); err != nil {
return nil, 0, err
}
out, err := s.queryMessages(`SELECT a.id, a.list_id, a.thread_id, a.message_id, a.in_reply_to, a.subject, a.sender, a.date, a.size
FROM archive_fts JOIN archive a ON a.id=archive_fts.rowid
WHERE archive_fts MATCH ? AND a.list_id=? ORDER BY a.date DESC, a.id DESC LIMIT ? OFFSET ?`,
match, listID, limit, offset)
return out, total, err
}
// ThreadMessages returns every message of a thread, oldest first, with bodies.
func (s *Store) ThreadMessages(listID, threadID int64) ([]Message, error) {
rows, err := s.db.Query(`SELECT `+msgCols+`, body FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Message
for rows.Next() {
m, err := scanMessage(rows, true)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}